diff --git a/.github/workflows/CI-cluster-simulator.yml b/.github/workflows/CI-cluster-simulator.yml new file mode 100644 index 0000000000..ff4235c510 --- /dev/null +++ b/.github/workflows/CI-cluster-simulator.yml @@ -0,0 +1,129 @@ +# Builds ProxySQL once with every cluster simulation flag enabled, then runs +# each registered cluster_sim_* TAP group as an independent matrix job. +# +# Maintenance notes: +# - Groups and TAP binaries are discovered from test/tap/groups/groups.json. +# - `testall` is intentional: every matrix job shares one ProxySQL binary built +# with all simulation flags. +# - Registering a new simulation group and TAP binary requires no YAML changes. +# - The exact-SHA cache contains only the runtime files used by matrix jobs. +# - Command details and local examples: test/infra/control/cluster-simulator-ci.bash help. + +name: CI-cluster-simulator +run-name: '${{ github.head_ref || github.ref_name }} ${{ github.workflow }} ${{ github.sha }}' + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }} + cancel-in-progress: true + +env: + BUILD_CACHE_KEY: cluster-simulator-v3-ubuntu22-${{ github.sha }} + RUNTIME_CACHE_DIR: .cluster-simulator-runtime + +jobs: + build: + name: build + runs-on: ubuntu-22.04 + outputs: + groups: ${{ steps.simulator-groups.outputs.groups }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Discover simulation groups + id: simulator-groups + run: test/infra/control/cluster-simulator-ci.bash discover + + - name: Restore simulation build + id: simulator-build + uses: actions/cache/restore@v4 + with: + key: ${{ env.BUILD_CACHE_KEY }} + path: ${{ env.RUNTIME_CACHE_DIR }} + + - name: Install cached simulation runtime + if: steps.simulator-build.outputs.cache-hit == 'true' + run: test/infra/control/cluster-simulator-ci.bash install + + - name: Build simulation test runtime + if: steps.simulator-build.outputs.cache-hit != 'true' + run: test/infra/control/cluster-simulator-ci.bash build + + - name: Verify simulation build + run: test/infra/control/cluster-simulator-ci.bash verify + + - name: Stage simulation runtime + if: steps.simulator-build.outputs.cache-hit != 'true' + run: test/infra/control/cluster-simulator-ci.bash stage + + - name: Save simulation build + if: steps.simulator-build.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + key: ${{ env.BUILD_CACHE_KEY }} + path: ${{ env.RUNTIME_CACHE_DIR }} + + test: + name: test / ${{ matrix.group }} + needs: build + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + group: ${{ fromJSON(needs.build.outputs.groups) }} + env: + INFRA_ID: ${{ matrix.group }}-${{ github.run_id }}-${{ github.run_attempt }} + TAP_GROUP: ${{ matrix.group }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Restore simulation build + uses: actions/cache/restore@v4 + with: + key: ${{ env.BUILD_CACHE_KEY }} + fail-on-cache-miss: true + path: ${{ env.RUNTIME_CACHE_DIR }} + + - name: Install simulation runtime + run: test/infra/control/cluster-simulator-ci.bash install + + - name: Verify simulation build + run: test/infra/control/cluster-simulator-ci.bash verify "${TAP_GROUP}" + + - name: Build CI base image + run: docker build --network host -t proxysql-ci-base:latest test/infra/docker-base + + - name: Start infrastructure + run: test/infra/control/ensure-infras.bash + + - name: Run simulation tests + run: test/infra/control/run-tests-isolated.bash + + - name: Cleanup + if: always() + run: | + test/infra/control/stop-proxysql-isolated.bash || true + test/infra/control/destroy-infras.bash || true + + - name: Archive failure logs + if: ${{ failure() && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.group }}-${{ github.sha }}-logs-run${{ github.run_number }} + path: ci_infra_logs/ diff --git a/.gitignore b/.gitignore index 0177cb872b..e27f4e6798 100644 --- a/.gitignore +++ b/.gitignore @@ -198,6 +198,8 @@ pkgroot/ #files generated during CI run proxysql-save.cfg +.cluster-simulator-binaries +.cluster-simulator-runtime/ test/tap/tests/test_cluster_sync_config/cluster_sync_node_stderr.txt test/tap/tests/test_cluster_sync_config/proxysql*.pem test/tap/tests/test_cluster_sync_config/test_cluster_sync.cnf diff --git a/Makefile b/Makefile index b7aa0c4d92..dcd4997c01 100644 --- a/Makefile +++ b/Makefile @@ -220,6 +220,10 @@ testreadonly: build_src_testreadonly build_cluster_simulator .PHONY: testreplicationlag testreplicationlag: build_src_testreplicationlag build_cluster_simulator +.PHONY: test_rds_bgd +test_rds_bgd: build_src_test_rds_bgd + cd test/tap && OPTZ="${O0} -ggdb -DDEBUG" CC=${CC} CXX=${CXX} ${MAKE} debug + .PHONY: testall testall: build_src_testall @@ -337,13 +341,21 @@ build_src_testreplicationlag: build_lib_testreplicationlag build_lib_testreplicationlag: build_deps_debug cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE} +.PHONY: build_src_test_rds_bgd +build_src_test_rds_bgd: build_lib_test_rds_bgd + cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_RDS_BGD" CC=${CC} CXX=${CXX} ${MAKE} + +.PHONY: build_lib_test_rds_bgd +build_lib_test_rds_bgd: build_deps_debug + cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_RDS_BGD" CC=${CC} CXX=${CXX} ${MAKE} + .PHONY: build_src_testall build_src_testall: build_lib_testall - cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE} + cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG -DTEST_RDS_BGD" CC=${CC} CXX=${CXX} ${MAKE} .PHONY: build_lib_testall build_lib_testall: build_deps_debug - cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE} + cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG -DTEST_RDS_BGD" CC=${CC} CXX=${CXX} ${MAKE} .PHONY: build_tap_test build_tap_test: build_tap_tests diff --git a/bgd-test-style.md b/bgd-test-style.md new file mode 100644 index 0000000000..3c34b2e211 --- /dev/null +++ b/bgd-test-style.md @@ -0,0 +1,157 @@ +# AWS RDS BGD TAP Style Rules + +These rules apply to every new or refactored AWS RDS BGD TAP file. + +## Test Scope And Documentation + +- Use one TAP executable per independently reportable behavior. +- Keep each TAP executable focused on the behavior named by the file. Do not + add connection-pool, TLS, server-status, or other property mutations unless + that behavior requires them. +- Move independently useful coverage into a dedicated TAP executable instead + of carrying it through an unrelated scenario. +- File and test headers must name the exact BGD configuration change, topology + state, and expected ProxySQL result. +- Use names from the BGD implementation and Admin tables. Avoid generic wording + such as "surface", "departure", or "input replacement". +- Document multi-step tests with setup, mutation, and verification bullets. +- Test public configuration and observable routing, runtime, connection-pool, + and probe behavior. Do not test or document non-public internal server states. + +## Local Test Harness + +- Every BGD TAP file defines local `setup()` and `cleanup()` functions. +- `setup()` initializes only the test harness: load the TAP environment, + connect to ProxySQL Admin, and connect to the SQLite simulator. +- `setup()` must not configure BGD topology, Admin rows, simulated writer + state, or other scenario prerequisites. +- `setup()` must not contain TAP assertions. +- `setup()` releases any partially created connection before returning a + failure. `main()` then returns `exit_status()` without calling `cleanup()`. +- `cleanup()` clears the ProxySQL Admin/runtime and simulator state created by + the test, then closes test connections. +- `cleanup()` is called only after `setup()` succeeds and may assume its + connections are valid. +- `cleanup()` attempts every cleanup operation, closes test connections, and + returns `EXIT_FAILURE` if any Admin or simulator cleanup operation fails. +- A cleanup failure must fail the TAP executable so the developer and CI can + see that the test did not leave a clean state. +- Store the Admin and simulator cleanup results separately. Log each failure + with an `Error:` diagnostic, continue with the remaining cleanup operations, + and return one combined result. +- Close the Admin connection immediately after its cleanup operation. Do not + skip connection closure because cleanup failed. +- Do not wait for probe quiescence during cleanup. +- After harness setup succeeds, failures must flow to a single + `exit_cleanup:` label in `main()`. That label always calls the local + `cleanup()` function. +- Declare the TAP plan before calling `setup()`. After `exit_cleanup:`, return + `EXIT_FAILURE` when cleanup fails; otherwise return `exit_status()`. A setup + failure is then reported as missing planned assertions instead of bypassing + TAP result handling. +- At `exit_cleanup:`, check `cleanup()` directly without a temporary result. + Return `EXIT_FAILURE` immediately when cleanup fails. +- Do not use `BAIL_OUT()` after test resources have been created or test state + has been changed. Log the failure, set the process result, and continue to + `exit_cleanup:`. +- Do not register global pointers or `atexit` handlers for test cleanup. +- A test assumes clean ProxySQL and simulator state at entry. Do not clear + state at the start of `setup()` or the test function. +- Scenario topology, ProxySQL configuration, mutations, waits, and TAP + assertions belong in the named test function. +- Keep `main()` limited to `plan()`, `setup()`, the named test, the + `exit_cleanup:` path, and the TAP exit status. + +## C++ Layout + +- Do not wrap TAP test files in an anonymous namespace. Each TAP file builds as + its own executable, so file-local namespace isolation is unnecessary. +- When phases share local state, name the struct `TestState`. Do not include + the test or file name in the local state type. +- Call the named test phases directly from `main()`. Do not add a wrapper test + function whose only job is to call the phases. +- The comment before each phase call in `main()` must make the phase + understandable without opening the function. State the concrete simulator + or ProxySQL configuration, the BGD status or server placement being + produced, and the observable result being verified. +- Do not write call-site comments that merely restate the function name or use + vague verbs such as "establish", "prepare", "handle", or "process". +- Keep call-site phase comments concise, but use exact BGD statuses, + hostgroups, tables, or server roles where they matter. +- Name simulator state helpers after the exact variable and value being set, + such as `set_writer_read_only_0()`. Avoid interpreted names such as + `set_writers_writable()`. +- Organize phase comments by the system being acted on, using labels such as + `Simulator:`, `ProxySQL:`, `Client:`, and `Verify:`. Include only the + applicable systems, and state the concrete action or expected result for + each one. +- Always use braces for `if` statements, including one-line bodies: + + ```cpp + if (condition) { + action(); + } + ``` + +- Keep complete function declarations, definitions, and calls on one line + whenever they remain readable. Use 120 characters as a guideline, not a + hard limit; prefer a small overrun to splitting a simple call across lines. + Wrap only when the complete statement is materially too long. +- Do not add `const` qualifiers to function parameters in these TAP files. + Prefer shorter, simpler signatures over strict const-correctness in + test-local helpers. +- Build SQL strings, expected-result text, and list arguments in named local + variables before calling a helper. Do not mix string concatenation, + temporary lists, and the function call in one statement. +- Format query helpers in four visible blocks separated by blank lines: + construct the query, construct related arguments, call the helper, and + return the stored result. +- When a helper call must wrap, group related arguments across as few lines as + possible and place the closing `);` on its own line. +- Keep return statements simple. Store a function result in a local variable + and return that variable instead of returning a large function call or a + heavily combined expression. +- Do not create duplicate query helpers that differ only by one expected + value. Pass that expected value as an argument to one clearly named helper. +- Do not place multiple function calls inside one `ok()` condition. Evaluate + each call into a clearly named local variable first, then combine those + boolean variables in the assertion. +- When a helper performs multiple function calls, execute them one at a time. + Return `EXIT_FAILURE` immediately after the specific call that fails, then + return `EXIT_SUCCESS` after all calls succeed. Do not combine calls with + `&&` and a ternary return. +- Apply the same sequential pattern inside test phases. Separate each + call-and-failure-check block with a blank line; do not conditionally invoke + later operations through ternary expressions. +- Before returning `EXIT_FAILURE` from setup, helper, phase, or `main()`, emit + a diagnostic beginning with `Error:` that names the failed operation. + Cleanup may log its individual failures before returning one combined + result. +- Shared condition and probe wait helpers must only return their result. They + must not dump timeout diagnostics; the calling phase logs the relevant + `Error:` message. +- Use `bgd_expect_no_table_check()` and `bgd_expect_no_metadata_probe()` for + negative probe checks instead of duplicating timeout logic in TAP files. + Pass the bounded timeout explicitly; these helpers return success only when + the unwanted probe wait returns `ETIMEDOUT`. +- Pass wait helpers only the values required to perform the wait. Do not pass + scenario names, phase names, expected text, probe sequences, or hostgroup + lists solely for generic diagnostics. +- Do not add `ok(false, ...)` to a failure-return path. Emit the `Error:` + diagnostic and return `EXIT_FAILURE`; the incomplete TAP plan will fail the + executable. +- Assertion messages and phase names must identify concrete BGD statuses, + hostgroups, tables, or server movements. Avoid undefined relational wording + such as "original definition", "current state", or "changed setup". +- In assertion messages, describe a runtime BGD status as + `BGD status for wHG ` instead of the longer + `runtime BGD row for writer hostgroup `. +- Separate environment loading, Admin connection, and simulator connection + into distinct blocks with blank lines. +- Separate setup, test execution, and cleanup calls in `main()` with blank + lines. +- Use `RDS_BGD_Cluster::get_endpoints()`, `get_blue_endpoints()`, and + `get_green_endpoints()` instead of rebuilding endpoint lists with reader + loops in individual tests. +- Keep positive condition and probe waits at three seconds or less. Use bounded + waits instead of fixed sleeps. diff --git a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md new file mode 100644 index 0000000000..0ac0555823 --- /dev/null +++ b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md @@ -0,0 +1,717 @@ +# AWS RDS Blue/Green Monitor + +**Document status:** FEATURE CONTRACT; IMPLEMENTED + +**Applies to:** Amazon RDS Multi-AZ DB instance blue/green deployment +monitoring + +**Primary implementation:** `include/MySQL_Monitor.hpp` and +`lib/MySQL_Monitor.cpp` + +**Simulator specification:** [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md) + +## Purpose And Scope + +This document defines the operational contract for ProxySQL monitoring of +Amazon RDS Multi-AZ DB instance blue/green deployments. It describes the AWS +topology observations consumed by the monitor, the resulting ProxySQL state +transitions and external effects, configuration and worker lifetime, +connection-retirement and cleanup semantics, and the verification surface for +the feature introduced by +[PR #5861](https://github.com/sysown/proxysql/pull/5861). + +The contract distinguishes among behavior defined by AWS, behavior observed in +a bounded deployment trace, intentional ProxySQL policy, and implementation +mechanics. A scoped observation is not promoted to a universal AWS guarantee, +and a ProxySQL policy is not presented as an AWS property. + +The monitor contract covers: + +- Detection of blue/green topology through `mysql.rds_topology`. +- Interpretation of source and target roles and switchover statuses. +- Mapping of configured blue writer and reader servers to green servers. +- Green address resolution and direct topology probing. +- Writer and reader switchover handling. +- DNS pinning, hostgroup placement, reader shunning, and connection draining. +- Successful completion, cancellation rollback, and topology disappearance. +- Explicit and automatic green-hostgroup configuration. +- In-place configuration refresh, worker detach and recreation, and process + restart. +- The simulator, TAP, unit-test, and CI surface used to verify the contract. +- Documented configuration limitations and administrator responsibilities. + +This document does not define: + +- Amazon Aurora, Group Replication, Galera, or PostgreSQL monitoring. +- RDS behavior that is not consumed by this monitor. +- A durable effect ledger or a replacement controller architecture. +- Persistence of in-progress BGD state across a ProxySQL process restart. +- Detailed simulator implementation, which is specified in + [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md). + +## Contract Basis + +The external behavior in this contract is based on: + +- An + [AWS-provided RDS topology metadata document](https://github.com/user-attachments/files/30019110/RDS_Topology_metadata.md) + describing `mysql.rds_topology`, roles, statuses, switchover stages, traffic + availability, and polling guidance. +- A + [timestamped topology trace](https://github.com/user-attachments/files/30019175/aws-rds-topology-watch.txt) + from one complete switchover, sampled at approximately 250 ms. +- A separately observed cancellation and operational observations supplied by + the feature author. +- The source implementation and tests referenced in this document. + +The captured deployment used RDS MySQL 8.4.x, a Multi-AZ DB instance with two +read replicas, and `eu-north-1`. Timing and row-lifecycle statements derived +only from this trace apply to that observation. They do not establish fixed +timing or universal AWS behavior. + +The following language identifies the authority for a statement: + +| Wording | Meaning | +|---|---| +| **AWS defines** | The AWS-provided metadata document specifies the behavior. | +| **The supplied trace observed** | The behavior occurred in the bounded trace described above. | +| **The author observed** | The feature author supplied an operational observation outside that trace. | +| **ProxySQL policy** | The behavior is an intentional product decision, including a decision made where AWS does not provide a stronger guarantee. | +| **The implementation** | The statement describes the source behavior on the feature branch. | + +## Terminology + +| Term | Meaning | +|---|---| +| Blue | The source deployment before switchover. | +| Green | The target deployment before switchover. | +| Topology observation | The result of one `mysql.rds_topology` existence check or metadata query. | +| Blue/green pair | A configured blue server and its name-matched green counterpart, together with the connection attributes needed by the monitor. | +| Direct probe | A topology query sent to the resolved green writer IP rather than to a configured blue hostname. | +| Topology drain | A successful metadata query returning no rows, or the topology table becoming unavailable. | +| Rollback cleanup | One-shot cleanup invoked for topology drain outside the reader phase, a recognized backward transition, configuration refresh at or after post-processing, or worker exit from any active phase. | +| Successful cleanup | One-shot cleanup selected when topology drains after the monitor has observed writer completion. | +| Configuration refresh | An in-place update of a running worker after its deployment checksum changes. | +| Worker detach | Termination of a worker because the deployment is disabled, removed, or no longer has an eligible blue writer. | + +## Configuration Model + +Each active deployment identifies a blue writer hostgroup and a blue reader +hostgroup. It also carries `writer_is_also_reader`, a baseline check interval, +and a check timeout. One worker owns the monitor state for one blue writer +hostgroup. + +Green hostgroup nullability depends on the origin of the row. It is not a +user-selectable mixed configuration: + +| Row origin and storage | Green writer hostgroup | Green reader hostgroup | Semantics | +|---|---|---|---| +| User row in persistent Admin configuration | Required | Required | Explicit green-hostgroup mode. The persistent table declares both columns `NOT NULL`. | +| User row materialized into runtime/HGM | Value | Value | The configured values are retained with `auto_generated=0`. | +| Runtime row created by automatic discovery | `NULL` | `NULL` | Automatic mode. The row carries `auto_generated=1` and exists only in runtime/HGM state. | +| User row with one or both values missing | Invalid | Invalid | A user cannot select automatic handling for only one green role. | + +The runtime Admin and Hostgroup Manager schemas permit nullable green +hostgroups so that they can represent automatically generated rows. This +runtime representation does not make a `NULL` green hostgroup valid in the +persistent user table. + +Saving runtime BGD configuration to the persistent Admin table skips every row +whose runtime `auto_generated` value is nonzero. An automatically generated row +with two `NULL` green hostgroups is therefore never inserted into the +persistent `NOT NULL` columns. User rows contain both values and are saved +normally. + +`OFFLINE_SOFT` and `OFFLINE_HARD` servers are not eligible for blue/green +mapping or for connection-drain actions. An explicit green writer in either +offline state is not selected as the active green endpoint. + +## AWS Topology + +### Topology Recognition + +`parse_aws_rds_topology()` classifies a result as blue/green topology from the +first fetched row. The result is classified as blue/green when the `role` and +`status` columns exist and both first-row cells are non-`NULL`. Empty strings +still satisfy this non-`NULL` test. Later rows do not change the initial +classification. + +AWS defines the following blue/green role values: + +```text +BLUE_GREEN_DEPLOYMENT_SOURCE +BLUE_GREEN_DEPLOYMENT_TARGET +``` + +AWS defines the following target status values: + +```text +AVAILABLE +SWITCHOVER_INITIATED +SWITCHOVER_IN_PROGRESS +SWITCHOVER_IN_POST_PROCESSING +SWITCHOVER_COMPLETED +``` + +If the query returns no rows, either column is absent, or either first-row cell +is `NULL`, `blue_green` remains false. The parser is shared with the Multi-AZ +Cluster discovery path, so other RDS topology shapes may be handled outside the +BGD state machine. + +### Observed Lifecycle + +The supplied trace observed this row and status sequence: + +```text +Two rows: + SOURCE = blue + TARGET = green + status = AVAILABLE + +Two rows: + repeated SWITCHOVER_INITIATED observations + -> repeated SWITCHOVER_IN_PROGRESS observations + -> repeated SWITCHOVER_IN_POST_PROCESSING observations + +One row: + TARGET = green + repeated SWITCHOVER_COMPLETED observations + +Zero rows: + observed approximately 44 seconds after SWITCHOVER_COMPLETED +``` + +Both rows carried the same status while both were present in the supplied +trace. The trace establishes that this occurred in the captured deployment; it +does not establish source/target status equality as a universal AWS guarantee. + +The trace also observed monotonic forward transitions and repeated observations +within each phase. At `SWITCHOVER_COMPLETED`, the source row disappeared and +the target row remained for approximately 44 seconds before the table became +empty. The 44-second duration is not fixed. The table remained present in +`information_schema`; the trace did not observe `ER_NO_SUCH_TABLE`. + +### AWS Completion And Cancellation Boundaries + +AWS defines `SWITCHOVER_COMPLETED` as completion of writer DNS propagation: the +original source endpoint points to the promoted target. + +AWS permits cancellation during `SWITCHOVER_INITIATED` and +`SWITCHOVER_IN_PROGRESS`. Rollback is no longer allowed after the deployment +enters `SWITCHOVER_IN_POST_PROCESSING`. The author separately observed a +cancellation returning the deployment to `AVAILABLE`. + +The author observed that the green hostname stopped resolving after completion +while the promoted IP remained reachable. The monitor consequently retains a +complete direct-probe target while direct probing is required rather than +depending on the continued resolvability of the green hostname. + +### Reader-Completion Policy + +The metadata table describes writer topology; it does not expose a reader +switchover status. The supplied trace did not measure reader DNS propagation +directly. The author observed reader errors before the table drained and normal +reader behavior afterward. + +ProxySQL therefore uses the following explicit policy: + +- `SWITCHOVER_COMPLETED` is the writer-completion signal. +- A topology drain after writer completion is the reader-cleanup signal. +- A topology drain before writer completion is a cancellation or rollback + signal. + +Using topology drain as the reader-cleanup signal is an accepted operational +correlation. It is not an AWS guarantee that an empty topology result proves +reader DNS propagation. + +### Topology Outcomes + +The worker distinguishes the following query outcomes: + +| Outcome | Detection | Monitor behavior | +|---|---|---| +| Table absent | The existence query returns no rows. | Apply phase-specific topology-drain handling. | +| Table vanished | A metadata query returns `ER_NO_SUCH_TABLE`. | Reset the query state to the table check, restore the baseline interval, and apply phase-specific topology-drain handling. | +| Table empty | A successful metadata query returns no rows. | Apply phase-specific topology-drain handling. | +| Metadata available | A successful metadata query returns rows. | Parse and pass the topology to the BGD state machine. | +| Query failure | The connection, timeout, existence query, or metadata query fails for another reason. | Log the error and retain the state for a later poll. A generic query failure is not a completion or cancellation signal. | + +Table absence and an empty table remain diagnostically distinct. ProxySQL +intentionally applies the same phase boundary to both because only the empty +table was present in the supplied lifecycle trace. + +## Monitor Architecture + +### Worker And Polling Model + +The dispatcher creates one BGD worker for each active deployment that has an +eligible blue writer. The worker keeps its state on its own stack and performs +the following polling cycle: + +```text +check whether mysql.rds_topology exists + -> fetch topology metadata + -> classify the observation + -> update the deployment state machine + -> apply phase actions + -> wait for the effective check interval +``` + +After the table has been observed, the worker normally continues with metadata +fetches. `ER_NO_SUCH_TABLE` returns it to the table-existence check. + +The configured check interval is the baseline. The state machine uses 250 ms +while the deployment is `AVAILABLE` and 100 ms during the active writer +switchover phases. It returns to the configured baseline after writer +completion or state cleanup. + +### Blue/Green Mapping + +The worker builds a `bg_map` from current runtime configuration and the +discovered topology: + +- The blue writer is matched to the target writer using the RDS green-hostname + naming relationship. +- In explicit mode, eligible configured green readers are name-matched to blue + readers. +- The topology contains writer endpoints only, so complete reader mapping is + not assumed. +- Blue readers without a mapped green counterpart are handled independently + and may be shunned during post-processing. +- `OFFLINE_SOFT` and `OFFLINE_HARD` servers do not participate. + +Each pair retains the blue hostname and connection attributes, the green +hostname, the resolved green IP and its expiry, and whether DNS pinning and +connection draining have already completed for that pair. + +### Direct-Probe Tuple + +The direct green-writer probe always uses one coherent host, port, and TLS +tuple derived from the matched writer pair: + +| Mode | Host or IP | Port | TLS | +|---|---|---|---| +| Automatic | Resolved IP of the target endpoint from `mysql.rds_topology`. | Configured port of the matched blue writer. | `use_ssl` from the matched blue writer. | +| Explicit | Resolved IP of the exact configured target writer. | Configured port of the matched blue writer. | `use_ssl` from the exact eligible green writer row. | + +The monitor does not consume or validate the target topology row's port. +Source/target pairs that use different ports are unsupported. This is a +ProxySQL constraint, not an AWS guarantee that the ports are always equal. +Different configured pairs may use different ports. + +Automatic mode has no independent green `mysql_servers` row from which to +derive TLS configuration, so it uses the matched blue writer's value. + +Explicit mode selects the green writer by exact target hostname and the +matched-blue port. It copies `use_ssl` from an existing eligible row. If +discovery creates a missing row or restores an `OFFLINE_HARD` row, the +successful add path obtains `use_ssl` from the resulting exact row after +hostgroup defaults are applied. An `OFFLINE_SOFT` row remains ineligible. + +### Direct-Probe Lifetime + +The worker resolves each eligible green endpoint through the DNS cache or a +live DNS lookup. Once the green writer IP is available, the worker directs +subsequent topology polls to that IP. This keeps the observation path available +through the source connectivity gap and the retirement of the green DNS name. + +The worker retries unresolved pairs on every eligible equal-phase observation +from `AVAILABLE` through `WRITER_SWITCHOVER_POST_PROCESSING`. During repeated +post-processing observations, it pins and drains only pairs whose green IP is +available and whose `green_ip_pinned` flag is false. A completed pair is not +drained again during that worker lifetime; unresolved pairs remain eligible +for a later retry. + +If three consecutive topology polls to the direct green IP fail, the worker +clears the direct target, removes its mapped blue DNS pins, purges the +corresponding monitor connections, and falls back to polling through the blue +configuration. + +## ProxySQL State Machine + +The monitor uses the following ordered states: + +```text +NONE + -> AVAILABLE + -> WRITER_SWITCHOVER_INITIATED + -> WRITER_SWITCHOVER_IN_PROGRESS + -> WRITER_SWITCHOVER_POST_PROCESSING + -> WRITER_SWITCHOVER_COMPLETED + -> READER_SWITCHOVER_IN_PROGRESS + -> SWITCHOVER_COMPLETED + -> NONE +``` + +The five states from `AVAILABLE` through +`WRITER_SWITCHOVER_COMPLETED` correspond to AWS target statuses. +`READER_SWITCHOVER_IN_PROGRESS` and `SWITCHOVER_COMPLETED` are ProxySQL states: + +- `READER_SWITCHOVER_IN_PROGRESS` records that writer completion has been + observed and defers reader cleanup until topology drains. +- `SWITCHOVER_COMPLETED` is a short-lived successful-cleanup state before the + worker returns to `NONE`. + +The arrows show nominal lifecycle order, not mandatory predecessor edges. +Forward transitions may skip states. A worker that first observes +`SWITCHOVER_IN_POST_PROCESSING`, for example, builds the required mapping and +performs post-processing setup directly. + +### Phase Actions + +| State or observation | Action | +|---|---| +| `AVAILABLE` | Set the next-check interval to 250 ms, build the blue/green map, resolve green IPs, and optionally add the green writer to its configured hostgroup. | +| `WRITER_SWITCHOVER_INITIATED` | Set the interval to 100 ms, rebuild the map, resolve green IPs, optionally add the green writer, and suppress ordinary read-only monitoring for the deployment servers. | +| `WRITER_SWITCHOVER_IN_PROGRESS` | Perform initiated-phase setup, sustain read-only suppression, and demote the mapped blue writer to read-only. | +| `WRITER_SWITCHOVER_POST_PROCESSING` | Perform setup even after late entry, sustain read-only suppression, pin mapped blue names to resolved green IPs, retire matching connections, configure writer placement, and shun unmapped blue readers. | +| `WRITER_SWITCHOVER_COMPLETED` | Advance immediately to `READER_SWITCHOVER_IN_PROGRESS`, remove the writer DNS-cache entry, retain mapped reader pins, and restore the baseline check interval. | +| Topology drain in `READER_SWITCHOVER_IN_PROGRESS` | Run successful cleanup, transition briefly through `SWITCHOVER_COMPLETED`, and return to `NONE`. | +| Topology drain in another active state | Run rollback cleanup and return to `NONE`. | +| Worker exit in an active state | Run rollback cleanup before discarding worker-local state. | + +### Equal And Repeated Phases + +When the converted target status equals the stored state, the worker does not +rebuild the configuration-derived map or repeat the complete phase action. +It performs only the eligible same-phase reconciliation: + +- Retry unresolved green IPs from `AVAILABLE` through post-processing. +- During post-processing, pin and drain newly resolved pairs once. +- Ignore repeated raw `SWITCHOVER_COMPLETED` observations after the local state + has advanced to `READER_SWITCHOVER_IN_PROGRESS`. + +### Backward And Unrecognized Phases + +A recognized target status whose enum value is lower than the stored state is a +backward transition. The worker runs rollback cleanup. If the new status is +`AVAILABLE`, it then re-enters `AVAILABLE`, restores the 250 ms interval, and +rebuilds mapping, resolution, and optional green-writer placement. Other +backward statuses leave the worker in `NONE`. + +An unknown nonempty target status maps to `NONE`. From an active higher state, +that value follows the backward-transition path and invokes rollback. An empty +target status, a missing target row, or a topology result not classified as +blue/green sets the phase to `NONE` and resets the interval without calling the +rollback helper. The latter path does not perform the rollback helper's map, +DNS, connection, or reader cleanup. + +## Switchover Effects + +### DNS Pinning And Reader Handling + +During writer post-processing, ProxySQL pins each mapped blue hostname to the +resolved green IP and retires existing connections for that blue endpoint. +New connections through the stable blue hostname then reach the green +instance. + +The metadata topology does not provide a complete reader mapping. During +post-processing, ProxySQL identifies eligible blue readers with no mapped +green counterpart and marks them `SHUNNED_AWS_BGD`. If shunning all blue +readers would leave the reader hostgroup empty, the worker temporarily makes +the writer available to the reader hostgroup. Final cleanup restores placement +according to `writer_is_also_reader` and unshuns the readers recorded by that +worker. + +After writer completion, the writer DNS-cache entry is removed immediately so +normal DNS resolution can resume for the stable writer name. Reader pins remain +until the topology drains and successful cleanup runs. + +### Connection Retirement + +`MySrvConnList::mark_connections_unhealthy()` deletes matching free +connections immediately. It marks matching used connections with +`healthy=false` and `reusable=false`; those connections finish their current +ownership and are destroyed when released. + +The retirement state flow is: + +```text +ACTIVE_BACKEND + healthy=true, reusable=true + | + | BGD drain selects a used connection + v +RETIRE_ON_RELEASE + healthy=false, reusable=false + | + | optional connection or session reset + | healthy remains false + v +POOL_RETURN + | + +--> push_MyConn_local: unhealthy -> global destruction path + | + `--> push_MyConn_to_pool: unhealthy -> delete + +No transition returns RETIRE_ON_RELEASE to a local or global free pool. +``` + +`MySQL_Connection::reset()` resets session state without restoring +`healthy=true`. `MySQL_Thread::push_MyConn_local()` rejects an unhealthy +connection before adding it to the thread-local cache. +`MySQL_HostGroups_Manager::push_MyConn_to_pool()` removes the connection from +the used list and destroys it before any free-pool insertion. Array pool return +is covered because reusable entries delegate to the same global return path, +while the existing non-reusable branch destroys the connection directly. + +These paths read `healthy` using ProxySQL's existing unlocked +connection-field convention. The feature accepts that project-level race model +and does not introduce a separate retirement flag or a broader locking policy. + +## Completion And Rollback + +### Phase Selection + +`aws_rds_bgd_handle_topology_absent()` selects cleanup from the last monitor +state: + +```text +topology drains + | + +--> state is READER_SWITCHOVER_IN_PROGRESS + | -> successful cleanup + | + +--> state is another non-NONE state + | -> rollback cleanup + | + `--> state is NONE + -> no cleanup +``` + +The same rollback helper also runs for a recognized backward phase and for +worker exit from an active state. + +### Rollback Cleanup + +Rollback performs the following one-shot actions: + +- Restore a blue writer demoted during + `WRITER_SWITCHOVER_IN_PROGRESS` or + `WRITER_SWITCHOVER_POST_PROCESSING`. +- Reconcile writer membership in the reader hostgroup according to + `writer_is_also_reader`. +- Unshun readers recorded by the worker. +- Remove DNS-cache entries and purge monitor-pool connections for recorded + shunned readers and mapped blue endpoints. +- Purge direct-probe monitor connections keyed by resolved green IPs. +- Clear worker mapping, probe, interval, and phase bookkeeping. +- Invoke read-only suppression cleanup and return to `NONE`. + +Rollback intentionally does not: + +- Drain application connections in configured green hostgroups. +- Remove green DNS entries. +- Change green server status. +- Remove user-configured or automatically added green rows. + +Purging a direct-probe monitor connection keyed by a green IP cleans up the +monitor's observation channel. It is not equivalent to draining application +connections from a green hostgroup. + +### Successful Cleanup + +Successful cleanup does not restore the obsolete blue writer. It reconciles +writer membership in the reader hostgroup according to +`writer_is_also_reader`, unshuns the readers recorded by the worker, removes +mapped and recorded-reader DNS entries, purges the corresponding monitor-pool +connections, invokes read-only suppression cleanup, and clears worker +bookkeeping. It also drains application connections for every eligible server +in the configured green writer and reader hostgroups and removes those green +hostnames from the DNS and monitor connection caches. + +`OFFLINE_SOFT` and `OFFLINE_HARD` green servers are excluded from that drain. +Successful cleanup leaves every green server row and status unchanged. +Green-hostgroup membership remains runtime configuration until an +administrator removes it, including membership added automatically by the BGD +monitor. + +### One-Shot Cleanup Policy + +Cleanup is best effort and one shot. The worker does not maintain a per-effect +completion ledger, does not verify every external postcondition, and clears its +local state after invoking the cleanup operations. + +This is an intentional ProxySQL policy. Process termination or an individual +operation failure can prevent the monitor from proving that every action +completed. The feature does not require a retained cleanup executor or durable +effect ownership. + +The final interval depends on the caller: + +- `ER_NO_SUCH_TABLE` restores the baseline interval before cleanup. +- Successful and rollback cleanup reset the interval. +- Topology absence observed while already in `NONE` performs no cleanup and + does not independently reset an existing interval override. + +## Worker And Configuration Lifetime + +### Deployment Checksum + +The dispatcher calculates a per-deployment checksum from the active BGD row and +eligible blue and green runtime server rows. An Admin `mysql_servers` commit +refreshes that checksum. + +A checksum change for an active deployment signals the existing worker through +`AWS_RDS_BGD_Worker::current_checksum`. The worker captures a consistent +candidate configuration and applies it in place. A checksum change alone does +not stop, join, or replace the worker thread. + +The checksum is a configuration-generation signal. It is not an effect ledger +and is not the mechanism used to retry transient DNS resolution. Runtime +publishes initiated by the BGD worker do not refresh the checksum, preventing +the worker's own hostgroup actions from triggering a configuration refresh. + +### In-Place Refresh + +Refresh behavior depends on the phase: + +1. Before `WRITER_SWITCHOVER_POST_PROCESSING`, the worker preserves its phase, + applies the candidate scalar and hostgroup configuration, clears the direct + probe and failure counter, and marks the map for reconstruction. The next + topology result rebuilds configuration-derived state. +2. If the reader hostgroup changes while read-only suppression is active, the + worker clears suppression for the old hostgroup and enables it for the new + hostgroup. +3. During `WRITER_SWITCHOVER_IN_PROGRESS`, map reconstruction restores a + replaced old writer and demotes the newly mapped writer. +4. At or after `WRITER_SWITCHOVER_POST_PROCESSING`, replacing only the map + cannot safely reconcile existing DNS pins, reader shuns, monitor + connections, and placement. The same worker runs one-shot rollback, applies + the candidate configuration, resets topology polling to the table check, + and restarts the state machine from `NONE`. + +The fourth case is rollback and state-machine restart within the same worker, +not worker replacement. + +### Worker Detach And Recreation + +If a deployment is disabled, removed, or loses its eligible blue writer, the +dispatcher requests worker stop. A worker exiting from a non-`NONE` state runs +one-shot rollback before discarding its local map, reader list, probe target, +and phase. + +If configuration later makes the deployment eligible again, the dispatcher +creates a new worker with fresh state. A fresh worker does not recover the +prior worker's effects or cleanup results. If its first observation is +`SWITCHOVER_COMPLETED`, it enters `READER_SWITCHOVER_IN_PROGRESS` without +reconstructing a prior map and waits for topology drain. + +### Process Restart + +A full ProxySQL process restart is a fresh start: + +- Worker state, blue/green maps, direct-probe targets, suppression state, and + cleanup bookkeeping are recreated empty. +- DNS cache and connection pools are recreated. +- Runtime placement and status are rebuilt from administrator configuration. +- User-configured BGD and server rows reload through the normal configuration + path. +- An automatically added runtime-only green row disappears unless it was + independently configured or synchronized into another restart input. + +No durable BGD progress or effect-ownership state is recovered. A +simulator-driven full process-restart fixture is not required by this contract. + +## Read-Only Suppression + +BGD suppresses ordinary read-only monitoring while it intentionally changes +writer and reader placement. Suppression is keyed by +`hostname:::port` in the shared `aws_rds_bgd_server_status` map. + +The required lifetime is: + +```text +UNSUPPRESSED + | + | BGD enters INITIATED, IN_PROGRESS, or POST_PROCESSING + | record every endpoint key owned by this deployment + v +SUPPRESSED + | + +--> new read-only work for an owned key is not admitted + | + +--> an earlier result revalidates suppression before applying placement + | + | BGD returns to NONE or changes configuration + v +OWNED_KEYS_REMOVED + | + `--> other deployments' keys remain unchanged +``` + +Suppression ownership is deployment-specific. Cleanup must erase the exact keys +inserted for that deployment even if the corresponding server has already been +removed from its current hostgroup. Concurrent deployments using distinct +endpoints must not clear each other's keys. + +Each server endpoint must belong to only one active blue/green deployment. +Administrators are responsible for avoiding overlapping endpoint assignments; +the monitor does not validate or protect against this unsupported configuration. + +Before a read-only result changes placement through +`read_only_action_v2()`, result application must revalidate that the endpoint is +not suppressed for an active BGD transition. Admission-time validation alone +is insufficient because a task may have been admitted before suppression was +enabled. + +## Verification + +The BGD verification surface consists of: + +- The SQLite3-server simulator compiled under `TEST_RDS_BGD`. +- The `cluster_sim_rds_bgd-g1` TAP group registered in + `test/tap/groups/groups.json`. +- The 22 `test_rds_bgd_*-t` scenario binaries under `test/tap/tests`. +- `connection_unhealthy_unit-t`, which verifies terminal retirement across + reset, local pool return, and global pool return. +- `.github/workflows/CI-cluster-simulator.yml`, which discovers registered + simulator groups, builds the combined simulator flavor, and runs each group + as an automatic pull-request check. + +The scenario suite covers: + +- Explicit startup and automatic discovery. +- Probe destination and TLS selection. +- Writer and reader switchover. +- Late entry into writer phases and first observation at completion. +- Cancellation, rollback, topology empty/absent, and query errors. +- Green membership persistence and green-pool cleanup. +- Offline server exclusions and reader policy. +- Configuration persistence, in-place refresh, disablement, removal, and + worker hostgroup changes. +- Repeated deployments and concurrent deployment isolation. + +The simulator cannot reproduce mutable DNS failure followed by recovery, so +same-phase DNS recovery is verified by source review rather than a mutable-DNS +simulator case. Full ProxySQL process restart is an accepted fresh-start +assumption and intentionally has no simulator fixture. + +## Implementation Anchors + +The monitor behavior is primarily implemented by: + +- `parse_aws_rds_topology()` +- `monitor_RDS_BGD_thread_HG()` +- `handle_aws_rds_bgd()` +- `aws_rds_bgd_resolve_green_ips()` +- `aws_rds_bgd_refresh_worker_config()` +- `aws_rds_bgd_config_refresh_action()` +- `aws_rds_bgd_handle_topology_absent()` +- `handle_aws_rds_bgd_post_switchover()` +- `aws_rds_bgd_drain_green_hg()` + +The broader configuration and effect surface includes: + +- `include/DNS_Cache.hpp` and `lib/DNS_Cache.cpp` +- `include/MySQL_HostGroups_Manager.h` and + `lib/MySQL_HostGroups_Manager.cpp` +- `include/mysql_connection.h` and `lib/mysql_connection.cpp` +- `lib/MySrvConnList.cpp` +- `lib/ProxySQL_Admin.cpp` +- `lib/ProxySQL_Config.cpp` +- `include/ProxySQL_Admin_Tables_Definitions.h` + +Changes to these entry points or their state, configuration, DNS, hostgroup, or +connection semantics should be reviewed against this contract and the +simulator specification. diff --git a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md new file mode 100644 index 0000000000..7c228a93bc --- /dev/null +++ b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md @@ -0,0 +1,607 @@ +# AWS RDS Blue/Green Deployment Simulator + +**Document status:** IMPLEMENTED + +**Applies to:** `TEST_RDS_BGD`, the SQLite3-server simulation surface, BGD TAP +helpers, local and GitHub runners, and supported simulator coverage + +**Related monitor contract:** [RDS_BGD_Monitor.md](RDS_BGD_Monitor.md) + +## Purpose + +This document defines the simulator used to test ProxySQL's AWS RDS Blue/Green +Deployment monitor. It combines the behavioral contract, SQLite3-server +changes, TAP helper API, network fixture, local runner, and supported coverage +into one implementation specification. + +## Architecture + +The TAP test is the scenario controller. It configures ProxySQL with AWS-style +hostnames, writes simulated backend state to ProxySQL's SQLite3 server, changes +that state to drive the BGD FSM, and verifies ProxySQL through runtime, +statistics, and simulator probe-log tables. Topology state is keyed by backend +IP and port, while read-only state is keyed by the configured hostname and port. + +No `test/deps/cluster_simulator` process or backend database container is +required. A common TAP helper owns reusable SQLite3-server operations, while a +BGD helper translates explicit test intent into topology state and probe-log +queries. Neither helper advances the FSM or owns scenario timing. + +## `TEST_RDS_BGD` Boundary + +Simulator tables, BGD response interception, listener changes, and supporting +members are compiled only under `TEST_RDS_BGD`. A production build contains no +BGD simulator surface and preserves the existing production monitor queries, +connection behavior, and DNS behavior. + +The flag reuses shared TEST-mode SQLite3-server infrastructure, including the +existing `READONLY_STATUS` mechanism, without changing the behavior of +`TEST_AURORA`, `TEST_GALERA`, `TEST_GROUPREP`, `TEST_READONLY`, or +`TEST_REPLICATIONLAG` builds. + +## Network Model + +The BGD TAP group injects a shared `/etc/hosts` map containing AWS-style blue +and green names. Every hostname resolves to a distinct loopback IP and uses +port 3306, preserving the address shape used by AWS while a single wildcard +SQLite3-server listener handles all simulated endpoints. + +Tests add servers to ProxySQL by hostname. They configure topology state using +the corresponding IP and read-only state using the configured hostname and +port. Distinct destination IPs retain the blue/green split when ProxySQL +resolves a hostname or directly probes the resolved green IP. + +The map reserves multiple clusters and two green endpoint sets for cluster 1. +Tests configure only the endpoints they need: separate clusters support +simultaneous switchovers, and the alternate cluster-1 green set supports a +second switchover after the previous FSM resets on empty or absent topology. + +## Backend Identity and Topology Ownership + +ProxySQL sends the production BGD queries unchanged. The SQLite3 server calls +`getsockname()` on the accepted connection and uses the resulting +`backend_ip, backend_port` as the simulator key; no hostname, address, port, or +comment is appended to the query. + +Topology belongs only to backend keys explicitly updated by the TAP test. A +normal scenario publishes metadata to the blue and green writer IPs; readers +have no topology table unless a test intentionally configures one. The helper +does not copy state between deployment members implicitly. + +Under `TEST_RDS_BGD`, use `sockaddr_storage` for IPv4 and IPv6-safe local +address extraction. Failure to resolve the accepted local address is a +simulator error and must not fall back to an arbitrary backend row. + +## SQLite3-Server Storage + +Create the following tables in `SQLite3_Server::init()` and store them in the +existing persistent `GloVars.sqlite3serverdb` database: + +```sql +CREATE TABLE RDS_BGD_CONTROL ( + backend_ip TEXT NOT NULL, + backend_port INTEGER NOT NULL, + topology_present INTEGER NOT NULL DEFAULT 0 CHECK (topology_present IN (0,1)), + error_code INTEGER NOT NULL DEFAULT 0, + error_msg TEXT NOT NULL DEFAULT '', + PRIMARY KEY (backend_ip, backend_port) +); + +CREATE TABLE RDS_BGD_TOPOLOGY ( + backend_ip TEXT NOT NULL, + backend_port INTEGER NOT NULL, + row_order INTEGER NOT NULL, + id TEXT NOT NULL, + endpoint TEXT NOT NULL, + topology_port INTEGER NOT NULL, + role TEXT NOT NULL, + status TEXT NOT NULL, + PRIMARY KEY (backend_ip, backend_port, row_order) +); + +CREATE TABLE RDS_BGD_PROBE_LOG ( + sequence_id INTEGER PRIMARY KEY AUTOINCREMENT, + backend_ip TEXT NOT NULL, + backend_port INTEGER NOT NULL, + probe_kind TEXT NOT NULL CHECK (probe_kind IN ('table_check','metadata')), + encrypted INTEGER NOT NULL CHECK (encrypted IN (0,1)) +); +``` + +Each SQLite3-server session already opens this database in WAL/FULLMUTEX mode. +TAP writes and monitor reads therefore share persistent state without an +attached in-memory schema or a control connection that keeps data alive. + +## Query Dispatch + +Run the existing SQL normalization first: collapse whitespace, remove trailing +spaces or semicolons, and compare case-insensitively. Intercept only a complete +match for one of the production BGD constants: + +```sql +SELECT 1 FROM information_schema.TABLES + WHERE TABLE_SCHEMA='mysql' AND TABLE_NAME='rds_topology' + +SELECT * FROM mysql.rds_topology +``` + +For either match, resolve the accepted backend key, append a probe-log row, +load its control row, and select the response described below. An +address-extraction failure returns a simulator error without selecting state or +logging an invalid backend identity. + +Simulated read-only checks follow the handling described below. All remaining +statements continue through normal SQLite3-server handling. TAP control and +inspection statements against the simulator tables are not rewritten or +recorded as BGD monitor probes. + +## Control-State Meaning + +The TAP helper publishes control and topology changes atomically. Monitor probes +read committed simulator state without holding a cross-query snapshot. The +supported states are: + +| `RDS_BGD_CONTROL` state | Topology rows | Meaning | +|---|---|---| +| No backend row | None | Backend is unconfigured; topology is absent. | +| `topology_present=1`, `error_code=0` | One or more | Return the configured topology. | +| `topology_present=1`, `error_code=0` | Empty | Table exists but contains no topology. | +| `topology_present=1`, `error_code!=0` | Unchanged | Table exists, but its metadata query fails. | +| `topology_present=0`, `error_code=1146` | Empty | Table has been dropped. | + +Topology update and delete operations clear `error_code` and `error_msg`. +Configured errors other than 1146 treat the table as present and retain its rows; +error 1146 marks it absent. Dropping topology also removes its rows. Other flag +combinations are invalid helper state. + +## Topology Responses + +The table check consults only `topology_present`. Metadata handling applies a +configured error before reading topology rows: + +| Query | Selected backend state | MySQL response | +|---|---|---| +| Table check | No control row or `topology_present=0` | Successful result with zero rows. | +| Table check | `topology_present=1` | One column named `1`, containing one row with value `1`. | +| Metadata | No control row | Error 1146: `Table 'mysql.rds_topology' doesn't exist`. | +| Metadata | `error_code!=0` | Stored `error_code` and `error_msg`. | +| Metadata | `error_code=0`, `topology_present=0` | Error 1146 as a defensive fallback. | +| Metadata | `error_code=0`, `topology_present=1` | Ordered backend rows; an empty set remains successful. | + +A successful metadata result exposes `id, endpoint, port, role, status`. +`topology_port` supplies the `port` result, and `row_order` determines row +order. Rows belonging to another backend key must never enter the result. + +## Error Packets and Probe Log + +The existing `send_MySQL_ERR()` always returns error 1045. Add an overload that +accepts an error code and message; error 1146 uses SQLSTATE `42S02`, while other +configured simulator errors may use `HY000` unless a test requires a specific +mapping. + +Every handled topology-table check or metadata query appends one row to +`RDS_BGD_PROBE_LOG`, including empty and error responses. `sequence_id` +preserves order, `probe_kind` identifies the query, `backend_ip, backend_port` +identify the destination, and `encrypted` records the accepted stream's TLS +state. + +The TAP test is the only probe-log consumer; ProxySQL never reads it. A test +reads the last sequence before changing state and then reads later rows +to verify the selected destination and TLS mode. A probe-log insertion failure +is a simulator failure and must not be silently reported as a normal backend +response. + +## Read-Only Simulation + +`TEST_RDS_BGD` builds the shared `READONLY_STATUS(hostname, port, read_only)` +table and the read-only cache, without calling `enable_readonly_testing()`. +The TAP test owns ProxySQL hostgroup and server configuration and writes each +read-only value using the AWS hostname configured in `mysql_servers`. + +Read-only monitor tasks send the simulation query +`SELECT @@global.read_only read_only :`. The SQLite3 server +uses the suffix to read the cached value populated from `READONLY_STATUS` and +returns one `read_only` column. Table writes refresh the cache, and a missing +entry returns the safe default `read_only=1`. + +BGD topology tasks send the production topology queries unchanged. Read-only +handling does not consult `RDS_BGD_CONTROL` or write `RDS_BGD_PROBE_LOG`. + +## TAP Helper API + +The API follows existing TAP conventions: write methods return `EXIT_SUCCESS` +or `EXIT_FAILURE`, and read methods return the existing `rc_t` type. The +interfaces below form the simulator design surface used by BGD scenarios. + +### Common Endpoint + +```cpp +struct Endpoint { + std::string host; + int port; +}; +``` + +Identifies one simulated backend. For BGD topology and probe-log operations, +`host` is the backend IP. For `read_only_update()`, `host` is the AWS hostname +configured in ProxySQL. + +### `Cluster_Simulator` + +```cpp +int connect( + char* host, + int port, + char* username, + char* password, + bool use_ssl = false); + +int read_only_update(Endpoint backend, bool read_only); +``` + +`connect()` opens the SQLite3-server control connection with the MySQL client +API; the helper closes it when destroyed. `read_only_update()` changes the +`READONLY_STATUS` row identified by configured hostname and port. + +### Topology and Host Types + +```cpp +struct RDS_BGD_Topology_Row { + std::string id; + std::string endpoint; + int port; + std::string role; + std::string status; +}; + +struct RDS_BGD_Host { + std::string hostname; + std::string ip; + int port; + + Endpoint endpoint(); + Endpoint host_endpoint(); +}; +``` + +`RDS_BGD_Topology_Row` represents one `mysql.rds_topology` row using +C++11-compatible field types. `RDS_BGD_Host` keeps the ProxySQL-facing +hostname and simulator-facing IP together. + +### Cluster Fixture + +```cpp +class RDS_BGD_Cluster { +public: + RDS_BGD_Host blue_writer; + RDS_BGD_Host green_writer; + std::vector blue_readers; + std::vector green_readers; + + std::vector get_writers(); + std::vector get_blue_endpoints(); + std::vector get_green_endpoints(); + std::vector get_endpoints(); + std::vector get_topology(std::string status); +}; +``` + +Each TAP test owns and initializes the cluster fixtures it uses. A fixture +keeps the selected `/etc/hosts` mapping together. `get_writers()` returns the +blue and green writer IP endpoints. `get_blue_endpoints()` and +`get_green_endpoints()` include the writer and readers for one deployment, +while `get_endpoints()` returns the complete cluster. `get_topology(status)` +returns the standard two-row SOURCE/TARGET writer topology using the configured +hostnames and status. Tests add reader rows explicitly when the scenario needs +reader mapping. + +### BGD Topology Operations + +```cpp +int topology_update( + std::vector backends, + std::vector rows); + +int topology_delete(std::vector backends); + +int topology_drop(std::vector backends); + +int topology_error( + std::vector backends, + int error_code, + std::string error_msg); + +int cleanup(); +``` + +`topology_update()` marks the table present, clears any configured error, and +replaces rows on only the supplied backends. `topology_delete()` clears rows +and errors while leaving the table present. + +`topology_drop()` clears rows, marks the table absent, and records error 1146 +with `Table 'mysql.rds_topology' doesn't exist`. `topology_error()` requires a +nonzero code; 1146 marks topology absent, while any other code marks it present +and leaves existing rows unchanged. + +`cleanup()` removes read-only state, topology rows, control rows, and probe-log +rows. Tests call it together with their ProxySQL Admin cleanup so scenarios do +not inherit simulator state from an earlier binary. + +### Probe-Log Operations + +```cpp +enum class RDS_BGD_Probe_Kind { + table_check, + metadata, +}; + +struct RDS_BGD_Probe_Log { + uint64_t sequence_id; + Endpoint backend; + RDS_BGD_Probe_Kind probe_kind; + bool encrypted; +}; + +rc_t probe_log_last_sequence(); + +rc_t> probe_log_since(uint64_t sequence_id); + +rc_t wait_for_probe_log( + uint64_t sequence_id, + Endpoint backend, + RDS_BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted = -1); +``` + +`probe_log_last_sequence()` returns zero for an empty log. `probe_log_since()` +returns rows after the supplied sequence. `wait_for_probe_log()` waits for one matching row; +`encrypted` is `-1` for either mode, `0` for plaintext, and `1` for TLS. + +## Typical TAP Test + +```cpp +int main() { + plan(3); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator simulator {}; + + if (setup(cl, admin, simulator) != EXIT_SUCCESS) + return exit_status(); + + TestState state {}; + + if (publish_available_topology(simulator, state) != EXIT_SUCCESS) + goto exit_cleanup; + + if (configure_bgd_available(admin, state) != EXIT_SUCCESS) + goto exit_cleanup; + + if (test_plaintext_green_writer_probe(simulator, state) != EXIT_SUCCESS) + goto exit_cleanup; + +exit_cleanup: + if (cleanup(admin, simulator) != EXIT_SUCCESS) + return EXIT_FAILURE; + return exit_status(); +} +``` + +Each test defines small setup, scenario, and cleanup functions around this +control flow. ProxySQL configuration remains test-local. The simulator changes +backend responses and reads probe evidence; assertions against ProxySQL use +Admin SQL. Cleanup removes both Admin and simulator state before returning. + +## Build Integration + +The build provides `build_lib_test_rds_bgd`, `build_src_test_rds_bgd`, and the +top-level `test_rds_bgd` target. The lib and src targets compile with +`-DDEBUG -DTEST_RDS_BGD`; none depends on `build_cluster_simulator`. + +`test_rds_bgd` depends on `build_src_test_rds_bgd` and then invokes `make +debug` in `test/tap`: + +```text +build_deps_debug -> build_lib_test_rds_bgd -> build_src_test_rds_bgd + -> TAP debug build +``` + +`test_rds_bgd` is the focused local entry point. Do not invoke +`build_tap_test_debug` afterward because its `build_src_debug` dependency +selects the normal debug daemon. `testall` includes `-DTEST_RDS_BGD` and is +used by the shared cluster-simulator CI build. + +## Local CI Group + +The `test/tap/groups/cluster_sim_rds_bgd/` group executes as +`cluster_sim_rds_bgd-g1`. + +| File | BGD-specific content | +|---|---| +| `env.sh` | Set the fixed host map, wait for the SQLite3-server port, and skip backend cluster startup. | +| `add-hosts` | Define the fixed hostname/IP map below. | +| `pre-proxysql.bash` | Keep the existing short startup wait before Admin writes. | +| `pre-proxysql.sql` | Add the simulator user and move the SQLite3 server to port 3306. | + +The group has no `infras.lst`, `CLUSTER_SIM_BINARY_PATH`, or +`CLUSTER_SIM_TESTS_ROOT`. The TAP binary controls the simulator directly. + +```bash +export CLUSTER_SIM_HOST_FILE="${WORKSPACE}/test/tap/groups/cluster_sim_rds_bgd/add-hosts" +export PROXYSQL_READY_PORTS_EXTRA="3306" +export SKIP_CLUSTER_START=1 +``` + +### Fixed Host Map + +All BGD TAP tests use this map. Green endpoints retain the blue endpoint's +first label and append `-green-` before the common domain. + +```text +# Cluster 1: blue endpoints +db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.11 +db-1-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.12 +db-1-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.13 + +# Cluster 1: green deployment A +db-1-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.14 +db-1-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.15 +db-1-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.16 + +# Cluster 1: green deployment B, for repeated switchovers +db-1-green-s7m2kx.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.17 +db-1-reader-1-green-v4n8qp.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.18 +db-1-reader-2-green-w6h3rz.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.19 + +# Cluster 2: reserved for multi-cluster tests +db-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.20 +db-2-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.21 +db-2-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.22 +db-2-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.23 +db-2-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.24 +db-2-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.25 + +# Cluster 3: reserved for multi-cluster tests +db-3.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.26 +db-3-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.27 +db-3-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.28 +db-3-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.29 +db-3-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.30 +db-3-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.31 +``` + +Every endpoint uses port 3306. Cluster 1 with green deployment A is sufficient +for normal FSM cases. A repeated-switchover test completes deployment A, waits +for empty or absent topology to reset the FSM, replaces A's green hostgroup +rows with deployment B, and publishes the next topology. Clusters 2 and 3 are +available for simultaneous switchovers. + +### SQLite3-Server Hook + +`pre-proxysql.sql` provisions the simulator credentials and changes the +SQLite3-server listener from the CI default to port 3306: + +```sql +INSERT OR REPLACE INTO mysql_users + (username, password, default_hostgroup, active) + VALUES ('testuser', 'testuser', 0, 1); +LOAD MYSQL USERS TO RUNTIME; +SAVE MYSQL USERS TO DISK; + +SET sqliteserver-mysql_ifaces='0.0.0.0:3306'; +LOAD SQLITESERVER VARIABLES TO RUNTIME; +SAVE SQLITESERVER VARIABLES TO DISK; +``` + +The hook does not populate `mysql_servers` or +`mysql_aws_rds_bgd_hostgroups`; each test owns its ProxySQL configuration and +simulator transitions. From the TAP container, the control connection uses +`proxysql:3306` on the existing isolated Docker network. + +### Group Registration and Local Run + +The BGD TAP binaries are registered in `test/tap/groups/groups.json` under +`cluster_sim_rds_bgd-g1`. The registry is the source of truth for both local +execution and GitHub CI. The simulator table in `test/infra/README.md` records +the group and its `make test_rds_bgd` requirement. + +Clean when switching compile flavors because Make does not track changed +preprocessor flags: + +```bash +make clean +make -j"$(nproc)" test_rds_bgd + +export INFRA_ID="rds-bgd-$(date +%s)" +export TAP_GROUP="cluster_sim_rds_bgd-g1" + +./test/infra/control/ensure-infras.bash +./test/infra/control/run-tests-isolated.bash +./test/infra/control/destroy-infras.bash +``` + +The existing runner injects the host aliases, starts ProxySQL with +`--sqlite3-server`, executes the registered TAP binaries in the test container, +and collects logs. No BGD branch is required in `ensure-infras.bash`, +`start-proxysql-isolated.bash`, or `run-tests-isolated.bash`. + +## GitHub CI + +`.github/workflows/CI-cluster-simulator.yml` builds and executes every registered +`cluster_sim_*` group. It discovers groups and their TAP binaries from +`test/tap/groups/groups.json`, so registration in `cluster_sim_rds_bgd-g1` +places the complete BGD suite in the workflow matrix without BGD-specific YAML. +The workflow runs for pull requests and `workflow_dispatch`. + +The build job uses `test/infra/control/cluster-simulator-ci.bash` to build +`testall`, the cluster-simulator binary, the TAP library, and every registered +simulation binary. `testall` is intentional: one ProxySQL executable contains +all simulation flags, including `TEST_RDS_BGD`, and is shared by the matrix +jobs. The verified runtime is staged in an exact-SHA cache. + +Each matrix job restores and verifies that runtime for its selected group, +builds the common runner image, and executes `ensure-infras.bash` followed by +`run-tests-isolated.bash`. Cleanup always stops ProxySQL and destroys the +isolated runner; failure logs are archived by group and SHA. + +The shared runtime includes `test/deps/cluster_simulator` for groups that need +it. The BGD group sets `SKIP_CLUSTER_START=1`, starts no backend infrastructure, +and drives ProxySQL's SQLite3-server simulator directly. + +## Supported Test Coverage + +The suite is behavior-driven rather than a unit test for every helper method. +Tests publish backend observations through the simulator and verify the BGD +monitor through Admin runtime tables, server placement, connection-pool state, +backend routing, read-only logs, and the simulator probe log. + +### Configuration and Discovery + +| Behavior | Coverage | +|---|---| +| Automatic discovery ordering | Topology before blue configuration and blue configuration before topology both converge on one runtime-only automatic BGD row. | +| Explicit startup ordering | A worker starts only after both an explicit BGD row and an eligible blue server exist, regardless of which is loaded first. | +| Green membership ordering | Configured green membership may arrive before `AVAILABLE`, after discovery, or after the worker starts. | +| Configuration ownership | Automatic rows remain runtime-only; explicit rows persist; invalid partial green-hostgroup configuration is rejected; automatic discovery does not overwrite administrator-owned rows. | +| Probe destination and TLS | Automatic and explicit configurations select the mapped writer tuple, apply the correct TLS source, and probe table check, blue metadata, and green metadata in order. | +| Active configuration refresh | Server TLS, membership, status, interval, timeout, hostgroups, and mapped-writer changes are incorporated while preserving the applicable BGD phase and probe policy. | +| Disablement and removal | Disabling or deleting an active BGD row performs phase-appropriate rollback and suppresses or removes the runtime worker. | + +### Switchover, Rollback, and Cleanup + +| Behavior | Coverage | +|---|---|---| +| Acceptance | An explicitly configured worker consumes TAP-controlled `AVAILABLE` topology and probes the green writer directly. | +| Writer switchover | `AVAILABLE`, `SWITCHOVER_INITIATED`, `SWITCHOVER_IN_PROGRESS`, and `SWITCHOVER_IN_POST_PROCESSING` drive the defined status, placement, suppression, pool-drain, and routing effects. Repeated post-processing does not redrain a post-cutover pool. | +| Reader switchover and cleanup | Target-only `SWITCHOVER_COMPLETED` enters reader switchover; terminal empty or absent topology restores reader policy, drains eligible green pools, retains configured green rows, and returns to `NONE`. | +| Cancellation rollback | Returning from initiated or in-progress topology to `AVAILABLE` restores blue routing and monitoring without removing explicit green rows or draining green pools. | +| Topology loss and errors | Empty topology, absent topology, metadata error 1146, and generic metadata errors retain their distinct effects before and after writer completion. | +| Late entry | Fresh workers starting at initiated, in-progress, post-processing, or completed observations apply only the state supported by the first observation. | +| Reader and pool policy | Matched and unmatched readers, writer fallback, and `ONLINE`, `SHUNNED`, `OFFLINE_SOFT`, and `OFFLINE_HARD` green pools follow their routing and cleanup policies. | +| Repeated and concurrent deployments | A second deployment reuses hostgroups without stale membership or probes, while three simultaneous workers retain independent topology, phase, placement, and TLS state. | + +The simulator does not claim to validate application traffic, AWS control-plane +timing, mutable DNS propagation, packet loss, or exact post-switchover address +movement. Those require separate integration infrastructure when a test's +assertion depends on them. + +## Code Boundaries + +| Area | Boundary | +|---|---| +| `Makefile` | Provides the BGD build targets and includes `TEST_RDS_BGD` in `testall`. | +| `include/SQLite3_Server.h` | Declares the TEST-mode table ownership and shared read-only simulator members. | +| `src/SQLite3_Server.cpp` | Handles endpoint extraction, table creation, BGD/read-only interception, and probe logging. | +| `test/tap` helpers | Provide the common simulator and BGD-specific API defined above. | +| `test/tap/groups/cluster_sim_rds_bgd` | Defines the fixed host map and SQLite3-server group configuration. | +| `test/tap/groups/groups.json` | Registers BGD TAP binaries in `cluster_sim_rds_bgd-g1`. | +| `test/infra/README.md` | Documents the group and its required `test_rds_bgd` build target. | +| `.github/workflows/CI-cluster-simulator.yml` | Builds the combined simulation flavor and executes each registered simulator group in its own matrix job. | +| `test/infra/control/cluster-simulator-ci.bash` | Discovers groups and binaries, builds and verifies the shared runtime, and stages the exact-SHA cache payload. | +| BGD production monitor | Reuses existing query constants without simulator query decoration or a test initializer. | + +Existing simulator builds retain their behavior. The scenario, not the helper, +owns topology publication, FSM timing, ProxySQL configuration, and expected +outcomes. diff --git a/include/Base_HostGroups_Manager.h b/include/Base_HostGroups_Manager.h index 9e1c7279df..9a47dcc68e 100644 --- a/include/Base_HostGroups_Manager.h +++ b/include/Base_HostGroups_Manager.h @@ -86,7 +86,7 @@ class MetricsCollector; "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ "comment VARCHAR , UNIQUE (reader_hostgroup))" -#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" +#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" WHEN 5 THEN \"SHUNNED_AWS_BGD\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" #define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" @@ -95,7 +95,7 @@ class MetricsCollector; /* * @brief Generates the 'runtime_mysql_servers' resultset exposed to other ProxySQL cluster members. - * @details Makes 'SHUNNED' and 'SHUNNED_REPLICATION_LAG' statuses equivalent to 'ONLINE'. 'SHUNNED' states + * @details Makes 'SHUNNED', 'SHUNNED_REPLICATION_LAG' and 'SHUNNED_AWS_BGD' statuses equivalent to 'ONLINE'. 'SHUNNED' states * are by definition local transitory states, this is why a 'mysql_servers' table reconfiguration isn't * normally performed when servers are internally imposed with these statuses. This means, that propagating * this state to other cluster members is undesired behavior, and so it's generating a different checksum, @@ -117,6 +117,7 @@ class MetricsCollector; " WHEN 2 THEN \"OFFLINE_SOFT\"" \ " WHEN 3 THEN \"OFFLINE_HARD\"" \ " WHEN 4 THEN \"ONLINE\" " \ + " WHEN 5 THEN \"ONLINE\" " \ "END status," \ "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ "FROM mysql_servers " \ @@ -127,7 +128,7 @@ class MetricsCollector; * @brief Generates the 'mysql_servers_v2' resultset exposed to other ProxySQL cluster members. * @details The generated resultset is used for the checksum computation of the runtime ProxySQL config * ('mysql_servers_v2' checksum), and it's also forwarded to other cluster members when querying the Admin - * interface with 'CLUSTER_QUERY_MYSQL_SERVERS_V2'. It makes 'SHUNNED' state equivalent to 'ONLINE', and also + * interface with 'CLUSTER_QUERY_MYSQL_SERVERS_V2'. It makes 'SHUNNED' and 'SHUNNED_AWS_BGD' states equivalent to 'ONLINE', and also * filters out any 'OFFLINE_HARD' entries. This is done because none of the statuses are valid configuration * statuses, they are local, transient status that ProxySQL uses during operation. */ @@ -136,6 +137,7 @@ class MetricsCollector; "hostgroup_id, hostname, port, gtid_port, " \ "CASE" \ " WHEN status=\"SHUNNED\" THEN \"ONLINE\"" \ + " WHEN status=\"SHUNNED_AWS_BGD\" THEN \"ONLINE\"" \ " ELSE status " \ "END AS status, " \ "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ @@ -570,11 +572,11 @@ class Base_HostGroups_Manager { PtrArray *MyHostGroups; std::unordered_mapMyHostGroups_map; - HGC * MyHGC_find(unsigned int); HGC * MyHGC_create(unsigned int); public: Base_HostGroups_Manager(); + HGC * MyHGC_find(unsigned int); HGC * MyHGC_lookup(unsigned int); SQLite3_result * execute_query(char *query, char **error); @@ -608,6 +610,7 @@ class MySQL_HostGroups_Manager { MYSQL_AWS_AURORA_HOSTGROUPS, MYSQL_HOSTGROUP_ATTRIBUTES, MYSQL_SERVERS_SSL_PARAMS, + MYSQL_AWS_RDS_BGD_HOSTGROUPS, MYSQL_SERVERS, HGM_TABLES_SIZE_ diff --git a/include/DNS_Cache.hpp b/include/DNS_Cache.hpp index a994a23b7f..44fb6dad44 100644 --- a/include/DNS_Cache.hpp +++ b/include/DNS_Cache.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "thread.h" @@ -82,11 +83,44 @@ class DNS_Cache { void remove(const std::string& hostname); void clear(); bool empty() const; - std::string lookup(const std::string& hostname, size_t* ip_count) const; + bool is_ip_valid(const std::string& hostname, const std::string& ip) const; + std::string lookup(const std::string& hostname, size_t* ip_count); + + /** + * @brief Pin a hostname to a fixed IP until it is explicitly unpinned. + * + * @param hostname Hostname whose cached resolution is overridden. + * @param ip IP address to serve for 'hostname' while pinned. + */ + void pin(const std::string& hostname, const std::string& ip); + + /** + * @brief Pin a hostname to a fixed IP for a bounded time. + * + * @details While the pin is active, lookup() serves 'ip' instead of the resolved + * address set. Once ttl_ms expires, lookup() serves the resolved address and + * clears the expired pin before returning. + * + * @param hostname Hostname whose cached resolution is overridden. + * @param ip IP address to serve for 'hostname' while pinned. + * @param ttl_ms Pin lifetime in milliseconds; 0 means no expiry. + */ + void pin(const std::string& hostname, const std::string& ip, unsigned long long ttl_ms); + + /** + * @brief Remove a pin set by pin(), restoring normal resolution (no-op if not pinned). + * + * @param hostname Hostname to unpin. + */ + void unpin(const std::string& hostname); private: struct IP_ADDR { std::vector ips; + // Pinned override: when non-empty, lookup() serves it instead of 'ips' + // while pinned_until is not expired. + std::string pinned_ip; + unsigned long long pinned_until = 0; // 'counter' is bumped by get_next_ip() (a const method) for // round-robin selection; the logical state of the cache record is // unchanged, so mutable is the right tool here and lets us drop a @@ -94,8 +128,23 @@ class DNS_Cache { mutable unsigned long counter = 0; }; - std::string get_next_ip(const IP_ADDR& ip_addr) const; - std::unordered_map records; + struct lookup_result_t { + std::string resolved_ip; + size_t ip_count = 0; + std::string pinned_ip; + unsigned long long pinned_until = 0; + }; + + /** + * @brief Next round-robin IP for 'ip_addr' and the size of the served set. + * + * @param ip_addr Cache record to select from. + * + * @return Selected resolved IP details plus current pin metadata. + */ + lookup_result_t get_next_ip(const IP_ADDR& ip_addr) const; + + mutable std::unordered_map records; std::atomic_bool enabled; mutable pthread_rwlock_t rwlock_; @@ -154,6 +203,16 @@ bool validate_ip(const std::string& ip); // failure / non-IP families. std::string get_connected_peer_ip_from_socket(int socket_fd); +/** +* @brief Resolve a hostname to its IP(s) via getaddrinfo. +* +* @param hostname Hostname to resolve. +* @param ai_family Address family for getaddrinfo (an AF_* value; AF_UNSPEC for OS default). +* +* @return The resolved IPs, or an empty vector on failure. +*/ +std::vector dns_resolve(const std::string& hostname, int ai_family); + // Helper: stringify a list of IPs for debug logging. Defined inline because // it's templated over the iterable type used by the various call sites. template diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index ef77d67429..6edc571723 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -39,8 +39,8 @@ // we have 2 versions of the same tables: with (debug) and without (no debug) checks #ifdef DEBUG -#define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" -#define MYHGM_MYSQL_SERVERS_INCOMING "CREATE TABLE mysql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port))" +#define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4, 5)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define MYHGM_MYSQL_SERVERS_INCOMING "CREATE TABLE mysql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4, 5)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port))" #else #define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT NOT NULL DEFAULT 1 , status INT NOT NULL DEFAULT 0 , compression INT NOT NULL DEFAULT 0 , max_connections INT NOT NULL DEFAULT 1000 , max_replication_lag INT NOT NULL DEFAULT 0 , use_ssl INT NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" #define MYHGM_MYSQL_SERVERS_INCOMING "CREATE TABLE mysql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT NOT NULL DEFAULT 1 , status INT NOT NULL DEFAULT 0 , compression INT NOT NULL DEFAULT 0 , max_connections INT NOT NULL DEFAULT 1000 , max_replication_lag INT NOT NULL DEFAULT 0 , use_ssl INT NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port))" @@ -64,7 +64,21 @@ "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ "comment VARCHAR , UNIQUE (reader_hostgroup))" -#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" +#define MYHGM_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE mysql_aws_rds_bgd_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0), " \ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0), " \ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0), " \ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , " \ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000, " \ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800, " \ + "comment VARCHAR NOT NULL DEFAULT '', " \ + "auto_generated INT CHECK (auto_generated IN (0,1)) NOT NULL DEFAULT 0, " \ + "status INT NOT NULL DEFAULT 0, " \ + "UNIQUE (reader_hostgroup))" + +#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" WHEN 5 THEN \"SHUNNED_AWS_BGD\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" #define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" @@ -73,7 +87,7 @@ /* * @brief Generates the 'runtime_mysql_servers' resultset exposed to other ProxySQL cluster members. - * @details Makes 'SHUNNED' and 'SHUNNED_REPLICATION_LAG' statuses equivalent to 'ONLINE'. 'SHUNNED' states + * @details Makes 'SHUNNED', 'SHUNNED_REPLICATION_LAG' and 'SHUNNED_AWS_BGD' statuses equivalent to 'ONLINE'. 'SHUNNED' states * are by definition local transitory states, this is why a 'mysql_servers' table reconfiguration isn't * normally performed when servers are internally imposed with these statuses. This means, that propagating * this state to other cluster members is undesired behavior, and so it's generating a different checksum, @@ -95,6 +109,7 @@ " WHEN 2 THEN \"OFFLINE_SOFT\"" \ " WHEN 3 THEN \"OFFLINE_HARD\"" \ " WHEN 4 THEN \"ONLINE\" " \ + " WHEN 5 THEN \"ONLINE\" " \ "END status," \ "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ "FROM mysql_servers " \ @@ -105,7 +120,7 @@ * @brief Generates the 'mysql_servers_v2' resultset exposed to other ProxySQL cluster members. * @details The generated resultset is used for the checksum computation of the runtime ProxySQL config * ('mysql_servers_v2' checksum), and it's also forwarded to other cluster members when querying the Admin - * interface with 'CLUSTER_QUERY_MYSQL_SERVERS_V2'. It makes 'SHUNNED' state equivalent to 'ONLINE', and also + * interface with 'CLUSTER_QUERY_MYSQL_SERVERS_V2'. It makes 'SHUNNED' and 'SHUNNED_AWS_BGD' states equivalent to 'ONLINE', and also * filters out any 'OFFLINE_HARD' entries. This is done because none of the statuses are valid configuration * statuses, they are local, transient status that ProxySQL uses during operation. */ @@ -114,6 +129,7 @@ "hostgroup_id, hostname, port, gtid_port, " \ "CASE" \ " WHEN status=\"SHUNNED\" THEN \"ONLINE\"" \ + " WHEN status=\"SHUNNED_AWS_BGD\" THEN \"ONLINE\"" \ " ELSE status " \ "END AS status, " \ "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ @@ -173,6 +189,7 @@ class MySrvConnList { void get_random_MyConn_inner_search(unsigned int start, unsigned int end, unsigned int& conn_found_idx, unsigned int& connection_quality_level, unsigned int& number_of_matching_session_variables, const MySQL_Connection * client_conn); unsigned int conns_length() { return conns->len; } void drop_all_connections(); + void mark_connections_unhealthy(); MySQL_Connection *index(unsigned int); }; @@ -202,6 +219,9 @@ class MySrvC { // MySQL Server Container unsigned long long queries_gtid_sync; unsigned long long bytes_sent; unsigned long long bytes_recv; + // shunned_automatic acts as a guard for server auto-recovery. When true, the shun recovery path + // (MyHGC::get_random_MySrvC) brings the server back online after shun_recovery_time; when false, + // the shun is held until an explicit unshun. bool shunned_automatic; bool shunned_and_kill_all_connections; // if a serious failure is detected, this will cause all connections to die even if the server is just shunned int32_t use_ssl; @@ -513,6 +533,7 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { MYSQL_AWS_AURORA_HOSTGROUPS, MYSQL_HOSTGROUP_ATTRIBUTES, MYSQL_SERVERS_SSL_PARAMS, + MYSQL_AWS_RDS_BGD_HOSTGROUPS, MYSQL_SERVERS, HGM_TABLES_SIZE_ @@ -697,6 +718,17 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { pthread_mutex_t AWS_Aurora_Info_mutex; std::map AWS_Aurora_Info_Map; + /** + * @brief Materializes the runtime `mysql_aws_rds_bgd_hostgroups` table from the staged + * `incoming_aws_rds_bgd_hostgroups` resultset. + * + * @details Inserts each staged row with `auto_generated=0` (config-loaded entries are + * user-defined) and NULL green hostgroups preserved, then clears the staging resultset. + * No-op when nothing is staged. + */ + void generate_mysql_aws_rds_bgd_hostgroups_table(); + SQLite3_result *incoming_aws_rds_bgd_hostgroups; + void generate_mysql_hostgroup_attributes_table(); SQLite3_result *incoming_hostgroup_attributes; @@ -1015,13 +1047,87 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { void replication_lag_action_inner(MyHGC *, const char*, unsigned int, int, bool); void replication_lag_action(const std::list& mysql_servers); -// void read_only_action(char *hostname, int port, int read_only); - void read_only_action_v2(const std::list& mysql_servers); + /** + * @brief Reconcile writer/reader hostgroup placement from read_only monitor results. + * + * @details New implementation of the read_only_action that does not depend on the admin table. + * Checks each server in the provided list and adjusts writer/reader hostgroup placement + * according to the corresponding read_only value. If any change occurs, the runtime + * mysql_servers table and checksum are regenerated. + * + * @param mysql_servers Servers and their observed/read-only state. + * @param ignore_aws_bgd True to apply the result while BGD switchover is in progress. + */ + void read_only_action_v2(const std::list& mysql_servers, bool ignore_aws_bgd = false); unsigned int get_servers_table_version(); void wait_servers_table_version(unsigned, unsigned); bool shun_and_killall(char *hostname, int port); void set_server_current_latency_us(char *hostname, int port, unsigned int _current_latency_us); void set_Readyset_status(char *hostname, int port, enum MySerStatus status); + /** + * @brief Set or clear AWS RDS BGD shun state for a matching server. + * + * @details When shunning, transitions an ONLINE server to SHUNNED_AWS_BGD, + * enables shun metadata, and drops free connections. When unshunning, + * transitions only SHUNNED_AWS_BGD back to ONLINE and clears shun metadata. + * Servers in other statuses are left unchanged. + * + * @param hostgroup_id Hostgroup to search. + * @param hostname Address of the server to match. + * @param port Port of the server to match. + * @param shun true to shun the server, false to unshun it. + * + * @return true if this call changed a server's status. + * + * @note Caller must hold wrlock(). + */ + bool aws_rds_bgd_set_shun_server(unsigned int hostgroup_id, const char *hostname, int port, bool shun); + /** + * @brief Configure the AWS RDS BGD writer's writer/reader hostgroup membership. + * + * @details Ensures the writer is present in its writer hostgroup, with optional reader + * hostgroup membership controlled by writer_is_also_reader. + * + * @param hostname Server hostname to configure. + * @param port Server port to configure. + * @param writer_is_also_reader Whether the writer should also be present in reader hostgroup. + * + * @return true if hostgroup membership changed. + * + * @note Caller must hold wrlock(). + */ + bool aws_rds_bgd_configure_writer(const char *hostname, int port, bool writer_is_also_reader); + /** + * @brief Set AWS RDS BGD switchover status in runtime mysql_aws_rds_bgd_hostgroups table + * + * @param writer_hg Writer hostgroup identifying the deployment. + * @param status AWS_RDS_BGD_Status underlying value. + */ + void aws_rds_bgd_set_runtime_status(unsigned int writer_hg, int status); + /** + * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. + * + * @details One-way alignment (in-memory -> runtime): regenerates the runtime 'mysql_servers' table + * from the current in-memory MyHGM state, recomputes/republishes the global checksum, and refreshes + * 'mysql_servers_to_monitor' for the regular monitor threads. + * + * @note Caller must hold wrlock(). + */ + void publish_mysql_servers_to_runtime(); + /** + * @brief Drain existing backend connections for a server in all hostgroups. + * + * @details Drops free connections immediately and marks used connections as unhealthy and non-reusable, + * so in-flight operations fail on their next backend step and the connection is never pooled again. + * + * @param hostname Address of the server to match. + * @param port Port of the server to match. + * @return true if a matching server was found. + * + * @note Caller must hold wrlock(). + */ + bool drain_server_connections(const char *hostname, int port); + unsigned long long Get_Memory_Stats(); void add_discovered_servers_to_mysql_servers_and_replication_hostgroups(const vector>& new_servers); @@ -1107,6 +1213,26 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * be taken or not. */ void update_aws_aurora_hosts_monitor_resultset(bool lock=false); + /** + * @brief Rebuilds the AWS RDS BGD monitor's host resultset. + * + * @details Rebuilds `GloMyMon->AWS_RDS_BGD_Hosts_resultset` and publishes both the full BGD hosts + * checksum and one checksum per writer hostgroup. + */ + void update_aws_rds_bgd_hosts_monitor_resultset(); + /** + * @brief Auto-generate a runtime `mysql_aws_rds_bgd_hostgroups` entry for a server's writer hostgroup. + * + * @details Called when the read_only monitor detects a blue/green deployment. The writer/reader + * hostgroups are derived from the server's `hostgroup_server_mapping`. Green hostgroups are + * stored NULL with `auto_generated=1`. Idempotent. + * + * @param hostname Hostname of the server that exposed the blue/green topology. + * @param port Port of the server. + * + * @return true if a new entry was added; false otherwise. + */ + bool add_aws_rds_bgd_hostgroup_entry(const std::string& hostname, int port); SQLite3_result * get_stats_mysql_gtid_executed(); void generate_mysql_gtid_executed_tables(); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 9b0f814a44..ad599493a5 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -40,7 +40,7 @@ #define MONITOR_SQLITE_TABLE_MYSQL_SERVER_AWS_AURORA_FAILOVERS "CREATE TABLE mysql_server_aws_aurora_failovers (writer_hostgroup INT NOT NULL , hostname VARCHAR NOT NULL , inserted_at VARCHAR NOT NULL)" -#define MONITOR_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , PRIMARY KEY (hostname, port) )" +#define MONITOR_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL , status INT CHECK (status IN (0, 1, 2, 3, 4, 5)) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , PRIMARY KEY (hostname, port) )" #define MONITOR_SQLITE_TABLE_PROXYSQL_SERVERS "CREATE TABLE proxysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port) )" @@ -60,7 +60,8 @@ struct cmp_str { #define N_L_ASE 16 #define AWS_ENDPOINT_SUFFIX_STRING "rds.amazonaws.com" -#define QUERY_READ_ONLY_AND_AWS_TOPOLOGY_DISCOVERY "SELECT @@global.read_only read_only, id, endpoint, port from mysql.rds_topology" +#define QUERY_AWS_RDS_TOPOLOGY_DISCOVERY "SELECT * FROM mysql.rds_topology" +#define QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK "SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA='mysql' AND TABLE_NAME='rds_topology'" /* @@ -204,12 +205,14 @@ enum MySQL_Monitor_State_Data_Task_Type { MON_REPLICATION_LAG, MON_GALERA, MON_AWS_AURORA, - MON_READ_ONLY__AND__AWS_RDS_TOPOLOGY_DISCOVERY + MON_AWS_RDS_BGD, + MON_AWS_RDS_TOPOLOGY_DISCOVERY }; enum class MySQL_Monitor_State_Data_Task_Result { TASK_RESULT_UNKNOWN, TASK_RESULT_TIMEOUT, + TASK_RESULT_TIMEOUT_STALE_IP, TASK_RESULT_FAILED, TASK_RESULT_SUCCESS, TASK_RESULT_PENDING @@ -296,6 +299,11 @@ class MySQL_Monitor_State_Data { return task_result_; } + inline + const char* get_query() const { + return query_.c_str(); + } + private: std::string query_; unsigned long long task_expiry_time_; // task expiry time (t1 + task_timeout_ * 1000) @@ -386,10 +394,211 @@ struct mon_metrics_map_idx { }; }; -// DNS_Cache, DNS_Cache_Record, DNS_Resolve_Data and the resolver helpers now -// live in DNS_Cache.hpp (included above) so the same machinery can back the -// independent PgSQL_Monitor DNS cache. +/** + * @brief Server hostname and port. + */ +struct srv_addr_t { + std::string host; + int port = 0; +}; + +/** +* @brief State of the per-host RDS topology probe. +*/ +enum RDS_BGD_Topology_Monitor_State { + TOPOLOGY_TABLE_CHECK, ///< verify mysql.rds_topology exists + TOPOLOGY_METADATA_FETCH ///< table confirmed present; fetch and branch on its metadata +}; + +/** + * @brief Column positions in `AWS_RDS_BGD_Hosts_resultset`. + */ +enum AWS_RDS_BGD_Hosts_Column { + AWS_RDS_BGD_HOSTNAME = 0, + AWS_RDS_BGD_PORT, + AWS_RDS_BGD_USE_SSL, + AWS_RDS_BGD_WRITER_HOSTGROUP, + AWS_RDS_BGD_READER_HOSTGROUP, + AWS_RDS_BGD_GREEN_WRITER_HOSTGROUP, + AWS_RDS_BGD_GREEN_READER_HOSTGROUP, + AWS_RDS_BGD_CHECK_INTERVAL_MS, + AWS_RDS_BGD_CHECK_TIMEOUT_MS, + AWS_RDS_BGD_WRITER_IS_ALSO_READER, + AWS_RDS_BGD_SRV_TYPE, + AWS_RDS_BGD_IS_WRITER, + AWS_RDS_BGD_HOSTS_COLUMNS +}; + +/** + * @brief Switchover phase for an RDS blue/green deployment. + * + * @details AWS's mysql.rds_topology status only captures the writer switchover. As of 2026/07/03 + * the table exposes no read-replica switchover status; ProxySQL infers that the replicas have + * switched over from the table draining to empty (or disappearing) after it last reported + * SWITCHOVER_COMPLETED. + * + * Observed table lifecycle across one switchover: + * - Steady state: two rows (SOURCE = blue, TARGET = green), both AVAILABLE. + * - Switching: both rows step through SWITCHOVER_INITIATED -> _IN_PROGRESS -> _IN_POST_PROCESSING. + * - Writer done: the SOURCE row drops; a lone TARGET row reports SWITCHOVER_COMPLETED. + * - Replicas done: the table drains to empty (blue-reader DNS has propagated). + * + * The WRITER_SWITCHOVER_* values map 1:1 onto the mysql.rds_topology status strings. + * READER_SWITCHOVER_IN_PROGRESS is a ProxySQL inferred status entered after + * WRITER_SWITCHOVER_COMPLETED; it defers reader/DNS cleanup until the topology table drains + * to empty. SWITCHOVER_COMPLETED is a short-lived status used for final cleanup before + * returning to NONE. + */ +enum class AWS_RDS_BGD_Status { + NONE = 0, ///< no BGD topology / baseline + AVAILABLE = 1, ///< "AVAILABLE" + WRITER_SWITCHOVER_INITIATED = 2, ///< "SWITCHOVER_INITIATED" + WRITER_SWITCHOVER_IN_PROGRESS = 3, ///< "SWITCHOVER_IN_PROGRESS" + WRITER_SWITCHOVER_POST_PROCESSING = 4, ///< "SWITCHOVER_IN_POST_PROCESSING" + WRITER_SWITCHOVER_COMPLETED = 5, ///< "SWITCHOVER_COMPLETED" + READER_SWITCHOVER_IN_PROGRESS = 6, ///< ProxySQL inferred status; awaiting topology drain + deferred cleanup + SWITCHOVER_COMPLETED = 7, ///< short-lived status used for final cleanup before returning to NONE +}; + +enum class AWS_RDS_BGD_Server_Status { + NONE = 0, + IN_PROGRESS = 1 +}; + +// AWS RDS blue/green role and switchover-status column values (mysql.rds_topology). +inline const char* const BGD_ROLE_SOURCE = "BLUE_GREEN_DEPLOYMENT_SOURCE"; // blue +inline const char* const BGD_ROLE_TARGET = "BLUE_GREEN_DEPLOYMENT_TARGET"; // green +inline const char* const BGD_STATUS_AVAILABLE = "AVAILABLE"; +inline const char* const BGD_STATUS_INITIATED = "SWITCHOVER_INITIATED"; +inline const char* const BGD_STATUS_IN_PROGRESS = "SWITCHOVER_IN_PROGRESS"; +inline const char* const BGD_STATUS_POST_PROC = "SWITCHOVER_IN_POST_PROCESSING"; +inline const char* const BGD_STATUS_COMPLETED = "SWITCHOVER_COMPLETED"; + +/** +* @brief BGD Monitor state for one AWS RDS BGD worker. +*/ +struct AWS_RDS_BGD_Worker { + int writer_hg = 0; + pthread_t thread {}; + std::atomic_bool worker_stop {false}; + std::atomic current_checksum {0}; +}; + +/** + * @brief A single node (row) of a 'SELECT * FROM mysql.rds_topology' result. + */ +struct AWS_RDS_Topology_Node { + std::string id; + std::string endpoint; + int port = 0; + std::string role; ///< empty when the column is absent or NULL + std::string status; ///< empty when the column is absent or NULL +}; + +/** + * @brief Parsed representation of a 'SELECT * FROM mysql.rds_topology' result, + * shared by the read_only monitor's discovery path and the AWS RDS BGD + * monitor thread. + */ +class AWS_RDS_Topology_Result { +public: + bool blue_green = false; ///< 'role' and 'status' present AND non-NULL + std::vector nodes; + + /** + * @brief Find the blue/green deployment TARGET node. + * + * @return The TARGET node, or nullptr when it is not present. + */ + AWS_RDS_Topology_Node* target() { + for (AWS_RDS_Topology_Node& node : nodes) { + if (strcasecmp(node.role.c_str(), BGD_ROLE_TARGET) == 0) { + return &node; + } + } + return nullptr; + } +}; + +/** + * @brief Mapping between one blue host and its name-matched green counterpart. + * + * @details The RDS BGD worker builds these pairs from the current blue + * writer/reader hostgroups and the discovered green topology. Each entry + * carries the blue server attributes needed to move the matching green + * server during switchover handling. + */ +struct AWS_RDS_BlueGreenPair { + std::string blue_host; ///< Blue hostname from the writer or reader hostgroup. + std::string green_host; ///< Matched green hostname using the RDS "-green-" naming pattern. + int port = 0; ///< Shared blue/green port; hostgroup manager keys servers by host and port. + int64_t blue_weight = 1; ///< Blue server weight mirrored onto the green server when it is added. + int64_t blue_max_conns = 1000; ///< Blue server max_connections mirrored onto the green server when it is added. + int32_t blue_use_ssl = 0; ///< Blue server SSL setting mirrored onto the green server when it is added. + int32_t green_use_ssl = -1; ///< Green server SSL; -1 means unset (use blue_use_ssl). + bool green_offline = false; ///< True when the configured green writer is OFFLINE_SOFT/OFFLINE_HARD. + std::string green_ip; ///< Green host IP resolved at SWITCHOVER_INITIATED and held warm. + unsigned long long green_ip_ttl = 0; ///< Expiry for green_ip when resolved by the BGD thread; 0 means DNS_Cache-sourced. + bool green_ip_pinned = false; ///< True after green_ip has been pinned and blue_host connections drained/purged. + bool is_writer = false; ///< True when this pair maps the blue writer. +}; + +/** + * @brief Host used by a BGD worker to probe `mysql.rds_topology`. + */ +struct AWS_RDS_BGD_Probe_Host { + std::string hostname; + int port = 0; + int use_ssl = 0; +}; + +/** + * @brief Switchover state carried by RDS BGD worker thread. + * + * @details One worker (monitor_RDS_BGD_thread_HG) owns one writer hostgroup == + * one blue/green deployment, so this struct lives on the worker's stack and is + * single-owner (no locking on the struct itself). It is passed by reference to + * handle_aws_rds_bgd, which runs the status-driven switchover FSM and mutates it + * across poll cycles. Config-derived fields can be refreshed in place; the rest + * carries resolved IPs and state for switchover actions and cleanup. + */ +struct AWS_RDS_BGD_State { + unsigned int writer_hg = 0; ///< blue/current writer hostgroup + unsigned int reader_hg = 0; ///< blue/current reader hostgroup + int green_writer_hg = -1; ///< -1 when NULL (auto-discovery path) + int green_reader_hg = -1; ///< -1 when NULL + int writer_is_also_reader = 0; ///< drives post-switchover writer cleanup + unsigned int check_interval_ms = 0; ///< configured baseline check interval + unsigned int check_timeout_ms = 0; ///< configured topology-check timeout + + std::vector bg_map; ///< [writer] always; [readers] only when green_reader_hg is configured + std::vector probe_hosts; ///< hosts eligible for topology probes + + std::vector shunned_readers; ///< readers we shunned + std::vector read_only_check_disabled; ///< servers whose read_only checks this worker disabled + AWS_RDS_BGD_Status bgd_status = AWS_RDS_BGD_Status::NONE; ///< drives the FSM and the deferred cleanup + + bool bgd_in_progress_set = false; ///< deployment's servers flagged in aws_rds_bgd_server_status + bool config_refresh_pending = false; ///< bg_map must be rebuilt from the next topology result + + unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline + std::string next_check_host; ///< FSM-pinned probe host; when set (the green IP), the worker + ///< polls it directly instead of selecting among the blue hosts + unsigned int next_check_host_failures = 0; ///< consecutive failures polling next_check_host; clears it after 3 +}; + +// Maps a switchover status enum to its stored/display string. +const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s); +// read_only monitor server-enumeration query. +// Every server that belongs to a replication hostgroup and status NOT IN (OFFLINE_SOFT, OFFLINE_HARD) +#define SELECT_SERVERS_FOR_READ_ONLY "SELECT hostname, port, MAX(use_ssl) use_ssl, check_type, reader_hostgroup FROM mysql_servers JOIN mysql_replication_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE status NOT IN (2,3) GROUP BY hostname, port ORDER BY RANDOM()" + +// Defined in MySQL_HostGroups_Manager.h; forward-declared here because the include cycle +// (Monitor.hpp -> HGM.h -> cpp.h -> Monitor.hpp) can leave them undefined at this point. Only +// used below via pointer, so a forward declaration is sufficient. +struct srv_info_t; +struct srv_opts_t; class MySQL_Monitor { public: @@ -397,9 +606,35 @@ class MySQL_Monitor { static std::string dns_lookup(const char* hostname, bool return_hostname_if_lookup_fails = true, size_t* ip_count = nullptr); static bool update_dns_cache_from_mysql_conn(const MYSQL* mysql); static void trigger_dns_cache_update(); + bool timeout_validate_ip_change(const MySQL_Monitor_State_Data* mmsd) const; - void process_discovered_topology(const std::string& originating_server_hostname, const vector& discovered_servers, int reader_hostgroup); - bool is_aws_rds_multi_az_db_cluster_topology(const std::vector& discovered_servers); + /** + * @brief Classify the parsed mysql.rds_topology result and dispatch. + * + * @details A blue/green deployment optionally auto-generates a runtime aws_rds_bgd_hostgroups + * entry (when 'mysql-aws_blue_green_deployment_auto_discovery' is enabled); otherwise the rows + * are treated as a Multi-AZ Cluster and handed to the existing auto-discovery path. + */ + void process_aws_rds_topology(MySQL_Monitor_State_Data* mmsd); + /** + * @brief Parse a 'SELECT * FROM mysql.rds_topology' result into an AWS_RDS_Topology_Result. + * + * @details Columns are resolved by name (they may be absent or differently ordered by RDS type). + * 'blue_green' is set when the 'role'/'status' columns are present and non-NULL on the first row. + * + * @return The parsed topology; empty 'nodes' if 'result' is NULL or has no rows. The result cursor is rewound before returning. + */ + AWS_RDS_Topology_Result parse_aws_rds_topology(MYSQL_RES* result); + /** + * @brief Processes the discovered servers to eventually add them to 'runtime_mysql_servers'. + * + * @details This method takes a vector of discovered servers, compares them against the existing servers, and adds the new servers to 'runtime_mysql_servers'. + * + * @param origin_server A string which denotes the hostname of the originating server, from which the discovered servers were queried and found. + * @param discovered_servers A vector of servers discovered when querying the cluster's topology. + * @param reader_hostgroup Reader hostgroup to which we will add the discovered servers. + */ + void handle_aws_rds_multi_az_cluster(const std::string& origin_server, const std::vector& discovered_servers, int reader_hostgroup); private: std::vector *tables_defs_monitor; @@ -413,6 +648,8 @@ class MySQL_Monitor { pthread_mutex_t group_replication_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t galera_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t aws_aurora_mutex; // for simplicity, a mutex instead of a rwlock + pthread_mutex_t aws_rds_bgd_mutex; + pthread_mutex_t aws_rds_bgd_hosts_mutex; pthread_mutex_t mysql_servers_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t proxysql_servers_mutex; //std::map Group_Replication_Hosts_Map; @@ -423,6 +660,10 @@ class MySQL_Monitor { std::map AWS_Aurora_Hosts_Map; SQLite3_result *AWS_Aurora_Hosts_resultset; uint64_t AWS_Aurora_Hosts_resultset_checksum; + std::unordered_map aws_rds_bgd_server_status; + std::shared_ptr AWS_RDS_BGD_Hosts_resultset; + uint64_t AWS_RDS_BGD_Hosts_checksum; + std::unordered_map AWS_RDS_BGD_Cluster_checksum; unsigned int num_threads; unsigned int aux_threads; unsigned int started_threads; @@ -470,6 +711,139 @@ class MySQL_Monitor { void * monitor_group_replication_2(); void * monitor_galera(); void * monitor_aws_aurora(); + /** + * @brief AWS RDS BGD monitor thread entry point. + * + * @details Maintains one worker (monitor_RDS_BGD_thread_HG) per active writer hostgroup. The parent starts + * and stops workers and signals configuration changes. Each worker selects a pingable probe host, + * probes 'mysql.rds_topology', and runs the switchover state machine. + */ + void * monitor_aws_rds_bgd(); + /** + * @brief Run an asynchronous query and store its result on a BGD monitor connection. + * + * @param mmsd Monitor state data holding the connection, timing, and result. + * @param query SQL text to execute. + * @param worker_stop Per-worker shutdown signal. + * + * @return 0 on success, 1 on timeout or query error, and 2 when shutdown is requested. + */ + int aws_rds_bgd_async_query( + MySQL_Monitor_State_Data* mmsd, const char* query, std::atomic_bool& worker_stop); + /** + * @brief Apply changed configuration to one running BGD worker. + * + * @details Before writer post-processing, applies the configuration and schedules mapping + * reconciliation after the next topology poll. At or after post-processing, rolls back the + * deployment and restarts its topology state machine. + * + * @param st Worker-owned BGD state. + * @param current_checksum Per-cluster checksum captured for this refresh. + * @param topology_state Current topology query state. + * @param next_loop_at Next scheduled worker iteration. + * + * @return true when the configuration was applied; false when it must be retried. + */ + bool aws_rds_bgd_refresh_worker_config( + AWS_RDS_BGD_State& st, uint64_t current_checksum, + RDS_BGD_Topology_Monitor_State& topology_state, unsigned long long& next_loop_at); + /** + * @brief Run the status-driven blue/green switchover FSM for one deployment. + * + * @details Invoked each poll cycle by the BGD worker after it fetches the + * mysql.rds_topology result. Dispatches on the deployment's switchover status + * (AVAILABLE -> SWITCHOVER_INITIATED -> IN_PROGRESS -> IN_POST_PROCESSING -> + * COMPLETED): builds the blue<->green map, pre-resolves green IPs, repoints the + * blue hostnames onto the green IPs in the DNS cache, drains blue free + * connections, and shuns/enforces reader handling. State carried across cycles + * lives in @p st. + * + * @param st BGD switchover state. + * @param topology Parsed mysql.rds_topology result for this cycle. + */ + void handle_aws_rds_bgd(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topology); + /** + * @brief Pin green IPs and drain existing blue-host connections. + * + * @param st BGD switchover state. + */ + void aws_rds_bgd_pin_green_ips(AWS_RDS_BGD_State& st); + /** + * @brief Run deferred switchover teardown or rollback cleanup. + * + * @details Restores post-switchover reader handling, unshuns readers, drops DNS pins, + * and clears BGD switchover state. Normal post-switchover cleanup also drains + * connections from green hosts; rollback leaves green rows and connections unchanged. + * + * When rollback is false (normal post-switchover), the caller must be in + * READER_SWITCHOVER_IN_PROGRESS; the function advances through + * SWITCHOVER_COMPLETED before clearing to NONE. + * + * When rollback is true (topology table disappeared or worker exit mid-switchover), + * the function accepts any non-NONE bgd_status, restores the blue writer to the + * writer hostgroup if it was demoted, then resets switchover state. + * + * @param st BGD switchover state. + * @param rollback True if called due to a rollback/cancellation, false for normal completion. + */ + void handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bool rollback = false); + /** + * @brief Drain connections from green hosts after switchover. + * + * @details Drains connections from every green host that is neither OFFLINE_SOFT nor + * OFFLINE_HARD. Server rows and statuses are left unchanged. + * + * @param st Switchover state. + */ + void aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st); + /** + * @brief Handle an absent, empty, or vanished mysql.rds_topology table. + * + * @details Routes to deferred cleanup when bgd_status is READER_SWITCHOVER_IN_PROGRESS; + * otherwise clears any in-progress switchover state for this deployment. + * + * @param st BGD switchover state. + */ + void aws_rds_bgd_handle_topology_absent(AWS_RDS_BGD_State& st); + /** + * @brief Apply BGD hostgroup changes for the current switchover status. + * + * @details POST_PROCESSING configures the writer placement and shuns unmapped readers. + * SWITCHOVER_COMPLETED unshuns readers and removes the writer from reader HG when + * writer_is_also_reader is false. Runtime mysql_servers and checksum are re-generated + * when server hostgroup membership changes. + * + * @param bgd_status Current BGD FSM status driving the action. + * @param writer Writer server to configure. + * @param writer_is_also_reader Whether the writer should also remain in reader_hg. + * @param reader_hg Reader hostgroup for reader shun/unshun and optional writer membership. + * @param readers Reader servers to shun or unshun. + */ + void aws_rds_bgd_hostgroup_action( + AWS_RDS_BGD_Status bgd_status, + srv_addr_t& writer, bool writer_is_also_reader, + unsigned int reader_hg, std::vector& readers); + /** + * @brief Check whether a server is flagged as BGD switchover-in-progress. + * + * @param hostname Server hostname. + * @param port Server port. + * + * @return true if the server is flagged IN_PROGRESS. + */ + bool is_aws_rds_bgd_server_in_progress(const std::string& hostname, int port); + /** + * @brief Flag/unflag every server in BGD hostgroups as switchover-in-progress. + * + * @details Called by the BGD worker at switchover initiation (INITIATED / IN_PROGRESS / + * POST_PROCESSING) and cleared after SWITCHOVER_COMPLETED. Saves the marked servers in the + * worker state so cleanup does not depend on the current hostgroup configuration. + * + * @param st BGD worker state. + * @param in_progress true to flag servers, false to clear. + */ + void set_aws_rds_bgd_server_in_progress(AWS_RDS_BGD_State& st, bool in_progress); + void * monitor_replication_lag(); void * monitor_dns_cache(); void * run(); @@ -500,6 +874,64 @@ class MySQL_Monitor { void monitor_gr_async_actions_handler(const vector>& mmsds); private: + /** + * @brief Load one BGD worker's configuration from the published host rows. + * + * @details Copies the cluster rows, verifies their checksum, copies configuration fields from + * the first row, and builds the probe host list. + * + * @param writer_hg Writer hostgroup identifying the deployment. + * @param current_checksum Per-cluster checksum captured for this refresh. + * @param candidate State populated from the published rows. + * + * @return true when the checksum matches and the rows contain a probe host. + */ + bool aws_rds_bgd_load_worker_config(int writer_hg, uint64_t current_checksum, AWS_RDS_BGD_State& candidate); + /** + * @brief Replace the configuration-derived fields in a live BGD worker state. + * + * @param st Live worker state. + * @param candidate Parsed configuration to apply. + */ + void aws_rds_bgd_apply_cluster_config(AWS_RDS_BGD_State& st, AWS_RDS_BGD_State& candidate); + /** + * @brief Rebuild the mapping and reconcile writer state after a configuration refresh. + * + * @details Called only when config_refresh_pending is set. + * + * @param st Worker-owned BGD state. + * @param topology Fresh topology used to rebuild the mapping. + */ + void aws_rds_bgd_config_refresh_action(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topology); + /** + * @brief Build the blue-to-green host mapping for a BGD worker. + * + * @param st Worker-owned BGD state. + * @param topology Parsed topology used to identify the green target. + */ + void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topology); + /** + * @brief Resolve green host IPs and select the next topology probe host. + * + * @param st Worker-owned BGD state. + */ + void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st); + /** + * @brief Add the green writer to its configured hostgroup. + * + * @param st Worker-owned BGD state. + */ + void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st); + /** + * @brief Find the writer pair in a blue-to-green host mapping. + * + * @param bg_map Host mapping to inspect. + * @param writer Writer address populated when a pair is found. + * + * @return true when the map contains a writer pair. + */ + bool aws_rds_bgd_find_writer(std::vector& bg_map, srv_addr_t& writer); + /** * @brief Handling of monitor tasks asyncronously * @details Basic workflow is same for all monitor_*_async methods: diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index b3faa621c2..47b6de5222 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -455,6 +455,8 @@ class MySQL_Threads_Handler int monitor_ping_timeout; //! Monitor aws rds topology discovery interval. Unit: 'one discovery check per X monitor_read_only checks'. int monitor_aws_rds_topology_discovery_interval; + //! Auto-generate runtime aws_rds_bgd_hostgroups entries when the read_only monitor detects a blue/green deployment. + int aws_blue_green_deployment_auto_discovery; //! Monitor read only timeout. Unit: 'ms'. int monitor_read_only_interval; //! Monitor read only timeout. Unit: 'ms'. diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index 5be64d5c49..4514853bd0 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -147,7 +147,7 @@ #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS "CREATE TABLE runtime_mysql_firewall_whitelist_sqli_fingerprints (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , fingerprint VARCHAR NOT NULL , PRIMARY KEY (fingerprint) )" -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS "CREATE TABLE runtime_mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , gtid_port INT CHECK ((gtid_port <> port OR gtid_port=0) AND gtid_port >= 0 AND gtid_port <= 65535) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS "CREATE TABLE runtime_mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , gtid_port INT CHECK ((gtid_port <> port OR gtid_port=0) AND gtid_port >= 0 AND gtid_port <= 65535) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','SHUNNED_AWS_BGD','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS_SSL_PARAMS "CREATE TABLE runtime_mysql_servers_ssl_params (hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , username VARCHAR NOT NULL DEFAULT '' , ssl_ca VARCHAR NOT NULL DEFAULT '' , ssl_cert VARCHAR NOT NULL DEFAULT '' , ssl_key VARCHAR NOT NULL DEFAULT '' , ssl_capath VARCHAR NOT NULL DEFAULT '' , ssl_crl VARCHAR NOT NULL DEFAULT '' , ssl_crlpath VARCHAR NOT NULL DEFAULT '' , ssl_cipher VARCHAR NOT NULL DEFAULT '' , tls_version VARCHAR NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port, username) )" @@ -239,6 +239,33 @@ #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup))" +// AWS RDS hostgroups; adds blue/green (green_*_hostgroup) over aurora. +// The runtime table carries one extra runtime-only column: auto_generated. +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE mysql_aws_rds_bgd_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ + "green_writer_hostgroup INT NOT NULL CHECK (green_writer_hostgroup>=0) , " \ + "green_reader_hostgroup INT NOT NULL CHECK (green_reader_hostgroup>=0) , " \ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , " \ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , " \ + "comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (reader_hostgroup))" + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_rds_bgd_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , " \ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , " \ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , "\ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , " \ + "comment VARCHAR NOT NULL DEFAULT '' , " \ + "auto_generated INT CHECK (auto_generated IN (0,1)) NOT NULL DEFAULT 0 , " \ + "status VARCHAR NOT NULL DEFAULT 'NONE' , "\ + "UNIQUE (reader_hostgroup))" + #define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" #define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_2 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index d81485151f..eaccc4bbb6 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -72,6 +72,9 @@ /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_aurora_hostgroups'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup" +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_rds_bgd_hostgroups'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_AWS_RDS_BGD "PROXY_SELECT writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, writer_is_also_reader, check_interval_ms, check_timeout_ms, comment, auto_generated, status FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE auto_generated=0 ORDER BY writer_hostgroup" + /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_galera_hostgroups'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_GALERA "PROXY_SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_galera_hostgroups ORDER BY writer_hostgroup" @@ -446,6 +449,8 @@ struct p_cluster_counter { pulled_mysql_servers_hostgroup_attributes_failure, pulled_mysql_servers_ssl_params_success, pulled_mysql_servers_ssl_params_failure, + pulled_mysql_servers_aws_rds_bgd_hostgroups_success, + pulled_mysql_servers_aws_rds_bgd_hostgroups_failure, pulled_mysql_servers_runtime_checks_success, pulled_mysql_servers_runtime_checks_failure, diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index 09fe3f9bc0..a2ceaabe2a 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -51,15 +51,18 @@ class SQLite3_Server { std::unordered_map grouprep_map; std::vector *tables_defs_grouprep; #endif // TEST_GROUPREP -#ifdef TEST_READONLY - std::unordered_map readonly_map; +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) std::vector *tables_defs_readonly; -#endif // TEST_READONLY + std::unordered_map readonly_map; +#endif // TEST_READONLY || TEST_RDS_BGD +#ifdef TEST_RDS_BGD + std::vector *tables_defs_rds_bgd; +#endif // TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG std::unordered_map> replicationlag_map; std::vector* tables_defs_replicationlag; #endif // TEST_REPLICATIONLAG -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) void insert_into_tables_defs(std::vector *, const char *table_name, const char *table_def); void drop_tables_defs(std::vector *tables_defs); void check_and_build_standard_tables(SQLite3DB *db, std::vector *tables_defs); @@ -94,14 +97,14 @@ class SQLite3_Server { void init_grouprep_ifaces_string(std::string& s); group_rep_status grouprep_test_value(const std::string& srv_addr); #endif // TEST_GROUPREP -#ifdef TEST_READONLY +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) pthread_mutex_t test_readonly_mutex; void load_readonly_table(MySQL_Session *sess); int readonly_test_value(char *p); int readonly_map_size() { return readonly_map.size(); } -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG pthread_mutex_t test_replicationlag_mutex; void load_replicationlag_table(MySQL_Session* sess); @@ -122,5 +125,6 @@ class SQLite3_Server { void wrunlock(); void send_MySQL_OK(MySQL_Protocol *myprot, char *msg, int rows=0, uint16_t status=2); void send_MySQL_ERR(MySQL_Protocol *myprot, char *msg); + void send_MySQL_ERR(MySQL_Protocol *myprot, uint16_t error_code, const char *msg); }; #endif // CLASS_PROXYSQL_SQLITE3_SERVER_H diff --git a/include/ServerSelection.h b/include/ServerSelection.h index 58a8352461..606381c43f 100644 --- a/include/ServerSelection.h +++ b/include/ServerSelection.h @@ -25,7 +25,8 @@ enum ServerSelectionStatus { SERVER_SHUNNED = 1, SERVER_OFFLINE_SOFT = 2, SERVER_OFFLINE_HARD = 3, - SERVER_SHUNNED_REPLICATION_LAG = 4 + SERVER_SHUNNED_REPLICATION_LAG = 4, + SERVER_SHUNNED_AWS_BGD = 5 }; /** diff --git a/include/mysql_connection.h b/include/mysql_connection.h index 25588e6d35..0e34da620a 100644 --- a/include/mysql_connection.h +++ b/include/mysql_connection.h @@ -166,6 +166,7 @@ class MySQL_Connection { my_bool ret_bool; bool async_fetch_row_start; bool send_quit; + bool healthy; bool reusable; bool processing_multi_statement; bool multiplex_delayed; diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 5ef4c980ca..a4b947bfb4 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -160,10 +160,11 @@ struct incoming_servers_t { SQLite3_result* incoming_aurora_hostgroups = NULL; SQLite3_result* incoming_hostgroup_attributes = NULL; SQLite3_result* incoming_mysql_servers_ssl_params = NULL; + SQLite3_result* incoming_aws_rds_bgd_hostgroups = NULL; SQLite3_result* runtime_mysql_servers = NULL; incoming_servers_t(); - incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); + incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); }; // Separate structs for runtime mysql server and mysql server v2 to avoid human error diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index c450dfb5c1..2a8cf047ef 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -19,7 +19,8 @@ enum MySerStatus { MYSQL_SERVER_STATUS_SHUNNED, MYSQL_SERVER_STATUS_OFFLINE_SOFT, MYSQL_SERVER_STATUS_OFFLINE_HARD, - MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG + MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG, + MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD }; enum log_event_type { @@ -1404,6 +1405,7 @@ __thread int mysql_thread___monitor_ping_interval; __thread int mysql_thread___monitor_ping_max_failures; __thread int mysql_thread___monitor_ping_timeout; __thread int mysql_thread___monitor_aws_rds_topology_discovery_interval; +__thread int mysql_thread___aws_blue_green_deployment_auto_discovery; __thread int mysql_thread___monitor_read_only_interval; __thread int mysql_thread___monitor_read_only_timeout; __thread int mysql_thread___monitor_read_only_max_timeout_count; @@ -1758,6 +1760,7 @@ extern __thread int mysql_thread___monitor_ping_interval; extern __thread int mysql_thread___monitor_ping_max_failures; extern __thread int mysql_thread___monitor_ping_timeout; extern __thread int mysql_thread___monitor_aws_rds_topology_discovery_interval; +extern __thread int mysql_thread___aws_blue_green_deployment_auto_discovery; extern __thread int mysql_thread___monitor_read_only_interval; extern __thread int mysql_thread___monitor_read_only_timeout; extern __thread int mysql_thread___monitor_read_only_max_timeout_count; diff --git a/include/proxysql_utils.h b/include/proxysql_utils.h index 7937635f76..24bb05bf02 100644 --- a/include/proxysql_utils.h +++ b/include/proxysql_utils.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,7 @@ #include #include +#include "mysql.h" #include "../deps/json/json.hpp" #ifndef ProxySQL_Checksum_Value_LENGTH @@ -424,4 +426,27 @@ static inline bool wait_for_glo_mth() { return false; } +/** + * @brief Pretty-print a MySQL result set into a string. + * + * @details Formats the full buffered result set as an ASCII table. The current row cursor is preserved: + * the function seeks to the first row for formatting and restores the original cursor before returning. + * + * @param result MySQL result set to format. + * + * @return Pretty-printed result set, or an empty string if the result is NULL or has no fields. + */ +std::string mysql_result_to_string(MYSQL_RES* result); + +/** + * @brief Pretty-print a MySQL result set to a file stream. + * + * @details Uses mysql_result_to_string() for formatting and writes the resulting string to the supplied + * file stream. The result set row cursor is preserved. + * + * @param file Destination file stream. + * @param result MySQL result set to format. + */ +void dump_mysql_result(FILE* file, MYSQL_RES* result); + #endif diff --git a/lib/Admin_Bootstrap.cpp b/lib/Admin_Bootstrap.cpp index d1a0448ae1..5290a54e15 100644 --- a/lib/Admin_Bootstrap.cpp +++ b/lib/Admin_Bootstrap.cpp @@ -750,6 +750,8 @@ bool ProxySQL_Admin::init(const bootstrap_info_t& bootstrap_info) { insert_into_tables_defs(tables_defs_admin,"runtime_mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"mysql_aws_rds_bgd_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_aws_rds_bgd_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_BGD_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES); insert_into_tables_defs(tables_defs_admin,"mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); @@ -834,6 +836,7 @@ bool ProxySQL_Admin::init(const bootstrap_info_t& bootstrap_info) { insert_into_tables_defs(tables_defs_config,"mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_config,"mysql_aws_rds_bgd_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); insert_into_tables_defs(tables_defs_config,"mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); insert_into_tables_defs(tables_defs_config,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); diff --git a/lib/Admin_Handler.cpp b/lib/Admin_Handler.cpp index 7ee304624c..8946c6d388 100644 --- a/lib/Admin_Handler.cpp +++ b/lib/Admin_Handler.cpp @@ -3206,6 +3206,8 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { tn = "mysql_hostgroup_attributes"; } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS_SSL_PARAMS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS_SSL_PARAMS))) { tn = "mysql_servers_ssl_params"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_AWS_RDS_BGD, query_no_space, strlen(CLUSTER_QUERY_MYSQL_AWS_RDS_BGD))) { + tn = "mysql_aws_rds_bgd_hostgroups"; } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS_V2, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS_V2))) { tn = "mysql_servers_v2"; } @@ -4476,6 +4478,15 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { tablename=(char *)"MYSQL AURORA HOSTGROUPS"; SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); } + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL RDS BGD HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL RDS BGD HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL RDS BGD HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL RDS BGD HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL RDS BGD HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL RDS BGD HOSTGROUPS", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_aws_rds_bgd_hostgroups ORDER BY writer_hostgroup"; + tablename=(char *)"MYSQL RDS BGD HOSTGROUPS"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space))) || (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MEM MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space))) diff --git a/lib/Base_HostGroups_Manager.cpp b/lib/Base_HostGroups_Manager.cpp index 7b15284b1e..06cef14d9f 100644 --- a/lib/Base_HostGroups_Manager.cpp +++ b/lib/Base_HostGroups_Manager.cpp @@ -1778,6 +1778,9 @@ void MySQL_HostGroups_Manager::generate_mysql_servers_table(int *_onlyhg) { case 4: st=(char *)"SHUNNED"; break; + case 5: + st=(char *)"SHUNNED_AWS_BGD"; + break; } fprintf(stderr,"HID: %d , address: %s , port: %d , gtid_port: %d , weight: %ld , status: %s , max_connections: %ld , max_replication_lag: %u , use_ssl: %u , max_latency_ms: %u , comment: %s\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, mysrvc->gtid_port, mysrvc->weight, st, mysrvc->max_connections, mysrvc->max_replication_lag, mysrvc->use_ssl, mysrvc->max_latency_us*1000, mysrvc->comment); } @@ -3140,6 +3143,9 @@ SQLite3_result * MySQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset, int case 4: pta[3]=strdup("SHUNNED_REPLICATION_LAG"); break; + case 5: + pta[3]=strdup("SHUNNED_AWS_BGD"); + break; default: // LCOV_EXCL_START assert(0); diff --git a/lib/Base_Session.cpp b/lib/Base_Session.cpp index 6a8a956e70..7e6e980cac 100644 --- a/lib/Base_Session.cpp +++ b/lib/Base_Session.cpp @@ -511,7 +511,7 @@ void Base_Session::housekeeping_before_pkts() { DS * myds = mybe->server_myds; if constexpr (std::is_same_v) { if (mysql_thread___autocommit_false_not_reusable && myds->myconn->IsAutoCommit() == false) { - if (mysql_thread___reset_connection_algorithm == 2) { + if (mysql_thread___reset_connection_algorithm == 2 && myds->myconn->healthy) { create_new_session_and_reset_connection(myds); } else { myds->destroy_MySQL_Connection_From_Pool(true); diff --git a/lib/DNS_Cache.cpp b/lib/DNS_Cache.cpp index 689e0a0279..8face0834c 100644 --- a/lib/DNS_Cache.cpp +++ b/lib/DNS_Cache.cpp @@ -61,12 +61,18 @@ std::string get_connected_peer_ip_from_socket(int socket_fd) { return result; } -void* monitor_dns_resolver_thread(const std::vector& dns_resolve_data_list) { - assert(!dns_resolve_data_list.empty()); - DNS_Resolve_Data* dns_resolve_data = dns_resolve_data_list.front(); +/** +* @brief Resolve a hostname to its IP(s) via getaddrinfo. +* +* @param hostname Hostname to resolve. +* @param ai_family Address family for getaddrinfo (an AF_* value; AF_UNSPEC for OS default). +* +* @return The resolved IPs, or an empty vector on failure. +*/ +std::vector dns_resolve(const std::string& hostname, int ai_family) { + std::vector ips; struct addrinfo hints, *res = NULL; - memset(&hints, 0, sizeof(hints)); hints.ai_protocol = IPPROTO_TCP; hints.ai_socktype = SOCK_STREAM; @@ -76,86 +82,79 @@ void* monitor_dns_resolver_thread(const std::vector& dns_reso // purpose. Useful on IPv4-only hosts so getaddrinfo() doesn't return IPv6 // addresses that connect/bind would always fail on. hints.ai_flags = AI_ADDRCONFIG; - hints.ai_family = dns_resolve_data->ai_family; - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, - "Resolving hostname:[%s] to its mapped IP address.\n", - dns_resolve_data->hostname.c_str()); - int gai_rc = getaddrinfo(dns_resolve_data->hostname.c_str(), NULL, &hints, &res); + hints.ai_family = ai_family; + int gai_rc = getaddrinfo(hostname.c_str(), NULL, &hints, &res); if (gai_rc != 0 || !res) { - proxy_error("An error occurred while resolving hostname: %s [%d]\n", - dns_resolve_data->hostname.c_str(), gai_rc); - goto __error; + proxy_error("An error occurred while resolving hostname: %s [%d]\n", hostname.c_str(), gai_rc); + return ips; } - try { - std::vector ips; - ips.reserve(64); + char ip_addr[INET6_ADDRSTRLEN]; + for (auto p = res; p != NULL; p = p->ai_next) { + if (p->ai_family == AF_INET) { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)p->ai_addr; + inet_ntop(p->ai_addr->sa_family, &ipv4->sin_addr, ip_addr, INET_ADDRSTRLEN); + ips.push_back(ip_addr); + } + else { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)p->ai_addr; + inet_ntop(p->ai_addr->sa_family, &ipv6->sin6_addr, ip_addr, INET6_ADDRSTRLEN); + ips.push_back(ip_addr); + } + } - char ip_addr[INET6_ADDRSTRLEN]; + freeaddrinfo(res); + return ips; +} - for (auto p = res; p != NULL; p = p->ai_next) { - if (p->ai_family == AF_INET) { - struct sockaddr_in* ipv4 = (struct sockaddr_in*)p->ai_addr; - inet_ntop(p->ai_addr->sa_family, &ipv4->sin_addr, ip_addr, INET_ADDRSTRLEN); - ips.push_back(ip_addr); - } - else { - struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)p->ai_addr; - inet_ntop(p->ai_addr->sa_family, &ipv6->sin6_addr, ip_addr, INET6_ADDRSTRLEN); - ips.push_back(ip_addr); - } - } +void* monitor_dns_resolver_thread(const std::vector& dns_resolve_data_list) { + assert(!dns_resolve_data_list.empty()); + DNS_Resolve_Data* data = dns_resolve_data_list.front(); - freeaddrinfo(res); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Resolving hostname:[%s] to its mapped IP address.\n", + data->hostname.c_str()); + try { + std::vector ips = dns_resolve(data->hostname, data->ai_family); if (!ips.empty()) { - - bool to_update_cache = false; - int cache_ttl = dns_resolve_data->ttl; - if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) { + unsigned int cache_ttl = data->ttl; + if (data->ttl > data->refresh_intv) { // NOSONAR cpp:S2245 — mt19937 used here only as a DNS-cache // TTL jitter source (non-cryptographic timing tweak); no // security boundary. Inline annotation on the construction // line because Sonar attributes the hotspot to it. thread_local std::mt19937 gen(std::random_device{}()); // NOSONAR cpp:S2245 - const int jitter = static_cast(dns_resolve_data->ttl * 0.025); + const int jitter = static_cast(data->ttl * 0.025); std::uniform_int_distribution dis(-jitter, jitter); cache_ttl += dis(gen); } - if (!dns_resolve_data->cached_ips.empty()) { - - if (dns_resolve_data->cached_ips.size() == ips.size()) { - for (const std::string& ip : ips) { - if (dns_resolve_data->cached_ips.find(ip) == dns_resolve_data->cached_ips.end()) { - to_update_cache = true; - break; - } - } - } - else - to_update_cache = true; - - if (!to_update_cache) { + bool to_update_cache = true; + unsigned long long expiry = monotonic_time() + (1000ULL * (unsigned long long)cache_ttl); + + if (!data->cached_ips.empty() + && data->cached_ips.size() == ips.size()) { + bool match_all = std::all_of( + ips.begin(), + ips.end(), + [&](const std::string& ip) { return data->cached_ips.count(ip) != 0; } + ); + if (match_all) { + // keep the existing record, just refresh its expiry + to_update_cache = false; proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache record already up-to-date. (Hostname:[%s] IP:[%s])\n", - dns_resolve_data->hostname.c_str(), - debug_iplisttostring(ips).c_str()); - dns_resolve_data->result.set_value(std::make_tuple<>(true, - DNS_Cache_Record(dns_resolve_data->hostname, - std::move(dns_resolve_data->cached_ips), - monotonic_time() + (1000ULL * static_cast(cache_ttl))))); + data->hostname.c_str(), debug_iplisttostring(ips).c_str()); + data->result.set_value(std::make_tuple<>(true, + DNS_Cache_Record(data->hostname, std::move(data->cached_ips), expiry))); } } - else - to_update_cache = true; if (to_update_cache) { - dns_resolve_data->result.set_value(std::make_tuple<>(true, - DNS_Cache_Record(dns_resolve_data->hostname, ips, - monotonic_time() + (1000ULL * static_cast(cache_ttl))))); - dns_resolve_data->dns_cache->add(dns_resolve_data->hostname, std::move(ips)); + data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(data->hostname, ips, expiry))); + data->dns_cache->add(data->hostname, std::move(ips)); } return NULL; @@ -163,15 +162,14 @@ void* monitor_dns_resolver_thread(const std::vector& dns_reso } catch (std::exception& ex) { proxy_error("An exception occurred while resolving hostname: %s [%s]\n", - dns_resolve_data->hostname.c_str(), ex.what()); + data->hostname.c_str(), ex.what()); } catch (...) { proxy_error("An unknown exception has occurred while resolving hostname: %s\n", - dns_resolve_data->hostname.c_str()); + data->hostname.c_str()); } -__error: - dns_resolve_data->result.set_value(std::make_tuple<>(false, DNS_Cache_Record())); + data->result.set_value(std::make_tuple<>(false, DNS_Cache_Record())); return NULL; } @@ -207,19 +205,58 @@ void* DNSResolverWorker::run() { return nullptr; } +bool DNS_Cache::is_ip_valid(const std::string& hostname, const std::string& ip) const { + if (!enabled || hostname.empty() || ip.empty()) { + return false; + } + + int rc = pthread_rwlock_rdlock(&rwlock_); + assert(rc == 0); -bool DNS_Cache::add(const std::string& hostname, std::vector&& ips) { + bool valid = false; + auto itr = records.find(hostname); + if (itr != records.end()) { + const unsigned long long now = monotonic_time(); + const bool pin_active = !itr->second.pinned_ip.empty() + && (itr->second.pinned_until == 0 || now <= itr->second.pinned_until); + if (pin_active) { + valid = ip == itr->second.pinned_ip; + } else { + valid = std::find(itr->second.ips.begin(), itr->second.ips.end(), ip) != itr->second.ips.end(); + } + } + + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + + return valid; +} +bool DNS_Cache::add(const std::string& hostname, std::vector&& ips) { if (!enabled) return false; proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Updating DNS cache. (Hostname:[%s] IP:[%s])\n", hostname.c_str(), debug_iplisttostring(ips).c_str()); + int rc = pthread_rwlock_wrlock(&rwlock_); assert(rc == 0); + auto& ip_addr = records[hostname]; ip_addr.ips = std::move(ips); + + // Check if IP pinning is no longer necessary. + if (!ip_addr.pinned_ip.empty() && + std::find(ip_addr.ips.begin(), ip_addr.ips.end(), ip_addr.pinned_ip) != ip_addr.ips.end()) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Unpinning DNS cache record because resolved IP matches pinned IP. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), ip_addr.pinned_ip.c_str()); + ip_addr.pinned_ip.clear(); + ip_addr.pinned_until = 0; + } + __sync_fetch_and_and(&ip_addr.counter, 0); + rc = pthread_rwlock_unlock(&rwlock_); assert(rc == 0); @@ -235,15 +272,29 @@ bool DNS_Cache::add_if_not_exist(const std::string& hostname, std::vectorsecond.ips.empty()) { proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Updating DNS cache. (Hostname:[%s] IP:[%s])\n", hostname.c_str(), debug_iplisttostring(ips).c_str()); auto& ip_addr = records[hostname]; ip_addr.ips = std::move(ips); + + // Check if IP pinning is no longer necessary. + if (!ip_addr.pinned_ip.empty() && + std::find(ip_addr.ips.begin(), ip_addr.ips.end(), ip_addr.pinned_ip) != ip_addr.ips.end()) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Unpinning DNS cache record because resolved IP matches pinned IP. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), ip_addr.pinned_ip.c_str()); + ip_addr.pinned_ip.clear(); + ip_addr.pinned_until = 0; + } + __sync_fetch_and_and(&ip_addr.counter, 0); inserted = true; } + rc = pthread_rwlock_unlock(&rwlock_); assert(rc == 0); @@ -253,17 +304,29 @@ bool DNS_Cache::add_if_not_exist(const std::string& hostname, std::vectorfetch_add(1, std::memory_order_relaxed); @@ -280,10 +344,23 @@ std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) con auto itr = records.find(hostname); if (itr != records.end()) { - ip = get_next_ip(itr->second); - - if (ip_count) - *ip_count = itr->second.ips.size(); + lookup_result_t result = get_next_ip(itr->second); + + const unsigned long long now = monotonic_time(); + const bool pin_active = !result.pinned_ip.empty() + && (result.pinned_until == 0 || now <= result.pinned_until); + clear_expired_pin = !result.pinned_ip.empty() + && result.pinned_until != 0 && now > result.pinned_until; + + if (pin_active) { + ip = result.pinned_ip; + if (ip_count) + *ip_count = 1; + } else { + ip = result.resolved_ip; + if (ip_count) + *ip_count = result.ip_count; + } proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache lookup success. (Hostname:[%s] IP returned:[%s])\n", @@ -299,9 +376,93 @@ std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) con if (!ip.empty() && counter_lookup_success_) counter_lookup_success_->fetch_add(1, std::memory_order_relaxed); + // cleanup expired pinned IP + if (clear_expired_pin) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Removing expired DNS cache pin. (Hostname:[%s])\n", hostname.c_str()); + unpin(hostname); + } + return ip; } +/** +* @brief Pin a hostname to a fixed IP until it is explicitly unpinned. +* +* @param hostname Hostname whose cached resolution is overridden. +* @param ip IP address to serve for 'hostname' while pinned. +*/ +void DNS_Cache::pin(const std::string& hostname, const std::string& ip) { + pin(hostname, ip, 0); +} + +/** +* @brief Pin a hostname to a fixed IP for a bounded time. +* +* @details While the pin is active, lookup() serves 'ip' instead of the resolved +* address set. Once ttl_ms expires, lookup() serves the resolved address and +* clears the expired pin before returning. +* +* @param hostname Hostname whose cached resolution is overridden. +* @param ip IP address to serve for 'hostname' while pinned. +* @param ttl_ms Pin lifetime in milliseconds; 0 means no expiry. +*/ +void DNS_Cache::pin(const std::string& hostname, const std::string& ip, unsigned long long ttl_ms) { + if (!enabled || hostname.empty() || ip.empty()) return; + + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Pinning DNS cache record. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), ip.c_str()); + + int rc = pthread_rwlock_wrlock(&rwlock_); + assert(rc == 0); + + // Store on the record's 'pinned_ip' so a concurrent resolver add() (which + // only rewrites 'ips') cannot drop the override on a TTL refresh. + auto& ip_addr = records[hostname]; + ip_addr.pinned_ip = ip; + ip_addr.pinned_until = ttl_ms ? monotonic_time() + (ttl_ms * 1000) : 0; + __sync_fetch_and_and(&ip_addr.counter, 0); + + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + + if (counter_record_updated_) + counter_record_updated_->fetch_add(1, std::memory_order_relaxed); +} + +/** +* @brief Remove a pin set by pin(), restoring normal resolution (no-op if not pinned). +* +* @param hostname Hostname to unpin. +*/ +void DNS_Cache::unpin(const std::string& hostname) { + bool item_removed = false; + + int rc = pthread_rwlock_wrlock(&rwlock_); + assert(rc == 0); + + auto itr = records.find(hostname); + if (itr != records.end() && !itr->second.pinned_ip.empty()) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Unpinning DNS cache record. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), itr->second.pinned_ip.c_str()); + itr->second.pinned_ip.clear(); + itr->second.pinned_until = 0; + // drop the record entirely if pinning was the only thing keeping it alive + // (e.g. the host is not otherwise resolved into the cache). + if (itr->second.ips.empty()) + records.erase(itr); + item_removed = true; + } + + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + + if (item_removed && counter_record_updated_) + counter_record_updated_->fetch_add(1, std::memory_order_relaxed); +} + void DNS_Cache::remove(const std::string& hostname) { bool item_removed = false; diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 8bd245e870..5a5ef1ad4f 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -660,7 +660,7 @@ hg_metrics_map = std::make_tuple( std::make_tuple ( p_hg_dyn_gauge::connection_pool_status, "proxysql_connpool_conns_status", - "The status of the backend server (1 - ONLINE, 2 - SHUNNED, 3 - OFFLINE_SOFT, 4 - OFFLINE_HARD, 5 - SHUNNED_REPLICATION_LAG).", + "The status of the backend server (1 - ONLINE, 2 - SHUNNED, 3 - OFFLINE_SOFT, 4 - OFFLINE_HARD, 5 - SHUNNED_REPLICATION_LAG, 6 - SHUNNED_AWS_BGD).", metric_tags { { "protocol", "mysql" } } @@ -727,6 +727,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { mydb->execute(MYHGM_MYSQL_GROUP_REPLICATION_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_GALERA_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS); + mydb->execute(MYHGM_MYSQL_AWS_RDS_BGD_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES); mydb->execute(MYHGM_MYSQL_SERVERS_SSL_PARAMS); mydb->execute("CREATE INDEX IF NOT EXISTS idx_mysql_servers_hostname_port ON mysql_servers (hostname,port)"); @@ -736,6 +737,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { incoming_group_replication_hostgroups=NULL; incoming_galera_hostgroups=NULL; incoming_aws_aurora_hostgroups = NULL; + incoming_aws_rds_bgd_hostgroups = NULL; incoming_hostgroup_attributes = NULL; incoming_mysql_servers_ssl_params = NULL; incoming_mysql_servers_v2 = NULL; @@ -1000,6 +1002,7 @@ void MySQL_HostGroups_Manager::commit_update_checksums_from_tables(SpookyHash& m CUCFT1(myhash,init,"mysql_aws_aurora_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_AWS_AURORA_HOSTGROUPS]); CUCFT1(myhash,init,"mysql_hostgroup_attributes","hostgroup_id", table_resultset_checksum[HGM_TABLES::MYSQL_HOSTGROUP_ATTRIBUTES]); CUCFT1(myhash,init,"mysql_servers_ssl_params","hostname,port,username", table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS_SSL_PARAMS]); + CUCFT1(myhash,init,"mysql_aws_rds_bgd_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_AWS_RDS_BGD_HOSTGROUPS]); } /** @@ -1546,6 +1549,12 @@ bool MySQL_HostGroups_Manager::commit( generate_mysql_aws_aurora_hostgroups_table(); } + // AWS RDS + if (incoming_aws_rds_bgd_hostgroups) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_aws_rds_bgd_hostgroups\n"); + generate_mysql_aws_rds_bgd_hostgroups_table(); + } + // hostgroup attributes if (incoming_hostgroup_attributes) { proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_hostgroup_attributes\n"); @@ -1607,6 +1616,8 @@ bool MySQL_HostGroups_Manager::commit( // NOTE: In order to guarantee the latest generated version, this should be kept after all the // calls to 'generate_mysql_servers'. update_table_mysql_servers_for_monitor(false); + // Refresh BGD monitoring after all runtime server changes are applied. + update_aws_rds_bgd_hosts_monitor_resultset(); wrunlock(); unsigned long long curtime2=monotonic_time(); @@ -1875,6 +1886,9 @@ void MySQL_HostGroups_Manager::generate_mysql_servers_table(int *_onlyhg) { case 4: st=(char *)"SHUNNED"; break; + case 5: + st=(char *)"SHUNNED_AWS_BGD"; + break; } fprintf(stderr,"HID: %u , address: %s , port: %d , gtid_port: %d , weight: %ld , status: %s , max_connections: %ld , max_replication_lag: %u , use_ssl: %d , max_latency_ms: %u , comment: %s\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, mysrvc->gtid_port, mysrvc->weight, st, mysrvc->max_connections, mysrvc->max_replication_lag, mysrvc->use_ssl, mysrvc->max_latency_us*1000, mysrvc->comment); } @@ -2247,6 +2261,9 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) if (name == "mysql_aws_aurora_hostgroups") { query=(char *)"SELECT writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups"; + } else if (name == "mysql_aws_rds_bgd_hostgroups") { + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader," + "check_interval_ms,check_timeout_ms,comment,auto_generated,status FROM mysql_aws_rds_bgd_hostgroups"; } else if (name == "mysql_galera_hostgroups") { query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_galera_hostgroups"; } else if (name == "mysql_group_replication_hostgroups") { @@ -2333,8 +2350,14 @@ void MySQL_HostGroups_Manager::push_MyConn_to_pool(MySQL_Connection *c, bool _lo goto __exit_push_MyConn_to_pool; } + if (!c->healthy) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying unhealthy MySQL_Connection %p, server %s:%d\n", c, mysrvc->address, mysrvc->port); + delete c; + goto __exit_push_MyConn_to_pool; + } + // If the largest query length exceeds the threshold, destroy the connection - if (GloMTH && c->largest_query_length > (unsigned int)GloMTH->variables.threshold_query_length) { + if (c->largest_query_length > (unsigned int)GloMTH->variables.threshold_query_length) { proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying MySQL_Connection %p, server %s:%d with status %d . largest_query_length = %lu\n", c, mysrvc->address, mysrvc->port, (int)mysrvc->get_status(), c->largest_query_length); delete c; goto __exit_push_MyConn_to_pool; @@ -2393,9 +2416,14 @@ void MySQL_HostGroups_Manager::push_MyConn_to_pool_array(MySQL_Connection **ca, wrlock(); // Iterate through the array of connections - while (ireusable) { + c->send_quit = false; + destroy_MyConn_from_pool(c, false); + } else { + // Push the current connection back to the pool without acquiring a lock for each individual push + push_MyConn_to_pool(c, false); + } i++; if (iparent; - if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE && c->send_quit && queue.size() < __sync_fetch_and_add(&GloMTH->variables.connpoll_reset_queue_length, 0)) { + if (c->healthy && mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE && c->send_quit && queue.size() < __sync_fetch_and_add(&GloMTH->variables.connpoll_reset_queue_length, 0)) { if (c->async_state_machine==ASYNC_IDLE) { // overall, the backend seems healthy and so it is the connection. Try to reset it int myerr=mysql_errno(c->mysql); @@ -3077,6 +3105,8 @@ void MySQL_HostGroups_Manager::save_incoming_mysql_table(SQLite3_result *s, cons SQLite3_result ** inc = NULL; if (name == "mysql_aws_aurora_hostgroups") { inc = &incoming_aws_aurora_hostgroups; + } else if (name == "mysql_aws_rds_bgd_hostgroups") { + inc = &incoming_aws_rds_bgd_hostgroups; } else if (name == "mysql_galera_hostgroups") { inc = &incoming_galera_hostgroups; } else if (name == "mysql_group_replication_hostgroups") { @@ -3151,6 +3181,8 @@ SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_table(const string& return this->incoming_hostgroup_attributes; } else if (name == "mysql_servers_ssl_params") { return this->incoming_mysql_servers_ssl_params; + } else if (name == "mysql_aws_rds_bgd_hostgroups") { + return this->incoming_aws_rds_bgd_hostgroups; } else if (name == "cluster_mysql_servers") { return this->runtime_mysql_servers; } else if (name == "mysql_servers_v2") { @@ -3488,6 +3520,9 @@ SQLite3_result * MySQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset, int case 4: pta[3]=strdup("SHUNNED_REPLICATION_LAG"); break; + case 5: + pta[3]=strdup("SHUNNED_AWS_BGD"); + break; default: // LCOV_EXCL_START assert(0); @@ -3548,20 +3583,37 @@ SQLite3_result * MySQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset, int } /** - * @brief New implementation of the read_only_action method that does not depend on the admin table. - * The method checks each server in the provided list and adjusts the servers according to their corresponding read_only value. - * If any change has occured, checksum is calculated. + * @brief Reconcile writer/reader hostgroup placement from read_only monitor results. * - * @param mysql_servers List of servers having hostname, port and read only value. - * + * @details New implementation of the read_only_action that does not depend on the admin table. + * Checks each server in the provided list and adjusts writer/reader hostgroup placement + * according to the corresponding read_only value. If any change occurs, the runtime + * mysql_servers table and checksum are regenerated. + * + * @param mysql_servers Servers and their observed/read-only state. + * @param ignore_aws_bgd True to apply the result while BGD switchover is in progress. */ -void MySQL_HostGroups_Manager::read_only_action_v2(const std::list& mysql_servers) { +void MySQL_HostGroups_Manager::read_only_action_v2(const std::list& mysql_servers, bool ignore_aws_bgd) { + // Skip read_only results for servers flagged as AWS RDS BGD switchover in progress. + std::list filtered_servers; + for (const auto& server : mysql_servers) { + const std::string& hostname = std::get(server); + const int port = std::get(server); + + if (!ignore_aws_bgd && GloMyMon->is_aws_rds_bgd_server_in_progress(hostname, port)) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring read_only result for '%s:%d' because AWS RDS BGD switchover is in progress\n", + hostname.c_str(), port); + continue; + } + filtered_servers.push_back(server); + } bool update_mysql_servers_table = false; unsigned long long curtime1 = monotonic_time(); wrlock(); - for (const auto& server : mysql_servers) { + for (const auto& server : filtered_servers) { bool is_writer = false; const std::string& hostname = std::get(server); const int port = std::get(server); @@ -3697,7 +3749,7 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listmysrvs) { + for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc = myhgc->mysrvs->idx(j); + if (mysrvc->port != port || strcmp(mysrvc->address, hostname) != 0) { + continue; + } + + time_t now = time(NULL); + + if (shun) { + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE) { + mysrvc->set_status(MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD); + mysrvc->shunned_automatic = true; + mysrvc->shunned_and_kill_all_connections = true; + mysrvc->time_last_detected_error = now; + mysrvc->ConnectionsFree->drop_all_connections(); + mysrvc->ConnectionsUsed->mark_connections_unhealthy(); + proxy_warning("AWS RDS BGD shunning server %s:%d in HG %u\n", + hostname, port, myhgc->hid); + changed = true; + } + } else { + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD) { + mysrvc->set_status(MYSQL_SERVER_STATUS_ONLINE); + mysrvc->shunned_automatic = false; + mysrvc->shunned_and_kill_all_connections = false; + mysrvc->time_last_detected_error = 0; + proxy_info("AWS RDS BGD unshunning server %s:%d in HG %u\n", + hostname, port, myhgc->hid); + changed = true; + } + } + } + } + + return changed; +} + +/** + * @brief Configure the AWS RDS BGD writer's writer/reader hostgroup membership. + * + * @details Ensures the writer is present in its writer hostgroup, with optional reader + * hostgroup membership controlled by writer_is_also_reader. + * + * @param hostname Server hostname to configure. + * @param port Server port to configure. + * @param writer_is_also_reader Whether the writer should also be present in reader hostgroup. + * + * @return true if hostgroup membership changed. + * + * @note Caller must hold wrlock(). + */ +bool MySQL_HostGroups_Manager::aws_rds_bgd_configure_writer(const char *hostname, int port, bool writer_is_also_reader) { + const std::string srv_id = std::string(hostname) + ":::" + std::to_string(port); + auto itr = hostgroup_server_mapping.find(srv_id); + + if (itr == hostgroup_server_mapping.end() || !itr->second) { + proxy_warning("AWS RDS BGD: server %s:%d not found in hostgroup_server_mapping\n", hostname, port); + return false; + } + + HostGroup_Server_Mapping* srv_map = itr->second.get(); + bool changed = false; + + if (srv_map->get(HostGroup_Server_Mapping::Type::WRITER).empty()) { + if (srv_map->get(HostGroup_Server_Mapping::Type::READER).empty()) { + proxy_warning("AWS RDS BGD: server %s:%d has no writer or reader hostgroup mapping\n", hostname, port); + return false; + } + + srv_map->copy_if_not_exists(HostGroup_Server_Mapping::Type::WRITER, HostGroup_Server_Mapping::Type::READER); + proxy_info("AWS RDS BGD: adding server %s:%d to writer hostgroup\n", hostname, port); + changed = true; + } + + if (writer_is_also_reader) { + if (srv_map->get(HostGroup_Server_Mapping::Type::READER).empty()) { + srv_map->copy_if_not_exists(HostGroup_Server_Mapping::Type::READER, HostGroup_Server_Mapping::Type::WRITER); + proxy_info("AWS RDS BGD: adding server %s:%d to reader hostgroup\n", hostname, port); + changed = true; + } + } else if (!srv_map->get(HostGroup_Server_Mapping::Type::READER).empty()) { + srv_map->clear(HostGroup_Server_Mapping::Type::READER); + proxy_info("AWS RDS BGD: removing server %s:%d from reader hostgroup\n", hostname, port); + changed = true; + } + + return changed; +} + +void MySQL_HostGroups_Manager::aws_rds_bgd_set_runtime_status(unsigned int writer_hg, int status) { + char query[128]; + snprintf(query, sizeof(query), + "UPDATE mysql_aws_rds_bgd_hostgroups SET status=%d WHERE writer_hostgroup=%u", status, writer_hg); + wrlock(); + mydb->execute(query); + wrunlock(); +} + +/** + * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. + * + * @details One-way alignment (in-memory -> runtime): regenerates the runtime 'mysql_servers' table + * from the current in-memory 'MyHGC'/'MySrvC' structures, recomputes/republishes the global + * 'mysql_servers' checksum for cluster sync, and refreshes 'mysql_servers_to_monitor' for the + * regular monitor threads. + * + * @note the caller MUST already hold 'wrlock()'. + */ +void MySQL_HostGroups_Manager::publish_mysql_servers_to_runtime() { + // update runtime table + purge_mysql_servers_table(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); + + // Update the global checksums after 'mysql_servers' regeneration + unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; + uint64_t raw_checksum = resultset ? resultset->raw_checksum() : 0; + hgsm_mysql_servers_checksum = raw_checksum; + string mysrvs_checksum { get_checksum_from_hash(raw_checksum) }; + save_runtime_mysql_servers(resultset.release()); + proxy_info("Checksum for table %s is %s\n", "mysql_servers", mysrvs_checksum.c_str()); + pthread_mutex_lock(&GloVars.checksum_mutex); + update_glovars_mysql_servers_checksum(mysrvs_checksum); + pthread_mutex_unlock(&GloVars.checksum_mutex); + + // update monitor table + update_table_mysql_servers_for_monitor(false); +} + +/** + * @brief Drain existing backend connections for a server in all hostgroups. + * + * @details Drops free connections immediately and marks used connections as unhealthy and non-reusable, + * so in-flight operations fail on their next backend step and the connection is never pooled again. + * + * @param hostname Address of the server to match. + * @param port Port of the server to match. + * @return true if a matching server was found. + * + * @note Caller must hold wrlock(). + */ +bool MySQL_HostGroups_Manager::drain_server_connections(const char *hostname, int port) { + bool found = false; + + for (unsigned int i = 0; i < MyHostGroups->len; i++) { + MyHGC *myhgc = (MyHGC *)MyHostGroups->index(i); + if (!myhgc || !myhgc->mysrvs) { + continue; + } + + for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc = myhgc->mysrvs->idx(j); + if (mysrvc->port != port || strcmp(mysrvc->address, hostname) != 0) { + continue; + } + + mysrvc->ConnectionsFree->drop_all_connections(); + mysrvc->ConnectionsUsed->mark_connections_unhealthy(); + proxy_warning("Draining existing connections for server %s:%d in HG %u\n", + hostname, port, myhgc->hid); + found = true; + } + } + + return found; +} + void MySQL_HostGroups_Manager::p_update_metrics() { p_update_counter(status.p_counter_array[p_hg_counter::servers_table_version], status.servers_table_version); // Update *server_connections* related metrics @@ -6273,6 +6511,158 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { pthread_mutex_unlock(&AWS_Aurora_Info_mutex); } +/** + * @brief Regenerates the runtime in-memory `mysql_aws_rds_bgd_hostgroups` table from `incoming_aws_rds_bgd_hostgroups`. + * + * @details The incoming resultset comes from the admin config table (11 columns, no `auto_generated`); config-loaded + * entries are user-defined, so `auto_generated` is stored as 0. `green_writer_hostgroup` and + * `green_reader_hostgroup` are optional and bound as SQL NULL when absent. + * + * @note Existing deployments preserve their runtime `status` while configured fields are reloaded. + */ +void MySQL_HostGroups_Manager::generate_mysql_aws_rds_bgd_hostgroups_table() { + if (incoming_aws_rds_bgd_hostgroups==NULL) { + return; + } + + struct RuntimeRow { + int reader_hostgroup; + int status; + }; + + std::map runtime_rows; + std::map incoming_reader_hostgroups; + + for (SQLite3_row* row : incoming_aws_rds_bgd_hostgroups->rows) { + incoming_reader_hostgroups.emplace(atoi(row->fields[0]), atoi(row->fields[1])); + } + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + const char* select_query = "SELECT writer_hostgroup, reader_hostgroup, status FROM mysql_aws_rds_bgd_hostgroups"; + mydb->execute_statement(select_query, &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error("Error on %s : %s\n", select_query, error); + free(error); + error = NULL; + assert(0); + } + if (resultset) { + for (SQLite3_row* row : resultset->rows) { + runtime_rows.emplace(atoi(row->fields[0]), RuntimeRow {atoi(row->fields[1]), atoi(row->fields[2])}); + } + delete resultset; + resultset = NULL; + } + + int rc; + const char* delete_query = "DELETE FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=?1"; + auto [delete_rc, delete_statement_unique] = mydb->prepare_v2(delete_query); + ASSERT_SQLITE_OK(delete_rc, mydb); + sqlite3_stmt* delete_statement = delete_statement_unique.get(); + + // Remove missing deployments and release changed reader hostgroups before inserting their replacements. + for (const auto& [writer_hostgroup, runtime_row] : runtime_rows) { + auto incoming_it = incoming_reader_hostgroups.find(writer_hostgroup); + bool removed = incoming_it == incoming_reader_hostgroups.end(); + bool reader_changed = !removed && incoming_it->second != runtime_row.reader_hostgroup; + if (!removed && !reader_changed) { + continue; + } + + rc=(*proxy_sqlite3_bind_int64)(delete_statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(delete_statement); + rc=(*proxy_sqlite3_clear_bindings)(delete_statement); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(delete_statement); ASSERT_SQLITE_OK(rc, mydb); + } + + const char* update_query = + "UPDATE mysql_aws_rds_bgd_hostgroups SET " + "reader_hostgroup=?1, green_writer_hostgroup=?2, green_reader_hostgroup=?3, active=?4, " + "writer_is_also_reader=?5, check_interval_ms=?6, check_timeout_ms=?7, comment=?8, auto_generated=?9 " + "WHERE writer_hostgroup=?10"; + auto [update_rc, update_statement_unique] = mydb->prepare_v2(update_query); + ASSERT_SQLITE_OK(update_rc, mydb); + sqlite3_stmt* update_statement = update_statement_unique.get(); + + const char* insert_query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active," + "writer_is_also_reader, check_interval_ms, check_timeout_ms, comment, auto_generated, status" + ") VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + auto [insert_rc, insert_statement_unique] = mydb->prepare_v2(insert_query); + ASSERT_SQLITE_OK(insert_rc, mydb); + sqlite3_stmt* insert_statement = insert_statement_unique.get(); + + proxy_info("New mysql_aws_rds_bgd_hostgroups table\n"); + + for (SQLite3_row* r : incoming_aws_rds_bgd_hostgroups->rows) { + int writer_hostgroup=atoi(r->fields[0]); + int reader_hostgroup=atoi(r->fields[1]); + const char *gw_str = r->fields[2]; + const char *gr_str = r->fields[3]; + int green_writer_hostgroup = (gw_str && gw_str[0]) ? atoi(gw_str) : -1; + int green_reader_hostgroup = (gr_str && gr_str[0]) ? atoi(gr_str) : -1; + int active=atoi(r->fields[4]); + int writer_is_also_reader = atoi(r->fields[5]); + int check_interval_ms = atoi(r->fields[6]); + int check_timeout_ms = atoi(r->fields[7]); + // entries loaded from the admin config table are always user-defined + int auto_generated = 0; + proxy_info("Loading AWS RDS info for (%d,%d,%d,%d,%s,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, + green_writer_hostgroup,green_reader_hostgroup,(active ? "on" : "off"),writer_is_also_reader, + check_interval_ms,check_timeout_ms,auto_generated,r->fields[8]); + + auto runtime_it = runtime_rows.find(writer_hostgroup); + bool update_existing = + runtime_it != runtime_rows.end() && + runtime_it->second.reader_hostgroup == reader_hostgroup; + sqlite3_stmt* statement = update_existing ? update_statement : insert_statement; + int field_offset = update_existing ? 0 : 1; + + if (!update_existing) { + rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + } + rc=(*proxy_sqlite3_bind_int64)(statement, 1 + field_offset, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + if (green_writer_hostgroup >= 0) { + rc=(*proxy_sqlite3_bind_int64)(statement, 2 + field_offset, green_writer_hostgroup); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 2 + field_offset); + } + ASSERT_SQLITE_OK(rc, mydb); + if (green_reader_hostgroup >= 0) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3 + field_offset, green_reader_hostgroup); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3 + field_offset); + } + ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 4 + field_offset, active); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5 + field_offset, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6 + field_offset, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 7 + field_offset, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 8 + field_offset, r->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9 + field_offset, auto_generated); ASSERT_SQLITE_OK(rc, mydb); + + if (update_existing) { + rc=(*proxy_sqlite3_bind_int64)(statement, 10, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + } else { + int status = runtime_it == runtime_rows.end() + ? static_cast(AWS_RDS_BGD_Status::NONE) + : runtime_it->second.status; + rc=(*proxy_sqlite3_bind_int64)(statement, 11, status); ASSERT_SQLITE_OK(rc, mydb); + } + + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mydb); + } + + delete incoming_aws_rds_bgd_hostgroups; + incoming_aws_rds_bgd_hostgroups=NULL; +} + //void MySQL_HostGroups_Manager::aws_aurora_replication_lag_action(int _whid, int _rhid, char *address, unsigned int port, float current_replication_lag, bool enable, bool verbose) { @@ -6849,6 +7239,153 @@ void MySQL_HostGroups_Manager::update_aws_aurora_hosts_monitor_resultset(bool lo } } +const char SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR[] { + "SELECT srv.hostname, srv.port, MAX(srv.use_ssl) AS use_ssl, " + "bgd.writer_hostgroup, bgd.reader_hostgroup, bgd.green_writer_hostgroup, bgd.green_reader_hostgroup, " + "bgd.check_interval_ms, bgd.check_timeout_ms, bgd.writer_is_also_reader, " + "'B' AS srv_type, MAX(srv.hostgroup_id=bgd.writer_hostgroup) AS is_writer " + "FROM mysql_servers AS srv " + "JOIN mysql_aws_rds_bgd_hostgroups AS bgd " + "ON srv.hostgroup_id=bgd.writer_hostgroup OR srv.hostgroup_id=bgd.reader_hostgroup " + "WHERE bgd.active=1 AND srv.status NOT IN (2,3) " + "GROUP BY bgd.writer_hostgroup, srv.hostname, srv.port " + "UNION ALL " + "SELECT srv.hostname, srv.port, srv.use_ssl, " + "bgd.writer_hostgroup, bgd.reader_hostgroup, bgd.green_writer_hostgroup, bgd.green_reader_hostgroup, " + "bgd.check_interval_ms, bgd.check_timeout_ms, bgd.writer_is_also_reader, " + "'G' AS srv_type, srv.hostgroup_id=bgd.green_writer_hostgroup AS is_writer " + "FROM mysql_servers AS srv " + "JOIN mysql_aws_rds_bgd_hostgroups AS bgd " + "ON srv.hostgroup_id=bgd.green_writer_hostgroup OR srv.hostgroup_id=bgd.green_reader_hostgroup " + "WHERE bgd.active=1 AND srv.status NOT IN (2,3) " + "ORDER BY writer_hostgroup, srv_type, is_writer DESC, hostname, port" +}; + +/** + * @brief Rebuilds the AWS RDS BGD monitor's host resultset. + * + * @details Rebuilds `GloMyMon->AWS_RDS_BGD_Hosts_resultset` and publishes both the full BGD hosts + * checksum and one checksum per writer hostgroup. The previous result remains active when the + * query fails. + */ +void MySQL_HostGroups_Manager::update_aws_rds_bgd_hosts_monitor_resultset() { + if (!GloMyMon) { + return; + } + + SQLite3_result* resultset = nullptr; + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + mydb->execute_statement(SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR, &error, &cols, &affected_rows, &resultset); + + if (error || !resultset) { + proxy_error("Error refreshing AWS RDS BGD hosts: %s\n", error ? error : "empty resultset"); + free(error); + delete resultset; + return; + } + free(error); + + std::unordered_map cluster_checksums; + std::unordered_map cluster_resultsets; + for (SQLite3_row* row : resultset->rows) { + const int writer_hg = atoi(row->fields[AWS_RDS_BGD_WRITER_HOSTGROUP]); + auto cluster_it = cluster_resultsets.find(writer_hg); + if (cluster_it == cluster_resultsets.end()) { + cluster_it = cluster_resultsets.emplace( + writer_hg, new SQLite3_result(resultset->columns)).first; + } + cluster_it->second->add_row(row); + } + for (const auto& [writer_hg, cluster_resultset] : cluster_resultsets) { + cluster_checksums[writer_hg] = cluster_resultset->raw_checksum(); + delete cluster_resultset; + } + + const uint64_t hosts_checksum = resultset->raw_checksum(); + std::shared_ptr hosts_resultset { resultset }; + + pthread_mutex_lock(&GloMyMon->aws_rds_bgd_hosts_mutex); + GloMyMon->AWS_RDS_BGD_Hosts_resultset.swap(hosts_resultset); + GloMyMon->AWS_RDS_BGD_Hosts_checksum = hosts_checksum; + GloMyMon->AWS_RDS_BGD_Cluster_checksum.swap(cluster_checksums); + pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_hosts_mutex); +} + +/** + * @brief Auto-generate a runtime `mysql_aws_rds_bgd_hostgroups` entry for a server's writer hostgroup. + * + * @details Called when the read_only monitor detects a blue/green deployment. The writer/reader + * hostgroups are derived from the server's `hostgroup_server_mapping`. Green hostgroups are + * stored NULL with `auto_generated=1`. Idempotent. + * + * @param hostname Hostname of the server that exposed the blue/green topology. + * @param port Port of the server. + * + * @return true if a new entry was added; false otherwise. + */ +bool MySQL_HostGroups_Manager::add_aws_rds_bgd_hostgroup_entry(const std::string& hostname, int port) { + bool added = false; + const std::string srv_id = hostname + ":::" + std::to_string(port); + + wrlock(); + + auto itr = hostgroup_server_mapping.find(srv_id); + if (itr != hostgroup_server_mapping.end() && itr->second) { + int writer_hg = -1, reader_hg = -1; + const auto& wmap = itr->second->get(HostGroup_Server_Mapping::Type::WRITER); + const auto& rmap = itr->second->get(HostGroup_Server_Mapping::Type::READER); + if (!wmap.empty()) { + writer_hg = (int)wmap[0].writer_hostgroup_id; + reader_hg = (int)wmap[0].reader_hostgroup_id; + } else if (!rmap.empty()) { + writer_hg = (int)rmap[0].writer_hostgroup_id; + reader_hg = (int)rmap[0].reader_hostgroup_id; + } + if (writer_hg >= 0 && reader_hg >= 0 && writer_hg != reader_hg) { + // only add when no runtime entry exists yet for this writer hostgroup + bool exists = false; + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* res = nullptr; + + std::string sel = "SELECT 1 FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + std::to_string(writer_hg); + mydb->execute_statement(sel.c_str(), &error, &cols, &affected_rows, &res); + if (res) { + exists = (res->rows_count > 0); + delete res; + } + + if (!exists) { + std::string ins = + "INSERT INTO mysql_aws_rds_bgd_hostgroups (" + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " + "active, writer_is_also_reader, check_interval_ms, check_timeout_ms, " + "comment, auto_generated" + ") VALUES (" + + std::to_string(writer_hg) + ", " + std::to_string(reader_hg) + + ", NULL, NULL, 1, 0, 1000, 800, '', 1)"; + mydb->execute(ins.c_str()); + added = true; + proxy_info( + "AWS RDS: auto-generated blue/green hostgroup entry (writer HG %d, reader HG %d) from server %s:%d\n", + writer_hg, reader_hg, hostname.c_str(), port + ); + } + } + } + + if (added) { + // publish the refreshed host list to the BGD monitor thread + update_aws_rds_bgd_hosts_monitor_resultset(); + } + + wrunlock(); + return added; +} + MySrvC* MySQL_HostGroups_Manager::find_server_in_hg(unsigned int _hid, const std::string& addr, int port) { MySrvC* f_server = nullptr; @@ -7044,9 +7581,11 @@ MySQLServers_SslParams * MySQL_HostGroups_Manager::get_Server_SSL_Params(char *h /** * @brief Updates replication hostgroups by adding autodiscovered mysql servers. +* * @details Adds each server from 'new_servers' to the 'runtime_mysql_servers' table. * We then rebuild the 'mysql_servers' table as well as the internal 'hostname_hostgroup_mapping'. -* @param new_servers A vector of tuples where each tuple contains the values needed to add each new server. +* +* @param new_servers A vector of tuples where each tuple contains the values needed to add each new server. */ void MySQL_HostGroups_Manager::add_discovered_servers_to_mysql_servers_and_replication_hostgroups( const vector>& new_servers diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 5207c07814..cbf56b9682 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -244,6 +244,16 @@ class MySQL_Monitor_Connection_Pool { MYSQL * get_connection(char *hostname, int port, MySQL_Monitor_State_Data *mmsd); void put_connection(char *hostname, MySQL_Monitor_State_Data* mmsd); void purge_some_connections(); + /** + * @brief Purge idle monitor connections for a server. + * + * @details Removes the idle monitor connection pool entry matching the supplied hostname and port. + * Active monitor tasks are not affected. + * + * @param hostname Server hostname to match. + * @param port Server port to match. + */ + void purge_connections(const char* hostname, int port); void purge_all_connections(); void destroy_mysql_connection(MySQL_Monitor_State_Data* mmsd); MySQL_Monitor_Connection_Pool() { @@ -337,6 +347,38 @@ void MySQL_Monitor_Connection_Pool::purge_all_connections() { #endif } +/** + * @brief Purge idle monitor connections for a server. + * + * @details Removes the idle monitor connection pool entry matching the supplied hostname and port. + * Active monitor tasks are not affected. + * + * @param hostname Server hostname to match. + * @param port Server port to match. + */ +void MySQL_Monitor_Connection_Pool::purge_connections(const char* hostname, int port) { + std::lock_guard lock(mutex); +#ifdef DEBUG + pthread_mutex_lock(&m2); +#endif + if (servers) { + for (unsigned int i = 0; i < servers->len; i++) { + MonMySrvC* srv = static_cast(servers->index(i)); + if (srv && srv->port == port && strcmp(hostname, srv->address) == 0) { + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "Purging %u idle monitor connections for server %s:%d\n", + srv->conns->len, hostname, port); + delete srv; + servers->remove_index_fast(i); + break; + } + } + } +#ifdef DEBUG + pthread_mutex_unlock(&m2); +#endif +} + void MySQL_Monitor_Connection_Pool::destroy_mysql_connection(MySQL_Monitor_State_Data* mmsd) { if (mmsd->mysql) { #ifdef DEBUG @@ -371,6 +413,7 @@ MYSQL * MySQL_Monitor_Connection_Pool::get_connection(char *hostname, int port, } } #endif // DEBUG + std::vector skipped_conn; while (srv->conns->len) { unsigned int idx = rand() % srv->conns->len; MYSQL* mysql = (MYSQL*)srv->conns->remove_index_fast(idx); @@ -386,9 +429,23 @@ MYSQL * MySQL_Monitor_Connection_Pool::get_connection(char *hostname, int port, continue; } + // The pool is grouped by hostname and port, but the same server can + // be monitored over plaintext and TLS. Keep connections that may + // match another monitor task and continue searching for this one. + bool connection_uses_ssl = mysql->options.use_ssl != 0; + if (mmsd && connection_uses_ssl != mmsd->use_ssl) { + skipped_conn.push_back(mysql); + continue; + } + my = mysql; break; } + + // Return skipped connections to the pool + for (MYSQL* mysql : skipped_conn) { + srv->conns->add(mysql); + } #ifdef DEBUG // 'my' can be NULL due to connection cleanup, and can cause crash if (my) { @@ -640,7 +697,7 @@ void MySQL_Monitor_State_Data::init_async() { task_timeout_ = mysql_thread___monitor_ping_timeout; task_handler_ = &MySQL_Monitor_State_Data::ping_handler; break; -#ifndef TEST_READONLY +#if !defined(TEST_READONLY) && !defined(TEST_RDS_BGD) case MON_READ_ONLY: query_ = "SELECT @@global.read_only read_only"; async_state_machine_ = ASYNC_QUERY_START; @@ -671,13 +728,7 @@ void MySQL_Monitor_State_Data::init_async() { task_timeout_ = mysql_thread___monitor_read_only_timeout; task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; break; - case MON_READ_ONLY__AND__AWS_RDS_TOPOLOGY_DISCOVERY: - query_ = QUERY_READ_ONLY_AND_AWS_TOPOLOGY_DISCOVERY; - async_state_machine_ = ASYNC_QUERY_START; - task_timeout_ = mysql_thread___monitor_read_only_timeout; - task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; - break; -#else // TEST_READONLY +#else // TEST_READONLY || TEST_RDS_BGD case MON_READ_ONLY: case MON_INNODB_READ_ONLY: case MON_SUPER_READ_ONLY: @@ -689,7 +740,13 @@ void MySQL_Monitor_State_Data::init_async() { task_timeout_ = mysql_thread___monitor_read_only_timeout; task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; break; -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD + case MON_AWS_RDS_TOPOLOGY_DISCOVERY: + query_ = QUERY_AWS_RDS_TOPOLOGY_DISCOVERY; + async_state_machine_ = ASYNC_QUERY_START; + task_timeout_ = mysql_thread___monitor_read_only_timeout; + task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; + break; case MON_GROUP_REPLICATION: async_state_machine_ = ASYNC_QUERY_START; #ifdef TEST_GROUPREP @@ -788,12 +845,17 @@ void MySQL_Monitor_State_Data::init_async() { break; case MON_AWS_AURORA: break; + case MON_AWS_RDS_BGD: + break; } } void MySQL_Monitor_State_Data::mark_task_as_timeout(unsigned long long time) { - task_result_ = MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT; + const bool stale_ip_timeout = GloMyMon && GloMyMon->timeout_validate_ip_change(this); + task_result_ = stale_ip_timeout + ? MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT_STALE_IP + : MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT; t2 = time; if (mysql_error_msg) @@ -801,10 +863,15 @@ void MySQL_Monitor_State_Data::mark_task_as_timeout(unsigned long long time) { if (task_id_ == MON_PING) { async_state_machine_ = ASYNC_PING_TIMEOUT; - mysql_error_msg = strdup("timeout during ping"); + mysql_error_msg = strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout during ping"); } else { async_state_machine_ = (async_state_machine_ == ASYNC_QUERY_CONT) ? ASYNC_QUERY_TIMEOUT : ASYNC_STORE_RESULT_TIMEOUT; - mysql_error_msg = strdup("timeout check"); + mysql_error_msg = strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + } + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring monitor timeout for %s:%d because resolved IP is no longer valid\n", + hostname, port); } } @@ -881,6 +948,17 @@ void * monitor_aws_aurora_pthread(void *arg) { return NULL; } +void * monitor_aws_rds_bgd_pthread(void *arg) { + set_thread_name("MonitorRdsBgd", GloVars.set_thread_name); + + // Wait for GloMTH to be initialized + if (!wait_for_glo_mth()) + return NULL; + + GloMyMon->monitor_aws_rds_bgd(); + return NULL; +} + void * monitor_replication_lag_pthread(void *arg) { #ifndef NOJEM bool cache=false; @@ -1077,10 +1155,15 @@ MySQL_Monitor::MySQL_Monitor() { Galera_Hosts_resultset=NULL; pthread_mutex_init(&aws_aurora_mutex,NULL); + pthread_mutex_init(&aws_rds_bgd_mutex,NULL); + pthread_mutex_init(&aws_rds_bgd_hosts_mutex,NULL); pthread_mutex_init(&mysql_servers_mutex,NULL); pthread_mutex_init(&proxysql_servers_mutex, NULL); AWS_Aurora_Hosts_resultset=NULL; AWS_Aurora_Hosts_resultset_checksum = 0; + AWS_RDS_BGD_Hosts_resultset.reset(); + AWS_RDS_BGD_Hosts_checksum = 0; + AWS_RDS_BGD_Cluster_checksum.clear(); shutdown=false; monitor_enabled=true; // default // create new SQLite datatabase @@ -1181,6 +1264,9 @@ MySQL_Monitor::~MySQL_Monitor() { delete AWS_Aurora_Hosts_resultset; AWS_Aurora_Hosts_resultset=NULL; } + AWS_RDS_BGD_Hosts_resultset.reset(); + AWS_RDS_BGD_Cluster_checksum.clear(); + pthread_mutex_destroy(&aws_rds_bgd_hosts_mutex); std::map::iterator it2; AWS_Aurora_monitor_node *node=NULL; for (it2 = AWS_Aurora_Hosts_Map.begin(); it2 != AWS_Aurora_Hosts_Map.end(); ++it2) { @@ -1650,6 +1736,8 @@ void * monitor_read_only_thread(const std::vector& da mysql_close(mysql_init(NULL)); bool timeout_reached = false; MySQL_Monitor_State_Data *mmsd = data.front(); + std::string monitor_query; + bool stale_ip_timeout = false; // Wait for GloMTH to be initialized if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart MySQL_Thread * mysql_thr = new MySQL_Thread(); @@ -1686,27 +1774,27 @@ void * monitor_read_only_thread(const std::vector& da mmsd->t1=monotonic_time(); mmsd->interr=0; // reset the value -#ifndef TEST_READONLY - if (mmsd->get_task_type() == MON_INNODB_READ_ONLY) { - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.innodb_read_only read_only"); - } else if (mmsd->get_task_type() == MON_SUPER_READ_ONLY) { - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.super_read_only read_only"); - } else if (mmsd->get_task_type() == MON_READ_ONLY__AND__INNODB_READ_ONLY) { - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.read_only&@@global.innodb_read_only read_only"); - } else if (mmsd->get_task_type() == MON_READ_ONLY__OR__INNODB_READ_ONLY) { - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.read_only|@@global.innodb_read_only read_only"); - } else if (mmsd->get_task_type() == MON_READ_ONLY__AND__AWS_RDS_TOPOLOGY_DISCOVERY) { - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql, QUERY_READ_ONLY_AND_AWS_TOPOLOGY_DISCOVERY); - } else { // default - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.read_only read_only"); - } -#else // TEST_READONLY - { - std::string s = "SELECT @@global.read_only read_only"; - s += " " + std::string(mmsd->hostname) + ":" + std::to_string(mmsd->port); - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,s.c_str()); - } -#endif // TEST_READONLY + if (mmsd->get_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY) { + monitor_query = QUERY_AWS_RDS_TOPOLOGY_DISCOVERY; + } else { +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) + monitor_query = "SELECT @@global.read_only read_only"; + monitor_query += " " + std::string(mmsd->hostname) + ":" + std::to_string(mmsd->port); +#else + if (mmsd->get_task_type() == MON_INNODB_READ_ONLY) { + monitor_query = "SELECT @@global.innodb_read_only read_only"; + } else if (mmsd->get_task_type() == MON_SUPER_READ_ONLY) { + monitor_query = "SELECT @@global.super_read_only read_only"; + } else if (mmsd->get_task_type() == MON_READ_ONLY__AND__INNODB_READ_ONLY) { + monitor_query = "SELECT @@global.read_only&@@global.innodb_read_only read_only"; + } else if (mmsd->get_task_type() == MON_READ_ONLY__OR__INNODB_READ_ONLY) { + monitor_query = "SELECT @@global.read_only|@@global.innodb_read_only read_only"; + } else { // default + monitor_query = "SELECT @@global.read_only read_only"; + } +#endif // TEST_READONLY || TEST_RDS_BGD + } + mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,monitor_query.c_str()); while (mmsd->async_exit_status) { mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); #ifdef DEBUG @@ -1715,9 +1803,16 @@ void * monitor_read_only_thread(const std::vector& da const unsigned long long now = monotonic_time(); #endif if (now > mmsd->t1 + mysql_thread___monitor_read_only_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on read_only check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_read_only_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_READ_ONLY_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring read_only timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on read_only check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_read_only_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_READ_ONLY_CHECK_TIMEOUT); + } timeout_reached = true; goto __exit_monitor_read_only_thread; } @@ -1749,9 +1844,16 @@ void * monitor_read_only_thread(const std::vector& da const unsigned long long now = monotonic_time(); #endif if (now > mmsd->t1 + mysql_thread___monitor_read_only_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on read_only check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_read_only_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_READ_ONLY_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring read_only timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on read_only check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_read_only_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_READ_ONLY_CHECK_TIMEOUT); + } timeout_reached = true; goto __exit_monitor_read_only_thread; } @@ -1769,7 +1871,14 @@ void * monitor_read_only_thread(const std::vector& da __exit_monitor_read_only_thread: mmsd->t2=monotonic_time(); - { + if (mmsd->get_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY) { + if (mmsd->interr == 0 && mmsd->result) { + GloMyMon->process_aws_rds_topology(mmsd); + mysql_free_result(mmsd->result); + mmsd->result = NULL; + read_only_success = true; + } + } else { /* handle read_only checks */ char *query=NULL; query=(char *)"INSERT OR REPLACE INTO mysql_server_read_only_log VALUES (?1 , ?2 , ?3 , ?4 , ?5 , ?6)"; auto [rc1, statement_unique] = mmsd->mondb->prepare_v2(query); @@ -1777,6 +1886,7 @@ void * monitor_read_only_thread(const std::vector& da sqlite3_stmt *statement = statement_unique.get(); int rc; int read_only=1; // as a safety mechanism , read_only=1 is the default + bool valid_result = true; rc=(*proxy_sqlite3_bind_text)(statement, 1, mmsd->hostname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mmsd->mondb); rc=(*proxy_sqlite3_bind_int)(statement, 2, mmsd->port); ASSERT_SQLITE_OK(rc, mmsd->mondb); unsigned long long time_now=realtime_time(); @@ -1813,8 +1923,11 @@ VALGRIND_ENABLE_ERROR_REPORTING; // rc=(*proxy_sqlite3_bind_null)(statement, 5); ASSERT_SQLITE_OK(rc, mmsd->mondb); // } } else { - proxy_error("mysql_fetch_fields returns NULL, or mysql_num_fields is incorrect. Server %s:%d . See bug #1994\n", mmsd->hostname, mmsd->port); + valid_result = false; rc=(*proxy_sqlite3_bind_null)(statement, 5); ASSERT_SQLITE_OK(rc, mmsd->mondb); + proxy_error("mysql_fetch_fields returns NULL, or mysql_num_fields is incorrect. Server %s:%d . See bug #1994\n", mmsd->hostname, mmsd->port); + proxy_info("Dumping read_only result for server %s:%d, query: %s\n", mmsd->hostname, mmsd->port, monitor_query.c_str()); + dump_mysql_result(stderr, mmsd->result); } mysql_free_result(mmsd->result); mmsd->result=NULL; @@ -1831,11 +1944,13 @@ VALGRIND_ENABLE_ERROR_REPORTING; rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mmsd->mondb); rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mmsd->mondb); - if (mmsd->mysql_error_msg == NULL) { + if (valid_result && mmsd->mysql_error_msg == NULL) { read_only_success = true; } - if (timeout_reached == false && mmsd->interr == 0) { + if (!valid_result || stale_ip_timeout) { + // Ignore; do not infer backend state. + } else if (timeout_reached == false && mmsd->interr == 0) { MyHGM->read_only_action_v2( std::list { read_only_server_t { mmsd->hostname, mmsd->port, read_only } } ); // default behavior @@ -1870,10 +1985,21 @@ VALGRIND_ENABLE_ERROR_REPORTING; free(buff); } } + + /* error handling for both read_only and rds_topology checks */ if (mmsd->interr || mmsd->mysql_error_msg) { // check failed if (mmsd->mysql) { - proxy_error("Got error: mmsd %p , MYSQL %p , FD %d : %s\n", mmsd, mmsd->mysql, mmsd->mysql->net.fd, mmsd->mysql_error_msg); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, mysql_errno(mmsd->mysql)); + // AWS RDS topology discovery probes every replication-hostgroup member, but + // mysql.rds_topology exists only where a blue/green deployment is active. + // Treat ER_NO_SUCH_TABLE (1146) as "no topology here" and skip quietly. + if (mmsd->get_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY && mysql_errno(mmsd->mysql) == 1146) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "mysql.rds_topology not present on %s:%d; skipping blue/green discovery\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Got error: mmsd %p , MYSQL %p , FD %d : %s\n", mmsd, mmsd->mysql, mmsd->mysql->net.fd, mmsd->mysql_error_msg); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, mysql_errno(mmsd->mysql)); + } GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); } } else { @@ -1883,6 +2009,7 @@ VALGRIND_ENABLE_ERROR_REPORTING; } } } + __fast_exit_monitor_read_only_thread: if (mmsd->mysql) { // if we reached here we didn't put the connection back @@ -1907,11 +2034,13 @@ VALGRIND_ENABLE_ERROR_REPORTING; } } } + if (read_only_success) { __sync_fetch_and_add(&GloMyMon->read_only_check_OK,1); } else { __sync_fetch_and_add(&GloMyMon->read_only_check_ERR,1); } + delete mysql_thr; return NULL; } @@ -1920,6 +2049,7 @@ void * monitor_group_replication_thread(const std::vector mmsd->t1 + mysql_thread___monitor_groupreplication_healthcheck_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_groupreplication_healthcheck_timeout. Assuming viable_candidate=NO and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GR_HEALTH_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring group replication timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_groupreplication_healthcheck_timeout. Assuming viable_candidate=NO and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GR_HEALTH_CHECK_TIMEOUT); + } goto __exit_monitor_group_replication_thread; } if (mmsd->interr) { @@ -2000,9 +2137,16 @@ void * monitor_group_replication_thread(const std::vector mmsd->t1 + mysql_thread___monitor_groupreplication_healthcheck_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_groupreplication_healthcheck_timeout. Assuming viable_candidate=NO and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GR_HEALTH_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring group replication timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_groupreplication_healthcheck_timeout. Assuming viable_candidate=NO and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GR_HEALTH_CHECK_TIMEOUT); + } goto __exit_monitor_group_replication_thread; } if (GloMyMon->shutdown==true) { @@ -2109,7 +2253,9 @@ void * monitor_group_replication_thread(const std::vectorgroup_replication_mutex); // NOTE: we update MyHGM outside the mutex group_replication_mutex - if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure GR + if (stale_ip_timeout) { + // Logged/counted; do not change GR state for stale DNS targets. + } else if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure GR if (num_timeouts == 0) { // it wasn't a timeout, reconfigure immediately MyHGM->update_group_replication_set_offline(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, mmsd->mysql_error_msg); @@ -2261,6 +2407,7 @@ void * monitor_galera_thread(const std::vector& data) assert(!data.empty()); mysql_close(mysql_init(NULL)); MySQL_Monitor_State_Data *mmsd = data.front(); + bool stale_ip_timeout = false; // Wait for GloMTH to be initialized if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart MySQL_Thread * mysql_thr = new MySQL_Thread(); @@ -2344,9 +2491,16 @@ void * monitor_galera_thread(const std::vector& data) const unsigned long long now = monotonic_time(); #endif if (now > mmsd->t1 + mysql_thread___monitor_galera_healthcheck_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on Galera health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GALERA_HEALTH_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring Galera timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on Galera health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GALERA_HEALTH_CHECK_TIMEOUT); + } goto __exit_monitor_galera_thread; } if (GloMyMon->shutdown==true) { @@ -2365,9 +2519,16 @@ void * monitor_galera_thread(const std::vector& data) const unsigned long long now = monotonic_time(); #endif if (now > mmsd->t1 + mysql_thread___monitor_galera_healthcheck_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on Galera health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GALERA_HEALTH_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring Galera timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on Galera health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GALERA_HEALTH_CHECK_TIMEOUT); + } goto __exit_monitor_galera_thread; } if (GloMyMon->shutdown==true) { @@ -2544,7 +2705,9 @@ void * monitor_galera_thread(const std::vector& data) pthread_mutex_unlock(&GloMyMon->galera_mutex); // NOTE: we update MyHGM outside the mutex galera_mutex - if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure Galera + if (stale_ip_timeout) { + // Logged/counted; do not change Galera state for stale DNS targets. + } else if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure Galera if (num_timeouts == 0) { // it wasn't a timeout, reconfigure immediately MyHGM->update_galera_set_offline(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, mmsd->mysql_error_msg); @@ -3382,13 +3545,13 @@ VALGRIND_ENABLE_ERROR_REPORTING; } /** -* @brief Processes the discovered servers to eventually add them to 'runtime_mysql_servers'. -* @details This method takes a vector of discovered servers, compares them against the existing servers, and adds the new servers to 'runtime_mysql_servers'. -* @param originating_server_hostname A string which denotes the hostname of the originating server, from which the discovered servers were queried and found. +* @brief Add discovered servers to 'runtime_mysql_servers' and reader hostgroup. +* +* @param origin_server A string which denotes the hostname of the originating server, from which the discovered servers were queried and found. * @param discovered_servers A vector of servers discovered when querying the cluster's topology. -* @param reader_hostgroup Reader hostgroup to which we will add the discovered servers. +* @param reader_hostgroup Reader hostgroup to which we will add the discovered servers. */ -void MySQL_Monitor::process_discovered_topology(const std::string& originating_server_hostname, const vector& discovered_servers, int reader_hostgroup) { +void MySQL_Monitor::handle_aws_rds_multi_az_cluster(const std::string& origin_server, const std::vector& discovered_servers, int reader_hostgroup) { char *error = NULL; int cols = 0; int affected_rows = 0; @@ -3403,7 +3566,7 @@ void MySQL_Monitor::process_discovered_topology(const std::string& originating_s } else { vector> new_servers; vector saved_hostnames; - saved_hostnames.push_back(originating_server_hostname); + saved_hostnames.push_back(origin_server); // Do an initial loop through the query results to save existing runtime server hostnames for (std::vector::iterator it = runtime_mysql_servers->rows.begin(); it != runtime_mysql_servers->rows.end(); it++) { @@ -3414,20 +3577,12 @@ void MySQL_Monitor::process_discovered_topology(const std::string& originating_s } // Loop through discovered servers and process the ones we haven't saved yet - for (MYSQL_ROW s : discovered_servers) { - string current_discovered_hostname = s[2]; - string current_discovered_port_string = s[3]; - int current_discovered_port_int; - - try { - current_discovered_port_int = stoi(s[3]); - } catch (...) { - proxy_error( - "Unable to parse port value coming from '%s' during topology discovery ('%s':%s). Terminating discovery early.\n", - originating_server_hostname.c_str(), current_discovered_hostname.c_str(), current_discovered_port_string.c_str() - ); - return; + for (const AWS_RDS_Topology_Node& s : discovered_servers) { + if (s.endpoint.empty()) { + continue; } + const string& current_discovered_hostname = s.endpoint; + int current_discovered_port_int = s.port; if (find(saved_hostnames.begin(), saved_hostnames.end(), current_discovered_hostname) == saved_hostnames.end()) { tuple new_server(current_discovered_hostname, current_discovered_port_int, reader_hostgroup); @@ -3444,32 +3599,98 @@ void MySQL_Monitor::process_discovered_topology(const std::string& originating_s } /** -* @brief Check if a list of servers is matching the description of an AWS RDS Multi-AZ DB Cluster. -* @details This method takes a vector of discovered servers and checks that there are exactly three which are named "instance-[1|2|3]" respectively, as expected on an AWS RDS Multi-AZ DB Cluster. -* @param discovered_servers A vector of servers discovered when querying the cluster's topology. -* @return Returns 'true' if all conditions are met and 'false' otherwise. +* @brief Parse a 'SELECT * FROM mysql.rds_topology' result into an AWS_RDS_Topology_Result. +* +* @details Columns are resolved by name (they may be absent or differently ordered by RDS type). +* 'blue_green' is set when the 'role'/'status' columns are present and non-NULL on the first row. +* +* @return The parsed topology; empty 'nodes' if 'result' is NULL or has no rows. The result cursor is rewound before returning. */ -bool MySQL_Monitor::is_aws_rds_multi_az_db_cluster_topology(const std::vector& discovered_servers) { - if (discovered_servers.size() != 3) { - return false; +AWS_RDS_Topology_Result MySQL_Monitor::parse_aws_rds_topology(MYSQL_RES* result) { + AWS_RDS_Topology_Result out; + if (result == NULL) { + return out; } - const std::vector instance_names = {"-instance-1", "-instance-2", "-instance-3"}; - int identified_hosts = 0; - for (const std::string& instance_str : instance_names) { - for (MYSQL_ROW server : discovered_servers) { - if (server[2] == NULL || (server[2][0] == '\0')) { - continue; - } + unsigned int num_fields = mysql_num_fields(result); + MYSQL_FIELD *fields = mysql_fetch_fields(result); + int id_idx = -1, endpoint_idx = -1, port_idx = -1, role_idx = -1, status_idx = -1; + for (unsigned int i = 0; i < num_fields; i++) { + if (fields[i].name == NULL) { + continue; + } + if (strcasecmp(fields[i].name, "id") == 0) { + id_idx = (int)i; + } else if (strcasecmp(fields[i].name, "endpoint") == 0) { + endpoint_idx = (int)i; + } else if (strcasecmp(fields[i].name, "port") == 0) { + port_idx = (int)i; + } else if (strcasecmp(fields[i].name, "role") == 0) { + role_idx = (int)i; + } else if (strcasecmp(fields[i].name, "status") == 0) { + status_idx = (int)i; + } + } - std::string current_discovered_hostname = server[2]; - if (current_discovered_hostname.find(instance_str) != std::string::npos) { - ++identified_hosts; - break; + bool first = true; + MYSQL_ROW row; + while ((row = mysql_fetch_row(result))) { + AWS_RDS_Topology_Node node; + if (id_idx >= 0 && row[id_idx]) { + node.id = row[id_idx]; + } + if (endpoint_idx >= 0 && row[endpoint_idx]) { + node.endpoint = row[endpoint_idx]; + } + if (port_idx >= 0 && row[port_idx]) { + try { + node.port = std::stoi(row[port_idx]); + } catch (...) { + node.port = 0; } } + if (role_idx >= 0 && row[role_idx]) { + node.role = row[role_idx]; + } + if (status_idx >= 0 && row[status_idx]) { + node.status = row[status_idx]; + } + if (first) { + // blue/green deployment exposes non-NULL role/status; absent or NULL => Multi-AZ Cluster + out.blue_green = (role_idx >= 0 && status_idx >= 0 + && row[role_idx] != NULL && row[status_idx] != NULL); + first = false; + } + out.nodes.push_back(std::move(node)); + } + mysql_data_seek(result, 0); // rewind for any subsequent reader + return out; +} + +/** +* @brief Classify the parsed mysql.rds_topology result and dispatch. +* +* @details A blue/green deployment optionally auto-generates a runtime aws_rds_bgd_hostgroups +* entry (when 'mysql-aws_blue_green_deployment_auto_discovery' is enabled); otherwise the rows +* are treated as a Multi-AZ Cluster and handed to the existing auto-discovery path. +*/ +void MySQL_Monitor::process_aws_rds_topology(MySQL_Monitor_State_Data* mmsd) { + if (mmsd->result == NULL) { + return; + } + + AWS_RDS_Topology_Result topology = parse_aws_rds_topology(mmsd->result); + if (topology.nodes.empty()) { + return; + } + + if (topology.blue_green) { + if (mysql_thread___aws_blue_green_deployment_auto_discovery) { + MyHGM->add_aws_rds_bgd_hostgroup_entry(mmsd->hostname, mmsd->port); + } + } else { + handle_aws_rds_multi_az_cluster(mmsd->hostname, topology.nodes, mmsd->reader_hostgroup); } - return (identified_hosts == 3); } void * MySQL_Monitor::monitor_read_only() { @@ -3496,7 +3717,7 @@ void * MySQL_Monitor::monitor_read_only() { char *error=NULL; SQLite3_result *resultset=NULL; // add support for SSL - char *query=(char *)"SELECT hostname, port, MAX(use_ssl) use_ssl, check_type, reader_hostgroup FROM mysql_servers JOIN mysql_replication_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE status NOT IN (2,3) GROUP BY hostname, port ORDER BY RANDOM()"; + char *query=(char *)SELECT_SERVERS_FOR_READ_ONLY; t1=monotonic_time(); if (!GloMTH) return NULL; // quick exit during shutdown/restart @@ -3507,7 +3728,6 @@ void * MySQL_Monitor::monitor_read_only() { next_loop_at=0; } - if (t1 < next_loop_at) { goto __sleep_monitor_read_only; } @@ -3519,7 +3739,7 @@ void * MySQL_Monitor::monitor_read_only() { proxy_error("Error on %s : %s\n", query, error); goto __end_monitor_read_only_loop; } - + if (resultset->rows_count == 0) { goto __end_monitor_read_only_loop; } @@ -3528,7 +3748,7 @@ void * MySQL_Monitor::monitor_read_only() { if (topology_loop >= topology_loop_max) { do_discovery_check = true; topology_loop = 0; - } + } topology_loop += 1; } @@ -3569,6 +3789,7 @@ void * MySQL_Monitor::monitor_read_only() { usleep(st); } } + if (mysql_thr) { delete mysql_thr; mysql_thr=NULL; @@ -5034,6 +5255,13 @@ void * MySQL_Monitor::run() { assert(0); // LCOV_EXCL_STOP } + pthread_t monitor_aws_rds_bgd_thread; + if (pthread_create(&monitor_aws_rds_bgd_thread, &attr, &monitor_aws_rds_bgd_pthread,NULL) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } pthread_t monitor_replication_lag_thread; if (pthread_create(&monitor_replication_lag_thread, &attr, &monitor_replication_lag_pthread,NULL) != 0) { // LCOV_EXCL_START @@ -5133,6 +5361,7 @@ void * MySQL_Monitor::run() { pthread_join(monitor_group_replication_thread,NULL); pthread_join(monitor_galera_thread,NULL); pthread_join(monitor_aws_aurora_thread,NULL); + pthread_join(monitor_aws_rds_bgd_thread,NULL); pthread_join(monitor_replication_lag_thread,NULL); My_Conn_Pool->purge_all_connections(); @@ -6406,99 +6635,1639 @@ void * MySQL_Monitor::monitor_aws_aurora() { return NULL; } -unsigned int MySQL_Monitor::estimate_lag(char* server_id, AWS_Aurora_status_entry** aase, unsigned int idx, unsigned int add_lag_ms, unsigned int min_lag_ms, unsigned int lag_num_checks) { - assert(aase); - assert(server_id); - assert(idx >= 0 && idx < N_L_ASE); +/** +* @brief Runs an async query + store_result on the monitor connection. +* +* @param mmsd Monitor state data holding the connection, timing, and result. +* @param query SQL text to execute. +* @param worker_stop Per-worker shutdown signal. +* +* @return 0 on success, 1 on timeout/query-error, 2 if global or worker shutdown was requested. +*/ +int MySQL_Monitor::aws_rds_bgd_async_query(MySQL_Monitor_State_Data *mmsd, const char *query, std::atomic_bool& worker_stop) { + mmsd->t1 = monotonic_time(); + mmsd->interr = 0; + mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, query); + while (mmsd->async_exit_status) { + mmsd->async_exit_status = wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); + const unsigned long long now = monotonic_time(); + if (now > mmsd->t1 + mmsd->aws_aurora_check_timeout_ms * 1000) { + mmsd->mysql_error_msg = strdup("timeout check"); + return 1; + } + if (shutdown == true || worker_stop.load()) { + return 2; + } + if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { + mmsd->async_exit_status = mysql_query_cont(&mmsd->interr, mmsd->mysql, mmsd->async_exit_status); + } + } + mmsd->async_exit_status = mysql_store_result_start(&mmsd->result, mmsd->mysql); + while (mmsd->async_exit_status) { + mmsd->async_exit_status = wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); + const unsigned long long now = monotonic_time(); + if (now > mmsd->t1 + mmsd->aws_aurora_check_timeout_ms * 1000) { + mmsd->mysql_error_msg = strdup("timeout check"); + return 1; + } + if (shutdown == true || worker_stop.load()) { + return 2; + } + if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { + mmsd->async_exit_status = mysql_store_result_cont(&mmsd->result, mmsd->mysql, mmsd->async_exit_status); + } + } + if (mmsd->interr) { // query failed (may be ER_NO_SUCH_TABLE 1146) + mmsd->mysql_error_msg = strdup(mysql_error(mmsd->mysql)); + return 1; + } + return 0; +} - if (lag_num_checks > N_L_ASE) lag_num_checks = N_L_ASE; - if (lag_num_checks <= 0) lag_num_checks = 1; +/** +* @brief Flag servers as switchover-in-progress so the read_only monitor skips them. +*/ +static void aws_rds_bgd_set_bgd_in_progress(AWS_RDS_BGD_State& st) { + if (st.bgd_in_progress_set) { + return; + } - unsigned int mlag = 0; - unsigned int lag = 0; + GloMyMon->set_aws_rds_bgd_server_in_progress(st, true); + st.bgd_in_progress_set = true; + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover in progress, suspending read_only monitor checks on writer/reader hostgroups until SWITCHOVER_COMPLETED\n", + st.writer_hg, st.reader_hg); +} - for (unsigned int i = 1; i <= lag_num_checks; i++) { - if (!aase[idx] || !aase[idx]->host_statuses) - break; - for (auto hse : *(aase[idx]->host_statuses)) { - if (strcmp(server_id, hse->server_id)==0 && (unsigned int)hse->replica_lag_ms != 0) { - unsigned int ms = std::max(((unsigned int)hse->replica_lag_ms + add_lag_ms), min_lag_ms); - if (ms > mlag) mlag = ms; - if (!lag) lag = ms; - } - } - if (idx == 0) idx = N_L_ASE; - idx--; +/** +* @brief Clear the switchover-in-progress flag from the aws_rds_bgd_server_status map +*/ +static void aws_rds_bgd_clear_bgd_in_progress(AWS_RDS_BGD_State& st) { + if (!st.bgd_in_progress_set) { + return; } - return mlag; + GloMyMon->set_aws_rds_bgd_server_in_progress(st, false); + st.bgd_in_progress_set = false; + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover completed, resuming read_only monitor checks on writer/reader hostgroups\n", + st.writer_hg, st.reader_hg); } -void print_aws_aurora_status_entry(AWS_Aurora_status_entry* aase) { - if (aase && aase->start_time) { - if (aase->host_statuses->size()) { - for (AWS_Aurora_replica_host_status_entry* hse : *aase->host_statuses) { - if (hse) { - fprintf(stderr,"%s %s %s %f %f\n", hse->server_id, hse->session_id, hse->last_update_timestamp, hse->replica_lag_ms , hse->cpu); - } - } - } +/** +* @brief Set the deployment's switchover status and persist it in runtime table. +*/ +static void aws_rds_bgd_set_status(AWS_RDS_BGD_State& st, AWS_RDS_BGD_Status status) { + if (st.bgd_status == status) { + return; } -} -void MySQL_Monitor::aws_aurora_autopurge_servers(unsigned int wHG, unsigned int rHG, AWS_Aurora_status_entry *ase, unsigned int threshold, std::map& autopurge_counter, const std::string& domain_name) { - bool server_purged = false; + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", + st.writer_hg, st.reader_hg, + aws_rds_bgd_status_str(st.bgd_status), aws_rds_bgd_status_str(status)); - std::set present_servers; - for (auto h : *(ase->host_statuses)) { - present_servers.insert(h->server_id); + st.bgd_status = status; + MyHGM->aws_rds_bgd_set_runtime_status(st.writer_hg, static_cast(status)); + + if (status == AWS_RDS_BGD_Status::NONE) { + aws_rds_bgd_clear_bgd_in_progress(st); } +} - MyHGM->wrlock(); +/** +* @brief Load one BGD worker's configuration from the published host rows. +* +* @details Copies the cluster rows, verifies their checksum, copies configuration fields from +* the first row, and builds the probe host list. FSM fields retain their defaults and must not +* replace the corresponding fields in the live state. +* +* @param writer_hg Writer hostgroup identifying the deployment. +* @param current_checksum Per-cluster checksum captured for this refresh. +* @param candidate State populated from the published rows. +* +* @return true when the checksum matches and the rows contain a blue writer; false otherwise. +*/ +bool MySQL_Monitor::aws_rds_bgd_load_worker_config(int writer_hg, uint64_t current_checksum, AWS_RDS_BGD_State& candidate) { + SQLite3_result result(AWS_RDS_BGD_HOSTS_COLUMNS); + std::shared_ptr hosts_resultset; - // Writer hostgroup - MyHGC *whgc = MyHGM->MyHGC_lookup(wHG); - if (whgc && whgc->mysrvs) { - for (unsigned int j = 0; j < whgc->mysrvs->cnt(); j++) { - MySrvC *mysrvc = whgc->mysrvs->idx(j); - if (mysrvc->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) continue; + pthread_mutex_lock(&aws_rds_bgd_hosts_mutex); + hosts_resultset = AWS_RDS_BGD_Hosts_resultset; + pthread_mutex_unlock(&aws_rds_bgd_hosts_mutex); - std::string server_id(mysrvc->address); - size_t pos = server_id.rfind(domain_name); - if (pos != std::string::npos) { - server_id.erase(pos); + if (hosts_resultset) { + for (SQLite3_row* row : hosts_resultset->rows) { + if (atoi(row->fields[AWS_RDS_BGD_WRITER_HOSTGROUP]) == writer_hg) { + result.add_row(row); } + } + } + if (result.raw_checksum() != current_checksum) { + return false; + } - std::string srv_key = std::to_string(wHG) + ":" + server_id; - if (present_servers.find(server_id) == present_servers.end()) { - if (++autopurge_counter[srv_key] >= (int)threshold) { - proxy_warning("Auto-purging server %s:%d from hostgroup %u (absent from REPLICA_HOST_STATUS for %d checks)\n", - mysrvc->address, mysrvc->port, wHG, autopurge_counter[srv_key]); - MyHGM->remove_server_in_hg(wHG, mysrvc->address, mysrvc->port); - autopurge_counter.erase(srv_key); - server_purged = true; - } - } else { - autopurge_counter.erase(srv_key); - } + candidate.writer_hg = writer_hg; + bool first_row = true; + + for (SQLite3_row* row : result.rows) { + unsigned int reader_hg = atoi(row->fields[AWS_RDS_BGD_READER_HOSTGROUP]); + int green_writer_hg = row->fields[AWS_RDS_BGD_GREEN_WRITER_HOSTGROUP] + && row->fields[AWS_RDS_BGD_GREEN_WRITER_HOSTGROUP][0] + ? atoi(row->fields[AWS_RDS_BGD_GREEN_WRITER_HOSTGROUP]) : -1; + int green_reader_hg = row->fields[AWS_RDS_BGD_GREEN_READER_HOSTGROUP] + && row->fields[AWS_RDS_BGD_GREEN_READER_HOSTGROUP][0] + ? atoi(row->fields[AWS_RDS_BGD_GREEN_READER_HOSTGROUP]) : -1; + unsigned int check_interval_ms = atoi(row->fields[AWS_RDS_BGD_CHECK_INTERVAL_MS]); + unsigned int check_timeout_ms = atoi(row->fields[AWS_RDS_BGD_CHECK_TIMEOUT_MS]); + int writer_is_also_reader = atoi(row->fields[AWS_RDS_BGD_WRITER_IS_ALSO_READER]); + + if (first_row) { + candidate.reader_hg = reader_hg; + candidate.green_writer_hg = green_writer_hg; + candidate.green_reader_hg = green_reader_hg; + candidate.check_interval_ms = check_interval_ms; + candidate.check_timeout_ms = check_timeout_ms; + candidate.writer_is_also_reader = writer_is_also_reader; + first_row = false; + } + + char* srv_type = row->fields[AWS_RDS_BGD_SRV_TYPE]; + if (srv_type[0] == 'B' && atoi(row->fields[AWS_RDS_BGD_IS_WRITER]) != 0) { + candidate.probe_hosts.push_back(AWS_RDS_BGD_Probe_Host { + row->fields[AWS_RDS_BGD_HOSTNAME], + atoi(row->fields[AWS_RDS_BGD_PORT]), + atoi(row->fields[AWS_RDS_BGD_USE_SSL]) + }); } } - // Reader hostgroup - if (rHG > 0) { - MyHGC *rhgc = MyHGM->MyHGC_lookup(rHG); - if (rhgc && rhgc->mysrvs) { - for (unsigned int j = 0; j < rhgc->mysrvs->cnt(); j++) { - MySrvC *mysrvc = rhgc->mysrvs->idx(j); - if (mysrvc->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) continue; + if (first_row || candidate.probe_hosts.empty()) { + proxy_error("AWS RDS BGD [wHG=%d]: no blue writer available for topology checks\n", writer_hg); + return false; + } - std::string server_id(mysrvc->address); - size_t pos = server_id.rfind(domain_name); - if (pos != std::string::npos) { - server_id.erase(pos); - } + return true; +} - std::string srv_key = std::to_string(rHG) + ":" + server_id; +/** +* @brief Replace only configuration-derived fields in a live BGD worker state. +* +* @param st Live worker state. +* @param candidate Parsed configuration to apply. +*/ +void MySQL_Monitor::aws_rds_bgd_apply_cluster_config(AWS_RDS_BGD_State& st, AWS_RDS_BGD_State& candidate) { + st.reader_hg = candidate.reader_hg; + st.green_writer_hg = candidate.green_writer_hg; + st.green_reader_hg = candidate.green_reader_hg; + st.writer_is_also_reader = candidate.writer_is_also_reader; + st.check_interval_ms = candidate.check_interval_ms; + st.check_timeout_ms = candidate.check_timeout_ms; + st.probe_hosts = candidate.probe_hosts; +} + +/** +* @brief Run the monitor loop for one AWS RDS BGD writer hostgroup. +* +* @param arg Pointer to the worker state owned by the parent monitor thread. +* +* @return nullptr when the worker exits. +*/ +void* monitor_RDS_BGD_thread_HG(void* arg) { + AWS_RDS_BGD_Worker* worker = static_cast(arg); + unsigned int wHG = worker->writer_hg; + unsigned int cur_host_idx = 0; + set_thread_name("MonitorRdsBgdHG", GloVars.set_thread_name); + proxy_info("Started Monitor thread for AWS RDS writer HG %u\n", wHG); + + // Wait for GloMTH to be initialized + if (!wait_for_glo_mth()) + return NULL; + + AWS_RDS_BGD_State st; + st.writer_hg = wHG; + + MyHGM->aws_rds_bgd_set_runtime_status(wHG, static_cast(AWS_RDS_BGD_Status::NONE)); + + unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; + MySQL_Thread * mysql_thr = new MySQL_Thread(); + mysql_thr->curtime = monotonic_time(); + MySQL_Monitor__thread_MySQL_Thread_Variables_version = GloMTH->get_global_version(); + mysql_thr->refresh_variables(); + + unsigned long long t1 = 0; + unsigned long long next_loop_at = 0; + bool crc = false; + uint64_t last_checksum = 0; + size_t rnd; + bool found_pingable_host = false; + MySQL_Monitor_State_Data *mmsd = NULL; + RDS_BGD_Topology_Monitor_State topology_state = TOPOLOGY_TABLE_CHECK; + + t1 = monotonic_time(); + + while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true + && worker->worker_stop.load()==false) { + unsigned int glover; + t1 = monotonic_time(); + bool poll_success = false; + + if (!GloMTH) + goto __exit_monitor_RDS_BGD_thread_HG_now; + + // if variables changed, refresh and force a new check + glover = GloMTH->get_global_version(); + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version = glover; + mysql_thr->refresh_variables(); + next_loop_at = 0; + } + + uint64_t current_checksum = worker->current_checksum.load(); + if (current_checksum != last_checksum) { + if (!GloMyMon->aws_rds_bgd_refresh_worker_config(st, current_checksum, topology_state, next_loop_at)) { + usleep(50000); + continue; + } + last_checksum = current_checksum; + if (cur_host_idx >= st.probe_hosts.size()) { + cur_host_idx = 0; + } + } + + if (st.probe_hosts.empty()) { + next_loop_at = t1 + (st.check_interval_ms ? st.check_interval_ms : 1000) * 1000; + usleep(50000); + continue; + } + + if (t1 < next_loop_at) { + unsigned long long st = next_loop_at - t1; + if (st > 50000) { + st = 50000; + } + usleep(st); + continue; + } + + // Determine the host to probe. If the FSM pinned a host (the green IP, during a + // switchover), poll it directly and skip ping/random selection; otherwise pick a + // pingable host, starting at a random position. + const char* poll_host; + int poll_port; + bool poll_use_ssl; + if (!st.next_check_host.empty()) { + bool found_writer = false; + for (const auto& p : st.bg_map) { + if (p.is_writer) { + poll_host = st.next_check_host.c_str(); + poll_port = p.port; + poll_use_ssl = (p.green_use_ssl >= 0) ? p.green_use_ssl : p.blue_use_ssl; + found_writer = true; + break; + } + } + if (!found_writer) { + // Highly unlikely: next_check_host is set but bg_map has no writer pair. + // Clear the green pin and fall through to blue host selection. + st.next_check_host.clear(); + st.next_check_host_failures = 0; + } + } + + if (st.next_check_host.empty()) { + found_pingable_host = false; + rnd = (size_t) rand(); + rnd %= st.probe_hosts.size(); + for (size_t i = 0; found_pingable_host == false && i < st.probe_hosts.size(); i++) { + size_t host_idx = (rnd + i) % st.probe_hosts.size(); + AWS_RDS_BGD_Probe_Host& host = st.probe_hosts[host_idx]; + if (GloMyMon->server_responds_to_ping(host.hostname.data(), host.port)) { + found_pingable_host = true; + cur_host_idx = host_idx; + } else { + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, wHG, host.hostname.data(), host.port, + ER_PROXYSQL_AWS_NO_PINGABLE_SRV + ); + } + } + if (found_pingable_host == false) { + proxy_error("No node is pingable for AWS RDS cluster with writer HG %u\n", wHG); + next_loop_at = t1 + st.check_interval_ms * 1000; + continue; + } + poll_host = st.probe_hosts[cur_host_idx].hostname.c_str(); + poll_port = st.probe_hosts[cur_host_idx].port; + poll_use_ssl = st.probe_hosts[cur_host_idx].use_ssl; + } + + mmsd = new MySQL_Monitor_State_Data( + MON_AWS_RDS_BGD, (char*)poll_host, poll_port, poll_use_ssl + ); + mmsd->writer_hostgroup = wHG; + mmsd->aws_aurora_check_timeout_ms = st.check_timeout_ms; + mmsd->mysql = GloMyMon->My_Conn_Pool->get_connection(mmsd->hostname, mmsd->port, mmsd); + mmsd->t1 = t1; + + crc = false; + if (mmsd->mysql == NULL) { // need a new connection + bool rc = mmsd->create_new_connection(); + if (mmsd->mysql) { + GloMyMon->My_Conn_Pool->conn_register(mmsd); + } + crc = true; + if (rc == false) { + proxy_error("Error on AWS RDS check for %s:%d. Unable to create a connection.\n", mmsd->hostname, mmsd->port); + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, + ER_PROXYSQL_AWS_HEALTH_CHECK_CONN_TIMEOUT + ); + goto __end_of_loop; + } + } + + if (topology_state == TOPOLOGY_TABLE_CHECK) { + // State TOPOLOGY_TABLE_CHECK: confirm mysql.rds_topology exists. Once seen + // we advance to TOPOLOGY_METADATA_FETCH and skip this check on subsequent + // iterations, until a fetch reports the table is gone. + + int qrc = GloMyMon->aws_rds_bgd_async_query(mmsd, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK, worker->worker_stop); + if (qrc == 2) { + goto __exit_monitor_RDS_BGD_thread_HG_now; + } + if (qrc != 0) { + proxy_error( + "AWS RDS topology availability check failed for %s:%d : %s\n", + mmsd->hostname, mmsd->port, mmsd->mysql_error_msg ? mmsd->mysql_error_msg : "unknown" + ); + goto __end_of_loop; + } + bool table_available = (mmsd->result && mysql_num_rows(mmsd->result) > 0); + if (mmsd->result) { + mysql_free_result(mmsd->result); + mmsd->result = NULL; + } + if (!table_available) { + // no blue/green deployment or multi-az cluster discovery in progress, or the + // post-switchover topology has fully drained; run any pending deferred cleanup. + GloMyMon->aws_rds_bgd_handle_topology_absent(st); + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "mysql.rds_topology not present on %s:%d (RDS writer HG %u); skipping\n", + mmsd->hostname, mmsd->port, wHG); + goto __end_of_loop; + } + topology_state = TOPOLOGY_METADATA_FETCH; + } else if (topology_state == TOPOLOGY_METADATA_FETCH) { + // State TOPOLOGY_METADATA_FETCH: fetch topology metadata. The column set + // differs by RDS type (the Multi-AZ Cluster topology table may not expose + // 'role'/'status' at all), so dump all columns and detect what is present. + + int qrc = GloMyMon->aws_rds_bgd_async_query(mmsd, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY, worker->worker_stop); + if (qrc == 2) { + goto __exit_monitor_RDS_BGD_thread_HG_now; + } + if (qrc != 0) { + unsigned int err = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; + if (err == 1146) { + // the table vanished (ER_NO_SUCH_TABLE), e.g. a blue/green deployment + // was cancelled or a post-switchover topology fully drained: re-check its + // existence on the next iteration and return to the baseline poll interval. + topology_state = TOPOLOGY_TABLE_CHECK; + st.next_check_interval_ms = 0; + GloMyMon->aws_rds_bgd_handle_topology_absent(st); + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "mysql.rds_topology vanished on %s:%d (RDS writer HG %u); rechecking availability\n", + mmsd->hostname, mmsd->port, wHG); + } else { + proxy_error( + "AWS RDS topology fetch failed for %s:%d : %s\n", + mmsd->hostname, mmsd->port, mmsd->mysql_error_msg ? mmsd->mysql_error_msg : "unknown" + ); + } + goto __end_of_loop; + } + + // the BGD thread only monitors blue/green hostgroups; parse the topology + // (shared with the read_only path) and hand the struct to the handler. + if (mmsd->result && mysql_num_rows(mmsd->result) > 0) { + poll_success = true; + AWS_RDS_Topology_Result topo = GloMyMon->parse_aws_rds_topology(mmsd->result); + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "AWS RDS BGD [wHG=%u]: topology probe on %s:%d (blue_green=%d, nodes=%zu)\n", + wHG, mmsd->hostname, mmsd->port, topo.blue_green ? 1 : 0, topo.nodes.size()); + GloMyMon->handle_aws_rds_bgd(st, topo); + } else { + poll_success = true; + // Query succeeded with no rows: mysql.rds_topology has drained (blue-reader + // DNS fully propagated). Run post-switchover cleanup. + GloMyMon->aws_rds_bgd_handle_topology_absent(st); + } + + if (mmsd->result) { + mysql_free_result(mmsd->result); + mmsd->result = NULL; + } + } + +__end_of_loop: + if (!st.next_check_host.empty()) { + if (poll_success) { + st.next_check_host_failures = 0; + } else { + st.next_check_host_failures++; + if (st.next_check_host_failures >= 3) { + proxy_warning("AWS RDS BGD [wHG=%u rHG=%u]: green probe host %s unreachable after %u attempts, falling back to blue and clearing DNS pins\n", + wHG, st.reader_hg, st.next_check_host.c_str(), st.next_check_host_failures); + st.next_check_host.clear(); + st.next_check_host_failures = 0; + for (const auto& p : st.bg_map) { + GloMyMon->dns_cache->remove(p.blue_host); + GloMyMon->My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); + } + } + } + } + + mmsd->t2 = monotonic_time(); + // the FSM tightens the interval to 100ms while a switchover is in flight + // (st.next_check_interval_ms); otherwise fall back to the configured baseline. + unsigned int eff = st.next_check_interval_ms ? st.next_check_interval_ms : st.check_interval_ms; + next_loop_at = t1 + (eff * 1000); + if (mmsd->t2 > t1) { + next_loop_at -= (mmsd->t2 - t1); + } + if (mmsd->mysql) { + if (mmsd->mysql_error_msg) { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } else if (crc) { + if (mmsd->set_wait_timeout()) { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } else { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } + } else { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } + } + delete mmsd; + mmsd = NULL; + } + +__exit_monitor_RDS_BGD_thread_HG_now: + if (st.bgd_status != AWS_RDS_BGD_Status::NONE) { + GloMyMon->handle_aws_rds_bgd_post_switchover(st, true); + } + + if (mmsd) { + if (mmsd->mysql) { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } + delete mmsd; + mmsd = NULL; + } + + if (mysql_thr) { + delete mysql_thr; + mysql_thr = NULL; + } + + proxy_info("Stopping Monitor thread for AWS RDS writer HG %u\n", wHG); + return NULL; +} + +// Split "." into the host (part before the first dot) and the remaining domain. +static void aws_rds_bgd_split_hostname(const std::string& hostname, std::string& host, std::string& domain) { + size_t dot = hostname.find('.'); + if (dot == std::string::npos) { + host = hostname; + domain.clear(); + return; + } + host = hostname.substr(0, dot); + domain = hostname.substr(dot + 1); +} + +// Given a green host "-green-", return ""; +// returns empty when the "-green-" suffix is absent. +static std::string aws_rds_bgd_strip_green_host_suffix(const std::string& green_host) { + size_t pos = green_host.find("-green-"); + if (pos == std::string::npos) { + return ""; + } + return green_host.substr(0, pos); +} + +// True when the green hostname is the blue/green TARGET counterpart of the blue hostname, i.e. +// green "-green-." maps to blue ".". +static bool aws_rds_bgd_match_host(const std::string& blue_hostname, const std::string& green_hostname) { + std::string b_host, b_domain, g_host, g_domain; + aws_rds_bgd_split_hostname(blue_hostname, b_host, b_domain); + aws_rds_bgd_split_hostname(green_hostname, g_host, g_domain); + std::string g_host_stripped = aws_rds_bgd_strip_green_host_suffix(g_host); + if (g_host_stripped.empty()) { + return false; + } + return g_host_stripped == b_host && g_domain == b_domain; +} + +/** +* @brief Build the blue-to-green host mapping for a BGD worker. +* +* @details Builds the map only when it is empty. The topology exposes only primaries, so the +* writer pair is always present; reader pairs are added when green_reader_hostgroup is configured. +* +* @param st Worker-owned BGD state. +* @param topo Parsed topology used to identify the green target. +*/ +void MySQL_Monitor::aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topo) { + if (!st.bg_map.empty()) { + return; + } + + AWS_RDS_Topology_Node* target = topo.target(); + if (!target || target->endpoint.empty()) { + return; + } + + std::string green_writer_host = target->endpoint; + + MyHGM->wrlock(); + + // blue writer: the writer_hostgroup member whose name matches the green TARGET. + MyHGC* whgc = MyHGM->MyHGC_find(st.writer_hg); + if (whgc && whgc->mysrvs) { + for (unsigned int j = 0; j < whgc->mysrvs->cnt(); j++) { + MySrvC* s = whgc->mysrvs->idx(j); + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD + || s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT) { + continue; + } + if (aws_rds_bgd_match_host(s->address, green_writer_host)) { + AWS_RDS_BlueGreenPair p; + p.blue_host = s->address; + p.port = s->port; + p.green_host = green_writer_host; + p.blue_weight = s->weight; + p.blue_max_conns = s->max_connections; + p.blue_use_ssl = s->use_ssl; + p.is_writer = true; + + // read the green writer's use_ssl config + if (st.green_writer_hg >= 0) { + MyHGC* gwhgc = MyHGM->MyHGC_find((unsigned int)st.green_writer_hg); + if (gwhgc && gwhgc->mysrvs) { + for (unsigned int k = 0; k < gwhgc->mysrvs->cnt(); k++) { + MySrvC* gs = gwhgc->mysrvs->idx(k); + if (strcasecmp(gs->address, green_writer_host.c_str()) != 0 || gs->port != p.port) { + continue; + } + + p.green_offline = + gs->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD + || gs->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT; + if (!p.green_offline) { + p.green_use_ssl = gs->use_ssl; + } + break; + } + } + } + + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u]: mapped blue writer '%s:%d' <-> green '%s'\n", + st.writer_hg, p.blue_host.c_str(), p.port, p.green_host.c_str()); + st.bg_map.push_back(std::move(p)); + break; + } + } + } + + // reader pairs: match blue readers to user-added green readers by name. + if (st.green_reader_hg >= 0) { + std::vector green_reader_hosts; + MyHGC* grhgc = MyHGM->MyHGC_find((unsigned int)st.green_reader_hg); + if (grhgc && grhgc->mysrvs) { + for (unsigned int j = 0; j < grhgc->mysrvs->cnt(); j++) { + MySrvC* s = grhgc->mysrvs->idx(j); + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD + || s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT) { + continue; + } + green_reader_hosts.push_back(s->address); + } + } + + MyHGC* rhgc = MyHGM->MyHGC_find(st.reader_hg); + if (rhgc && rhgc->mysrvs) { + for (unsigned int j = 0; j < rhgc->mysrvs->cnt(); j++) { + MySrvC* s = rhgc->mysrvs->idx(j); + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD + || s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT) { + continue; + } + for (const std::string& green_reader_host : green_reader_hosts) { + if (aws_rds_bgd_match_host(s->address, green_reader_host)) { + AWS_RDS_BlueGreenPair p; + p.blue_host = s->address; + p.port = s->port; + p.green_host = green_reader_host; + p.blue_weight = s->weight; + p.blue_max_conns = s->max_connections; + p.blue_use_ssl = s->use_ssl; + p.is_writer = false; + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u]: mapped blue reader '%s:%d' <-> green '%s'\n", + st.writer_hg, p.blue_host.c_str(), p.port, p.green_host.c_str()); + st.bg_map.push_back(std::move(p)); + break; + } + } + } + } + } + + MyHGM->wrunlock(); +} + +/** +* @brief Resolve the green IPs and pin the worker's probe to the green writer's IP. +* +* @details Resolves each pair's green host (DNS_Cache first, then a live lookup tracking TTL), +* then sets 'st.next_check_host' to the green writer's IP. From then on the worker polls the +* green primary BY IP: green stays reachable through the entire cutover (blue has a connectivity +* gap), and the green IP survives the post-COMPLETED name swap (it becomes the promoted primary), +* whereas the green DNS name is retired. 'next_check_host' is cleared at COMPLETED. +* +* @param st Worker-owned BGD state. +*/ +void MySQL_Monitor::aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { + int ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); + for (auto &p : st.bg_map) { + if (p.green_offline) { + continue; + } + + // Always check the cache first: a green host that is a monitored server may be there. + size_t n = 0; + std::string ip = MySQL_Monitor::dns_lookup(p.green_host, false, &n); + if (!ip.empty()) { + p.green_ip = ip; + p.green_ip_ttl = 0; + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u rHG=%u]: green '%s' IP %s (DNS_Cache)\n", + st.writer_hg, st.reader_hg, p.green_host.c_str(), p.green_ip.c_str()); + continue; + } + // Cache miss (green is not a monitored server): resolve DNS now and track its TTL. + if (p.green_ip.empty() || (p.green_ip_ttl != 0 && monotonic_time() > p.green_ip_ttl)) { + std::vector ips = dns_resolve(p.green_host, ai_family); + if (!ips.empty()) { + p.green_ip = ips.front(); + p.green_ip_ttl = monotonic_time() + + (1000ULL * (unsigned long long)mysql_thread___monitor_local_dns_cache_ttl); + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u rHG=%u]: green '%s' IP %s (resolved, ttl=%lus)\n", + st.writer_hg, st.reader_hg, p.green_host.c_str(), p.green_ip.c_str(), + (unsigned long)mysql_thread___monitor_local_dns_cache_ttl); + } + } + } + + // Pin the worker's next probe to the green writer's IP (observe the switchover from green). + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer && !p.green_offline && !p.green_ip.empty()) { + if (st.next_check_host != p.green_ip) { + st.next_check_host = p.green_ip; + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: pinning rds_topology probe to green IP %s\n", + st.writer_hg, st.reader_hg, p.green_ip.c_str()); + } + break; + } + } +} + +/** +* @brief Add the green writer to green_writer_hostgroup, when that hostgroup is configured. +* +* @param st Worker-owned BGD state. +*/ +void MySQL_Monitor::aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { + if (st.green_writer_hg < 0) { + return; + } + for (AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (!p.is_writer) { + continue; + } + + srv_info_t srv_info { p.green_host, (uint16_t)p.port, "AWS RDS BGD green writer" }; + srv_opts_t srv_opts { -1, -1, -1 }; + MyHGM->wrlock(); + int rc = MyHGM->create_new_server_in_hg((uint32_t)st.green_writer_hg, srv_info, srv_opts); + if (rc == 0) { + MySrvC* s = MyHGM->find_server_in_hg((unsigned int)st.green_writer_hg, p.green_host, p.port); + if (s) { + p.green_use_ssl = s->use_ssl; + p.green_offline = false; + } + MyHGM->publish_mysql_servers_to_runtime(); + } + MyHGM->wrunlock(); + break; + } +} + +/** +* @brief Find the writer pair in a blue/green map. +* +* @param bg_map Blue/green host mapping. +* @param writer Writer address populated when a pair is found. +* +* @return true when the map contains a writer pair; false otherwise. +*/ +bool MySQL_Monitor::aws_rds_bgd_find_writer(std::vector& bg_map, srv_addr_t& writer) { + for (AWS_RDS_BlueGreenPair& p : bg_map) { + if (p.is_writer) { + writer = srv_addr_t { p.blue_host, p.port }; + return true; + } + } + return false; +} + +/** +* @brief Apply a changed configuration to one running BGD worker. +* +* @details Before writer post-processing, applies the configuration and schedules mapping +* reconciliation after the next topology poll. At or after post-processing, rolls back the +* deployment and restarts its topology FSM without replacing the worker thread. +* +* @param st Worker-owned BGD state. +* @param current_checksum Per-cluster checksum captured for this refresh. +* @param topology_state Current topology query state. +* @param next_loop_at Next scheduled worker iteration. +* +* @return true when the captured configuration was applied; false when it must be retried. +*/ +bool MySQL_Monitor::aws_rds_bgd_refresh_worker_config( + AWS_RDS_BGD_State& st, uint64_t current_checksum, + RDS_BGD_Topology_Monitor_State& topology_state, unsigned long long& next_loop_at +) { + AWS_RDS_BGD_State candidate; + if (!aws_rds_bgd_load_worker_config(st.writer_hg, current_checksum, candidate)) { + return false; + } + + // Changes at or after writer post-processing require a full rollback and FSM restart. + if (st.bgd_status >= AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { + AWS_RDS_BGD_Status old_status = st.bgd_status; + handle_aws_rds_bgd_post_switchover(st, true); + aws_rds_bgd_apply_cluster_config(st, candidate); + topology_state = TOPOLOGY_TABLE_CHECK; + next_loop_at = 0; + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: applied checksum %llu with full rollback from %s\n", + st.writer_hg, st.reader_hg, (unsigned long long)current_checksum, + aws_rds_bgd_status_str(old_status)); + return true; + } + + AWS_RDS_BGD_Status status = st.bgd_status; + unsigned int old_reader_hg = st.reader_hg; + bool refresh_in_progress = st.bgd_in_progress_set; + bool hostgroups_changed = old_reader_hg != candidate.reader_hg; + + // Clear the in-progress marker from the old reader hostgroup before applying the new configuration. + if (refresh_in_progress && hostgroups_changed) { + aws_rds_bgd_clear_bgd_in_progress(st); + } + + // Apply the new configuration. + aws_rds_bgd_apply_cluster_config(st, candidate); + st.next_check_host.clear(); + st.next_check_host_failures = 0; + next_loop_at = 0; + // Rebuild bg_map from the next topology probe result. + st.config_refresh_pending = true; + + // Apply the in-progress marker to the new reader hostgroup after the refresh. + if (refresh_in_progress && hostgroups_changed) { + aws_rds_bgd_set_bgd_in_progress(st); + } + + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: applied checksum %llu with in-place refresh at %s\n", + st.writer_hg, st.reader_hg, (unsigned long long)current_checksum, + aws_rds_bgd_status_str(status)); + return true; +} + +/** +* @brief Rebuild the mapping and reconcile writer state after a configuration refresh. +* +* @details Called only when config_refresh_pending is set. +* +* @param st Worker-owned BGD state. +* @param topology Fresh topology used to rebuild the mapping. +*/ +void MySQL_Monitor::aws_rds_bgd_config_refresh_action(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topology) { + srv_addr_t old_writer; + bool had_old_writer = aws_rds_bgd_find_writer(st.bg_map, old_writer); + st.bg_map.clear(); + aws_rds_bgd_build_map(st, topology); + + srv_addr_t new_writer; + bool has_new_writer = aws_rds_bgd_find_writer(st.bg_map, new_writer); + aws_rds_bgd_add_green_writer_in_hg(st); + + // Reapply the in-progress demotion after the configuration reload restores configured placement. + if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS) { + bool writer_changed = + had_old_writer != has_new_writer || + (had_old_writer && (old_writer.host != new_writer.host || old_writer.port != new_writer.port)); + if (writer_changed && had_old_writer) { + MyHGM->read_only_action_v2(std::list { + read_only_server_t { old_writer.host, (port_t)old_writer.port, 0 } + }, true); + } + if (has_new_writer) { + MyHGM->read_only_action_v2(std::list { + read_only_server_t { new_writer.host, (port_t)new_writer.port, 1 } + }, true); + } + } +} + +// Map a raw mysql.rds_topology TARGET status string onto BGD phase enum. +static AWS_RDS_BGD_Status aws_rds_bgd_status_from_topology(const std::string& status) { + if (strcasecmp(status.c_str(), BGD_STATUS_AVAILABLE) == 0) { + return AWS_RDS_BGD_Status::AVAILABLE; + } else if (strcasecmp(status.c_str(), BGD_STATUS_INITIATED) == 0) { + return AWS_RDS_BGD_Status::WRITER_SWITCHOVER_INITIATED; + } else if (strcasecmp(status.c_str(), BGD_STATUS_IN_PROGRESS) == 0) { + return AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS; + } else if (strcasecmp(status.c_str(), BGD_STATUS_POST_PROC) == 0) { + return AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING; + } else if (strcasecmp(status.c_str(), BGD_STATUS_COMPLETED) == 0) { + return AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED; + } else { + return AWS_RDS_BGD_Status::NONE; + } +} + +// Human-readable name for a phase enum, for logging and (later) the runtime status column. +const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s) { + switch (s) { + case AWS_RDS_BGD_Status::NONE: + return "NONE"; + case AWS_RDS_BGD_Status::AVAILABLE: + return "AVAILABLE"; + case AWS_RDS_BGD_Status::WRITER_SWITCHOVER_INITIATED: + return "WRITER_SWITCHOVER_INITIATED"; + case AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS: + return "WRITER_SWITCHOVER_IN_PROGRESS"; + case AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING: + return "WRITER_SWITCHOVER_POST_PROCESSING"; + case AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED: + return "WRITER_SWITCHOVER_COMPLETED"; + case AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS: + return "READER_SWITCHOVER_IN_PROGRESS"; + case AWS_RDS_BGD_Status::SWITCHOVER_COMPLETED: + return "SWITCHOVER_COMPLETED"; + } + return "UNKNOWN"; +} + +/** +* @brief Run the status-driven blue/green switchover FSM for one deployment. +* +* @details Invoked each poll cycle by the BGD worker after it fetches the +* mysql.rds_topology result. Dispatches on the deployment's switchover status +* (AVAILABLE -> SWITCHOVER_INITIATED -> IN_PROGRESS -> IN_POST_PROCESSING -> +* COMPLETED): builds the blue<->green map, pre-resolves green IPs, repoints the +* blue hostnames onto the green IPs in the DNS cache, drains blue free +* connections, and shuns/enforces reader handling. State carried across cycles +* lives in @p st. +* +* @param st BGD switchover state (worker-owned, mutated here). +* @param topology Parsed mysql.rds_topology result for this cycle. +*/ +void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topology) { + if (!topology.blue_green) { + st.next_check_interval_ms = 0; + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); + return; + } + + AWS_RDS_Topology_Node* target = topology.target(); + if (!target || target->status.empty()) { + st.next_check_interval_ms = 0; + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); + return; + } + + AWS_RDS_BGD_Status topology_status = aws_rds_bgd_status_from_topology(target->status); + + // Once we advance to READER_SWITCHOVER_IN_PROGRESS phase, AWS keeps reporting + // WRITER_SWITCHOVER_COMPLETED (a single green row) until mysql.rds_topology drains. Ignore + // those repeats: the deferred cleanup fires from aws_rds_bgd_handle_topology_absent() when + // the table empties/vanishes, not from a status change here. + if (topology_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED + && st.bgd_status == AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { + return; + } + + // Detect backwards transition: the topology status moved to an earlier + // phase than what we've already processed. This happens when a user + // cancels the switchover from the AWS side, reverting to AVAILABLE, + // or when AWS aborts the switchover due to an error. Roll back all + // accumulated side effects, then re-enter the target state's setup. + if (topology_status < st.bgd_status) { + handle_aws_rds_bgd_post_switchover(st, true); + if (topology_status == AWS_RDS_BGD_Status::AVAILABLE) { + aws_rds_bgd_set_status(st, topology_status); + st.next_check_interval_ms = 250; + aws_rds_bgd_build_map(st, topology); + aws_rds_bgd_resolve_green_ips(st); + aws_rds_bgd_add_green_writer_in_hg(st); + } + return; + } + + // Rebuild a refreshed worker's mapping only after receiving this current topology result. + if (st.config_refresh_pending) { + aws_rds_bgd_config_refresh_action(st, topology); + st.config_refresh_pending = false; + } + + if (topology_status == st.bgd_status) { + // Refresh or retry green IP resolution on every eligible same-phase observation. + if (topology_status >= AWS_RDS_BGD_Status::AVAILABLE + && topology_status <= AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { + aws_rds_bgd_resolve_green_ips(st); + } + + // Retry pinning pairs whose green IP became available while remaining in POST_PROCESSING. + if (topology_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { + aws_rds_bgd_pin_green_ips(st); + } + return; + } + + aws_rds_bgd_set_status(st, topology_status); + + if (st.bgd_status == AWS_RDS_BGD_Status::AVAILABLE) { + st.next_check_interval_ms = 250; + + aws_rds_bgd_build_map(st, topology); + aws_rds_bgd_resolve_green_ips(st); + aws_rds_bgd_add_green_writer_in_hg(st); + } + else if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_INITIATED + || st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS) { + st.next_check_interval_ms = 100; + + aws_rds_bgd_build_map(st, topology); + aws_rds_bgd_resolve_green_ips(st); + aws_rds_bgd_add_green_writer_in_hg(st); + aws_rds_bgd_set_bgd_in_progress(st); + + if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS) { + // Demote the blue writer (RO=1) + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + auto srv = read_only_server_t{ p.blue_host, (port_t)p.port, 1 }; + MyHGM->read_only_action_v2(std::list{srv}, true); + break; + } + } + } + } + else if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { + st.next_check_interval_ms = 100; + + // Run setup here too: the thread may observe POST_PROCESSING directly, without having + // seen AVAILABLE/INITIATED first. All three are idempotent, so the repoint/shun logic + // below always runs against a built map, resolved green IPs, and an added green writer. + aws_rds_bgd_build_map(st, topology); + aws_rds_bgd_resolve_green_ips(st); + aws_rds_bgd_add_green_writer_in_hg(st); + aws_rds_bgd_set_bgd_in_progress(st); + + // Repoint each mapped blue host onto its green IP and drain existing + // connections so new backend work resolves to green. + aws_rds_bgd_pin_green_ips(st); + + // Blue readers without a green counterpart must stop serving reads. + + srv_addr_t writer; + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + writer = srv_addr_t{ p.blue_host, p.port }; + break; + } + } + + std::vector blue_readers; + MyHGM->wrlock(); + MyHGC* rhgc = MyHGM->MyHGC_lookup(st.reader_hg); + if (rhgc && rhgc->mysrvs) { + for (unsigned int j = 0; j < rhgc->mysrvs->cnt(); j++) { + MySrvC* s = rhgc->mysrvs->idx(j); + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT + || s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; + } + if (writer.host == s->address && writer.port == s->port) { + continue; + } + blue_readers.push_back(srv_addr_t{ std::string(s->address), s->port }); + } + } + MyHGM->wrunlock(); + + std::vector unmapped_readers; + for (const srv_addr_t& br : blue_readers) { + bool mapped = false; + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.blue_host == br.host && p.port == br.port) { + mapped = true; + break; + } + } + if (!mapped) { + unmapped_readers.push_back(br); + } + } + + bool writer_is_also_reader = (st.writer_is_also_reader != 0); + if (!unmapped_readers.empty() && unmapped_readers.size() == blue_readers.size()) { + // All blue readers would be transitioned to SHUNNED_AWS_BGD, leaving the reader HG empty. + // Temporarily enforce writer_is_also_reader until the reader switchover completes. + writer_is_also_reader = true; + } + + aws_rds_bgd_hostgroup_action(st.bgd_status, writer, writer_is_also_reader, st.reader_hg, unmapped_readers); + + st.shunned_readers.insert(st.shunned_readers.end(), unmapped_readers.begin(), unmapped_readers.end()); + } + else if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED) { + // Writer switchover done, but the topology table lingers with a single green row until the blue + // readers' DNS propagates and it drains. Defer reader teardown: advance to READER_SWITCHOVER phase + // and let the drain (aws_rds_bgd_handle_topology_absent) trigger it. + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS); + + // Drop the writer's DNS_Cache entry: this clears the IP pin and lets regular DNS + // resolution take over from here. Readers stay pinned until their DNS propagates. + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + dns_cache->remove(p.blue_host); + break; + } + } + + // release BGD monitor worker from fast-polling + st.next_check_interval_ms = 0; + } + else { + // unrecognized status: take no action, stay at baseline interval + st.next_check_interval_ms = 0; + } +} + +/** +* @brief Pin green IPs and drain existing blue-host connections. +* +* @param st BGD switchover state. +*/ +void MySQL_Monitor::aws_rds_bgd_pin_green_ips(AWS_RDS_BGD_State& st) { + for (AWS_RDS_BlueGreenPair& pair : st.bg_map) { + if (pair.green_ip_pinned) { + continue; + } + + if (pair.green_ip.empty()) { + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u rHG=%u]: green host '%s' remains unresolved; " + "deferring pin/drain for blue '%s:%d'\n", + st.writer_hg, st.reader_hg, pair.green_host.c_str(), pair.blue_host.c_str(), pair.port); + continue; + } + + dns_cache->pin(pair.blue_host, pair.green_ip); + MyHGM->wrlock(); + MyHGM->drain_server_connections(pair.blue_host.c_str(), pair.port); + MyHGM->wrunlock(); + My_Conn_Pool->purge_connections(pair.blue_host.c_str(), pair.port); + pair.green_ip_pinned = true; + + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: repointed blue '%s' to green IP %s\n", + st.writer_hg, st.reader_hg, pair.blue_host.c_str(), pair.green_ip.c_str()); + } +} + +/** +* @brief Apply BGD hostgroup changes for the current switchover status. +* +* @details POST_PROCESSING configures the writer placement and shuns unmapped readers. +* SWITCHOVER_COMPLETED unshuns readers and removes the writer from reader HG when +* writer_is_also_reader is false. Runtime mysql_servers and checksum are re-generated +* when server hostgroup membership changes. +* +* @param bgd_status Current BGD FSM status driving the action. +* @param writer Writer server to configure. +* @param writer_is_also_reader Whether the writer should also remain in reader_hg. +* @param reader_hg Reader hostgroup for reader shun/unshun and optional writer membership. +* @param readers Reader servers to shun or unshun. +*/ +void MySQL_Monitor::aws_rds_bgd_hostgroup_action( + AWS_RDS_BGD_Status bgd_status, + srv_addr_t& writer, bool writer_is_also_reader, + unsigned int reader_hg, std::vector& readers) +{ + bool changed = false; + bool shun_readers = false; + + MyHGM->wrlock(); + + if (bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { + changed |= MyHGM->aws_rds_bgd_configure_writer(writer.host.c_str(), writer.port, writer_is_also_reader); + shun_readers = true; + } else if (bgd_status == AWS_RDS_BGD_Status::SWITCHOVER_COMPLETED) { + changed |= MyHGM->aws_rds_bgd_configure_writer(writer.host.c_str(), writer.port, writer_is_also_reader); + } else { + MyHGM->wrunlock(); + return; + } + + for (srv_addr_t& s : readers) { + MyHGM->aws_rds_bgd_set_shun_server(reader_hg, s.host.c_str(), s.port, shun_readers); + } + + if (changed) { + MyHGM->publish_mysql_servers_to_runtime(); + } + + MyHGM->wrunlock(); +} + +/** +* @brief Run deferred switchover teardown or rollback cleanup. +* +* @details Restores post-switchover reader handling, unshuns readers, drops DNS pins, +* and clears BGD switchover state. Normal post-switchover cleanup also drains +* connections from green hosts; rollback leaves green rows and connections unchanged. +* +* When rollback is false (normal post-switchover), the caller must be in +* READER_SWITCHOVER_IN_PROGRESS; the function advances through +* SWITCHOVER_COMPLETED before clearing to NONE. +* +* When rollback is true (topology disappeared or worker exit mid-switchover), +* the function accepts any non-NONE bgd_status, restores the blue writer to the +* writer hostgroup if it was demoted, then resets switchover state. +* +* @param st BGD switchover state. +* @param rollback True if called due to a rollback/cancellation, false for normal completion. +*/ +void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bool rollback) { + if (st.bgd_status == AWS_RDS_BGD_Status::NONE) { + return; + } + + if (!rollback && st.bgd_status != AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { + return; + } + + if (rollback) { + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: rolling back from %s\n", + st.writer_hg, st.reader_hg, aws_rds_bgd_status_str(st.bgd_status)); + + // Restore the blue writer to the writer hostgroup. + // If writer exists in writer hostgroup, this is a no-op. + if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS + || st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + auto srv = read_only_server_t{ p.blue_host, (port_t)p.port, 0 }; + MyHGM->read_only_action_v2(std::list{srv}, true); + break; + } + } + } + } else { + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::SWITCHOVER_COMPLETED); + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: running post-switchover cleanup\n", st.writer_hg, st.reader_hg); + } + + // Restore the writer's original reader role based on writer_is_also_reader config and + // unshun the previously shunned blue readers. + srv_addr_t writer; + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + writer = srv_addr_t{ p.blue_host, p.port }; + break; + } + } + bool writer_is_also_reader = (st.writer_is_also_reader != 0); + aws_rds_bgd_hostgroup_action(AWS_RDS_BGD_Status::SWITCHOVER_COMPLETED, writer, writer_is_also_reader, st.reader_hg, st.shunned_readers); + + // Drop DNS cache + purge connections for the previously shunned readers + // so their blue names re-resolve to the promoted (green) instances. + if (!st.shunned_readers.empty()) { + for (const srv_addr_t& br : st.shunned_readers) { + dns_cache->remove(br.host); + My_Conn_Pool->purge_connections(br.host.c_str(), br.port); + } + st.shunned_readers.clear(); + } + + // Drop DNS pins for all mapped pairs + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + dns_cache->remove(p.blue_host); + My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); + if (!p.green_ip.empty()) { + My_Conn_Pool->purge_connections(p.green_ip.c_str(), p.port); + } + } + + if (!rollback) { + aws_rds_bgd_drain_green_hg(st); + } + + // state cleanup + st.bg_map.clear(); + st.config_refresh_pending = false; + st.next_check_host.clear(); + st.next_check_interval_ms = 0; + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); + + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: switchover cleanup complete; state cleared\n", + st.writer_hg, st.reader_hg); +} + +/** +* @brief Drain connections from green hosts after switchover. +* +* @details Drains connections from every green host that is neither OFFLINE_SOFT nor +* OFFLINE_HARD. Server rows and statuses are left unchanged. +* +* @param st Switchover state. +*/ +void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st) { + struct hg_srv_t { + int hostgroup; + srv_addr_t server; + }; + std::vector targets; + + MyHGM->wrlock(); + + for (int hg : { st.green_writer_hg, st.green_reader_hg }) { + if (hg < 0) { + continue; + } + MyHGC* hgc = MyHGM->MyHGC_find(hg); + if (!hgc || !hgc->mysrvs) { + continue; + } + for (unsigned int j = 0; j < hgc->mysrvs->cnt(); j++) { + MySrvC* s = hgc->mysrvs->idx(j); + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT + || s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; + } + + targets.push_back(hg_srv_t{ + hg, srv_addr_t{ std::string(s->address), s->port } }); + } + } + + for (const hg_srv_t& target : targets) { + MyHGM->drain_server_connections( + target.server.host.c_str(), target.server.port); + } + + MyHGM->wrunlock(); + + for (const hg_srv_t& target : targets) { + dns_cache->remove(target.server.host); + My_Conn_Pool->purge_connections( + target.server.host.c_str(), target.server.port); + + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: connections drained from green HG %d server '%s:%d'\n", + st.writer_hg, st.reader_hg, target.hostgroup, target.server.host.c_str(), + target.server.port); + } +} + +/** +* @brief Handle an absent, empty, or vanished mysql.rds_topology table. +* +* @details Routes to deferred cleanup when bgd_status is READER_SWITCHOVER_IN_PROGRESS; +* for any other non-NONE state, runs rollback cleanup to reverse accumulated side +* effects before resetting to NONE. +* +* @param st BGD switchover state. +*/ +void MySQL_Monitor::aws_rds_bgd_handle_topology_absent(AWS_RDS_BGD_State& st) { + if (st.bgd_status == AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { + handle_aws_rds_bgd_post_switchover(st); + } else if (st.bgd_status != AWS_RDS_BGD_Status::NONE) { + handle_aws_rds_bgd_post_switchover(st, true); + } +} + + +/** +* @brief Check whether a server is flagged as BGD switchover-in-progress. +* +* @param hostname Server hostname. +* @param port Server port. +* +* @return true if the server is flagged IN_PROGRESS. +*/ +bool MySQL_Monitor::is_aws_rds_bgd_server_in_progress(const std::string& hostname, int port) { + std::string key = hostname + ":::" + std::to_string(port); + pthread_mutex_lock(&aws_rds_bgd_mutex); + auto it = aws_rds_bgd_server_status.find(key); + bool r = (it != aws_rds_bgd_server_status.end() + && it->second == AWS_RDS_BGD_Server_Status::IN_PROGRESS); + pthread_mutex_unlock(&aws_rds_bgd_mutex); + return r; +} + +/** +* @brief Flag/unflag every server in BGD hostgroups as switchover-in-progress. +* +* @details Called by the BGD worker at switchover initiation (INITIATED / IN_PROGRESS / +* POST_PROCESSING) and cleared after SWITCHOVER_COMPLETED. Saves the marked servers in the +* worker state so cleanup does not depend on the current hostgroup configuration. +* +* @param st BGD worker state. +* @param in_progress true to flag servers, false to clear. +*/ +void MySQL_Monitor::set_aws_rds_bgd_server_in_progress(AWS_RDS_BGD_State& st, bool in_progress) { + if (in_progress) { + st.read_only_check_disabled.clear(); + + MyHGM->wrlock(); + unsigned int hgs[2] = { st.writer_hg, st.reader_hg }; + for (unsigned int i = 0; i < 2; i++) { + MyHGC* myhgc = MyHGM->MyHGC_find(hgs[i]); + if (myhgc == nullptr || myhgc->mysrvs == nullptr) { + continue; + } + for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { + MySrvC* s = myhgc->mysrvs->idx(j); + st.read_only_check_disabled.push_back(std::string(s->address) + ":::" + std::to_string(s->port)); + } + } + MyHGM->wrunlock(); + } + + pthread_mutex_lock(&aws_rds_bgd_mutex); + if (in_progress) { + for (const auto& k : st.read_only_check_disabled) { + aws_rds_bgd_server_status[k] = AWS_RDS_BGD_Server_Status::IN_PROGRESS; + } + } else { + for (const auto& k : st.read_only_check_disabled) { + aws_rds_bgd_server_status.erase(k); + } + } + pthread_mutex_unlock(&aws_rds_bgd_mutex); + + if (!in_progress) { + st.read_only_check_disabled.clear(); + } +} + +/** +* @brief AWS RDS BGD monitor thread entry point. +* +* @details Maintains one worker (monitor_RDS_BGD_thread_HG) per active writer hostgroup. The parent starts +* and stops workers and signals configuration changes. Each worker selects a pingable probe host, +* probes 'mysql.rds_topology', and runs the switchover state machine. +*/ +void * MySQL_Monitor::monitor_aws_rds_bgd() { + // Wait for GloMTH to be initialized + if (!wait_for_glo_mth()) + return NULL; + + unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; + MySQL_Thread * mysql_thr = new MySQL_Thread(); + mysql_thr->curtime = monotonic_time(); + MySQL_Monitor__thread_MySQL_Thread_Variables_version = GloMTH->get_global_version(); + mysql_thr->refresh_variables(); + + uint64_t last_checksum = 0; + std::unordered_map> workers; + + while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { + unsigned int glover; + if (!GloMTH) + break; + + glover = GloMTH->get_global_version(); + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version = glover; + mysql_thr->refresh_variables(); + } + + uint64_t new_checksum = 0; + std::shared_ptr hosts_resultset; + std::unordered_map cluster_checksums; + + pthread_mutex_lock(&aws_rds_bgd_hosts_mutex); + new_checksum = AWS_RDS_BGD_Hosts_checksum; + if (new_checksum != last_checksum && AWS_RDS_BGD_Hosts_resultset) { + hosts_resultset = AWS_RDS_BGD_Hosts_resultset; + cluster_checksums = AWS_RDS_BGD_Cluster_checksum; + } + pthread_mutex_unlock(&aws_rds_bgd_hosts_mutex); + + std::unordered_map active_cluster_checksums; + if (hosts_resultset) { + for (SQLite3_row* row : hosts_resultset->rows) { + char* srv_type = row->fields[AWS_RDS_BGD_SRV_TYPE]; + if (srv_type && srv_type[0] == 'B' + && atoi(row->fields[AWS_RDS_BGD_IS_WRITER]) != 0) { + int writer_hg = atoi(row->fields[AWS_RDS_BGD_WRITER_HOSTGROUP]); + auto checksum_it = cluster_checksums.find(writer_hg); + if (checksum_it != cluster_checksums.end()) { + active_cluster_checksums[writer_hg] = checksum_it->second; + } + } + } + } + + if (new_checksum != last_checksum) { + proxy_info("Detected changed definition for AWS RDS Blue Green monitoring\n"); + last_checksum = new_checksum; + std::vector stopped_workers; + + for (auto& [writer_hg, worker] : workers) { + auto cluster_it = active_cluster_checksums.find(writer_hg); + if (cluster_it == active_cluster_checksums.end()) { + worker->worker_stop.store(true); + stopped_workers.push_back(writer_hg); + proxy_info( + "AWS RDS BGD [wHG=%d]: stopping worker; deployment is inactive, removed, or has no blue writer\n", + writer_hg); + continue; + } + + uint64_t old_cluster_checksum = worker->current_checksum.load(); + if (old_cluster_checksum != cluster_it->second) { + worker->current_checksum.store(cluster_it->second); + proxy_info( + "AWS RDS BGD [wHG=%d]: signaling config refresh, checksum %llu -> %llu\n", + writer_hg, (unsigned long long)old_cluster_checksum, + (unsigned long long)cluster_it->second); + } + } + + for (auto& [writer_hg, checksum] : active_cluster_checksums) { + if (workers.find(writer_hg) != workers.end()) { + continue; + } + + std::unique_ptr worker(new AWS_RDS_BGD_Worker); + worker->writer_hg = writer_hg; + worker->current_checksum.store(checksum); + AWS_RDS_BGD_Worker* worker_arg = worker.get(); + workers.emplace(writer_hg, std::move(worker)); + proxy_info("Starting Monitor thread for AWS RDS writer HG %d\n", writer_hg); + if (pthread_create(&worker_arg->thread, NULL, monitor_RDS_BGD_thread_HG, worker_arg) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } + } + + for (int writer_hg : stopped_workers) { + auto worker_it = workers.find(writer_hg); + if (worker_it == workers.end()) { + continue; + } + pthread_join(worker_it->second->thread, NULL); + proxy_info("Stopped Monitor thread for AWS RDS writer HG %d\n", writer_hg); + workers.erase(worker_it); + } + } + + usleep(10000); + } + for (auto& [writer_hg, worker] : workers) { + worker->worker_stop.store(true); + } + for (auto& [writer_hg, worker] : workers) { + pthread_join(worker->thread, NULL); + proxy_info("Stopped Monitor thread for AWS RDS writer HG %d\n", writer_hg); + } + workers.clear(); + if (mysql_thr) { + delete mysql_thr; + mysql_thr = NULL; + } + return NULL; +} + +unsigned int MySQL_Monitor::estimate_lag(char* server_id, AWS_Aurora_status_entry** aase, unsigned int idx, unsigned int add_lag_ms, unsigned int min_lag_ms, unsigned int lag_num_checks) { + assert(aase); + assert(server_id); + assert(idx >= 0 && idx < N_L_ASE); + + if (lag_num_checks > N_L_ASE) lag_num_checks = N_L_ASE; + if (lag_num_checks <= 0) lag_num_checks = 1; + + unsigned int mlag = 0; + unsigned int lag = 0; + + for (unsigned int i = 1; i <= lag_num_checks; i++) { + if (!aase[idx] || !aase[idx]->host_statuses) + break; + for (auto hse : *(aase[idx]->host_statuses)) { + if (strcmp(server_id, hse->server_id)==0 && (unsigned int)hse->replica_lag_ms != 0) { + unsigned int ms = std::max(((unsigned int)hse->replica_lag_ms + add_lag_ms), min_lag_ms); + if (ms > mlag) mlag = ms; + if (!lag) lag = ms; + } + } + if (idx == 0) idx = N_L_ASE; + idx--; + } + + return mlag; +} + +void print_aws_aurora_status_entry(AWS_Aurora_status_entry* aase) { + if (aase && aase->start_time) { + if (aase->host_statuses->size()) { + for (AWS_Aurora_replica_host_status_entry* hse : *aase->host_statuses) { + if (hse) { + fprintf(stderr,"%s %s %s %f %f\n", hse->server_id, hse->session_id, hse->last_update_timestamp, hse->replica_lag_ms , hse->cpu); + } + } + } + } +} + +void MySQL_Monitor::aws_aurora_autopurge_servers(unsigned int wHG, unsigned int rHG, AWS_Aurora_status_entry *ase, unsigned int threshold, std::map& autopurge_counter, const std::string& domain_name) { + bool server_purged = false; + + std::set present_servers; + for (auto h : *(ase->host_statuses)) { + present_servers.insert(h->server_id); + } + + MyHGM->wrlock(); + + // Writer hostgroup + MyHGC *whgc = MyHGM->MyHGC_lookup(wHG); + if (whgc && whgc->mysrvs) { + for (unsigned int j = 0; j < whgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc = whgc->mysrvs->idx(j); + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) continue; + + std::string server_id(mysrvc->address); + size_t pos = server_id.rfind(domain_name); + if (pos != std::string::npos) { + server_id.erase(pos); + } + + std::string srv_key = std::to_string(wHG) + ":" + server_id; + if (present_servers.find(server_id) == present_servers.end()) { + if (++autopurge_counter[srv_key] >= (int)threshold) { + proxy_warning("Auto-purging server %s:%d from hostgroup %u (absent from REPLICA_HOST_STATUS for %d checks)\n", + mysrvc->address, mysrvc->port, wHG, autopurge_counter[srv_key]); + MyHGM->remove_server_in_hg(wHG, mysrvc->address, mysrvc->port); + autopurge_counter.erase(srv_key); + server_purged = true; + } + } else { + autopurge_counter.erase(srv_key); + } + } + } + + // Reader hostgroup + if (rHG > 0) { + MyHGC *rhgc = MyHGM->MyHGC_lookup(rHG); + if (rhgc && rhgc->mysrvs) { + for (unsigned int j = 0; j < rhgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc = rhgc->mysrvs->idx(j); + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) continue; + + std::string server_id(mysrvc->address); + size_t pos = server_id.rfind(domain_name); + if (pos != std::string::npos) { + server_id.erase(pos); + } + + std::string srv_key = std::to_string(rHG) + ":" + server_id; if (present_servers.find(server_id) == present_servers.end()) { if (++autopurge_counter[srv_key] >= (int)threshold) { proxy_warning("Auto-purging server %s:%d from hostgroup %u (absent from REPLICA_HOST_STATUS for %d checks)\n", @@ -6668,6 +8437,23 @@ std::string MySQL_Monitor::dns_lookup(const char* hostname, bool return_hostname return MySQL_Monitor::dns_lookup(std::string(hostname), return_hostname_if_lookup_fails, ip_count); } +bool MySQL_Monitor::timeout_validate_ip_change(const MySQL_Monitor_State_Data* mmsd) const { + if (!mmsd || !mmsd->mysql || !mmsd->hostname || !dns_cache) { + return false; + } + + if (mmsd->port == 0 || validate_ip(mmsd->hostname)) { + return false; + } + + const std::string connected_ip = get_connected_peer_ip_from_socket(mmsd->mysql->net.fd); + if (connected_ip.empty()) { + return false; + } + + return !dns_cache->is_ip_valid(mmsd->hostname, connected_ip); +} + bool MySQL_Monitor::update_dns_cache_from_mysql_conn(const MYSQL* mysql) { assert(mysql); @@ -7018,9 +8804,10 @@ MySQL_Monitor_State_Data_Task_Result MySQL_Monitor_State_Data::task_handler(shor assert(task_handler_); if (event_ != -1) { - - if (task_result_ == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT) - return MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT; + if (task_result_ == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT || + task_result_ == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT_STALE_IP) { + return task_result_; + } #ifdef DEBUG const unsigned long long now = (GloMyMon->proxytest_forced_timeout == false) ? monotonic_time() : ULLONG_MAX; #else @@ -7349,15 +9136,51 @@ MySQL_Monitor_State_Data_Task_Result MySQL_Monitor_State_Data::generic_handler(s } bool MySQL_Monitor::monitor_read_only_process_ready_tasks(const std::vector& mmsds) { - std::list mysql_servers; for (auto& mmsd : mmsds) { string originating_server_hostname = mmsd->hostname; const auto task_result = mmsd->get_task_result(); + const bool stale_ip_timeout = task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT_STALE_IP; assert(task_result != MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_PENDING); + // AWS RDS blue/green topology discovery is a standalone task: it does not + // write a read_only log entry. Classify the result and move on. + if (mmsd->get_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY) { + if (task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { + __sync_fetch_and_add(&read_only_check_OK, 1); + if (mmsd->interr == 0 && mmsd->result) { + process_aws_rds_topology(mmsd); + } + if (mmsd->result) { + mysql_free_result(mmsd->result); + mmsd->result = NULL; + } + My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } else { + __sync_fetch_and_add(&read_only_check_ERR, 1); + unsigned int err = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; + if (err == 1146) { + // mysql.rds_topology absent (no active blue/green deployment); expected, skip quietly + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "mysql.rds_topology not present on %s:%d; skipping blue/green discovery\n", + mmsd->hostname, mmsd->port); + } else { + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, + err ? err : ER_PROXYSQL_READ_ONLY_CHECK_TIMEOUT + ); + proxy_error( + "Error on AWS RDS blue/green topology discovery for %s:%d : %s\n", + mmsd->hostname, mmsd->port, (mmsd->mysql_error_msg ? mmsd->mysql_error_msg : "") + ); + } + My_Conn_Pool->destroy_mysql_connection(mmsd); + } + continue; + } + if (task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { __sync_fetch_and_add(&read_only_check_OK, 1); My_Conn_Pool->put_connection(mmsd->hostname, mmsd); @@ -7388,6 +9211,7 @@ bool MySQL_Monitor::monitor_read_only_process_ready_tasks(const std::vectorhostname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mmsd->mondb); rc = (*proxy_sqlite3_bind_int)(statement, 2, mmsd->port); ASSERT_SQLITE_OK(rc, mmsd->mondb); unsigned long long time_now = realtime_time(); @@ -7420,41 +9244,12 @@ VALGRIND_ENABLE_ERROR_REPORTING; } rc = (*proxy_sqlite3_bind_int64)(statement, 5, read_only); ASSERT_SQLITE_OK(rc, mmsd->mondb); - } else if (fields && mmsd->get_task_type() == MON_READ_ONLY__AND__AWS_RDS_TOPOLOGY_DISCOVERY) { - // Process the read_only field as above and store the first server - vector discovered_servers; - for (k = 0; k < num_fields; k++) { - if (strcmp((char*)"read_only", (char*)fields[k].name) == 0) { - j = k; - } - } - if (j > -1) { - MYSQL_ROW row = mysql_fetch_row(mmsd->result); - if (row) { - discovered_servers.push_back(row); -VALGRIND_DISABLE_ERROR_REPORTING; - if (row[j]) { - if (!strcmp(row[j], "0") || !strcasecmp(row[j], "OFF")) - read_only = 0; - } -VALGRIND_ENABLE_ERROR_REPORTING; - } - } - - // Store the remaining servers - int num_rows = mysql_num_rows(mmsd->result); - for (int i = 1; i < num_rows; i++) { - MYSQL_ROW row = mysql_fetch_row(mmsd->result); - discovered_servers.push_back(row); - } - - // Process the discovered servers and add them to 'runtime_mysql_servers' (process only for AWS RDS Multi-AZ DB Clusters) - if (!discovered_servers.empty() && is_aws_rds_multi_az_db_cluster_topology(discovered_servers)) { - process_discovered_topology(originating_server_hostname, discovered_servers, mmsd->reader_hostgroup); - } } else { - proxy_error("mysql_fetch_fields returns NULL, or mysql_num_fields is incorrect. Server %s:%d . See bug #1994\n", mmsd->hostname, mmsd->port); + valid_result = false; rc = (*proxy_sqlite3_bind_null)(statement, 5); ASSERT_SQLITE_OK(rc, mmsd->mondb); + proxy_error("mysql_fetch_fields returns NULL, or mysql_num_fields is incorrect. Server %s:%d . See bug #1994\n", mmsd->hostname, mmsd->port); + proxy_info("Dumping read_only result for server %s:%d, query: %s\n", mmsd->hostname, mmsd->port, mmsd->get_query()); + dump_mysql_result(stderr, mmsd->result); } mysql_free_result(mmsd->result); mmsd->result = NULL; @@ -7471,7 +9266,9 @@ VALGRIND_ENABLE_ERROR_REPORTING; rc = (*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mmsd->mondb); rc = (*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mmsd->mondb); - if (task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { + if (!valid_result || stale_ip_timeout) { + // Ignore; do not infer backend state. + } else if (task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { //MyHGM->read_only_action_v2(mmsd->hostname, mmsd->port, read_only); // default behavior mysql_servers.push_back( std::tuple { mmsd->hostname, mmsd->port, read_only }); } else { @@ -7520,6 +9317,14 @@ void MySQL_Monitor::monitor_read_only_async(SQLite3_result* resultset, bool do_d for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { const SQLite3_row* r = *it; + + if (is_aws_rds_bgd_server_in_progress(r->fields[0], atoi(r->fields[1]))) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Skipping read_only check for '%s:%d' because AWS RDS BGD switchover is in progress\n", + r->fields[0], atoi(r->fields[1])); + continue; + } + bool rc_ping = server_responds_to_ping(r->fields[0], atoi(r->fields[1])); if (rc_ping) { // only if server is responding to pings MySQL_Monitor_State_Data_Task_Type task_type = MON_READ_ONLY; @@ -7535,11 +9340,6 @@ void MySQL_Monitor::monitor_read_only_async(SQLite3_result* resultset, bool do_d task_type = MON_READ_ONLY__OR__INNODB_READ_ONLY; } - // Change task type if it's time to do discovery check. Only for aws rds endpoints - string hostname = r->fields[0]; - if (do_discovery_check && hostname.find(AWS_ENDPOINT_SUFFIX_STRING) != std::string::npos) { - task_type = MON_READ_ONLY__AND__AWS_RDS_TOPOLOGY_DISCOVERY; - } } std::unique_ptr mmsd( @@ -7554,10 +9354,29 @@ void MySQL_Monitor::monitor_read_only_async(SQLite3_result* resultset, bool do_d monitor_poll.add((POLLIN|POLLOUT|POLLPRI), mmsd.get()); mmsds.push_back(std::move(mmsd)); } else { - WorkItem* item = + WorkItem* item = new WorkItem(mmsd.release(), monitor_read_only_thread); queue->add(item); } + + // On discovery cycles, enqueue an additional standalone topology-discovery + // task for AWS RDS endpoints. The read_only check above is unaffected. + string hostname = r->fields[0]; + if (do_discovery_check && hostname.find(AWS_ENDPOINT_SUFFIX_STRING) != std::string::npos) { + std::unique_ptr tmmsd( + new MySQL_Monitor_State_Data(MON_AWS_RDS_TOPOLOGY_DISCOVERY, r->fields[0], atoi(r->fields[1]), atoi(r->fields[2]))); + tmmsd->reader_hostgroup = atoi(r->fields[4]); + tmmsd->mondb = monitordb; + tmmsd->mysql = My_Conn_Pool->get_connection(tmmsd->hostname, tmmsd->port, tmmsd.get()); + if (tmmsd->mysql) { + monitor_poll.add((POLLIN|POLLOUT|POLLPRI), tmmsd.get()); + mmsds.push_back(std::move(tmmsd)); + } else { + WorkItem* item = + new WorkItem(tmmsd.release(), monitor_read_only_thread); + queue->add(item); + } + } } if (shutdown) return; @@ -7581,11 +9400,10 @@ void MySQL_Monitor::monitor_read_only_async(SQLite3_result* resultset, bool do_d } bool MySQL_Monitor::monitor_group_replication_process_ready_tasks(const std::vector& mmsds) { - for (auto& mmsd : mmsds) { - const auto task_result = mmsd->get_task_result(); - + const bool stale_ip_timeout = task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT_STALE_IP; + assert(task_result != MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_PENDING); if (task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { @@ -7686,7 +9504,9 @@ bool MySQL_Monitor::monitor_group_replication_process_ready_tasks(const std::vec pthread_mutex_unlock(&group_replication_mutex); // NOTE: we update MyHGM outside the mutex group_replication_mutex - if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure GR + if (stale_ip_timeout) { + // Logged/counted; do not change GR state for stale DNS targets. + } else if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure GR if (num_timeouts == 0) { // it wasn't a timeout, reconfigure immediately MyHGM->update_group_replication_set_offline(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, mmsd->mysql_error_msg); @@ -8066,10 +9886,9 @@ void MySQL_Monitor::monitor_replication_lag_async(SQLite3_result* resultset) { } bool MySQL_Monitor::monitor_galera_process_ready_tasks(const std::vector& mmsds) { - for (auto& mmsd : mmsds) { - const auto task_result = mmsd->get_task_result(); + const bool stale_ip_timeout = task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT_STALE_IP; assert(task_result != MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_PENDING); @@ -8247,7 +10066,9 @@ bool MySQL_Monitor::monitor_galera_process_ready_tasks(const std::vectormysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure Galera + if (stale_ip_timeout) { + // Logged/counted; do not change Galera state for stale DNS targets. + } else if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure Galera if (num_timeouts == 0) { // it wasn't a timeout, reconfigure immediately MyHGM->update_galera_set_offline(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, mmsd->mysql_error_msg); diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 707b3b5a9c..bf143b39dd 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -5601,7 +5601,7 @@ bool MySQL_Session::handler_minus1_HandleErrorCodes(MySQL_Data_Stream *myds, int myds->destroy_MySQL_Connection_From_Pool(false); break; default: - if (mysql_thread___reset_connection_algorithm == 2) { + if (mysql_thread___reset_connection_algorithm == 2 && myds->myconn->healthy) { create_new_session_and_reset_connection(myds); } else { myds->destroy_MySQL_Connection_From_Pool(true); @@ -5696,7 +5696,7 @@ void MySQL_Session::handler_minus1_HandleBackendConnection(MySQL_Data_Stream *my if (mysql_thread___multiplexing && (myds->myconn->reusable==true) && myds->myconn->IsActiveTransaction()==false && myds->myconn->MultiplexDisabled()==false) { myds->DSS=STATE_NOT_INITIALIZED; if (mysql_thread___autocommit_false_not_reusable && myds->myconn->IsAutoCommit()==false) { - if (mysql_thread___reset_connection_algorithm == 2) { + if (mysql_thread___reset_connection_algorithm == 2 && myds->myconn->healthy) { create_new_session_and_reset_connection(myds); } else { myds->destroy_MySQL_Connection_From_Pool(true); @@ -9224,7 +9224,7 @@ void MySQL_Session::finishQuery(MySQL_Data_Stream *myds, MySQL_Connection *mycon myds->wait_until=0; myds->DSS=STATE_NOT_INITIALIZED; if (mysql_thread___autocommit_false_not_reusable && myds->myconn->IsAutoCommit()==false) { - if (mysql_thread___reset_connection_algorithm == 2) { + if (mysql_thread___reset_connection_algorithm == 2 && myds->myconn->healthy) { create_new_session_and_reset_connection(myds); } else { myds->destroy_MySQL_Connection_From_Pool(true); diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index aae511c0d8..801cfeac3a 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -401,6 +401,7 @@ static char * mysql_thread_variables_names[]= { (char *)"monitor_ping_max_failures", (char *)"monitor_ping_timeout", (char *)"monitor_aws_rds_topology_discovery_interval", + (char *)"aws_blue_green_deployment_auto_discovery", (char *)"monitor_read_only_interval", (char *)"monitor_read_only_timeout", (char *)"monitor_read_only_max_timeout_count", @@ -1296,6 +1297,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() { variables.monitor_ping_max_failures=3; variables.monitor_ping_timeout=1000; variables.monitor_aws_rds_topology_discovery_interval=0; + variables.aws_blue_green_deployment_auto_discovery=1; variables.monitor_read_only_interval=1000; variables.monitor_read_only_timeout=800; variables.monitor_read_only_max_timeout_count=3; @@ -2065,7 +2067,17 @@ bool MySQL_Threads_Handler::set_variable(char *name, const char *value) { // thi } bool special_variable = std::get<3>(it->second); // if special_variable is true, min and max values are ignored, and more input validation is needed if (special_variable == false) { - int intv=atoi(value); + // This option is stored as an integer for compatibility with the + // existing variable interface, but is documented and commonly set + // using the same true/false spelling as boolean variables. + int intv; + if (nameS == "aws_blue_green_deployment_auto_discovery" && strcasecmp(value, "true") == 0) { + intv = 1; + } else if (nameS == "aws_blue_green_deployment_auto_discovery" && strcasecmp(value, "false") == 0) { + intv = 0; + } else { + intv = atoi(value); + } if (intv >= std::get<1>(it->second) && intv <= std::get<2>(it->second)) { int * v = std::get<0>(it->second); *v = intv; @@ -2690,6 +2702,7 @@ char ** MySQL_Threads_Handler::get_variables_list() { // it is safe to do it here because get_variables_list() is the first function called during start time if (VariablesPointers_int.size() == 0) { // Monitor variables + VariablesPointers_int["aws_blue_green_deployment_auto_discovery"] = make_tuple(&variables.aws_blue_green_deployment_auto_discovery, 0, 1, false); VariablesPointers_int["monitor_history"] = make_tuple(&variables.monitor_history, 1000, 7*24*3600*1000, false); VariablesPointers_int["monitor_connect_interval"] = make_tuple(&variables.monitor_connect_interval, 100, 7*24*3600*1000, false); @@ -4796,6 +4809,7 @@ void MySQL_Thread::refresh_variables() { REFRESH_VARIABLE_INT(monitor_ping_max_failures); REFRESH_VARIABLE_INT(monitor_ping_timeout); REFRESH_VARIABLE_INT(monitor_aws_rds_topology_discovery_interval); + REFRESH_VARIABLE_INT(aws_blue_green_deployment_auto_discovery); REFRESH_VARIABLE_INT(monitor_read_only_interval); REFRESH_VARIABLE_INT(monitor_read_only_timeout); REFRESH_VARIABLE_INT(monitor_read_only_max_timeout_count); @@ -6549,6 +6563,11 @@ MySQL_Connection * MySQL_Thread::get_MyConn_local(unsigned int _hid, MySQL_Sessi for (i=0; ilen; i++) { c = (MySQL_Connection *) cached_connections->index(i); + // Skip unhealthy or non-reusable connections + if (!c->healthy || !c->reusable) { + continue; + } + // Skip cached connections whose parent server is inside the session-tracking // capability backoff window. See 'MySrvC::session_track_backoff_until' for the // full rationale; reads are relaxed because the deadline is compared against @@ -6623,6 +6642,11 @@ MySQL_Connection * MySQL_Thread::get_MyConn_local(unsigned int _hid, MySQL_Sessi * @param c Pointer to the MySQL_Connection object to be pushed to the local connection pool. */ void MySQL_Thread::push_MyConn_local(MySQL_Connection *c) { + if (!c->healthy) { + MyHGM->push_MyConn_to_pool(c); + return; + } + // Bounded local cache: cache 1-in-N releases (N = mysql_threads), push the // rest to the shared HGM pool so peer workers can pick them up. // At N=1 always cache (no sibling to share with). diff --git a/lib/MySrvConnList.cpp b/lib/MySrvConnList.cpp index 2ee3bf1db7..779fd804c1 100644 --- a/lib/MySrvConnList.cpp +++ b/lib/MySrvConnList.cpp @@ -48,6 +48,14 @@ void MySrvConnList::drop_all_connections() { } } +void MySrvConnList::mark_connections_unhealthy() { + for (unsigned int i = 0; i < conns_length(); i++) { + MySQL_Connection *conn = index(i); + conn->healthy=false; + conn->reusable=false; + } +} + unsigned int calculate_eviction_count(unsigned int conns_free, unsigned int conns_used, unsigned int max_connections) { if (conns_free < 1) return 0; unsigned int pct_max_connections = (3 * max_connections) / 4; @@ -303,4 +311,3 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff } return NULL; // never reach here } - diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 81300650ee..84c7215e2d 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -144,6 +144,7 @@ static const vector mysql_servers_tablenames = { "mysql_group_replication_hostgroups", "mysql_galera_hostgroups", "mysql_aws_aurora_hostgroups", + "mysql_aws_rds_bgd_hostgroups", "mysql_hostgroup_attributes", "mysql_servers_ssl_params", }; @@ -875,6 +876,7 @@ incoming_servers_t::incoming_servers_t( SQLite3_result* incoming_aurora_hostgroups, SQLite3_result* incoming_hostgroup_attributes, SQLite3_result* incoming_mysql_servers_ssl_params, + SQLite3_result* incoming_aws_rds_bgd_hostgroups, SQLite3_result* runtime_mysql_servers ) : incoming_mysql_servers_v2(incoming_mysql_servers_v2), @@ -884,6 +886,7 @@ incoming_servers_t::incoming_servers_t( incoming_aurora_hostgroups(incoming_aurora_hostgroups), incoming_hostgroup_attributes(incoming_hostgroup_attributes), incoming_mysql_servers_ssl_params(incoming_mysql_servers_ssl_params), + incoming_aws_rds_bgd_hostgroups(incoming_aws_rds_bgd_hostgroups), runtime_mysql_servers(runtime_mysql_servers) {} @@ -1512,6 +1515,8 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign || strstr(query_no_space,"runtime_mysql_aws_aurora_hostgroups") || + strstr(query_no_space,"runtime_mysql_aws_rds_bgd_hostgroups") + || strstr(query_no_space,"runtime_mysql_hostgroup_attributes") || strstr(query_no_space,"runtime_mysql_servers_ssl_params") @@ -7336,13 +7341,17 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { max_bulk_row_idx=max_bulk_row_idx*32; for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r1=*it; + const char *status = r1->fields[4]; + if (_runtime == false && (strcmp(status,"SHUNNED") == 0 || strcmp(status,"SHUNNED_AWS_BGD") == 0)) { + status = "ONLINE"; + } int idx=row_idx%32; if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_text)(statement32, (idx*12)+2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*12)+5, ( _runtime ? r1->fields[4] : ( strcmp(r1->fields[4],"SHUNNED")==0 ? "ONLINE" : r1->fields[4] ) ), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*12)+5, status, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, admindb); @@ -7360,7 +7369,7 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 5, ( _runtime ? r1->fields[4] : ( strcmp(r1->fields[4],"SHUNNED")==0 ? "ONLINE" : r1->fields[4] ) ), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 5, status, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, admindb); @@ -7556,6 +7565,79 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { if(resultset) delete resultset; resultset=NULL; + // dump mysql_aws_rds_bgd_hostgroups + // The runtime table carries the extra runtime-only 'auto_generated' column; the config table + // does not. 'dump_table_mysql' always returns 10 columns (last is 'auto_generated'); we bind + // 10 for the runtime table and only the first 9 for the config table. 'green_writer_hostgroup' + // and 'green_reader_hostgroup' (fields 2,3) are nullable and bound as NULL when absent. + + if (_runtime) { + query=(char *)"DELETE FROM main.runtime_mysql_aws_rds_bgd_hostgroups"; + } else { + query=(char *)"DELETE FROM main.mysql_aws_rds_bgd_hostgroups"; + } + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute(query); + resultset=MyHGM->dump_table_mysql("mysql_aws_rds_bgd_hostgroups"); + if (resultset) { + int rc; + sqlite3_stmt *statement=NULL; + + char *query=NULL; + if (_runtime) { + query=(char *)"INSERT INTO runtime_mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment,auto_generated,status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + } else { + query=(char *)"INSERT INTO mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; + } + + auto [rc1, statement_unique] = admindb->prepare_v2(query); + rc = rc1; + statement = statement_unique.get(); + ASSERT_SQLITE_OK(rc, admindb); + + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + // auto_generated (field 9) entries are created at runtime by the monitor; they are NOT + // user configuration, so they must not be persisted to the memory config table. They are + // still written to the runtime table. + if (!_runtime && r->fields[9] && atoi(r->fields[9]) != 0) { + continue; + } + rc=(*proxy_sqlite3_bind_int64)(statement, 1, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 2, atoi(r->fields[1])); ASSERT_SQLITE_OK(rc, admindb); + if (r->fields[2] && r->fields[2][0]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(r->fields[2])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3); + } + ASSERT_SQLITE_OK(rc, admindb); + if (r->fields[3] && r->fields[3][0]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(r->fields[3])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5, atoi(r->fields[4])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6, atoi(r->fields[5])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 7, atoi(r->fields[6])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, atoi(r->fields[7])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement, 9, r->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (_runtime) { + rc=(*proxy_sqlite3_bind_int64)(statement, 10, atoi(r->fields[9])); ASSERT_SQLITE_OK(rc, admindb); + // 'status' (field 10) is the AWS_RDS_BGD_Status underlying int; we store it as text in the runtime table. + const char *bgd_status_str = + aws_rds_bgd_status_str(static_cast(r->fields[10] ? atoi(r->fields[10]) : 0)); + rc=(*proxy_sqlite3_bind_text)(statement, 11, bgd_status_str, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + } + + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, admindb); + } + } + if(resultset) delete resultset; + resultset=NULL; + // dump mysql_hostgroup_attributes StrQuery = "DELETE FROM main."; @@ -7887,6 +7969,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc SQLite3_result *resultset_group_replication=NULL; SQLite3_result *resultset_galera=NULL; SQLite3_result *resultset_aws_aurora=NULL; + SQLite3_result *resultset_aws_rds_bgd=NULL; SQLite3_result *resultset_hostgroup_attributes=NULL; SQLite3_result *resultset_mysql_servers_ssl_params=NULL; @@ -7897,6 +7980,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc SQLite3_result* incoming_aurora_hostgroups = incoming_servers.incoming_aurora_hostgroups; SQLite3_result* incoming_hostgroup_attributes = incoming_servers.incoming_hostgroup_attributes; SQLite3_result* incoming_mysql_servers_ssl_params = incoming_servers.incoming_mysql_servers_ssl_params; + SQLite3_result* incoming_aws_rds_bgd_hostgroups = incoming_servers.incoming_aws_rds_bgd_hostgroups; SQLite3_result* incoming_mysql_servers_v2 = incoming_servers.incoming_mysql_servers_v2; const char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port"; @@ -8050,6 +8134,21 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc MyHGM->save_incoming_mysql_table(resultset_aws_aurora,"mysql_aws_aurora_hostgroups"); } + // support for AWS RDS, table mysql_aws_rds_bgd_hostgroups + query=(char *)"SELECT a.* FROM mysql_aws_rds_bgd_hostgroups a LEFT JOIN mysql_aws_rds_bgd_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + if (incoming_aws_rds_bgd_hostgroups == nullptr) { + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_rds_bgd); + } else { + resultset_aws_rds_bgd = incoming_aws_rds_bgd_hostgroups; + } + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } else { + // Pass the resultset to MyHGM + MyHGM->save_incoming_mysql_table(resultset_aws_rds_bgd,"mysql_aws_rds_bgd_hostgroups"); + } + // support for hostgroup attributes, table mysql_hostgroup_attributes query = (char *)"SELECT * FROM mysql_hostgroup_attributes ORDER BY hostgroup_id"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); @@ -8109,6 +8208,10 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc //delete resultset_aws_aurora; // do not delete, resultset is stored in MyHGM resultset_aws_aurora=NULL; } + if (resultset_aws_rds_bgd) { + //delete resultset_aws_rds_bgd; // do not delete, resultset is stored in MyHGM + resultset_aws_rds_bgd=NULL; + } if (resultset_hostgroup_attributes) { resultset_hostgroup_attributes = NULL; } diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 55fbcb26a9..9fe540c361 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -91,6 +91,7 @@ namespace SQLQueries { const char* const DELETE_MYSQL_AWS_AURORA_HOSTGROUPS = "DELETE FROM mysql_aws_aurora_hostgroups"; const char* const DELETE_MYSQL_HOSTGROUP_ATTRIBUTES = "DELETE FROM mysql_hostgroup_attributes"; const char* const DELETE_MYSQL_SERVERS_SSL_PARAMS = "DELETE FROM mysql_servers_ssl_params"; + const char* const DELETE_MYSQL_AWS_RDS_BGD_HOSTGROUPS = "DELETE FROM mysql_aws_rds_bgd_hostgroups"; const char* const DELETE_PGSQL_SERVERS = "DELETE FROM pgsql_servers"; const char* const DELETE_PGSQL_REPLICATION_HOSTGROUPS = "DELETE FROM pgsql_replication_hostgroups"; const char* const DELETE_PGSQL_HOSTGROUP_ATTRIBUTES = "DELETE FROM pgsql_hostgroup_attributes"; @@ -1821,12 +1822,14 @@ int ProxySQL_Cluster::fetch_and_store(MYSQL* conn, const fetch_query& f_query, M /** * @brief Generates a hash from the received resultsets from executing the following queries in the specified * order: - * - CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS. + * - CLUSTER_QUERY_MYSQL_SERVERS_V2. * - CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS. * - CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS. * - CLUSTER_QUERY_MYSQL_GALERA. * - CLUSTER_QUERY_MYSQL_AWS_AURORA. * - CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES. + * - CLUSTER_QUERY_MYSQL_SERVERS_SSL_PARAMS. + * - CLUSTER_QUERY_MYSQL_AWS_RDS_BGD. * * IMPORTANT: It's assumed that the previous queries were successful and that the resultsets are received in * the specified order. @@ -1871,6 +1874,7 @@ incoming_servers_t convert_mysql_servers_resultsets(const std::vector results(8,nullptr); + std::vector results(9,nullptr); // servers messages std::string fetch_servers_done = ""; @@ -2122,6 +2126,12 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch std::string fetch_mysql_servers_ssl_params_err = ""; string_format("Cluster: Fetching 'MySQL Servers SSL Params' from peer %s:%d failed: \n", fetch_mysql_servers_ssl_params_err, hostname, port); + // AWS RDS BGD hostgroups messages + std::string fetch_aws_rds_bgd_start = ""; + string_format("Cluster: Fetching 'MySQL AWS RDS BGD Hostgroups' from peer %s:%d\n", fetch_aws_rds_bgd_start, hostname, port); + std::string fetch_aws_rds_bgd_err = ""; + string_format("Cluster: Fetching 'MySQL AWS RDS BGD Hostgroups' from peer %s:%d failed: \n", fetch_aws_rds_bgd_err, hostname, port); + // Create fetching queries /** @@ -2170,6 +2180,12 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch p_cluster_counter::pulled_mysql_servers_ssl_params_success, p_cluster_counter::pulled_mysql_servers_ssl_params_failure, { fetch_mysql_servers_ssl_params_start, "", fetch_mysql_servers_ssl_params_err } + }, + { + CLUSTER_QUERY_MYSQL_AWS_RDS_BGD, + p_cluster_counter::pulled_mysql_servers_aws_rds_bgd_hostgroups_success, + p_cluster_counter::pulled_mysql_servers_aws_rds_bgd_hostgroups_failure, + { fetch_aws_rds_bgd_start, "", fetch_aws_rds_bgd_err } } }; @@ -2205,22 +2221,22 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch MYSQL_RES* fetch_res = nullptr; if (fetch_and_store(conn, query, &fetch_res) == 0) { - results[7] = fetch_res; + results[8] = fetch_res; } else { fetching_error = true; } } if (fetching_error == false) { - const uint64_t servers_hash = compute_servers_tables_raw_checksum(results, 7); // ignore runtime_mysql_servers in checksum calculation + const uint64_t servers_hash = compute_servers_tables_raw_checksum(results, 8); // ignore runtime_mysql_servers in checksum calculation const string computed_checksum{ get_checksum_from_hash(servers_hash) }; proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Servers v2 from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); proxy_info("Cluster: Computed checksum for MySQL Servers v2 from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); bool runtime_checksum_matches = true; - if (results[7]) { - const uint64_t runtime_mysql_server_hash = mysql_raw_checksum(results[7]); + if (results[8]) { + const uint64_t runtime_mysql_server_hash = mysql_raw_checksum(results[8]); const std::string runtime_mysql_server_computed_checksum = get_checksum_from_hash(runtime_mysql_server_hash); proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, runtime_mysql_server_computed_checksum.c_str()); proxy_info("Cluster: Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, runtime_mysql_server_computed_checksum.c_str()); @@ -2248,7 +2264,11 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch char* o = escape_string_single_quotes(row[11], false); char* query = (char*)malloc(strlen(q) + l + strlen(o) + 64); - sprintf(query, q, row[0], row[1], row[2], row[3], (strcmp(row[4], "SHUNNED") == 0 ? "ONLINE" : row[4]), row[5], row[6], row[7], row[8], row[9], row[10], o); + const char *status = row[4]; + if (strcmp(status, "SHUNNED") == 0 || strcmp(status, "SHUNNED_AWS_BGD") == 0) { + status = "ONLINE"; + } + sprintf(query, q, row[0], row[1], row[2], row[3], status, row[5], row[6], row[7], row[8], row[9], row[10], o); if (o != row[11]) { // there was a copy free(o); } @@ -2480,6 +2500,49 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch resultset->dump_to_stderr(); delete resultset; + // sync mysql_aws_rds_bgd_hostgroups + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Writing mysql_aws_rds_bgd_hostgroups table\n"); + proxy_info("Cluster: Writing mysql_aws_rds_bgd_hostgroups table\n"); + GloAdmin->admindb->execute(SQLQueries::DELETE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); + { + const char* q = (const char*)"INSERT INTO mysql_aws_rds_bgd_hostgroups (" + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " + "active, writer_is_also_reader, check_interval_ms, check_timeout_ms, comment) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; + auto [rc, statement1_unique] = GloAdmin->admindb->prepare_v2(q); + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + sqlite3_stmt *statement1 = statement1_unique.get(); + + while ((row = mysql_fetch_row(results[7]))) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atol(row[0])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // writer_hostgroup + rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atol(row[1])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // reader_hostgroup + if (row[2]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atol(row[2])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // green_writer_hostgroup + } else { + rc=(*proxy_sqlite3_bind_null)(statement1, 3); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + } + if (row[3]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atol(row[3])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // green_reader_hostgroup + } else { + rc=(*proxy_sqlite3_bind_null)(statement1, 4); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + } + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atol(row[4])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // active + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atol(row[5])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // writer_is_also_reader + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atol(row[6])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // check_interval_ms + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atol(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // check_timeout_ms + rc=(*proxy_sqlite3_bind_text)(statement1, 9, row[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + } + } + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Dumping fetched 'mysql_aws_rds_bgd_hostgroups'\n"); + proxy_info("Dumping fetched 'mysql_aws_rds_bgd_hostgroups'\n"); + GloAdmin->admindb->execute_statement((char*)"SELECT * FROM mysql_aws_rds_bgd_hostgroups", &error, &cols, &affected_rows, &resultset); + resultset->dump_to_stderr(); + delete resultset; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime MySQL Servers v2 from peer %s:%d\n", hostname, port); proxy_info("Cluster: Loading to runtime MySQL Servers v2 from peer %s:%d\n", hostname, port); GloAdmin->load_mysql_servers_to_runtime(incoming_servers, peer_runtime_mysql_server, peer_mysql_server_v2); @@ -4978,6 +5041,27 @@ cluster_metrics_map = std::make_tuple( ), // ==================================================================== + // ==================================================================== + std::make_tuple ( + p_cluster_counter::pulled_mysql_servers_aws_rds_bgd_hostgroups_success, + "proxysql_cluster_pulled_total", + "Number of times a 'module' have been pulled from a peer.", + metric_tags { + { "module_name", "mysql_servers_aws_rds_bgd_hostgroups" }, + { "status", "success" } + } + ), + std::make_tuple ( + p_cluster_counter::pulled_mysql_servers_aws_rds_bgd_hostgroups_failure, + "proxysql_cluster_pulled_total", + "Number of times a 'module' have been pulled from a peer.", + metric_tags { + { "module_name", "mysql_servers_aws_rds_bgd_hostgroups" }, + { "status", "failure" } + } + ), + // ==================================================================== + // ==================================================================== std::make_tuple ( p_cluster_counter::pulled_mysql_servers_runtime_checks_success, diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index 6b0376ec1b..a9b8fb9b05 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -1228,6 +1228,40 @@ int ProxySQL_Config::Write_MySQL_Servers_to_configfile(std::string& data) { sqlite_resultset = NULL; } + query=(char *)"SELECT * FROM mysql_aws_rds_bgd_hostgroups"; + admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); + if (error) { + proxy_error("Error on read from mysql_aws_rds_bgd_hostgroups: %s\n", error); + return -1; + } else { + if (sqlite_resultset) { + data += "mysql_aws_rds_bgd_hostgroups:\n(\n"; + bool isNext = false; + for (auto r : sqlite_resultset->rows) { + if (isNext) + data += ",\n"; + data += "\t{\n"; + addField(data, "writer_hostgroup", r->fields[0], ""); + addField(data, "reader_hostgroup", r->fields[1], ""); + // green_writer_hostgroup / green_reader_hostgroup are nullable; addField skips NULLs + addField(data, "green_writer_hostgroup", r->fields[2], ""); + addField(data, "green_reader_hostgroup", r->fields[3], ""); + addField(data, "active", r->fields[4], ""); + addField(data, "writer_is_also_reader", r->fields[5], ""); + addField(data, "check_interval_ms", r->fields[6], ""); + addField(data, "check_timeout_ms", r->fields[7], ""); + addField(data, "comment", r->fields[8]); + + data += "\t}"; + isNext = true; + } + data += "\n)\n"; + } + } + + if (sqlite_resultset) + delete sqlite_resultset; + query = (char *)"SELECT * FROM mysql_hostgroup_attributes"; admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); if (error) { @@ -1619,6 +1653,60 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { rows++; } } + + if (root.exists("mysql_aws_rds_bgd_hostgroups")==true) { + const Setting &mysql_aws_rds_bgd_hostgroups = root["mysql_aws_rds_bgd_hostgroups"]; + int count = mysql_aws_rds_bgd_hostgroups.getLength(); + // green_writer_hostgroup / green_reader_hostgroup are nullable -> passed as %s ("NULL" or an integer) + char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_rds_bgd_hostgroups (writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, writer_is_also_reader, check_interval_ms, check_timeout_ms, comment ) VALUES (%d, %d, %s, %s, %d, %d, %d, %d, '%s')"; + for (i=0; i< count; i++) { + const Setting &line = mysql_aws_rds_bgd_hostgroups[i]; + int writer_hostgroup; + int reader_hostgroup; + int green_writer_hostgroup; + int green_reader_hostgroup; + int active=1; // default + int writer_is_also_reader; + int check_interval_ms; + int check_timeout_ms; + std::string comment=""; + if (line.lookupValue("writer_hostgroup", writer_hostgroup)==false) { + proxy_error("Admin: detected a mysql_aws_rds_bgd_hostgroups in config file without a mandatory writer_hostgroup\n"); + continue; + } + if (line.lookupValue("reader_hostgroup", reader_hostgroup)==false) { + proxy_error("Admin: detected a mysql_aws_rds_bgd_hostgroups in config file without a mandatory reader_hostgroup\n"); + continue; + } + char green_writer_str[24]; + char green_reader_str[24]; + if (line.lookupValue("green_writer_hostgroup", green_writer_hostgroup)==false) { + strcpy(green_writer_str, "NULL"); + } else { + snprintf(green_writer_str, sizeof(green_writer_str), "%d", green_writer_hostgroup); + } + if (line.lookupValue("green_reader_hostgroup", green_reader_hostgroup)==false) { + strcpy(green_reader_str, "NULL"); + } else { + snprintf(green_reader_str, sizeof(green_reader_str), "%d", green_reader_hostgroup); + } + if (line.lookupValue("active", active)==false) active=1; + if (line.lookupValue("writer_is_also_reader", writer_is_also_reader)==false) writer_is_also_reader=0; + if (line.lookupValue("check_interval_ms", check_interval_ms)==false) check_interval_ms=1000; + if (line.lookupValue("check_timeout_ms", check_timeout_ms)==false) check_timeout_ms=800; + line.lookupValue("comment", comment); + char *o1=strdup(comment.c_str()); + char *o=escape_string_single_quotes(o1, false); + char *query=(char *)malloc(strlen(q)+strlen(o)+256); // 128 vs sizeof(int)*8 + sprintf(query,q, writer_hostgroup, reader_hostgroup, green_writer_str, green_reader_str, active, writer_is_also_reader, check_interval_ms, check_timeout_ms, o); + admindb->execute(query); + if (o!=o1) free(o); + free(o1); + free(query); + rows++; + } + } + if (root.exists("mysql_hostgroup_attributes") == true) { const Setting &mysql_hostgroup_attributes = root["mysql_hostgroup_attributes"]; int count = mysql_hostgroup_attributes.getLength(); diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index a48c36f40e..d3fe351a80 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -428,6 +428,7 @@ MySQL_Connection::MySQL_Connection() { async_state_machine=ASYNC_CONNECT_START; ret_mysql=NULL; send_quit=true; + healthy=true; myds=NULL; inserted_into_pool=0; reusable=false; @@ -2138,6 +2139,15 @@ int MySQL_Connection::async_connect(short event) { creation_time = monotonic_time(); return 0; } + + // Abort if the server went offline or was marked unhealthy while waiting to connect. + // The server status can change (shunned by monitor, AWS BGD switchover, manual OFFLINE) + // or the connection can be marked unhealthy (AWS BGD drain) between server selection and + // connection completion. + if (IsServerOffline()) { + return -1; + } + handler(event); switch (async_state_machine) { case ASYNC_CONNECT_SUCCESSFUL: @@ -2160,21 +2170,28 @@ int MySQL_Connection::async_connect(short event) { bool MySQL_Connection::IsServerOffline() { - bool ret=false; - if (parent==NULL) + bool ret = false; + if (parent == NULL) return ret; - server_status=parent->get_status(); // we copy it here to avoid race condition. The caller will see this + + if (healthy == false) + return true; + + server_status = parent->get_status(); // we copy it here to avoid race condition. The caller will see this + bool server_shunned = (server_status == MYSQL_SERVER_STATUS_SHUNNED) || (server_status == MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD); + if ( - (server_status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user + (server_status == MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user || - (server_status==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_automatic==true && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue + (server_shunned && parent->shunned_automatic == true && parent->shunned_and_kill_all_connections == true) // the server is SHUNNED due to a serious issue || - (server_status==MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) // slave is lagging! see #774 + (server_status == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) // slave is lagging! see #774 || (parent->myhgc->online_servers_within_threshold() == false) // number of online servers in a hostgroup exceeds the configured maximum servers ) { - ret=true; + ret = true; } + return ret; } @@ -3031,10 +3048,13 @@ int MySQL_Connection::async_send_simple_command(short event, char *stmt, unsigne assert(mysql); assert(ret_mysql); server_status=parent->get_status(); // we copy it here to avoid race condition. The caller will see this + bool server_shunned = (server_status == MYSQL_SERVER_STATUS_SHUNNED) || (server_status == MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD); if ( - (parent->get_status()==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user + (healthy == false) + || + (server_status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user || - (parent->get_status()==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_automatic==true && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue + (server_shunned && parent->shunned_automatic == true && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue ) { return -1; } diff --git a/lib/mysql_data_stream.cpp b/lib/mysql_data_stream.cpp index fe8d10806c..bd4d971fc5 100644 --- a/lib/mysql_data_stream.cpp +++ b/lib/mysql_data_stream.cpp @@ -1785,12 +1785,19 @@ void MySQL_Data_Stream::setDSS_STATE_QUERY_SENT_NET() { void MySQL_Data_Stream::return_MySQL_Connection_To_Pool() { MySQL_Connection *mc=myconn; mc->last_time_used=sess->thread->curtime; + // before detaching, check if last_HG_affected_rows matches . if yes, set it back to -1 if (mybe) { if (mybe->hostgroup_id == sess->last_HG_affected_rows) { sess->last_HG_affected_rows = -1; } } + + if (!mc->reusable) { + destroy_MySQL_Connection_From_Pool(true); + return; + } + unsigned long long intv = mysql_thread___connection_max_age_ms; intv *= 1000; if ( @@ -1805,7 +1812,7 @@ void MySQL_Data_Stream::return_MySQL_Connection_To_Pool() { // is used outside 'PINGING_SERVER' operation. For more context see #3502. sess->status != PINGING_SERVER ) { - if (mysql_thread___reset_connection_algorithm == 2) { + if (mysql_thread___reset_connection_algorithm == 2 && mc->healthy) { sess->create_new_session_and_reset_connection(this); } else { destroy_MySQL_Connection_From_Pool(true); @@ -1850,7 +1857,7 @@ bool MySQL_Data_Stream::data_in_rbio() { void MySQL_Data_Stream::reset_connection() { if (myconn) { - if (mysql_thread___multiplexing && (DSS == STATE_MARIADB_GENERIC || DSS == STATE_READY) && myconn->reusable == true && myconn->IsActiveTransaction() == false && myconn->MultiplexDisabled() == false && myconn->async_state_machine == ASYNC_IDLE) { + if (mysql_thread___multiplexing && (DSS == STATE_MARIADB_GENERIC || DSS == STATE_READY) && myconn->healthy == true && myconn->reusable == true && myconn->IsActiveTransaction() == false && myconn->MultiplexDisabled() == false && myconn->async_state_machine == ASYNC_IDLE) { myconn->last_time_used = sess->thread->curtime; return_MySQL_Connection_To_Pool(); } diff --git a/lib/proxysql_utils.cpp b/lib/proxysql_utils.cpp index 331902586c..3e92bb338b 100644 --- a/lib/proxysql_utils.cpp +++ b/lib/proxysql_utils.cpp @@ -835,3 +835,107 @@ int calculate_percentile_from_histogram( return thresholds.back(); } + +/** + * @brief Pretty-print a MySQL result set into a string. + * + * @details Formats the full buffered result set as an ASCII table. The current row cursor is preserved: + * the function seeks to the first row for formatting and restores the original cursor before returning. + * + * @param result MySQL result set to format. + * + * @return Pretty-printed result set, or an empty string if the result is NULL or has no fields. + */ +std::string mysql_result_to_string(MYSQL_RES* result) { + if (!result) return ""; + + MYSQL_ROW_OFFSET original_row = mysql_row_tell(result); + mysql_data_seek(result, 0); + + int num_fields = mysql_num_fields(result); + MYSQL_FIELD* fields = mysql_fetch_fields(result); + if (!fields || num_fields == 0) { + mysql_row_seek(result, original_row); + return ""; + } + + std::vector> rows; + MYSQL_ROW row; + while ((row = mysql_fetch_row(result))) { + unsigned long* lens = mysql_fetch_lengths(result); + std::vector r; + r.reserve(num_fields); + for (int i = 0; i < num_fields; i++) { + r.emplace_back(row[i] ? std::string(row[i], lens[i]) : "NULL"); + } + rows.push_back(std::move(r)); + } + + std::vector widths(num_fields); + for (int i = 0; i < num_fields; i++) { + widths[i] = strlen(fields[i].name); + } + for (const auto& r : rows) { + for (int i = 0; i < num_fields; i++) { + if (r[i].size() > widths[i]) widths[i] = r[i].size(); + } + } + + std::string s; + std::string out; + + auto append_border = [&]() { + s = "+"; + for (int i = 0; i < num_fields; i++) { + for (size_t j = 0; j < widths[i] + 2; j++) s += "-"; + s += "+"; + } + out += s; + out += "\n"; + }; + + append_border(); + s = "|"; + for (int i = 0; i < num_fields; i++) { + size_t len = strlen(fields[i].name); + s += " "; s += fields[i].name; + for (size_t j = 0; j < widths[i] - len + 1; j++) s += " "; + s += "|"; + } + out += s; + out += "\n"; + append_border(); + + for (const auto& r : rows) { + s = "|"; + for (int i = 0; i < num_fields; i++) { + s += " "; s += r[i]; + for (size_t j = 0; j < widths[i] - r[i].size() + 1; j++) s += " "; + s += "|"; + } + out += s; + out += "\n"; + } + append_border(); + + mysql_row_seek(result, original_row); + return out; +} + +/** + * @brief Pretty-print a MySQL result set to a file stream. + * + * @details Uses mysql_result_to_string() for formatting and writes the resulting string to the supplied + * file stream. The result set row cursor is preserved. + * + * @param file Destination file stream. + * @param result MySQL result set to format. + */ +void dump_mysql_result(FILE* file, MYSQL_RES* result) { + if (!file) return; + + std::string result_string = mysql_result_to_string(result); + if (!result_string.empty()) { + fputs(result_string.c_str(), file); + } +} diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index 25ad6b6128..ee3ebb4085 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -11,6 +11,9 @@ #include "proxysql_utils.h" #include "MySQL_Query_Processor.h" #include "SQLite3_Server.h" +#ifdef TEST_RDS_BGD +#include "MySQL_Monitor.hpp" +#endif #include #include @@ -358,7 +361,6 @@ vector get_hgs_info(SQLite3DB* db) { #endif void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *pkt) { - char *error=NULL; int cols; int affected_rows; @@ -373,37 +375,32 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p memcpy(query,(char *)pkt->ptr+sizeof(mysql_hdr)+1,query_length-1); query[query_length-1]=0; -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) if (sess->client_myds->proxy_addr.addr == NULL) { - struct sockaddr addr; - socklen_t addr_len=sizeof(struct sockaddr); + struct sockaddr_storage addr; + socklen_t addr_len=sizeof(addr); memset(&addr,0,addr_len); - int rc; - rc=getsockname(sess->client_myds->fd, &addr, &addr_len); - if (rc==0) { - char buf[512]; - switch (addr.sa_family) { - case AF_INET: { - struct sockaddr_in *ipv4 = (struct sockaddr_in *)&addr; - inet_ntop(addr.sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); - sess->client_myds->proxy_addr.addr = strdup(buf); - } - break; - case AF_INET6: { - struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)&addr; - inet_ntop(addr.sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); - sess->client_myds->proxy_addr.addr = strdup(buf); - } - break; - default: - sess->client_myds->proxy_addr.addr = strdup("unknown"); - break; + if (getsockname(sess->client_myds->fd, (struct sockaddr *)&addr, &addr_len)==0) { + char buf[INET6_ADDRSTRLEN]; + const void *src=NULL; + if (addr.ss_family == AF_INET) { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)&addr; + src = &ipv4->sin_addr; + sess->client_myds->proxy_addr.port = ntohs(ipv4->sin_port); + } else if (addr.ss_family == AF_INET6) { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)&addr; + src = &ipv6->sin6_addr; + sess->client_myds->proxy_addr.port = ntohs(ipv6->sin6_port); } - } else { + if (src && inet_ntop(addr.ss_family, src, buf, sizeof(buf))) { + sess->client_myds->proxy_addr.addr = strdup(buf); + } + } + if (sess->client_myds->proxy_addr.addr == NULL) { sess->client_myds->proxy_addr.addr = strdup("unknown"); } } -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD char *query_no_space=(char *)l_alloc(query_length); memcpy(query_no_space,query,query_length); @@ -574,13 +571,13 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p if (query_no_space_length==SELECT_VERSION_COMMENT_LEN) { if (!strncasecmp(SELECT_VERSION_COMMENT, query_no_space, query_no_space_length)) { l_free(query_length,query); -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) char *a = (char *)"SELECT '(ProxySQL Automated Test Server) - %s'"; query = (char *)malloc(strlen(a)+strlen(sess->client_myds->proxy_addr.addr)); sprintf(query,a,sess->client_myds->proxy_addr.addr); #else query=l_strdup("SELECT '(ProxySQL SQLite3 Server)'"); -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD query_length=strlen(query)+1; goto __run_query; } @@ -778,8 +775,99 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p __run_query: if (run_query) { -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT",query_no_space,6)==0) { +#ifdef TEST_RDS_BGD + const bool rds_bgd_table_check = + strcasecmp(query_no_space, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK) == 0; + const bool rds_bgd_metadata = + strcasecmp(query_no_space, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY) == 0; + if (rds_bgd_table_check || rds_bgd_metadata) { + if (sess->client_myds->proxy_addr.addr == NULL || + sess->client_myds->proxy_addr.port <= 0) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "RDS BGD simulator could not identify the accepted backend address"); + run_query=false; + } else { + SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; + const std::string backend_ip { sess->client_myds->proxy_addr.addr }; + const int backend_port = sess->client_myds->proxy_addr.port; + const std::string predicate { + "backend_ip='" + backend_ip + "' AND backend_port=" + + std::to_string(backend_port) + }; + const std::string log_query { + "INSERT INTO RDS_BGD_PROBE_LOG" + "(backend_ip,backend_port,probe_kind,encrypted) VALUES ('" + + backend_ip + "'," + std::to_string(backend_port) + ",'" + + (rds_bgd_table_check ? "table_check" : "metadata") + "'," + + (sess->client_myds->encrypted ? "1" : "0") + ")" + }; + if (!sqlite_sess->sessdb->execute(log_query.c_str())) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "RDS BGD simulator failed to record the topology probe"); + run_query=false; + } else { + char *control_error=NULL; + int control_cols=0; + int control_affected_rows=0; + SQLite3_result *control_result=NULL; + const std::string control_query { + "SELECT topology_present,error_code,error_msg FROM RDS_BGD_CONTROL WHERE " + + predicate + }; + sqlite_sess->sessdb->execute_statement( + control_query.c_str(), &control_error, &control_cols, + &control_affected_rows, &control_result); + + if (control_error != NULL) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, control_error); + free(control_error); + run_query=false; + } + + bool topology_present=false; + unsigned int configured_error=0; + std::string configured_error_msg {}; + if (run_query && control_result && control_result->rows_count == 1) { + SQLite3_row *row=control_result->rows.front(); + topology_present=atoi(row->fields[0]) != 0; + configured_error=static_cast(atoi(row->fields[1])); + configured_error_msg=row->fields[2] ? row->fields[2] : ""; + } + delete control_result; + + if (run_query && rds_bgd_table_check) { + l_free(query_length,query); + query=l_strdup(topology_present ? "SELECT 1" : "SELECT 1 WHERE 0"); + query_length=strlen(query)+1; + } else if (run_query && (configured_error != 0 || !topology_present)) { + const uint16_t error_code = configured_error + ? static_cast(configured_error) : 1146; + const char *error_msg = configured_error + ? configured_error_msg.c_str() + : "Table 'mysql.rds_topology' doesn't exist"; + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, error_code, error_msg); + run_query=false; + } else if (run_query) { + const std::string topology_query { + "SELECT id,endpoint,topology_port AS port,role,status " + "FROM RDS_BGD_TOPOLOGY WHERE " + predicate + + " ORDER BY row_order" + }; + l_free(query_length,query); + query=l_strdup(topology_query.c_str()); + query_length=strlen(query)+1; + } + } + } + } + +#endif // TEST_RDS_BGD #ifdef TEST_AURORA if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { pthread_mutex_lock(&GloSQLite3Server->aurora_mutex); @@ -860,7 +948,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } #endif // TEST_GROUPREP -#ifdef TEST_READONLY +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT @@global.read_only read_only ",query_no_space, strlen("SELECT @@global.read_only read_only "))==0) { if (strlen(query_no_space) > strlen("SELECT @@global.read_only read_only ")+5) { pthread_mutex_lock(&GloSQLite3Server->test_readonly_mutex); @@ -877,7 +965,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); } } -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG if ( strncasecmp("SELECT SLAVE STATUS ", query_no_space, strlen("SELECT SLAVE STATUS ")) == 0 @@ -913,7 +1001,12 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p sprintf(query,a,rand()%30+10); } } -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD + if (!run_query) { + l_free(pkt->size-sizeof(mysql_hdr),query_no_space); + l_free(query_length,query); + return; + } SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; if (sess->autocommit==false) { sqlite3 *db = sqlite_sess->sessdb->get_db(); @@ -977,7 +1070,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p bool deprecate_eof = sess->client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot, in_trans, deprecate_eof); delete resultset; -#ifdef TEST_READONLY +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT",query_no_space,6)) { if (strstr(query_no_space,(char *)"READONLY_STATUS")) { // the table is writable @@ -986,7 +1079,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); } } -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG if (strncasecmp("SELECT", query_no_space, 6)) { if (strstr(query_no_space, (char*)"REPLICATIONLAG_HOST_STATUS")) { @@ -1295,6 +1388,16 @@ SQLite3_Server::~SQLite3_Server() { drop_tables_defs(tables_defs_grouprep); delete tables_defs_grouprep; #endif // TEST_GROUPREP + +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) + drop_tables_defs(tables_defs_readonly); + delete tables_defs_readonly; +#endif + +#ifdef TEST_RDS_BGD + drop_tables_defs(tables_defs_rds_bgd); + delete tables_defs_rds_bgd; +#endif // TEST_RDS_BGD }; #ifdef TEST_AURORA @@ -1382,7 +1485,7 @@ SQLite3_Server::SQLite3_Server() { variables.read_only=false; -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) string s = ""; #ifdef TEST_AURORA @@ -1396,13 +1499,13 @@ SQLite3_Server::SQLite3_Server() { #ifdef TEST_GROUPREP init_grouprep_ifaces_string(s); #endif // TEST_GROUPREP -#ifdef TEST_READONLY - // for readonly test we listen on all IPs because we simulate a lot of clusters +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) + // Read-only simulation listens on all IPs because it can simulate many clusters. if (!s.empty()) s += ";"; s += "0.0.0.0:3306"; pthread_mutex_init(&test_readonly_mutex, NULL); -#endif //TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG // for replication test we listen on all IPs if (!s.empty()) @@ -1410,12 +1513,11 @@ SQLite3_Server::SQLite3_Server() { s += "0.0.0.0:3306"; pthread_mutex_init(&test_replicationlag_mutex, NULL); #endif //TEST_REPLICATIONLAG - variables.mysql_ifaces=strdup(s.c_str()); #else variables.mysql_ifaces=strdup("127.0.0.1:6030"); -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD }; @@ -1778,7 +1880,7 @@ void SQLite3_Server::populate_grouprep_table(MySQL_Session *sess, int txs_behind #endif // TEST_GALERA -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) void SQLite3_Server::insert_into_tables_defs(std::vector *tables_defs, const char *table_name, const char *table_def) { table_def_t *td = new table_def_t; td->table_name=strdup(table_name); @@ -1808,7 +1910,7 @@ void SQLite3_Server::drop_tables_defs(std::vector *tables_defs) { delete td; } }; -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD void SQLite3_Server::wrlock() { pthread_rwlock_wrlock(&rwlock); @@ -1859,14 +1961,41 @@ bool SQLite3_Server::init() { check_and_build_standard_tables(sessdb, tables_defs_grouprep); GloAdmin->enable_grouprep_testing(); #endif // TEST_GALERA -#ifdef TEST_READONLY +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) tables_defs_readonly = new std::vector; insert_into_tables_defs(tables_defs_readonly, (const char *)"READONLY_STATUS", (const char*)"CREATE TABLE READONLY_STATUS (hostname VARCHAR NOT NULL , port INT NOT NULL , read_only INT NOT NULL CHECK (read_only IN (0, 1)) DEFAULT 1 , PRIMARY KEY (hostname, port))"); check_and_build_standard_tables(sessdb, tables_defs_readonly); +#ifdef TEST_READONLY GloAdmin->enable_readonly_testing(); #endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD +#ifdef TEST_RDS_BGD + tables_defs_rds_bgd = new std::vector; + insert_into_tables_defs(tables_defs_rds_bgd, + (const char *)"RDS_BGD_CONTROL", + (const char *)"CREATE TABLE RDS_BGD_CONTROL (" + "backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, " + "topology_present INTEGER NOT NULL DEFAULT 0 CHECK (topology_present IN (0,1)), " + "error_code INTEGER NOT NULL DEFAULT 0, error_msg TEXT NOT NULL DEFAULT '', " + "PRIMARY KEY (backend_ip, backend_port))"); + insert_into_tables_defs(tables_defs_rds_bgd, + (const char *)"RDS_BGD_TOPOLOGY", + (const char *)"CREATE TABLE RDS_BGD_TOPOLOGY (" + "backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, row_order INTEGER NOT NULL, " + "id TEXT NOT NULL, endpoint TEXT NOT NULL, topology_port INTEGER NOT NULL, " + "role TEXT NOT NULL, status TEXT NOT NULL, " + "PRIMARY KEY (backend_ip, backend_port, row_order))"); + insert_into_tables_defs(tables_defs_rds_bgd, + (const char *)"RDS_BGD_PROBE_LOG", + (const char *)"CREATE TABLE RDS_BGD_PROBE_LOG (" + "sequence_id INTEGER PRIMARY KEY AUTOINCREMENT, backend_ip TEXT NOT NULL, " + "backend_port INTEGER NOT NULL, probe_kind TEXT NOT NULL " + "CHECK (probe_kind IN ('table_check','metadata')), encrypted INTEGER NOT NULL " + "CHECK (encrypted IN (0,1)))"); + check_and_build_standard_tables(sessdb, tables_defs_rds_bgd); +#endif // TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG tables_defs_replicationlag = new std::vector; insert_into_tables_defs(tables_defs_replicationlag, @@ -1987,7 +2116,16 @@ void SQLite3_Server::send_MySQL_ERR(MySQL_Protocol *myprot, char *msg) { myds->DSS=STATE_SLEEP; } -#ifdef TEST_READONLY +void SQLite3_Server::send_MySQL_ERR(MySQL_Protocol *myprot, uint16_t error_code, const char *msg) { + assert(myprot); + MySQL_Data_Stream *myds=myprot->get_myds(); + myds->DSS=STATE_QUERY_SENT_DS; + char *sqlstate = error_code == 1146 ? (char *)"42S02" : (char *)"HY000"; + myprot->generate_pkt_ERR(true,NULL,NULL,1,error_code,sqlstate,msg); + myds->DSS=STATE_SLEEP; +} + +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) void SQLite3_Server::load_readonly_table(MySQL_Session *sess) { // this function needs to be called with lock on mutex readonly_mutex already acquired GloAdmin->mysql_servers_wrlock(); @@ -2028,7 +2166,7 @@ int SQLite3_Server::readonly_test_value(char *p) { } return rc; } -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG void SQLite3_Server::load_replicationlag_table(MySQL_Session* sess) { diff --git a/test/infra/README.md b/test/infra/README.md index 194ad5b005..44efef3ce5 100644 --- a/test/infra/README.md +++ b/test/infra/README.md @@ -39,7 +39,11 @@ This will: ## 0.2. Simulator-backed TAP groups -Groups whose name starts with `cluster_sim_` (e.g. `cluster_sim_aurora-g1`, `cluster_sim_galera-g1`) drive ProxySQL via the in-repo `cluster_simulator` under `test/deps/cluster_simulator/`. The simulator mutates ProxySQL's internal cluster state through code paths gated by compile-time `#ifdef` flags, so the ProxySQL binary **must** be built with the matching flag or state mutations become no-ops and tests fail silently. +Groups whose name starts with `cluster_sim_` exercise simulator-specific +ProxySQL paths gated by compile-time `#ifdef` flags. State is driven either by +the in-repo `test/deps/cluster_simulator` process or directly by TAP helpers +through SQLite3-server, as in RDS BGD. The ProxySQL binary **must** be built with +the matching flag or state mutations become no-ops and tests fail silently. | Simulator group | Required build target | |-----------------------------|---------------------------| @@ -48,8 +52,12 @@ Groups whose name starts with `cluster_sim_` (e.g. `cluster_sim_aurora-g1`, `clu | `cluster_sim_group_repl-g` | `make testgrouprep` | | `cluster_sim_read_only-g` | `make testreadonly` | | `cluster_sim_repl_lag-g` | `make testreplicationlag` | +| `cluster_sim_rds_bgd-g` | `make test_rds_bgd` | -Each target sets the corresponding `-DTEST_` flag on the ProxySQL src and lib build and triggers the simulator binary build. A plain `make` is **not sufficient** for these groups. +Each target sets the corresponding `-DTEST_` flag on the ProxySQL src +and lib builds and builds the TAP artifacts. Targets backed by +`test/deps/cluster_simulator` build that process as well. A plain `make` is +**not sufficient** for these groups. --- ## 1. Core Concepts diff --git a/test/infra/control/cluster-simulator-ci.bash b/test/infra/control/cluster-simulator-ci.bash new file mode 100755 index 0000000000..a55f0d30e2 --- /dev/null +++ b/test/infra/control/cluster-simulator-ci.bash @@ -0,0 +1,339 @@ +#!/usr/bin/env bash +# +# Centralizes cluster simulation workflow operations so build and runtime +# packaging behavior is readable, reusable locally, and kept out of YAML. + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd -P)" +GROUPS_FILE="${REPO_ROOT}/test/tap/groups/groups.json" +SIMULATOR_BINARIES_FILE="${REPO_ROOT}/.cluster-simulator-binaries" +RUNTIME_CACHE_DIR_VALUE="${RUNTIME_CACHE_DIR:-.cluster-simulator-runtime}" + +if [[ "${RUNTIME_CACHE_DIR_VALUE}" = /* ]]; then + RUNTIME_CACHE_PATH="$(realpath -m -- "${RUNTIME_CACHE_DIR_VALUE}")" +else + RUNTIME_CACHE_PATH="$(realpath -m -- "${REPO_ROOT}/${RUNTIME_CACHE_DIR_VALUE}")" +fi + +case "${RUNTIME_CACHE_PATH}" in + "${REPO_ROOT}/"*) ;; + *) + echo "ERROR: RUNTIME_CACHE_DIR must resolve inside ${REPO_ROOT}." >&2 + exit 1 + ;; +esac + +STAGE_TEMP_DIR="" + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +usage_error() { + echo "ERROR: $*" >&2 + echo "Run '$0 help' for usage." >&2 + exit 2 +} + +expect_no_arguments() { + local command="${1}" + local argument_count="${2}" + + [[ "${argument_count}" -eq 0 ]] || + usage_error "'${command}' does not accept arguments." +} + +require_command() { + command -v "${1}" >/dev/null 2>&1 || + die "Required command '${1}' was not found." +} + +require_executable() { + [[ -x "${1}" ]] || die "Required executable is missing: ${1}" +} + +require_directory() { + [[ -d "${1}" ]] || die "Required directory is missing: ${1}" +} + +discover_groups_json() { + jq -ce ' + [.[] | .[] | select(type == "string" and startswith("cluster_sim_"))] + | unique + | if length > 0 then . else error("no cluster simulation groups found") end + ' "${GROUPS_FILE}" +} + +refresh_binaries_manifest() { + local temporary_manifest + + temporary_manifest="$(mktemp "${SIMULATOR_BINARIES_FILE}.tmp.XXXXXX")" + if ! jq -er ' + [ + to_entries[] + | select(any(.value[]; type == "string" and startswith("cluster_sim_"))) + | .key + ] + | unique + | if length > 0 then .[] else error("no cluster simulation TAP binaries found") end + ' "${GROUPS_FILE}" > "${temporary_manifest}"; then + rm -f -- "${temporary_manifest}" + die "Failed to discover cluster simulation TAP binaries from ${GROUPS_FILE}." + fi + + mv -- "${temporary_manifest}" "${SIMULATOR_BINARIES_FILE}" +} + +load_manifest_binaries() { + [[ -s "${SIMULATOR_BINARIES_FILE}" ]] || + die "Simulation binary manifest is missing: ${SIMULATOR_BINARIES_FILE}" + mapfile -t SIMULATOR_BINARIES < "${SIMULATOR_BINARIES_FILE}" + [[ "${#SIMULATOR_BINARIES[@]}" -gt 0 ]] || + die "No TAP binaries were written to ${SIMULATOR_BINARIES_FILE}." +} + +load_group_binaries() { + local group="${1}" + local binaries_json + + [[ "${group}" == cluster_sim_* ]] || + die "'${group}' is not a cluster simulation group." + + binaries_json="$(jq -ce --arg group "${group}" ' + [ + to_entries[] + | select(.value | index($group)) + | .key + ] + | unique + | if length > 0 then . else error("group is not registered") end + ' "${GROUPS_FILE}")" || + die "Cluster simulation group '${group}' is not registered in ${GROUPS_FILE}." + + mapfile -t SIMULATOR_BINARIES < <(jq -r '.[]' <<< "${binaries_json}") +} + +verify_runtime_paths() { + local root="${1}" + shift + local binary + + require_executable "${root}/src/proxysql" + require_executable "${root}/test/deps/cluster_simulator/cluster_simulator" + require_directory "${root}/test/tap/tap" + + for binary in "$@"; do + require_executable "${root}/test/tap/tests/${binary}" + done +} + +cleanup_stage_temp() { + if [[ -n "${STAGE_TEMP_DIR}" && -d "${STAGE_TEMP_DIR}" ]]; then + rm -rf -- "${STAGE_TEMP_DIR}" + fi +} + +trap cleanup_stage_temp EXIT + +# discover +# Purpose: Generate the matrix group JSON and the TAP-binary build manifest. +# Local use: Run `cluster-simulator-ci.bash discover` to inspect registry output. +# GitHub use: Supplies the build job's matrix output before cache restoration. +handle_discover() { + expect_no_arguments "discover" "$#" + require_command jq + + local groups_json + local group_count + local binary_count + + groups_json="$(discover_groups_json)" || + die "Failed to discover cluster simulation groups from ${GROUPS_FILE}." + refresh_binaries_manifest + + group_count="$(jq 'length' <<< "${groups_json}")" + binary_count="$(wc -l < "${SIMULATOR_BINARIES_FILE}")" + + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + printf 'groups=%s\n' "${groups_json}" >> "${GITHUB_OUTPUT}" + fi + + printf 'Discovered %s simulation groups and %s TAP binaries.\n' \ + "${group_count}" "${binary_count}" + jq -r '.[] | " group: \(.)"' <<< "${groups_json}" + sed 's/^/ binary: /' "${SIMULATOR_BINARIES_FILE}" +} + +# build +# Purpose: Build ProxySQL runtime with all simulation flags and every registered TAP binary. +# Local use: Run `cluster-simulator-ci.bash build` to reproduce the CI build. +# GitHub use: Invoked on an exact-SHA cache miss in the build job. +handle_build() { + expect_no_arguments "build" "$#" + require_command docker + require_command git + require_command jq + refresh_binaries_manifest + + local git_version + git_version="$(git -C "${REPO_ROOT}" describe --long --abbrev=7 2>/dev/null || + git -C "${REPO_ROOT}" describe --long --abbrev=7 --always)" || + die "Failed to derive the ProxySQL build version from Git." + + ( + cd "${REPO_ROOT}" + docker compose run --rm --no-deps \ + --env "GIT_VERSION_BASE=${git_version}" \ + --entrypoint /opt/proxysql/test/infra/control/cluster-simulator-ci.bash \ + --workdir /opt/proxysql \ + ubuntu22_build _build + ) +} + +# _build +# Purpose: Execute the compiler commands inside the Ubuntu 22 packaging image. +# Local use: Internal only; use the public `build` command from the host. +# GitHub use: Called by `build` as the packaging container entrypoint. +handle_internal_build() { + expect_no_arguments "_build" "$#" + load_manifest_binaries + [[ -n "${GIT_VERSION_BASE:-}" ]] || + die "GIT_VERSION_BASE was not provided by the host build command." + + cd "${REPO_ROOT}" + make -j"$(nproc)" GIT_VERSION_BASE="${GIT_VERSION_BASE}" testall + make -j"$(nproc)" GIT_VERSION_BASE="${GIT_VERSION_BASE}" build_cluster_simulator + make -C test/tap -j"$(nproc)" GIT_VERSION="${GIT_VERSION_BASE}" tap + make -C test/tap/tests -j"$(nproc)" \ + GIT_VERSION="${GIT_VERSION_BASE}" "${SIMULATOR_BINARIES[@]}" +} + +# verify +# Purpose: Check the complete runtime, or only the TAP binaries for one group. +# Local use: Run `verify` after a build, optionally with a cluster_sim_* group. +# GitHub use: Checks the build job runtime and each restored matrix-job runtime. +handle_verify() { + [[ "$#" -le 1 ]] || + usage_error "'verify' accepts at most one simulation group." + require_command jq + + local group="${1:-}" + + if [[ -n "${group}" ]]; then + load_group_binaries "${group}" + else + refresh_binaries_manifest + load_manifest_binaries + fi + + verify_runtime_paths "${REPO_ROOT}" "${SIMULATOR_BINARIES[@]}" + + if [[ -n "${group}" ]]; then + printf 'Verified simulation runtime for %s.\n' "${group}" + else + printf 'Verified simulation runtime for all registered groups.\n' + fi +} + +# stage +# Purpose: Assemble only the runtime files that matrix jobs need in the cache. +# Local use: Optional; run after `build` to inspect the cache payload locally. +# GitHub use: Creates the exact-SHA cache payload after a successful build. +handle_stage() { + expect_no_arguments "stage" "$#" + require_command jq + handle_verify + + local binary + local runtime_parent + + runtime_parent="$(dirname -- "${RUNTIME_CACHE_PATH}")" + mkdir -p -- "${runtime_parent}" + STAGE_TEMP_DIR="$(mktemp -d "${RUNTIME_CACHE_PATH}.tmp.XXXXXX")" + + install -D -m 0755 \ + "${REPO_ROOT}/src/proxysql" \ + "${STAGE_TEMP_DIR}/src/proxysql" + install -D -m 0755 \ + "${REPO_ROOT}/test/deps/cluster_simulator/cluster_simulator" \ + "${STAGE_TEMP_DIR}/test/deps/cluster_simulator/cluster_simulator" + install -d "${STAGE_TEMP_DIR}/test/tap" + cp -a "${REPO_ROOT}/test/tap/tap" "${STAGE_TEMP_DIR}/test/tap/" + + for binary in "${SIMULATOR_BINARIES[@]}"; do + install -D -m 0755 \ + "${REPO_ROOT}/test/tap/tests/${binary}" \ + "${STAGE_TEMP_DIR}/test/tap/tests/${binary}" + done + + if [[ -e "${RUNTIME_CACHE_PATH}" || -L "${RUNTIME_CACHE_PATH}" ]]; then + rm -rf -- "${RUNTIME_CACHE_PATH}" + fi + mv -- "${STAGE_TEMP_DIR}" "${RUNTIME_CACHE_PATH}" + STAGE_TEMP_DIR="" + + printf 'Staged simulation runtime in %s.\n' "${RUNTIME_CACHE_PATH}" +} + +# install +# Purpose: Restore a staged simulation runtime into the current checkout. +# Local use: Usually unnecessary; use it only to validate a staged cache payload. +# GitHub use: Installs files immediately after actions/cache restores the payload. +handle_install() { + expect_no_arguments "install" "$#" + require_command jq + require_directory "${RUNTIME_CACHE_PATH}" + refresh_binaries_manifest + load_manifest_binaries + verify_runtime_paths "${RUNTIME_CACHE_PATH}" "${SIMULATOR_BINARIES[@]}" + + cp -a "${RUNTIME_CACHE_PATH}/." "${REPO_ROOT}/" + printf 'Installed simulation runtime from %s.\n' "${RUNTIME_CACHE_PATH}" +} + +# help +# Purpose: Document the command interface, generated files, and common examples. +# Local use: Run `cluster-simulator-ci.bash help` when reproducing workflow steps. +# GitHub use: Not called by the workflow; it is maintainer-facing documentation. +handle_help() { + expect_no_arguments "help" "$#" + + cat < [arguments] + +Commands: + discover Print registered simulation groups and write: + ${SIMULATOR_BINARIES_FILE} + build Build ProxySQL with simulation support in ubuntu22_build. + verify [group] Verify all runtime files, or one matrix group. + stage Create the cache payload at: + ${RUNTIME_CACHE_PATH} + install Restore that cache payload into the checkout. + help Show this help. + +Examples: + $0 discover + $0 build + $0 verify + $0 verify cluster_sim_galera-g1 +EOF +} + +command_name="${1:-help}" +if [[ "$#" -gt 0 ]]; then + shift +fi + +case "${command_name}" in + discover) handle_discover "$@" ;; + build) handle_build "$@" ;; + _build) handle_internal_build "$@" ;; + verify) handle_verify "$@" ;; + stage) handle_stage "$@" ;; + install) handle_install "$@" ;; + help|-h|--help) handle_help "$@" ;; + *) usage_error "Unknown command '${command_name}'." ;; +esac diff --git a/test/infra/control/ensure-infras.bash b/test/infra/control/ensure-infras.bash index b1baa5e8f9..3a7d865380 100755 --- a/test/infra/control/ensure-infras.bash +++ b/test/infra/control/ensure-infras.bash @@ -6,6 +6,7 @@ set -o pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" export WORKSPACE="${REPO_ROOT}" +source "${SCRIPT_DIR}/readiness.bash" # Default INFRA_ID if not provided export INFRA_ID="${INFRA_ID:-dev-$USER}" @@ -101,6 +102,13 @@ for EXT in bash sql; do fi done +PROXYSQL_READY_PORTS=(6032 6033 6132 6133) +if [ -n "${PROXYSQL_READY_PORTS_EXTRA:-}" ]; then + read -r -a EXTRA_READY_PORTS <<< "${PROXYSQL_READY_PORTS_EXTRA}" + PROXYSQL_READY_PORTS+=("${EXTRA_READY_PORTS[@]}") +fi +wait_for_proxysql_ports "${PROXY_CONTAINER}" 30 "${PROXYSQL_READY_PORTS[@]}" + # 4. Ensure Docker Compose helper is available COMPOSE_CMD="docker compose" if ! $COMPOSE_CMD version &>/dev/null; then COMPOSE_CMD="docker-compose"; fi @@ -159,4 +167,4 @@ if [ -f "${SETUP_HOOK}" ]; then "${SETUP_HOOK}" fi -# ensure-infras.bash completed successfully \ No newline at end of file +# ensure-infras.bash completed successfully diff --git a/test/infra/control/readiness.bash b/test/infra/control/readiness.bash new file mode 100644 index 0000000000..5813346814 --- /dev/null +++ b/test/infra/control/readiness.bash @@ -0,0 +1,42 @@ +#!/bin/bash + +wait_for_proxysql_ports() { + local container="$1" + local timeout_seconds="$2" + shift 2 + + local port + local attempt + + if [[ ! "${timeout_seconds}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: Invalid ProxySQL readiness timeout: ${timeout_seconds}" >&2 + return 1 + fi + + echo ">>> Running readiness checks for ProxySQL ports: $*" + + for port in "$@"; do + if [[ ! "${port}" =~ ^[0-9]+$ ]]; then + echo "ERROR: Invalid ProxySQL readiness port: ${port}" >&2 + return 1 + fi + + echo -n ">>> Waiting for ${container}:${port} " + for ((attempt = 0; attempt < timeout_seconds; attempt++)); do + if docker exec "${container}" \ + bash -c "exec 3<>/dev/tcp/127.0.0.1/${port}" \ + >/dev/null 2>&1; then + echo "Ready." + break + fi + echo -n "." + sleep 1 + done + + if [ "${attempt}" -ge "${timeout_seconds}" ]; then + echo " TIMEOUT" + docker logs --tail=60 "${container}" >&2 || true + return 1 + fi + done +} diff --git a/test/tap/groups/cluster_sim_aurora/env.sh b/test/tap/groups/cluster_sim_aurora/env.sh index 45220ddd6e..17560db941 100644 --- a/test/tap/groups/cluster_sim_aurora/env.sh +++ b/test/tap/groups/cluster_sim_aurora/env.sh @@ -9,6 +9,7 @@ export CLUSTER_SIM_HOST_FILE="${WORKSPACE}/test/tap/groups/cluster_sim_aurora/ad # username/password match what enable_aurora_testing() inserts. export AURORA_HOSTNAME=proxysql export AURORA_PORT=3306 +export PROXYSQL_READY_PORTS_EXTRA="3306" # Skip the background cluster nodes: they are built without TEST_AURORA and # their empty mysql_users sync back to the primary, wiping aurora1/2/3. diff --git a/test/tap/groups/cluster_sim_galera/env.sh b/test/tap/groups/cluster_sim_galera/env.sh index e7b9ea3fff..2a37bfd682 100644 --- a/test/tap/groups/cluster_sim_galera/env.sh +++ b/test/tap/groups/cluster_sim_galera/env.sh @@ -8,6 +8,7 @@ export CLUSTER_SIM_TESTS_ROOT="${WORKSPACE}/test/deps/cluster_simulator/tests" # username/password match what enable_galera_testing() inserts (galera1/pass1). export GALERA_HOSTNAME=proxysql export GALERA_PORT=3306 +export PROXYSQL_READY_PORTS_EXTRA="3306" # Skip the background cluster nodes so their empty mysql_users do not sync # back to the primary and wipe galera1/2/galera. diff --git a/test/tap/groups/cluster_sim_group_repl/env.sh b/test/tap/groups/cluster_sim_group_repl/env.sh index 5a82453574..6aae7caa17 100644 --- a/test/tap/groups/cluster_sim_group_repl/env.sh +++ b/test/tap/groups/cluster_sim_group_repl/env.sh @@ -8,6 +8,7 @@ export CLUSTER_SIM_TESTS_ROOT="${WORKSPACE}/test/deps/cluster_simulator/tests" # for username/password (grouprep1/pass1) match what enable_grouprep_testing() inserts. export GROUPREP_HOSTNAME=proxysql export GROUPREP_PORT=3306 +export PROXYSQL_READY_PORTS_EXTRA="3306" # Skip the background cluster nodes so their empty mysql_users do not sync # back to the primary and wipe grouprep1. diff --git a/test/tap/groups/cluster_sim_rds_bgd/add-hosts b/test/tap/groups/cluster_sim_rds_bgd/add-hosts new file mode 100644 index 0000000000..7147ba38e2 --- /dev/null +++ b/test/tap/groups/cluster_sim_rds_bgd/add-hosts @@ -0,0 +1,36 @@ +# AWS RDS BGD simulator endpoint aliases. +# Format: " " per line; '#' comments allowed. +# These are injected into the ProxySQL container's /etc/hosts via Docker +# --add-host by test/infra/control/start-proxysql-isolated.bash when +# CLUSTER_SIM_HOST_FILE points at this file (see this group's env.sh). + +# Cluster 1: blue endpoints +db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.11 +db-1-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.12 +db-1-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.13 + +# Cluster 1: green deployment A +db-1-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.14 +db-1-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.15 +db-1-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.16 + +# Cluster 1: green deployment B +db-1-green-s7m2kx.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.17 +db-1-reader-1-green-v4n8qp.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.18 +db-1-reader-2-green-w6h3rz.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.19 + +# Cluster 2 +db-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.20 +db-2-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.21 +db-2-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.22 +db-2-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.23 +db-2-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.24 +db-2-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.25 + +# Cluster 3 +db-3.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.26 +db-3-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.27 +db-3-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.28 +db-3-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.29 +db-3-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.30 +db-3-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.31 diff --git a/test/tap/groups/cluster_sim_rds_bgd/env.sh b/test/tap/groups/cluster_sim_rds_bgd/env.sh new file mode 100644 index 0000000000..898b34322e --- /dev/null +++ b/test/tap/groups/cluster_sim_rds_bgd/env.sh @@ -0,0 +1,13 @@ +# shellcheck shell=bash +# AWS RDS BGD simulator TAP group environment + +# Inject AWS-style endpoint aliases into the ProxySQL container. +export CLUSTER_SIM_HOST_FILE="${WORKSPACE}/test/tap/groups/cluster_sim_rds_bgd/add-hosts" +export PROXYSQL_READY_PORTS_EXTRA="3306" + +# Skip background cluster nodes: the TAP test drives the primary ProxySQL's +# built-in SQLite3-server simulator directly. +export SKIP_CLUSTER_START=1 + +# No backend infra: the TAP test controls simulated backend state through the +# SQLite3 server. Intentionally NOT setting DEFAULT_MYSQL_INFRA / DEFAULT_PGSQL_INFRA. diff --git a/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.bash b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.bash new file mode 100755 index 0000000000..d195d60ff0 --- /dev/null +++ b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.bash @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -e +# ProxySQL's admin port goes live before its startup is done. Wait for all +# init__variables() to complete before running pre-proxysql.sql; +# concurrent writes return SQLITE_LOCKED, which flush-variables functions +# treat as fatal (assert on rc != 0). +sleep 5 diff --git a/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.sql b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.sql new file mode 100644 index 0000000000..40399d7dbd --- /dev/null +++ b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.sql @@ -0,0 +1,12 @@ +-- Create and persist the client account used by the BGD TAP tests. +INSERT OR REPLACE INTO mysql_users (username, password, default_hostgroup, active) + VALUES ('testuser', 'testuser', 0, 1); +LOAD MYSQL USERS TO RUNTIME; +SAVE MYSQL USERS TO DISK; + +-- When compiled with TEST_RDS_BGD, ProxySQL's monitor reaches its own SQLite3 +-- server on :3306. The default proxysql-ci.cnf pins the server to :6030, so +-- rebind it to :3306 here. +SET sqliteserver-mysql_ifaces='0.0.0.0:3306'; +LOAD SQLITESERVER VARIABLES TO RUNTIME; +SAVE SQLITESERVER VARIABLES TO DISK; diff --git a/test/tap/groups/cluster_sim_read_only/env.sh b/test/tap/groups/cluster_sim_read_only/env.sh index 39d25fd56b..a5fee6bb39 100644 --- a/test/tap/groups/cluster_sim_read_only/env.sh +++ b/test/tap/groups/cluster_sim_read_only/env.sh @@ -8,6 +8,7 @@ export CLUSTER_SIM_TESTS_ROOT="${WORKSPACE}/test/deps/cluster_simulator/tests" # defaults for username/password are 'root/root' (provisioned by pre-proxysql.sql). export READONLY_HOSTNAME=proxysql export READONLY_PORT=3306 +export PROXYSQL_READY_PORTS_EXTRA="3306" # Skip the background cluster nodes so their empty mysql_users do not sync # back to the primary and wipe the root user we inject. diff --git a/test/tap/groups/cluster_sim_repl_lag/env.sh b/test/tap/groups/cluster_sim_repl_lag/env.sh index 584475fba8..4ef92d774f 100644 --- a/test/tap/groups/cluster_sim_repl_lag/env.sh +++ b/test/tap/groups/cluster_sim_repl_lag/env.sh @@ -8,6 +8,7 @@ export CLUSTER_SIM_TESTS_ROOT="${WORKSPACE}/test/deps/cluster_simulator/tests" # defaults for username/password are 'root/root' (provisioned by pre-proxysql.sql). export REPL_LAG_HOSTNAME=proxysql export REPL_LAG_PORT=3306 +export PROXYSQL_READY_PORTS_EXTRA="3306" # Skip the background cluster nodes so their empty mysql_users do not sync # back to the primary and wipe the root user we inject. diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index d159a47af8..028563807c 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -18,9 +18,11 @@ "charset_find_unit-t" : [ "unit-tests-g1" ], "charset_unsigned_int-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], "clickhouse_php_conn-t" : [ "legacy-clickhouse-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], + "cluster_sync_unit-t" : [ "unit-tests-g1" ], "config_validation_unit-t" : [ "unit-tests-g1" ], "config_write_unit-t" : [ "unit-tests-g1" ], "connection_pool_unit-t" : [ "unit-tests-g1" ], + "connection_unhealthy_unit-t" : [ "unit-tests-g1" ], "deprecate_eof_cache-t" : [ "legacy-g4","mariadb10-galera-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql84-gr-g4","mysql90-g4","mysql95-g4" ], "envvars-t" : [ "legacy-g1","mariadb10-galera-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql84-gr-g1","mysql90-g1","mysql90-gr-g1","mysql93-g1","mysql93-gr-g1","mysql95-g1","mysql95-gr-g1" ], "eof_cache_mixed_flags-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], @@ -105,8 +107,8 @@ "mysql-watchdog_test-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], "mysql-zstd_compression_level-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], "mysql-zstd_compression_level_libmysql-t" : [ "mysql84-g1","mysql90-g1","mysql95-g1" ], - "mysql_encode_unit-t" : [ "unit-tests-g1" ], "mysql_decompress_payload_unit-t" : [ "unit-tests-g1" ], + "mysql_encode_unit-t" : [ "unit-tests-g1" ], "mysql_error_classifier_unit-t" : [ "unit-tests-g1" ], "mysql_hostgroup_attributes-servers_defaults-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], "mysql_hostgroup_attributes_config_file-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], @@ -116,6 +118,7 @@ "mysql_resolution_unit-t" : [ "unit-tests-g1" ], "mysql_stmt_send_long_data-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], "mysql_stmt_send_long_data_large-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], + "mysql_variables_unit-t" : [ "unit-tests-g1" ], "mysqlx_admin_commands_unit-t" : [ "unit-tests-g1","@proxysql_min_version:4.0" ], "mysqlx_admin_disk_commands_unit-t" : [ "mysqlx-tsan-g1","unit-tests-g1","@proxysql_min_version:4.0" ], "mysqlx_admin_schema_unit-t" : [ "mysqlx-tsan-g1","unit-tests-g1","@proxysql_min_version:4.0" ], @@ -431,6 +434,28 @@ "test_query_rules_fast_routing_algorithm-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], "test_query_rules_routing-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], "test_query_timeout-t" : [ "legacy-g9","mariadb10-galera-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql84-gr-g9","mysql90-g4","mysql95-g4" ], + "test_rds_bgd_automatic_discovery-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_concurrent_isolation-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_config_refresh_after_writer_completion-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_configuration_persistence-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_disable_during_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_explicit_startup-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_green_membership_ordering-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_green_pool_cleanup-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_late_entry_completed-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_late_entry_writer_phases-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_probe_tls-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_reader_policy-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_reader_switchover_cleanup-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_remove_during_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_repeated_deployment-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_rollback-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_smoke-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_topology_empty_absent-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_topology_errors-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_worker_config_refresh-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_worker_hostgroup_refresh-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_writer_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], "test_read_only_actions_offline_hard_servers-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g5","mysql84-g9","mysql90-g4","mysql90-g5","mysql95-g4","mysql95-g5" ], "test_rw_binary_data-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], "test_server_sess_status-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], diff --git a/test/tap/tap/Makefile b/test/tap/tap/Makefile index d7245f406e..ead866e075 100644 --- a/test/tap/tap/Makefile +++ b/test/tap/tap/Makefile @@ -80,8 +80,14 @@ noise_utils_mysql8.o: noise_utils.cpp noise_utils.h utils.h command_line.h cpp-d mcp_client.o: mcp_client.cpp mcp_client.h libcurl$(SHLIB_EXT) $(CXX) -fPIC -c mcp_client.cpp $(IDIRS) $(OPT) -libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a - $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo +cluster_simulator.o: cluster_simulator.cpp cluster_simulator.h + $(CXX) -fPIC -c cluster_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) + +rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h cluster_simulator.h utils.h + $(CXX) -fPIC -c rds_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) + +libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a + $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo libtap_mysql57.a: tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a $(AR) rcs libtap_mysql57.a tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo diff --git a/test/tap/tap/cluster_simulator.cpp b/test/tap/tap/cluster_simulator.cpp new file mode 100644 index 0000000000..666d897ee0 --- /dev/null +++ b/test/tap/tap/cluster_simulator.cpp @@ -0,0 +1,74 @@ +#include "cluster_simulator.h" + +#include + +#include "tap.h" + +using namespace std; + +int Cluster_Simulator::connect(char* host, int port, char* username, char* password, bool use_ssl) { + if (mysql_ != nullptr) { + mysql_close(mysql_); + mysql_ = nullptr; + } + + mysql_ = mysql_init(nullptr); + if (mysql_ == nullptr) { + diag("Failed to initialize the cluster simulator connection"); + return EXIT_FAILURE; + } + + unsigned long client_flags = 0; + if (use_ssl) { + mysql_ssl_set(mysql_, nullptr, nullptr, nullptr, nullptr, nullptr); + client_flags |= CLIENT_SSL; + } + + auto ret = mysql_real_connect(mysql_, host, username, password, nullptr, port, nullptr, client_flags); + if (ret == nullptr) { + diag("Failed to connect to cluster simulator at %s:%d: %s", host, port, mysql_error(mysql_)); + mysql_close(mysql_); + mysql_ = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int Cluster_Simulator::read_only_update(Endpoint backend, bool read_only) { + string query { + "INSERT OR REPLACE INTO READONLY_STATUS(hostname,port,read_only) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + (read_only ? "1" : "0") + ")" + }; + return execute(query); +} + +int Cluster_Simulator::execute(string query) { + if (mysql_ == nullptr) { + diag("Cluster simulator connection is not open"); + return EXIT_FAILURE; + } + + if (mysql_query(mysql_, query.c_str()) != 0) { + diag( + "Cluster simulator query failed (%u): %s; query: %s", + mysql_errno(mysql_), mysql_error(mysql_), query.c_str() + ); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +string Cluster_Simulator::sql_quote(string value) { + string quoted { "'" }; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} diff --git a/test/tap/tap/cluster_simulator.h b/test/tap/tap/cluster_simulator.h new file mode 100644 index 0000000000..f769c254f0 --- /dev/null +++ b/test/tap/tap/cluster_simulator.h @@ -0,0 +1,92 @@ +#ifndef TAP_CLUSTER_SIMULATOR_H +#define TAP_CLUSTER_SIMULATOR_H + +#include + +#include "mysql.h" + +using namespace std; + +/** + * @brief Identifies a simulated backend by address and listener port. + */ +struct Endpoint { + string host; ///< Hostname or IP address used to identify the simulated backend. + int port; ///< MySQL listener port of the simulated backend. +}; + +/** + * @brief Provides common control operations for TAP-driven cluster simulators. + * + * @details Owns the MySQL control connection to the SQLite3-server simulator and exposes + * backend state updates shared by technology-specific simulators. + */ +class Cluster_Simulator { +public: + Cluster_Simulator() : mysql_(nullptr) {} + virtual ~Cluster_Simulator() { + if (mysql_ != nullptr) { + mysql_close(mysql_); + mysql_ = nullptr; + } + } + + Cluster_Simulator(const Cluster_Simulator&) = delete; + Cluster_Simulator& operator=(const Cluster_Simulator&) = delete; + + /** + * @brief Opens the simulator control connection. + * + * @details Replaces any existing control connection, optionally enables MySQL client + * TLS, and reports connection failures through TAP diagnostics. + * + * @param host Simulator hostname or IP address. + * @param port Simulator MySQL listener port. + * @param username MySQL username used by the control connection. + * @param password MySQL password used by the control connection. + * @param use_ssl Whether the control connection must use TLS. + * + * @return EXIT_SUCCESS when the connection is established; EXIT_FAILURE otherwise. + */ + int connect(char* host, int port, char* username, char* password, bool use_ssl = false); + + /** + * @brief Sets the simulated read-only state for a backend. + * + * @details Upserts `READONLY_STATUS` using the endpoint as its key. The state is + * consumed through the hostname-suffixed monitor-query path shared with + * `TEST_READONLY`. + * + * @param backend Hostname and port identifying the backend. + * @param read_only Whether the backend must report itself as read-only. + * + * @return EXIT_SUCCESS when the state is updated; EXIT_FAILURE otherwise. + */ + int read_only_update(Endpoint backend, bool read_only); + +protected: + MYSQL* connection() { return mysql_; } + + /** + * @brief Executes a query on the simulator control connection. + * + * @param query SQL statement to execute. + * + * @return EXIT_SUCCESS when the query succeeds; EXIT_FAILURE otherwise. + */ + int execute(string query); + + /** + * @brief Quotes a string value for use in simulator control SQL. + * + * @param value String value to quote. + * + * @return Single-quoted SQL literal with embedded quotes escaped. + */ + static string sql_quote(string value); + +private: + MYSQL* mysql_; +}; + +#endif // TAP_CLUSTER_SIMULATOR_H diff --git a/test/tap/tap/rds_bgd_simulator.cpp b/test/tap/tap/rds_bgd_simulator.cpp new file mode 100644 index 0000000000..5b53f0aad4 --- /dev/null +++ b/test/tap/tap/rds_bgd_simulator.cpp @@ -0,0 +1,261 @@ +#include "rds_bgd_simulator.h" + +#include +#include +#include +#include + +#include "tap.h" + +using namespace std; + +const char* probe_kind_string(RDS_BGD_Probe_Kind kind) { + return kind == RDS_BGD_Probe_Kind::table_check ? "table_check" : "metadata"; +} + +rc_t parse_probe_kind(string value) { + if (value == "table_check") { + return { EXIT_SUCCESS, RDS_BGD_Probe_Kind::table_check }; + } + if (value == "metadata") { + return { EXIT_SUCCESS, RDS_BGD_Probe_Kind::metadata }; + } + return { EXIT_FAILURE, RDS_BGD_Probe_Kind::table_check }; +} + +Endpoint RDS_BGD_Host::endpoint() { + return { ip, port }; +} + +Endpoint RDS_BGD_Host::host_endpoint() { + return { hostname, port }; +} + +vector RDS_BGD_Cluster::get_writers() { + return { blue_writer.endpoint(), green_writer.endpoint() }; +} + +vector RDS_BGD_Cluster::get_blue_endpoints() { + vector endpoints { blue_writer.endpoint() }; + for (RDS_BGD_Host& host : blue_readers) endpoints.push_back(host.endpoint()); + return endpoints; +} + +vector RDS_BGD_Cluster::get_green_endpoints() { + vector endpoints { green_writer.endpoint() }; + for (RDS_BGD_Host& host : green_readers) endpoints.push_back(host.endpoint()); + return endpoints; +} + +vector RDS_BGD_Cluster::get_endpoints() { + vector endpoints = get_blue_endpoints(); + vector green_endpoints = get_green_endpoints(); + endpoints.insert(endpoints.end(), green_endpoints.begin(), green_endpoints.end()); + return endpoints; +} + +vector RDS_BGD_Cluster::get_topology(string status) { + return { + { blue_writer.hostname, blue_writer.hostname, blue_writer.port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", status }, + { green_writer.hostname, green_writer.hostname, green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", status }, + }; +} + +int RDS_BGD_Simulator::topology_update(vector backends, vector rows) { + if (backends.empty()) { + return EXIT_FAILURE; + } + + vector statements {}; + for (Endpoint& backend : backends) { + string predicate { backend_predicate(backend) }; + statements.push_back("DELETE FROM RDS_BGD_TOPOLOGY WHERE " + predicate); + statements.push_back( + "INSERT OR REPLACE INTO RDS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); + + for (size_t row_order = 0; row_order < rows.size(); ++row_order) { + RDS_BGD_Topology_Row& row = rows[row_order]; + statements.push_back( + "INSERT INTO RDS_BGD_TOPOLOGY" + "(backend_ip,backend_port,row_order,id,endpoint,topology_port,role,status) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + to_string(row_order) + "," + sql_quote(row.id) + "," + + sql_quote(row.endpoint) + "," + to_string(row.port) + "," + + sql_quote(row.role) + "," + sql_quote(row.status) + ")"); + } + } + + return execute_transaction(statements); +} + +int RDS_BGD_Simulator::topology_delete(vector backends) { + if (backends.empty()) { + return EXIT_FAILURE; + } + + vector statements {}; + for (Endpoint& backend : backends) { + statements.push_back( + "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); + statements.push_back( + "INSERT OR REPLACE INTO RDS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); + } + return execute_transaction(statements); +} + +int RDS_BGD_Simulator::topology_drop(vector backends) { + return topology_error(backends, 1146, "Table 'mysql.rds_topology' doesn't exist"); +} + +int RDS_BGD_Simulator::topology_error(vector backends, int error_code, string error_msg) { + if (backends.empty() || error_code == 0) { + return EXIT_FAILURE; + } + + bool topology_present = error_code != 1146; + vector statements {}; + for (Endpoint& backend : backends) { + if (!topology_present) { + statements.push_back( + "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); + } + statements.push_back( + "INSERT OR REPLACE INTO RDS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + (topology_present ? "1" : "0") + "," + to_string(error_code) + "," + + sql_quote(error_msg) + ")"); + } + return execute_transaction(statements); +} + +int RDS_BGD_Simulator::cleanup() { + vector statements { + "DELETE FROM READONLY_STATUS", + "DELETE FROM RDS_BGD_TOPOLOGY", + "DELETE FROM RDS_BGD_CONTROL", + "DELETE FROM RDS_BGD_PROBE_LOG", + }; + return execute_transaction(statements); +} + +rc_t RDS_BGD_Simulator::probe_log_last_sequence() { + if (connection() == nullptr) { + return { EXIT_FAILURE, 0 }; + } + + auto [rc, rows] = mysql_query_ext_rows( + connection(), "SELECT COALESCE(MAX(sequence_id),0) FROM RDS_BGD_PROBE_LOG"); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return { EXIT_FAILURE, 0 }; + } + + return { + EXIT_SUCCESS, + static_cast(strtoull(rows.front().front().c_str(), nullptr, 10)) + }; +} + +rc_t> RDS_BGD_Simulator::probe_log_since( + uint64_t sequence_id) +{ + if (connection() == nullptr) { + return { EXIT_FAILURE, {} }; + } + + string query { + "SELECT sequence_id,backend_ip,backend_port,probe_kind,encrypted " + "FROM RDS_BGD_PROBE_LOG WHERE sequence_id>" + to_string(sequence_id) + + " ORDER BY sequence_id" + }; + auto [rc, rows] = mysql_query_ext_rows(connection(), query); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + + vector logs {}; + for (mysql_res_row& row : rows) { + if (row.size() != 5) { + return { EXIT_FAILURE, {} }; + } + auto [kind_rc, kind] = parse_probe_kind(row[3]); + if (kind_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + logs.push_back({ + static_cast(strtoull(row[0].c_str(), nullptr, 10)), + { row[1], atoi(row[2].c_str()) }, + kind, + atoi(row[4].c_str()) != 0, + }); + } + + return { EXIT_SUCCESS, move(logs) }; +} + +rc_t RDS_BGD_Simulator::wait_for_probe_log( + uint64_t sequence_id, + Endpoint backend, + RDS_BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted) +{ + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = probe_log_since(sequence_id); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + for (RDS_BGD_Probe_Log& log : logs) { + if (log.backend.host == backend.host && log.backend.port == backend.port && + log.probe_kind == probe_kind && + (encrypted < 0 || log.encrypted == (encrypted != 0))) { + return { EXIT_SUCCESS, log }; + } + } + usleep(50000); + } while (monotonic_time() < deadline); + + auto [rc, logs] = probe_log_since(sequence_id); + if (rc == EXIT_SUCCESS) { + for (RDS_BGD_Probe_Log& log : logs) { + diag( + "Observed BGD probe sequence=%llu backend=%s:%d kind=%s encrypted=%d", + static_cast(log.sequence_id), + log.backend.host.c_str(), log.backend.port, + probe_kind_string(log.probe_kind), log.encrypted ? 1 : 0); + } + } + diag( + "Timed out waiting for BGD probe backend=%s:%d kind=%s encrypted=%d", + backend.host.c_str(), backend.port, probe_kind_string(probe_kind), encrypted); + return { ETIMEDOUT, {} }; +} + +int RDS_BGD_Simulator::execute_transaction(vector& statements) { + if (execute("START TRANSACTION") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + for (string& statement : statements) { + if (execute(statement) != EXIT_SUCCESS) { + (void)execute("ROLLBACK"); + return EXIT_FAILURE; + } + } + if (execute("COMMIT") != EXIT_SUCCESS) { + (void)execute("ROLLBACK"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +string RDS_BGD_Simulator::backend_predicate(Endpoint backend) { + return "backend_ip=" + sql_quote(backend.host) + + " AND backend_port=" + to_string(backend.port); +} diff --git a/test/tap/tap/rds_bgd_simulator.h b/test/tap/tap/rds_bgd_simulator.h new file mode 100644 index 0000000000..2827d39435 --- /dev/null +++ b/test/tap/tap/rds_bgd_simulator.h @@ -0,0 +1,258 @@ +#ifndef TAP_RDS_BGD_SIMULATOR_H +#define TAP_RDS_BGD_SIMULATOR_H + +#include +#include +#include + +#include "cluster_simulator.h" +#include "utils.h" + +using namespace std; + +/** + * @brief Represents one row returned by the simulated `mysql.rds_topology` table. + */ +struct RDS_BGD_Topology_Row { + string id; ///< RDS topology node identifier. + string endpoint; ///< RDS hostname exposed by the topology row. + int port; ///< MySQL port exposed by the topology row. + string role; ///< Blue/green deployment role reported by RDS. + string status; ///< Blue/green deployment status reported by RDS. +}; + +/** + * @brief Describes one RDS BGD host and its fixed simulator address. + */ +struct RDS_BGD_Host { + string hostname; ///< AWS-style RDS hostname configured in ProxySQL. + string ip; ///< Fixed loopback address used by the simulator. + int port; ///< MySQL listener port shared by the hostname and IP. + + /** + * @brief Returns the IP/port endpoint used for topology simulation. + * + * @return Simulator endpoint containing this host's IP address and port. + */ + Endpoint endpoint(); + + /** + * @brief Returns the hostname/port endpoint used for read-only simulation. + * + * @return Simulator endpoint containing this host's RDS hostname and port. + */ + Endpoint host_endpoint(); +}; + +/** + * @brief Holds the blue and green hosts participating in one simulated RDS BGD cluster. + * + * @details TAP tests populate the cluster with the deployment topology required by each + * scenario. Helper methods derive writer endpoints and AWS topology rows from + * the configured hosts. + */ +class RDS_BGD_Cluster { +public: + RDS_BGD_Host blue_writer; ///< Source writer configured in ProxySQL. + RDS_BGD_Host green_writer; ///< Target writer discovered from the topology. + vector blue_readers; ///< Source readers configured in ProxySQL. + vector green_readers; ///< Target readers discovered from the topology. + + /** + * @brief Returns both writer IP/port endpoints for topology simulation. + * + * @return Blue and green writer endpoints keyed by simulator IP address. + */ + vector get_writers(); + + /** + * @brief Returns the blue writer and configured blue readers. + * + * @return Blue deployment endpoints keyed by simulator IP address. + */ + vector get_blue_endpoints(); + + /** + * @brief Returns the green writer and configured green readers. + * + * @return Green deployment endpoints keyed by simulator IP address. + */ + vector get_green_endpoints(); + + /** + * @brief Returns every simulator IP/port endpoint in this cluster. + * + * @details Includes both writers and all configured blue and green readers. + * Tests use this list when resetting or publishing topology for a complete + * simulated deployment. + * + * @return Writer and reader endpoints keyed by simulator IP address. + */ + vector get_endpoints(); + + /** + * @brief Builds the topology rows published by the simulated writers. + * + * @details Creates one source row for the blue writer and one target row for the green + * writer. The supplied deployment status is applied to both rows. + * + * @param status RDS blue/green deployment status to publish. + * + * @return Source and target rows for the simulated topology table. + */ + vector get_topology(string status); +}; + +/** + * @brief Identifies the RDS BGD monitor query recorded in the simulator probe log. + */ +enum class RDS_BGD_Probe_Kind { + table_check, ///< Query checking whether `mysql.rds_topology` exists. + metadata, ///< Query fetching rows from `mysql.rds_topology`. +}; + +/** + * @brief Describes one RDS BGD monitor query observed by the simulator. + */ +struct RDS_BGD_Probe_Log { + uint64_t sequence_id; ///< Monotonically increasing probe-log sequence. + Endpoint backend; ///< Accepted backend IP address and port. + RDS_BGD_Probe_Kind probe_kind; ///< Type of topology query observed. + bool encrypted; ///< Whether the monitor connection used TLS. +}; + +/** + * @brief Controls RDS BGD topology responses and inspects monitor probes from TAP tests. + * + * @details Publishes per-backend topology rows, empty results, missing tables, or MySQL + * errors. It also reads the ordered probe log generated by the SQLite3-server + * simulator and reuses `Cluster_Simulator` for shared control operations. + */ +class RDS_BGD_Simulator : public Cluster_Simulator { +public: + /** + * @brief Replaces the simulated topology returned by each backend. + * + * @details Deletes existing topology rows before inserting the supplied rows. The + * topology table is marked present, configured errors are cleared, and the + * complete update is applied atomically across all supplied backends. + * + * @param backends Backend IP/port endpoints that must return the topology. + * @param rows Topology rows to publish on each backend. + * + * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. + */ + int topology_update(vector backends, vector rows); + + /** + * @brief Configures each backend to return an empty topology result. + * + * @details Deletes all topology rows associated with the supplied backends while keeping + * the topology table present and clearing any configured metadata error. + * + * @param backends Backend IP/port endpoints that must return an empty result. + * + * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. + */ + int topology_delete(vector backends); + + /** + * @brief Configures each backend to report that the topology table does not exist. + * + * @param backends Backend IP/port endpoints that must return MySQL error 1146. + * + * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. + */ + int topology_drop(vector backends); + + /** + * @brief Configures a MySQL error for topology queries on each backend. + * + * @details Stores the nonzero error code and message returned by subsequent metadata + * probes. Error 1146 marks the topology table absent and removes its existing + * rows; other error codes leave the table marked present. + * + * @param backends Backend IP/port endpoints that must return the error. + * @param error_code Nonzero MySQL error code to return. + * @param error_msg MySQL error message to return. + * + * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. + */ + int topology_error(vector backends, int error_code, string error_msg); + + /** + * @brief Removes all read-only, topology-control, topology-row, and probe state. + * + * @return EXIT_SUCCESS when the simulator state is empty; EXIT_FAILURE otherwise. + */ + int cleanup(); + + /** + * @brief Reads the latest sequence from the RDS BGD probe log. + * + * @return EXIT_SUCCESS and the latest sequence, or zero when the log is empty; + * EXIT_FAILURE and zero when the query fails. + */ + rc_t probe_log_last_sequence(); + + /** + * @brief Returns probe-log records newer than a sequence. + * + * @details Selects records with `sequence_id` strictly greater than the supplied value + * and preserves database sequence order in the returned vector. + * + * @param sequence_id Last probe-log sequence already observed by the TAP test. + * + * @return EXIT_SUCCESS and the matching records; EXIT_FAILURE and an empty vector + * when the query or record parsing fails. + */ + rc_t> probe_log_since(uint64_t sequence_id); + + /** + * @brief Waits for a matching RDS BGD probe-log record. + * + * @details Matches records newer than `sequence_id` by backend and probe kind. TLS state + * is matched when `encrypted` is zero or one; `-1` accepts either state. Observed + * probes are emitted through TAP diagnostics when the wait expires. + * + * @param sequence_id Last probe-log sequence observed before the expected probe. + * @param backend Backend IP/port endpoint expected to receive the probe. + * @param probe_kind Type of topology query expected. + * @param timeout_ms Maximum time to wait in milliseconds. + * @param encrypted Expected TLS state, or -1 to accept either state. + * + * @return EXIT_SUCCESS and the matching record; ETIMEDOUT and an empty record when + * the deadline expires; EXIT_FAILURE and an empty record when log retrieval fails. + */ + rc_t wait_for_probe_log( + uint64_t sequence_id, + Endpoint backend, + RDS_BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted = -1 + ); + +private: + /** + * @brief Builds the SQL predicate identifying one simulated backend. + * + * @param backend Backend IP/port endpoint to match. + * + * @return SQL predicate matching the backend control-table key. + */ + static string backend_predicate(Endpoint backend); + + /** + * @brief Executes simulator control statements in one transaction. + * + * @details Executes the supplied statements in order and commits only after every + * statement succeeds. A statement or commit failure triggers a rollback. + * + * @param statements SQL statements to execute atomically. + * + * @return EXIT_SUCCESS when the transaction commits; EXIT_FAILURE otherwise. + */ + int execute_transaction(vector& statements); +}; + +#endif // TAP_RDS_BGD_SIMULATOR_H diff --git a/test/tap/tap/rds_bgd_tap.h b/test/tap/tap/rds_bgd_tap.h new file mode 100644 index 0000000000..619450c7af --- /dev/null +++ b/test/tap/tap/rds_bgd_tap.h @@ -0,0 +1,558 @@ +#ifndef TAP_TESTS_RDS_BGD_TAP_H +#define TAP_TESTS_RDS_BGD_TAP_H + +#include +#include +#include +#include +#include + +#include "rds_bgd_simulator.h" +#include "tap.h" + +using namespace std; + +inline int execute_all(MYSQL* admin, vector queries); + +inline RDS_BGD_Cluster bgd_cluster_init() { + RDS_BGD_Cluster cluster { + { "db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.11", 3306 }, + { "db-1-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.14", 3306 }, + { + { "db-1-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.12", 3306 }, + { "db-1-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.13", 3306 }, + }, + { + { "db-1-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.15", 3306 }, + { "db-1-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.16", 3306 }, + }, + }; + return cluster; +} + +inline RDS_BGD_Cluster bgd_cluster_1_deployment_b_init() { + RDS_BGD_Cluster cluster { + { "db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.11", 3306 }, + { "db-1-green-s7m2kx.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.17", 3306 }, + { + { "db-1-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.12", 3306 }, + { "db-1-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.13", 3306 }, + }, + { + { "db-1-reader-1-green-v4n8qp.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.18", 3306 }, + { "db-1-reader-2-green-w6h3rz.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.19", 3306 }, + }, + }; + return cluster; +} + +inline RDS_BGD_Cluster bgd_cluster_2_init() { + RDS_BGD_Cluster cluster { + { "db-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.20", 3306 }, + { "db-2-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.23", 3306 }, + { + { "db-2-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.21", 3306 }, + { "db-2-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.22", 3306 }, + }, + { + { "db-2-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.24", 3306 }, + { "db-2-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.25", 3306 }, + }, + }; + return cluster; +} + +inline RDS_BGD_Cluster bgd_cluster_3_init() { + RDS_BGD_Cluster cluster { + { "db-3.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.26", 3306 }, + { "db-3-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.29", 3306 }, + { + { "db-3-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.27", 3306 }, + { "db-3-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.28", 3306 }, + }, + { + { "db-3-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.30", 3306 }, + { "db-3-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.31", 3306 }, + }, + }; + return cluster; +} + +enum class BGD_Admin_Mode { + automatic, + explicit_configuration, +}; + +struct BGD_Hostgroups { + int blue_writer; + int blue_reader; + int green_writer; + int green_reader; +}; + +inline vector bgd_topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + for (RDS_BGD_Host& host : cluster.blue_readers) { + rows.push_back({ host.hostname, host.hostname, host.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", status }); + } + for (RDS_BGD_Host& host : cluster.green_readers) { + rows.push_back({ host.hostname, host.hostname, host.port, "BLUE_GREEN_DEPLOYMENT_TARGET", status }); + } + return rows; +} + +inline int bgd_set_writer_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (sim.read_only_update(cluster.blue_writer.host_endpoint(), false) != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue writer"); + return EXIT_FAILURE; + } + + if (sim.read_only_update(cluster.green_writer.host_endpoint(), false) != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated green writer"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +inline int bgd_set_host_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Host& host) { + int rc = sim.read_only_update(host.host_endpoint(), false); + return rc; +} + +inline int bgd_set_host_read_only_1(RDS_BGD_Simulator& sim, RDS_BGD_Host& host) { + int rc = sim.read_only_update(host.host_endpoint(), true); + return rc; +} + +inline string bgd_sql_quote(string value) { + string quoted { "'" }; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} + +inline int bgd_admin_cleanup(MYSQL* admin) { + vector config_queries { + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "DELETE FROM mysql_aws_rds_bgd_hostgroups", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int config_rc = execute_all(admin, config_queries); + + vector state_queries { + "DELETE FROM mysql_servers", + "DELETE FROM mysql_replication_hostgroups", + "UPDATE mysql_users SET default_hostgroup=0 WHERE username='testuser'", + "LOAD MYSQL SERVERS TO RUNTIME", + "LOAD MYSQL USERS TO RUNTIME", + }; + int state_rc = execute_all(admin, state_queries); + + if (config_rc != EXIT_SUCCESS || state_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +inline int bgd_admin_add_servers( + MYSQL* admin, RDS_BGD_Cluster cluster, BGD_Hostgroups hostgroups, + vector hosts, bool green, int use_ssl) +{ + vector queries {}; + for (RDS_BGD_Host& host : hosts) { + int hostgroup = hostgroups.blue_reader; + if (!green && host.hostname == cluster.blue_writer.hostname) { + hostgroup = hostgroups.blue_writer; + } else if (green && host.hostname == cluster.green_writer.hostname) { + hostgroup = hostgroups.green_writer; + } else if (green) { + hostgroup = hostgroups.green_reader; + } + + string color = green ? "green " : "blue "; + string comment = bgd_sql_quote("BGD TAP " + color + host.ip); + string query = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hostgroup) + "," + bgd_sql_quote(host.hostname) + "," + + to_string(host.port) + ",'ONLINE'," + to_string(use_ssl) + "," + + comment + ")"; + queries.push_back(query); + } + + int rc = execute_all(admin, queries); + return rc; +} + +inline int bgd_admin_setup( + MYSQL* admin, RDS_BGD_Cluster cluster, BGD_Hostgroups hostgroups, + BGD_Admin_Mode mode, vector blue_hosts, + vector green_hosts = {}, int blue_use_ssl = 0, int green_use_ssl = 0) +{ + string auto_discovery = mode == BGD_Admin_Mode::automatic ? "true" : "false"; + vector queries { + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hostgroups.blue_writer) + "," + to_string(hostgroups.blue_reader) + ")", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "SET mysql-monitor_read_only_interval=100", + "SET mysql-monitor_aws_rds_topology_discovery_interval=1", + "SET mysql-aws_blue_green_deployment_auto_discovery='" + auto_discovery + "'", + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroups.blue_writer) + + " WHERE username='testuser'", + }; + + if (mode == BGD_Admin_Mode::explicit_configuration) { + string bgd_query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hostgroups.blue_writer) + "," + to_string(hostgroups.blue_reader) + "," + + to_string(hostgroups.green_writer) + "," + to_string(hostgroups.green_reader) + + ",1,0,100,800,'BGD TAP explicit configuration')"; + queries.push_back(bgd_query); + } + + int config_rc = execute_all(admin, queries); + if (config_rc != EXIT_SUCCESS) { + diag("Error: failed to configure ProxySQL BGD variables and hostgroups"); + return EXIT_FAILURE; + } + + int blue_rc = bgd_admin_add_servers(admin, cluster, hostgroups, blue_hosts, false, blue_use_ssl); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to configure blue servers"); + return EXIT_FAILURE; + } + + int green_rc = bgd_admin_add_servers(admin, cluster, hostgroups, green_hosts, true, green_use_ssl); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to configure green servers"); + return EXIT_FAILURE; + } + + vector load_queries { + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL USERS TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load ProxySQL BGD configuration to runtime"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +inline rc_t> bgd_runtime_rows(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "auto_generated,status FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup); + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +inline rc_t> bgd_runtime_servers(MYSQL* admin, vector hostgroups) { + string predicate {}; + for (size_t i = 0; i < hostgroups.size(); ++i) { + if (i != 0) { + predicate += ","; + } + predicate += to_string(hostgroups[i]); + } + + string query = + "SELECT hostgroup_id,hostname,port,status,use_ssl FROM runtime_mysql_servers WHERE hostgroup_id IN (" + + predicate + ") ORDER BY hostgroup_id,hostname,port"; + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +inline rc_t bgd_connection_pool_count(MYSQL* admin, int hostgroup, string hostname = "") { + string query = + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool WHERE hostgroup=" + + to_string(hostgroup); + if (!hostname.empty()) { + query += " AND srv_host=" + bgd_sql_quote(hostname); + } + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + rc_t result { EXIT_FAILURE, 0 }; + return result; + } + + int64_t count = strtoll(rows[0][0].c_str(), nullptr, 10); + rc_t result { EXIT_SUCCESS, count }; + return result; +} + +inline rc_t bgd_backend_ip_echo(MYSQL* proxy) { + string query = "SELECT @@version_comment LIMIT 1"; + + auto [rc, rows] = mysql_query_ext_rows(proxy, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result { EXIT_SUCCESS, rows[0][0] }; + return result; +} + +inline rc_t bgd_probe_count_since( + RDS_BGD_Simulator& sim, uint64_t sequence, Endpoint backend, RDS_BGD_Probe_Kind kind) +{ + auto [rc, logs] = sim.probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + rc_t result { EXIT_FAILURE, 0 }; + return result; + } + + uint64_t count = 0; + for (const RDS_BGD_Probe_Log& log : logs) { + bool backend_matches = + log.backend.host == backend.host && + log.backend.port == backend.port; + bool kind_matches = log.probe_kind == kind; + if (backend_matches && kind_matches) { + ++count; + } + } + + rc_t result { EXIT_SUCCESS, count }; + return result; +} + +inline void bgd_diag_runtime_state(MYSQL* admin) { + const string runtime_query = + "SELECT writer_hostgroup,reader_hostgroup,IFNULL(green_writer_hostgroup,'NULL')," + "IFNULL(green_reader_hostgroup,'NULL'),auto_generated,status " + "FROM runtime_mysql_aws_rds_bgd_hostgroups ORDER BY writer_hostgroup"; + auto [runtime_rc, runtime_rows] = mysql_query_ext_rows(admin, runtime_query); + if (runtime_rc != EXIT_SUCCESS) { + diag("RDS BGD diagnostic query failed with error %d", runtime_rc); + } else if (runtime_rows.empty()) { + diag("RDS BGD runtime hostgroup table is empty"); + } else { + diag("RDS BGD runtime hostgroup rows:"); + for (const mysql_res_row& row : runtime_rows) { + string row_text {}; + for (size_t i = 0; i < row.size(); ++i) { + if (i != 0) { + row_text += ","; + } + row_text += row[i]; + } + diag(" %s", row_text.c_str()); + } + } + + const string monitor_query = + "SELECT hostname,port,IFNULL(read_only,'NULL'),IFNULL(error,'') FROM mysql_server_read_only_log " + "ORDER BY time_start_us DESC LIMIT 10"; + auto [monitor_rc, monitor_rows] = mysql_query_ext_rows(admin, monitor_query); + if (monitor_rc != EXIT_SUCCESS) { + diag("RDS read_only diagnostic query failed with error %d", monitor_rc); + } else if (monitor_rows.empty()) { + diag("RDS read_only log has no rows"); + } else { + diag("Latest RDS read_only monitor rows:"); + for (const mysql_res_row& row : monitor_rows) { + string row_text {}; + for (size_t i = 0; i < row.size(); ++i) { + if (i != 0) { + row_text += ","; + } + row_text += row[i]; + } + diag(" %s", row_text.c_str()); + } + } +} + +inline int bgd_wait_for_condition(MYSQL* admin, string query, uint32_t timeout_seconds) { + int rc = wait_for_cond(admin, query, timeout_seconds); + if (rc != EXIT_SUCCESS) { + diag("RDS BGD wait timed out or failed for condition: %s", query.c_str()); + bgd_diag_runtime_state(admin); + } + return rc; +} + +inline int bgd_wait_for_status(MYSQL* admin, BGD_Hostgroups& hostgroups, string status, uint32_t timeout_seconds) { + string query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hostgroups.blue_writer) + " AND status=" + bgd_sql_quote(status); + + int rc = bgd_wait_for_condition(admin, query, timeout_seconds); + return rc; +} + +inline int bgd_wait_for_server_placement( + MYSQL* admin, int writer_hostgroup, int reader_hostgroup, RDS_BGD_Host& host, + bool in_reader_hostgroup, uint32_t timeout_seconds) +{ + string writer_count = in_reader_hostgroup ? "0" : "1"; + string reader_count = in_reader_hostgroup ? "1" : "0"; + + string query = "SELECT " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(writer_hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + ")=" + + writer_count + " AND " + + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(reader_hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + ")=" + + reader_count; + + int rc = bgd_wait_for_condition(admin, query, timeout_seconds); + return rc; +} + +inline rc_t bgd_wait_for_probe_from_backends( + RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, + RDS_BGD_Probe_Kind kind, uint32_t timeout_ms, int encrypted = -1) +{ + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = sim.probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + for (const RDS_BGD_Probe_Log& log : logs) { + for (const Endpoint& backend : backends) { + bool backend_matches = + log.backend.host == backend.host && + log.backend.port == backend.port; + bool kind_matches = log.probe_kind == kind; + bool encryption_matches = + encrypted < 0 || + log.encrypted == (encrypted != 0); + if (backend_matches && kind_matches && encryption_matches) { + rc_t result { EXIT_SUCCESS, log }; + return result; + } + } + } + + usleep(50000); + } while (monotonic_time() < deadline); + + rc_t result { ETIMEDOUT, {} }; + return result; +} + +/** + * Verify that a configuration change does not restart BGD discovery. + * + * The expected result is ETIMEDOUT because no table-check probe should appear + * after the given sequence. + */ +inline int bgd_expect_no_table_check( + RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) +{ + auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( + sim, sequence, backends, RDS_BGD_Probe_Kind::table_check, timeout_ms + ); + + if (probe_rc == ETIMEDOUT) { + return EXIT_SUCCESS; + } + return EXIT_FAILURE; +} + +/** + * Verify that one endpoint does not receive metadata probes. + * + * The expected result is ETIMEDOUT because no metadata probe should reach the + * endpoint after the given sequence. + */ +inline int bgd_expect_no_metadata_probe( + RDS_BGD_Simulator& sim, uint64_t sequence, Endpoint backend, uint32_t timeout_ms) +{ + vector backends { backend }; + + auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( + sim, sequence, backends, RDS_BGD_Probe_Kind::metadata, timeout_ms + ); + + if (probe_rc == ETIMEDOUT) { + return EXIT_SUCCESS; + } + return EXIT_FAILURE; +} + +/** + * Verify that none of the supplied endpoints receives a metadata probe. + * + * The expected result is ETIMEDOUT because no metadata probe should reach any + * endpoint after the given sequence. + */ +inline int bgd_expect_no_metadata_probe_from_backends( + RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) +{ + auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( + sim, sequence, backends, RDS_BGD_Probe_Kind::metadata, timeout_ms + ); + + if (probe_rc == ETIMEDOUT) { + return EXIT_SUCCESS; + } + return EXIT_FAILURE; +} + +/** + * Verify that read_only monitoring remains suppressed for the full observation window. + * + * The helper fails immediately if a new read_only log row appears after the + * supplied baseline. + */ +inline int bgd_expect_no_read_only_log(MYSQL* admin, RDS_BGD_Host& host, int64_t baseline, uint32_t timeout_ms) { + if (baseline < 0) { + return EXIT_FAILURE; + } + + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + string query = + "SELECT COUNT(*) FROM mysql_server_read_only_log WHERE hostname=" + + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND time_start_us>" + to_string(baseline); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return EXIT_FAILURE; + } + + if (rows[0][0] != "0") { + return EXIT_FAILURE; + } + + usleep(50000); + } while (monotonic_time() < deadline); + + return EXIT_SUCCESS; +} + +inline int execute_all(MYSQL* admin, vector queries) { + for (string& query : queries) { + if (mysql_query(admin, query.c_str()) != 0) { + diag("Error: Admin query failed (%u): %s; query: %s", + mysql_errno(admin), mysql_error(admin), query.c_str()); + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + +#endif // TAP_TESTS_RDS_BGD_TAP_H diff --git a/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp new file mode 100644 index 0000000000..f7ac46e125 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp @@ -0,0 +1,328 @@ +/** + * @file test_rds_bgd_automatic_discovery-t.cpp + * @brief Automatic BGD row creation from AVAILABLE topology. + * + * Steps: + * + * 1. Publish AVAILABLE topology before loading blue hostgroups 810 and 811. + * 2. Verify one runtime-only BGD row with derived blue hostgroups and NULL + * green hostgroups. + * 3. Load blue hostgroups 820 and 821 while topology is absent. + * 4. Verify no BGD row exists until AVAILABLE topology is published. + * 5. Verify repeated discovery keeps one runtime-only BGD row. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +// Automatic discovery is asynchronous and starts after the monitor observes the +// runtime server. Allow the monitor and the BGD worker to become ready on slower CI runners. +const uint32_t kTimeoutSeconds = 15; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster topology_first { bgd_cluster_init() }; + RDS_BGD_Cluster absent_first { bgd_cluster_2_init() }; + BGD_Hostgroups topology_first_hg { 810, 811, 812, 813 }; + BGD_Hostgroups absent_first_hg { 820, 821, 822, 823 }; + vector topology_first_endpoints { topology_first.get_endpoints() }; + vector absent_first_endpoints { absent_first.get_endpoints() }; + uint64_t absent_available_sequence { 0 }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +bool runtime_auto_row_matches(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hg.blue_writer) + " AND reader_hostgroup=" + to_string(hg.blue_reader) + + " AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL AND auto_generated=1"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +bool runtime_bgd_row_absent(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool absent = rows[0][0] == "0"; + return absent; +} + +bool persistent_bgd_row_absent(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT COUNT(*) FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool absent = rows[0][0] == "0"; + return absent; +} + +bool runtime_bgd_row_count_matches(MYSQL* admin, int writer_hostgroup, int expected_count) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == to_string(expected_count); + return matches; +} + +/** + * Discover a deployment whose AVAILABLE topology exists before its blue writer. + * + * - Set read_only=0 for both simulated writers. + * - Publish AVAILABLE topology before loading blue hostgroups 810 and 811. + * - Verify one auto-generated runtime row with NULL green hostgroups. + * - Verify automatic discovery does not create a persistent BGD row. + */ +int test_topology_before_writer(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.topology_first; + BGD_Hostgroups& hg = state.topology_first_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure topology-first simulated writers"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_first_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish topology-first AVAILABLE topology"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer }; + vector green_servers {}; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::automatic, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic discovery for blue hostgroups 810 and 811"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 810 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + bool row_matches = runtime_auto_row_matches(admin, hg); + ok(row_matches, "automatic discovery derives hostgroups 810 and 811 with NULL green hostgroups"); + + bool persistent_absent = persistent_bgd_row_absent(admin, hg.blue_writer); + ok(persistent_absent, "automatic discovery keeps wHG 810 out of mysql_aws_rds_bgd_hostgroups"); + return EXIT_SUCCESS; +} + +/** + * Start automatic discovery while topology is absent. + * + * - Set read_only=0 for both simulated writers. + * - Remove topology for blue hostgroups 820 and 821. + * - Load the blue writer and wait for the absent-table metadata probe. + * - Verify no runtime or persistent BGD row is created. + * - Publish AVAILABLE topology and verify automatic row creation. + */ +int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.absent_first; + BGD_Hostgroups& hg = state.absent_first_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure topology-absent simulated writers"); + return EXIT_FAILURE; + } + + int drop_rc = sim.topology_drop(state.absent_first_endpoints); + if (drop_rc != EXIT_SUCCESS) { + diag("Error: failed to publish absent topology for blue hostgroups 820 and 821"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before topology-absent discovery"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers {}; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::automatic, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic discovery for blue hostgroups 820 and 821"); + return EXIT_FAILURE; + } + + auto [absent_probe_rc, absent_probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (absent_probe_rc != EXIT_SUCCESS) { + diag("Error: automatic discovery did not issue the absent-table metadata probe"); + return EXIT_FAILURE; + } + + bool runtime_absent = runtime_bgd_row_absent(admin, hg.blue_writer); + bool persistent_absent = persistent_bgd_row_absent(admin, hg.blue_writer); + ok(runtime_absent && persistent_absent, "absent topology creates no runtime or persistent BGD row for wHG 820"); + + auto [available_seq_rc, available_seq] = sim.probe_log_last_sequence(); + if (available_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before AVAILABLE topology"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.absent_first_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 820"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 820 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + auto [green_probe_rc, green_probe] = + sim.wait_for_probe_log(available_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (green_probe_rc != EXIT_SUCCESS) { + diag("Error: wHG 820 did not probe the AVAILABLE green writer"); + return EXIT_FAILURE; + } + + state.absent_available_sequence = green_probe.sequence_id; + bool row_matches = runtime_auto_row_matches(admin, hg); + ok(row_matches, "AVAILABLE topology creates the derived automatic BGD row for wHG 820"); + return EXIT_SUCCESS; +} + +/** + * Observe steady metadata polling after automatic discovery reaches AVAILABLE. + * + * - Wait for another green-writer metadata probe. + * - Verify runtime contains one auto-generated BGD row. + * - Verify the automatic row remains absent from persistent configuration. + */ +int test_repeated_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.absent_first; + BGD_Hostgroups& hg = state.absent_first_hg; + + auto [probe_rc, probe] = sim.wait_for_probe_log( + state.absent_available_sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: automatic wHG 820 did not continue green-writer metadata polling"); + return EXIT_FAILURE; + } + + bool one_runtime_row = runtime_bgd_row_count_matches(admin, hg.blue_writer, 1); + bool persistent_absent = persistent_bgd_row_absent(admin, hg.blue_writer); + ok(one_runtime_row && persistent_absent, "steady metadata polling keeps one runtime-only BGD row for wHG 820"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish AVAILABLE topology before configuring blue hostgroups 810 and 811. + // ProxySQL: enable automatic discovery and load only the blue writer. + // Verify: one runtime-only auto-generated row uses derived blue hostgroups and NULL green hostgroups. + if (test_topology_before_writer(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish absent topology, then AVAILABLE topology for the second deployment. + // ProxySQL: load blue hostgroups 820 and 821 while topology is absent. + // Verify: no row exists while absent; AVAILABLE creates one auto-generated runtime row. + if (test_topology_absent_then_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: allow another table-check for the AVAILABLE deployment. + // Verify: repeated discovery keeps one runtime-only BGD row for wHG 820. + if (test_repeated_discovery(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp b/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp new file mode 100644 index 0000000000..14329cd04f --- /dev/null +++ b/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp @@ -0,0 +1,495 @@ +/** + * @file test_rds_bgd_concurrent_isolation-t.cpp + * @brief Isolating three concurrent BGD workers in hostgroups 1410-1433. + * + * Steps: + * + * 1. Configure three BGD rows with separate hostgroups, topology, and green + * metadata targets. + * 2. Move each worker to a different writer-switchover phase and verify that + * the other two workers keep their status and blue-writer placement. + * 3. Replace only cluster 1 green membership with a TLS-enabled deployment. + * 4. Verify that cluster 1 uses the new target while clusters 2 and 3 keep + * their own phases, placement, metadata targets, and TLS values. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 500; + +struct TestState { + RDS_BGD_Cluster cluster_1 { bgd_cluster_init() }; + RDS_BGD_Cluster cluster_1_b { bgd_cluster_1_deployment_b_init() }; + RDS_BGD_Cluster cluster_2 { bgd_cluster_2_init() }; + RDS_BGD_Cluster cluster_3 { bgd_cluster_3_init() }; + BGD_Hostgroups cluster_1_hg { 1410, 1411, 1412, 1413 }; + BGD_Hostgroups cluster_2_hg { 1420, 1421, 1422, 1423 }; + BGD_Hostgroups cluster_3_hg { 1430, 1431, 1432, 1433 }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int configure_available( + MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, int green_use_ssl) +{ + if (configure_read_only_values(sim, cluster) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); + if (sim.topology_update(cluster.get_endpoints(), topology) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, + blue_servers, green_servers, 0, green_use_ssl + ); + if (admin_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + return status_rc; +} + +bool worker_matches(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster, string status, bool demoted) { + string writer_count = demoted ? "0" : "1"; + string reader_count = demoted ? "1" : "0"; + string query = "SELECT " + "(SELECT COUNT(*) FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hg.blue_writer) + " AND status=" + bgd_sql_quote(status) + ")=1 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.blue_writer) + + " AND hostname=" + bgd_sql_quote(cluster.blue_writer.hostname) + " AND port=3306)=" + writer_count + " AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_writer.hostname) + " AND port=3306)=" + reader_count; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +bool runtime_green_membership_matches( + MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& present, RDS_BGD_Cluster& absent) +{ + string query = "SELECT " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(present.green_writer.hostname) + " AND port=3306 AND use_ssl=1)=1 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(present.green_readers[0].hostname) + " AND port=3306 AND use_ssl=1)=1 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(absent.green_writer.hostname) + " AND port=3306)=0 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(absent.green_readers[0].hostname) + " AND port=3306)=0"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +int replace_cluster_1_green_membership(MYSQL* admin, TestState& state) { + RDS_BGD_Cluster& old_deployment = state.cluster_1; + RDS_BGD_Cluster& new_deployment = state.cluster_1_b; + BGD_Hostgroups& hg = state.cluster_1_hg; + + vector delete_queries { + "DELETE FROM mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(old_deployment.green_writer.hostname) + " AND port=3306", + "DELETE FROM mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(old_deployment.green_readers[0].hostname) + " AND port=3306", + }; + if (execute_all(admin, delete_queries) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector green_servers { new_deployment.green_writer, new_deployment.green_readers[0] }; + if (bgd_admin_add_servers(admin, new_deployment, hg, green_servers, true, 1) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int rc = execute_all(admin, load_queries); + return rc; +} + +/** + * Start three BGD workers in AVAILABLE. + * + * - Configure hostgroups 1410-1413, 1420-1423, and 1430-1433. + * - Use plaintext green metadata for clusters 1 and 3 and TLS for cluster 2. + * - Verify that each BGD row reaches AVAILABLE through its own green writer. + */ +int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before starting three BGD workers"); + return EXIT_FAILURE; + } + + int cluster_1_rc = configure_available(admin, sim, state.cluster_1, state.cluster_1_hg, 0); + if (cluster_1_rc != EXIT_SUCCESS) { + diag("Error: failed to configure AVAILABLE for BGD wHG 1410"); + return EXIT_FAILURE; + } + + int cluster_2_rc = configure_available(admin, sim, state.cluster_2, state.cluster_2_hg, 1); + if (cluster_2_rc != EXIT_SUCCESS) { + diag("Error: failed to configure AVAILABLE for BGD wHG 1420"); + return EXIT_FAILURE; + } + + int cluster_3_rc = configure_available(admin, sim, state.cluster_3, state.cluster_3_hg, 0); + if (cluster_3_rc != EXIT_SUCCESS) { + diag("Error: failed to configure AVAILABLE for BGD wHG 1430"); + return EXIT_FAILURE; + } + + auto [cluster_1_probe_rc, cluster_1_probe] = sim.wait_for_probe_log( + seq, state.cluster_1.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (cluster_1_probe_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1410 did not probe its plaintext green writer"); + return EXIT_FAILURE; + } + ok(true, "BGD wHG 1410 reports AVAILABLE from its own plaintext green writer"); + + auto [cluster_2_probe_rc, cluster_2_probe] = sim.wait_for_probe_log( + seq, state.cluster_2.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (cluster_2_probe_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1420 did not probe its TLS green writer"); + return EXIT_FAILURE; + } + ok(true, "BGD wHG 1420 reports AVAILABLE from its own TLS green writer"); + + auto [cluster_3_probe_rc, cluster_3_probe] = sim.wait_for_probe_log( + seq, state.cluster_3.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (cluster_3_probe_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1430 did not probe its plaintext green writer"); + return EXIT_FAILURE; + } + ok(true, "BGD wHG 1430 reports AVAILABLE from its own plaintext green writer"); + return EXIT_SUCCESS; +} + +/** + * Move each BGD worker to a different writer-switchover phase. + * + * - Move wHG 1410 to WRITER_SWITCHOVER_IN_PROGRESS. + * - Move wHG 1420 to WRITER_SWITCHOVER_POST_PROCESSING. + * - Move wHG 1430 to WRITER_SWITCHOVER_INITIATED. + * - After each change, verify that the other two statuses and blue-writer + * placements remain unchanged. + */ +int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + vector cluster_1_topology = + topology_with_reader_pair(state.cluster_1, "SWITCHOVER_IN_PROGRESS"); + if (sim.topology_update(state.cluster_1.get_endpoints(), cluster_1_topology) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS for BGD wHG 1410"); + return EXIT_FAILURE; + } + + int cluster_1_status_rc = + bgd_wait_for_status(admin, state.cluster_1_hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (cluster_1_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1410 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int cluster_1_placement_rc = bgd_wait_for_server_placement( + admin, state.cluster_1_hg.blue_writer, state.cluster_1_hg.blue_reader, + state.cluster_1.blue_writer, true, kTimeoutSeconds + ); + if (cluster_1_placement_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1410 did not move its blue writer to reader hostgroup 1411"); + return EXIT_FAILURE; + } + ok(true, "advancing wHG 1410 moves only its blue writer from hostgroup 1410 to 1411"); + + bool cluster_2_available = worker_matches(admin, state.cluster_2_hg, state.cluster_2, "AVAILABLE", false); + bool cluster_3_available = worker_matches(admin, state.cluster_3_hg, state.cluster_3, "AVAILABLE", false); + ok(cluster_2_available && cluster_3_available, + "advancing wHG 1410 leaves wHG 1420 and wHG 1430 in AVAILABLE with unchanged blue placement"); + + vector cluster_2_topology = + topology_with_reader_pair(state.cluster_2, "SWITCHOVER_IN_POST_PROCESSING"); + if (sim.topology_update(state.cluster_2.get_endpoints(), cluster_2_topology) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING for BGD wHG 1420"); + return EXIT_FAILURE; + } + + int cluster_2_status_rc = + bgd_wait_for_status(admin, state.cluster_2_hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (cluster_2_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1420 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + int cluster_2_placement_rc = bgd_wait_for_server_placement( + admin, state.cluster_2_hg.blue_writer, state.cluster_2_hg.blue_reader, + state.cluster_2.blue_writer, false, kTimeoutSeconds + ); + if (cluster_2_placement_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1420 did not retain its blue writer in hostgroup 1420"); + return EXIT_FAILURE; + } + ok(true, "advancing wHG 1420 applies post-processing only to its blue writer"); + + bool cluster_1_in_progress = + worker_matches(admin, state.cluster_1_hg, state.cluster_1, "WRITER_SWITCHOVER_IN_PROGRESS", true); + bool cluster_3_still_available = worker_matches(admin, state.cluster_3_hg, state.cluster_3, "AVAILABLE", false); + ok(cluster_1_in_progress && cluster_3_still_available, + "advancing wHG 1420 preserves wHG 1410 progress and wHG 1430 availability"); + + vector cluster_3_topology = + topology_with_reader_pair(state.cluster_3, "SWITCHOVER_INITIATED"); + if (sim.topology_update(state.cluster_3.get_endpoints(), cluster_3_topology) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED for BGD wHG 1430"); + return EXIT_FAILURE; + } + + int cluster_3_status_rc = + bgd_wait_for_status(admin, state.cluster_3_hg, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds); + if (cluster_3_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1430 did not reach WRITER_SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + int cluster_3_placement_rc = bgd_wait_for_server_placement( + admin, state.cluster_3_hg.blue_writer, state.cluster_3_hg.blue_reader, + state.cluster_3.blue_writer, false, kTimeoutSeconds + ); + if (cluster_3_placement_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1430 did not retain its blue writer in hostgroup 1430"); + return EXIT_FAILURE; + } + ok(true, "advancing wHG 1430 records INITIATED without changing its blue writer placement"); + + bool cluster_1_still_in_progress = + worker_matches(admin, state.cluster_1_hg, state.cluster_1, "WRITER_SWITCHOVER_IN_PROGRESS", true); + bool cluster_2_post = worker_matches( + admin, state.cluster_2_hg, state.cluster_2, "WRITER_SWITCHOVER_POST_PROCESSING", false + ); + ok(cluster_1_still_in_progress && cluster_2_post, + "advancing wHG 1430 preserves wHG 1410 progress and wHG 1420 post-processing"); + return EXIT_SUCCESS; +} + +/** + * Refresh only cluster 1 green membership. + * + * - Replace cluster 1 green rows with TLS-enabled deployment B rows. + * - Keep wHG 1410 in WRITER_SWITCHOVER_IN_PROGRESS. + * - Verify that cluster 1 stops probing its removed target while clusters 2 + * and 3 keep their status, placement, metadata target, and TLS value. + */ +int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + if (configure_read_only_values(sim, state.cluster_1_b) != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for cluster 1 deployment B"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before refreshing BGD wHG 1410"); + return EXIT_FAILURE; + } + + int replace_rc = replace_cluster_1_green_membership(admin, state); + if (replace_rc != EXIT_SUCCESS) { + diag("Error: failed to replace green membership for BGD wHG 1410"); + return EXIT_FAILURE; + } + + vector topology = + topology_with_reader_pair(state.cluster_1_b, "SWITCHOVER_IN_PROGRESS"); + if (sim.topology_update(state.cluster_1_b.get_endpoints(), topology) != EXIT_SUCCESS) { + diag("Error: failed to publish deployment B topology for BGD wHG 1410"); + return EXIT_FAILURE; + } + + auto [cluster_1_probe_rc, cluster_1_probe] = sim.wait_for_probe_log( + seq, state.cluster_1_b.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (cluster_1_probe_rc != EXIT_SUCCESS) { + diag("Error: refreshed BGD wHG 1410 did not probe its TLS deployment B green writer"); + return EXIT_FAILURE; + } + + int stale_probe_rc = bgd_expect_no_metadata_probe( + sim, cluster_1_probe.sequence_id, state.cluster_1.green_writer.endpoint(), kNegativeProbeTimeoutMs + ); + if (stale_probe_rc != EXIT_SUCCESS) { + diag("Error: refreshed BGD wHG 1410 continued probing its removed green writer"); + return EXIT_FAILURE; + } + + auto [cluster_2_probe_rc, cluster_2_probe] = sim.wait_for_probe_log( + cluster_1_probe.sequence_id, state.cluster_2.green_writer.endpoint(), + RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (cluster_2_probe_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1420 did not continue probing its TLS green writer"); + return EXIT_FAILURE; + } + + auto [cluster_3_probe_rc, cluster_3_probe] = sim.wait_for_probe_log( + cluster_1_probe.sequence_id, state.cluster_3.green_writer.endpoint(), + RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (cluster_3_probe_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1430 did not continue probing its plaintext green writer"); + return EXIT_FAILURE; + } + + bool cluster_1_membership = + runtime_green_membership_matches(admin, state.cluster_1_hg, state.cluster_1_b, state.cluster_1); + bool cluster_1_phase = + worker_matches(admin, state.cluster_1_hg, state.cluster_1_b, "WRITER_SWITCHOVER_IN_PROGRESS", true); + bool cluster_2_phase = + worker_matches(admin, state.cluster_2_hg, state.cluster_2, "WRITER_SWITCHOVER_POST_PROCESSING", false); + bool cluster_3_phase = + worker_matches(admin, state.cluster_3_hg, state.cluster_3, "WRITER_SWITCHOVER_INITIATED", false); + ok(cluster_1_membership && cluster_1_phase && cluster_2_phase && cluster_3_phase, + "refreshing wHG 1410 changes only its target while wHG 1420 and wHG 1430 keep their phases and probes"); + return EXIT_SUCCESS; +} + +int main() { + plan(10); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish separate AVAILABLE topology for clusters 1, 2, and 3. + // ProxySQL: configure BGD wHGs 1410, 1420, and 1430 with distinct green targets and TLS values. + // Verify: each BGD row reports AVAILABLE through its own configured green writer. + if (test_three_workers_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS for wHG 1410, SWITCHOVER_IN_POST_PROCESSING for wHG 1420, + // and SWITCHOVER_INITIATED for wHG 1430. + // Verify: each BGD status and blue-writer placement changes without affecting the other two workers. + if (test_independent_phase_changes(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: keep cluster 1 in progress with deployment B topology. + // ProxySQL: replace only wHG 1410 green membership with TLS-enabled deployment B rows. + // Verify: wHG 1410 uses the new target while wHGs 1420 and 1430 keep their phases, targets, and TLS. + if (test_independent_config_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp b/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp new file mode 100644 index 0000000000..0d375e13a1 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp @@ -0,0 +1,281 @@ +/** + * @file test_rds_bgd_config_refresh_after_writer_completion-t.cpp + * @brief BGD configuration refresh during reader switchover. + * + * Steps: + * + * 1. Configure BGD hostgroups 1360-1363 and reach + * WRITER_SWITCHOVER_IN_PROGRESS. + * 2. Publish target-only SWITCHOVER_COMPLETED and verify + * READER_SWITCHOVER_IN_PROGRESS. + * 3. Change check_timeout_ms in mysql_aws_rds_bgd_hostgroups and load the + * configuration to runtime. + * 4. Verify that the refresh performs a blue table check before blue metadata + * and republishes READER_SWITCHOVER_IN_PROGRESS. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_3_init() }; + BGD_Hostgroups hostgroups { 1360, 1361, 1362, 1363 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + for (RDS_BGD_Host& host : cluster.blue_readers) { + rows.push_back({ + host.hostname, + host.hostname, + host.port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + } + for (RDS_BGD_Host& host : cluster.green_readers) { + rows.push_back({ + host.hostname, + host.hostname, + host.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + } + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +/** + * Reach reader switchover before changing the BGD configuration. + * + * - Publish SWITCHOVER_IN_PROGRESS before loading BGD hostgroups 1360-1363. + * - Require WRITER_SWITCHOVER_IN_PROGRESS. + * - Publish target-only SWITCHOVER_COMPLETED. + * - Verify BGD status READER_SWITCHOVER_IN_PROGRESS. + */ +int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1360"); + return EXIT_FAILURE; + } + + vector progress = topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + int topology_rc = sim.topology_update(state.topology_endpoints, progress); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG 1360"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1360-1363"); + return EXIT_FAILURE; + } + + int progress_status_rc = + bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1360 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + vector completed = target_only_completed(cluster); + int completed_rc = sim.topology_update(state.topology_endpoints, completed); + if (completed_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG 1360"); + return EXIT_FAILURE; + } + + int reader_status_rc = + bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (reader_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1360 did not reach READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "target-only SWITCHOVER_COMPLETED sets BGD status for wHG 1360 to READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_SUCCESS; +} + +/** + * Refresh the BGD configuration during reader switchover. + * + * - Change check_timeout_ms for wHG 1360 and load it to runtime. + * - Verify that the refresh starts with a blue-writer table check. + * - Verify that blue-writer metadata follows the table check. + * - Verify BGD status returns to READER_SWITCHOVER_IN_PROGRESS. + */ +int test_config_refresh_after_writer_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before refreshing wHG 1360"); + return EXIT_FAILURE; + } + + string update_query = + "UPDATE mysql_aws_rds_bgd_hostgroups SET check_timeout_ms=950 WHERE writer_hostgroup=" + + to_string(hg.blue_writer); + vector queries { + update_query, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int refresh_rc = execute_all(admin, queries); + if (refresh_rc != EXIT_SUCCESS) { + diag("Error: failed to refresh check_timeout_ms for wHG 1360"); + return EXIT_FAILURE; + } + + auto [table_rc, table] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (table_rc != EXIT_SUCCESS) { + diag("Error: post-completion refresh did not start with a blue-writer table check for wHG 1360"); + return EXIT_FAILURE; + } + + auto [blue_rc, blue] = sim.wait_for_probe_log( + table.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: post-completion refresh did not probe blue-writer metadata for wHG 1360"); + return EXIT_FAILURE; + } + + bool probe_order = table.sequence_id < blue.sequence_id; + ok(probe_order, "post-completion refresh checks the table before blue-writer metadata for wHG 1360"); + + int status_rc = + bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: refreshed wHG 1360 did not republish READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "post-completion refresh republishes READER_SWITCHOVER_IN_PROGRESS for wHG 1360"); + return EXIT_SUCCESS; +} + +int main() { + plan(3); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish SWITCHOVER_IN_PROGRESS, then target-only SWITCHOVER_COMPLETED. + // ProxySQL: configure BGD hostgroups 1360-1363. + // Verify: BGD status for wHG 1360 reports READER_SWITCHOVER_IN_PROGRESS. + if (test_reader_switchover_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: change check_timeout_ms and load the BGD configuration to runtime. + // Verify: refresh runs blue table check, then blue metadata, and republishes READER_SWITCHOVER_IN_PROGRESS. + if (test_config_refresh_after_writer_completion(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp new file mode 100644 index 0000000000..2d01d467b4 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp @@ -0,0 +1,652 @@ +/** + * @file test_rds_bgd_configuration_persistence-t.cpp + * @brief BGD runtime and persistent configuration ownership. + * + * Steps: + * + * 1. Convert an auto-generated row for wHG 890 into explicit configuration. + * 2. Verify persistent BGD rows reject NULL green hostgroups and accept a + * complete row. + * 3. SAVE runtime BGD state and verify only the explicit row is persisted. + * 4. Run automatic discovery beside administrator-owned configuration and + * verify its BGD row and green-server status remain unchanged. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +// Automatic discovery is asynchronous and starts after the monitor observes the +// runtime server. Allow the monitor and the BGD worker to become ready on slower CI runners. +const uint32_t kTimeoutSeconds = 15; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster conversion { bgd_cluster_2_init() }; + RDS_BGD_Cluster explicit_save { bgd_cluster_3_init() }; + RDS_BGD_Cluster automatic_save { bgd_cluster_1_deployment_b_init() }; + RDS_BGD_Cluster admin_owned { bgd_cluster_init() }; + BGD_Hostgroups conversion_hg { 890, 891, 892, 893 }; + BGD_Hostgroups valid_hg { 910, 911, 912, 913 }; + BGD_Hostgroups explicit_save_hg { 920, 921, 922, 923 }; + BGD_Hostgroups automatic_save_hg { 930, 931, 932, 933 }; + BGD_Hostgroups admin_owned_hg { 1310, 1311, 1312, 1313 }; + vector conversion_endpoints { conversion.get_endpoints() }; + vector explicit_save_endpoints { explicit_save.get_endpoints() }; + vector automatic_save_endpoints { automatic_save.get_endpoints() }; + vector admin_owned_endpoints { admin_owned.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int configure_monitor(MYSQL* admin, BGD_Hostgroups& hg, bool automatic) { + string automatic_value = automatic ? "true" : "false"; + vector queries { + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + ")", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "SET mysql-monitor_read_only_interval=100", + "SET mysql-monitor_aws_rds_topology_discovery_interval=1", + "SET mysql-aws_blue_green_deployment_auto_discovery='" + automatic_value + "'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int insert_explicit_bgd_row(MYSQL* admin, BGD_Hostgroups& hg, const string& comment, int active = 1) { + string query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + "," + + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + "," + + to_string(active) + ",0,100,800," + bgd_sql_quote(comment) + ")"; + + int rc = mysql_query(admin, query.c_str()); + if (rc != 0) { + diag("Error: failed to insert mysql_aws_rds_bgd_hostgroups row for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int add_all_servers(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + int blue_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to add blue servers for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + int green_rc = bgd_admin_add_servers(admin, cluster, hg, green_servers, true, 0); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to add green servers for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load servers for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +bool persistent_bgd_row_matches(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup " + "FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + to_string(hg.blue_writer); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 4) { + return false; + } + + bool matches = + rows[0][0] == to_string(hg.blue_writer) && + rows[0][1] == to_string(hg.blue_reader) && + rows[0][2] == to_string(hg.green_writer) && + rows[0][3] == to_string(hg.green_reader); + return matches; +} + +bool persistent_bgd_row_absent(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT COUNT(*) FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool absent = rows[0][0] == "0"; + return absent; +} + +bool runtime_explicit_bgd_row_matches(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hg.blue_writer) + " AND reader_hostgroup=" + to_string(hg.blue_reader) + + " AND green_writer_hostgroup=" + to_string(hg.green_writer) + + " AND green_reader_hostgroup=" + to_string(hg.green_reader) + " AND auto_generated=0"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +rc_t> bgd_admin_snapshot(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment " + "FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + to_string(writer_hostgroup); + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +rc_t> runtime_bgd_ownership_snapshot(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT green_writer_hostgroup,green_reader_hostgroup,active,auto_generated " + "FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + to_string(writer_hostgroup); + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +rc_t> green_server_snapshot(MYSQL* admin, const string& table, BGD_Hostgroups& hg) { + string query = + "SELECT hostgroup_id,hostname,port,status,use_ssl,weight,max_connections FROM " + table + + " WHERE hostgroup_id IN (" + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ") ORDER BY hostgroup_id,hostname,port"; + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +/** + * Convert the automatic runtime row for wHG 890 to explicit configuration. + * + * - Enable automatic discovery with only the blue writer configured. + * - Verify the runtime row has NULL green hostgroups and auto_generated=1. + * - Disable automatic discovery and load explicit hostgroups 890-893. + * - Verify explicit values replace the automatic row and persist. + */ +int test_automatic_to_explicit(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.conversion; + BGD_Hostgroups& hg = state.conversion_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic-conversion simulated writers"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.conversion_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 890"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg, true); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to enable automatic discovery for wHG 890"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer }; + int server_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (server_rc != EXIT_SUCCESS) { + diag("Error: failed to add the blue writer for wHG 890"); + return EXIT_FAILURE; + } + + vector load_server_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_server_rc = execute_all(admin, load_server_queries); + if (load_server_rc != EXIT_SUCCESS) { + diag("Error: failed to load the blue writer for wHG 890"); + return EXIT_FAILURE; + } + + string automatic_query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=890 " + "AND auto_generated=1 AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL"; + int automatic_rc = bgd_wait_for_condition(admin, automatic_query, kTimeoutSeconds); + if (automatic_rc != EXIT_SUCCESS) { + diag("Error: automatic discovery did not create the nullable runtime row for wHG 890"); + return EXIT_FAILURE; + } + + ok(true, "automatic discovery records NULL green hostgroups for wHG 890"); + + vector disable_queries { + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + }; + int disable_rc = execute_all(admin, disable_queries); + if (disable_rc != EXIT_SUCCESS) { + diag("Error: failed to disable automatic discovery before converting wHG 890"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "converted automatic BGD row"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert explicit configuration for wHG 890"); + return EXIT_FAILURE; + } + + vector load_row_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_row_rc = execute_all(admin, load_row_queries); + if (load_row_rc != EXIT_SUCCESS) { + diag("Error: failed to load explicit configuration for wHG 890"); + return EXIT_FAILURE; + } + + string explicit_query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=890 " + "AND auto_generated=0 AND green_writer_hostgroup=892 AND green_reader_hostgroup=893"; + int explicit_rc = bgd_wait_for_condition(admin, explicit_query, kTimeoutSeconds); + if (explicit_rc != EXIT_SUCCESS) { + diag("Error: explicit configuration did not replace the automatic row for wHG 890"); + return EXIT_FAILURE; + } + + bool runtime_matches = runtime_explicit_bgd_row_matches(admin, hg); + bool persistent_matches = persistent_bgd_row_matches(admin, hg); + ok(runtime_matches && persistent_matches, "explicit hostgroups 890-893 replace and persist the automatic row"); + return EXIT_SUCCESS; +} + +/** + * Validate persistent green-hostgroup requirements. + * + * - Attempt persistent rows with a NULL green writer or reader hostgroup. + * - Verify both invalid rows are rejected. + * - Load a complete row for hostgroups 910-913. + * - Verify it exists in persistent and runtime configuration. + */ +int test_persistent_row_validation(MYSQL* admin, TestState& state) { + string null_writer_query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup) " + "VALUES (900,901,NULL,903)"; + int null_writer_rc = mysql_query(admin, null_writer_query.c_str()); + bool null_writer_absent = persistent_bgd_row_absent(admin, 900); + ok(null_writer_rc != 0 && null_writer_absent, "persistent BGD configuration rejects a NULL green writer hostgroup"); + + string null_reader_query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup) " + "VALUES (904,905,906,NULL)"; + int null_reader_rc = mysql_query(admin, null_reader_query.c_str()); + bool null_reader_absent = persistent_bgd_row_absent(admin, 904); + ok(null_reader_rc != 0 && null_reader_absent, "persistent BGD configuration rejects a NULL green reader hostgroup"); + + BGD_Hostgroups& hg = state.valid_hg; + int monitor_rc = configure_monitor(admin, hg, false); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for hostgroups 910-913"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "valid persistent BGD row"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert valid persistent BGD row for hostgroups 910-913"); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load valid BGD row for hostgroups 910-913"); + return EXIT_FAILURE; + } + + bool persistent_matches = persistent_bgd_row_matches(admin, hg); + bool runtime_matches = runtime_explicit_bgd_row_matches(admin, hg); + ok(persistent_matches && runtime_matches, "complete persistent BGD configuration loads hostgroups 910-913"); + return EXIT_SUCCESS; +} + +/** + * Save explicit and automatic runtime rows back to persistent configuration. + * + * - Run explicit wHG 920 and automatic wHG 930 together. + * - Remove the persistent explicit row. + * - Execute SAVE MYSQL SERVERS FROM RUNTIME. + * - Verify SAVE restores wHG 920 and skips auto-generated wHG 930. + */ +int test_save_from_runtime(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& explicit_cluster = state.explicit_save; + RDS_BGD_Cluster& automatic_cluster = state.automatic_save; + BGD_Hostgroups& explicit_hg = state.explicit_save_hg; + BGD_Hostgroups& automatic_hg = state.automatic_save_hg; + + int explicit_writer_rc = bgd_set_writer_read_only_0(sim, explicit_cluster); + if (explicit_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure explicit SAVE simulated writers"); + return EXIT_FAILURE; + } + + int automatic_writer_rc = bgd_set_writer_read_only_0(sim, automatic_cluster); + if (automatic_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic SAVE simulated writers"); + return EXIT_FAILURE; + } + + vector explicit_topology = bgd_topology_with_readers(explicit_cluster, "AVAILABLE"); + int explicit_topology_rc = sim.topology_update(state.explicit_save_endpoints, explicit_topology); + if (explicit_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 920"); + return EXIT_FAILURE; + } + + vector automatic_topology = bgd_topology_with_readers(automatic_cluster, "AVAILABLE"); + int automatic_topology_rc = sim.topology_update(state.automatic_save_endpoints, automatic_topology); + if (automatic_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 930"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, explicit_hg, true); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure SAVE monitoring for wHG 920"); + return EXIT_FAILURE; + } + + int explicit_row_rc = insert_explicit_bgd_row(admin, explicit_hg, "explicit SAVE row"); + if (explicit_row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert explicit SAVE row for wHG 920"); + return EXIT_FAILURE; + } + + int explicit_servers_rc = add_all_servers(admin, explicit_cluster, explicit_hg); + if (explicit_servers_rc != EXIT_SUCCESS) { + diag("Error: failed to load servers for wHG 920"); + return EXIT_FAILURE; + } + + string replication_query = + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(automatic_hg.blue_writer) + "," + to_string(automatic_hg.blue_reader) + ")"; + vector automatic_config_queries { replication_query }; + int automatic_config_rc = execute_all(admin, automatic_config_queries); + if (automatic_config_rc != EXIT_SUCCESS) { + diag("Error: failed to configure replication hostgroups 930 and 931"); + return EXIT_FAILURE; + } + + vector automatic_blue_servers { automatic_cluster.blue_writer }; + int automatic_server_rc = bgd_admin_add_servers(admin, automatic_cluster, automatic_hg, automatic_blue_servers, false, 0); + if (automatic_server_rc != EXIT_SUCCESS) { + diag("Error: failed to add the automatic blue writer for wHG 930"); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load explicit and automatic SAVE scenarios"); + return EXIT_FAILURE; + } + + string explicit_runtime_query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups " + "WHERE writer_hostgroup=920 AND auto_generated=0"; + int explicit_runtime_rc = bgd_wait_for_condition(admin, explicit_runtime_query, kTimeoutSeconds); + if (explicit_runtime_rc != EXIT_SUCCESS) { + diag("Error: explicit wHG 920 did not reach runtime before SAVE"); + return EXIT_FAILURE; + } + + string automatic_runtime_query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups " + "WHERE writer_hostgroup=930 AND auto_generated=1"; + int automatic_runtime_rc = bgd_wait_for_condition(admin, automatic_runtime_query, kTimeoutSeconds); + if (automatic_runtime_rc != EXIT_SUCCESS) { + diag("Error: automatic wHG 930 did not reach runtime before SAVE"); + return EXIT_FAILURE; + } + + string delete_query = + "DELETE FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(explicit_hg.blue_writer); + vector save_queries { + delete_query, + "SAVE MYSQL SERVERS FROM RUNTIME", + }; + int save_rc = execute_all(admin, save_queries); + if (save_rc != EXIT_SUCCESS) { + diag("Error: failed to save runtime BGD rows to persistent configuration"); + return EXIT_FAILURE; + } + + bool explicit_persisted = persistent_bgd_row_matches(admin, explicit_hg); + bool automatic_absent = persistent_bgd_row_absent(admin, automatic_hg.blue_writer); + ok(explicit_persisted && automatic_absent, "SAVE restores explicit wHG 920 and skips auto-generated wHG 930"); + return EXIT_SUCCESS; +} + +/** + * Run automatic discovery beside administrator-owned BGD and server rows. + * + * - Configure inactive explicit hostgroups 1310-1313. + * - Set the configured green writer to SHUNNED. + * - Enable automatic discovery and publish AVAILABLE topology. + * - Verify the BGD row and green-server status remain unchanged. + */ +int test_admin_server_status_preserved(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.admin_owned; + BGD_Hostgroups& hg = state.admin_owned_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure administrator-owned simulated writers"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg, false); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for administrator-owned wHG 1310"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "administrator-owned inactive BGD row", 0); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert administrator-owned wHG 1310"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0] }; + int blue_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to load administrator-owned blue servers"); + return EXIT_FAILURE; + } + + vector green_servers { cluster.green_writer }; + int green_rc = bgd_admin_add_servers(admin, cluster, hg, green_servers, true, 0); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to load administrator-owned green writer"); + return EXIT_FAILURE; + } + + string shun_query = + "UPDATE mysql_servers SET status='SHUNNED' WHERE hostgroup_id=" + + to_string(hg.green_writer) + " AND hostname=" + bgd_sql_quote(cluster.green_writer.hostname) + + " AND port=3306"; + vector ownership_queries { + shun_query, + "LOAD MYSQL SERVERS TO RUNTIME", + "SET mysql-aws_blue_green_deployment_auto_discovery='true'", + "LOAD MYSQL VARIABLES TO RUNTIME", + }; + int ownership_rc = execute_all(admin, ownership_queries); + if (ownership_rc != EXIT_SUCCESS) { + diag("Error: failed to enable automatic discovery beside administrator-owned wHG 1310"); + return EXIT_FAILURE; + } + + auto [bgd_before_rc, bgd_before] = bgd_admin_snapshot(admin, hg.blue_writer); + auto [runtime_bgd_before_rc, runtime_bgd_before] = runtime_bgd_ownership_snapshot(admin, hg.blue_writer); + auto [admin_before_rc, admin_before] = green_server_snapshot(admin, "mysql_servers", hg); + auto [runtime_before_rc, runtime_before] = green_server_snapshot(admin, "runtime_mysql_servers", hg); + if (bgd_before_rc != EXIT_SUCCESS || runtime_bgd_before_rc != EXIT_SUCCESS || + admin_before_rc != EXIT_SUCCESS || runtime_before_rc != EXIT_SUCCESS) { + diag("Error: failed to snapshot administrator-owned BGD and green rows"); + return EXIT_FAILURE; + } + + bool explicit_runtime_bgd = + runtime_bgd_before.size() == 1 && + runtime_bgd_before[0].size() == 4 && + runtime_bgd_before[0][0] == "1312" && + runtime_bgd_before[0][1] == "1313" && + runtime_bgd_before[0][2] == "0" && + runtime_bgd_before[0][3] == "0"; + if (!explicit_runtime_bgd) { + diag("Error: runtime BGD row does not contain the administrator-owned hostgroups and flags"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before administrator-owned discovery"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.admin_owned_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology beside administrator-owned wHG 1310"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: automatic discovery did not probe beside administrator-owned wHG 1310"); + return EXIT_FAILURE; + } + + auto [bgd_after_rc, bgd_after] = bgd_admin_snapshot(admin, hg.blue_writer); + auto [runtime_bgd_after_rc, runtime_bgd_after] = runtime_bgd_ownership_snapshot(admin, hg.blue_writer); + auto [admin_after_rc, admin_after] = green_server_snapshot(admin, "mysql_servers", hg); + auto [runtime_after_rc, runtime_after] = green_server_snapshot(admin, "runtime_mysql_servers", hg); + if (bgd_after_rc != EXIT_SUCCESS || runtime_bgd_after_rc != EXIT_SUCCESS || + admin_after_rc != EXIT_SUCCESS || runtime_after_rc != EXIT_SUCCESS) { + diag("Error: failed to read administrator-owned rows after discovery"); + return EXIT_FAILURE; + } + + bool bgd_unchanged = bgd_before == bgd_after; + bool runtime_bgd_unchanged = runtime_bgd_before == runtime_bgd_after; + bool admin_servers_unchanged = admin_before == admin_after; + bool runtime_servers_unchanged = runtime_before == runtime_after; + ok(bgd_unchanged && runtime_bgd_unchanged && admin_servers_unchanged && runtime_servers_unchanged, + "automatic discovery preserves administrator-owned wHG 1310 and its SHUNNED green writer"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish AVAILABLE topology for the blue writer in wHG 890. + // ProxySQL: create an automatic row, disable discovery, and load explicit hostgroups 890-893. + // Verify: explicit green hostgroups replace the nullable automatic row and persist. + if (test_automatic_to_explicit(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: insert two BGD rows with one NULL green hostgroup, then one complete row. + // Verify: invalid rows are rejected and complete hostgroups 910-913 load as explicit configuration. + if (test_persistent_row_validation(admin, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: run explicit wHG 920 and auto-generated wHG 930, then SAVE runtime state. + // Verify: SAVE persists only the explicit BGD row. + if (test_save_from_runtime(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: configure inactive administrator-owned wHG 1310 with a SHUNNED green writer. + // Simulator: publish AVAILABLE while automatic discovery is enabled. + // Verify: the BGD row and green-server status remain unchanged. + if (test_admin_server_status_preserved(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp b/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp new file mode 100644 index 0000000000..07c7327bf9 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp @@ -0,0 +1,228 @@ +/** + * @file test_rds_bgd_disable_during_switchover-t.cpp + * @brief Disabling BGD during writer switchover. + * + * Steps: + * + * 1. Configure BGD hostgroups 1340-1343 and reach `AVAILABLE`. + * 2. Publish `SWITCHOVER_IN_PROGRESS` and verify that the blue writer moves + * from hostgroup 1340 to hostgroup 1341. + * 3. Set `active=0` without changing the configured hostgroups. + * 4. Verify that the blue writer returns from hostgroup 1341 to hostgroup 1340. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_init() }; + BGD_Hostgroups hostgroups { 1340, 1341, 1342, 1343 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Configure BGD hostgroups 1340-1343. + * + * - Set `read_only=0` for the simulated blue and green writers. + * - Publish `AVAILABLE` topology. + * - Configure `mysql_servers` and `mysql_aws_rds_bgd_hostgroups`. + * - Verify that the runtime BGD row reaches `AVAILABLE`. + */ +int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Set read_only=0 for the simulated blue and green writers. + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated writer read_only values"); + return EXIT_FAILURE; + } + + // Publish AVAILABLE topology. + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology"); + return EXIT_FAILURE; + } + + // Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure mysql_servers and mysql_aws_rds_bgd_hostgroups"); + return EXIT_FAILURE; + } + + // Wait for the runtime BGD row to report AVAILABLE. + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime BGD status did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1340 reports AVAILABLE"); + return EXIT_SUCCESS; +} + +/** + * Move the BGD row for writer hostgroup 1340 into writer switchover. + * + * - Publish `SWITCHOVER_IN_PROGRESS`. + * - Verify `WRITER_SWITCHOVER_IN_PROGRESS`. + * - Verify that the blue writer moves to the blue reader hostgroup. + */ +int test_writer_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Publish SWITCHOVER_IN_PROGRESS and wait for the runtime BGD status. + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime BGD status did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1340 reports WRITER_SWITCHOVER_IN_PROGRESS"); + + // Verify the blue writer was moved from the writer to the reader hostgroup. + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not move to the blue reader hostgroup"); + return EXIT_FAILURE; + } + + ok(true, "SWITCHOVER_IN_PROGRESS moves the blue writer from hostgroup 1340 to 1341"); + return EXIT_SUCCESS; +} + +/** + * Disable BGD during writer switchover. + * + * - Set `active=0` in `mysql_aws_rds_bgd_hostgroups`. + * - Load the configuration to runtime without changing any hostgroups. + * - Verify that the blue writer returns from hostgroup 1341 to hostgroup 1340. + */ +int test_disable_during_switchover(MYSQL* admin, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Disable BGD without changing mysql_servers or the configured hostgroups. + string update_bgd = "UPDATE mysql_aws_rds_bgd_hostgroups SET active=0 WHERE writer_hostgroup=" + to_string(hg.blue_writer); + vector queries { update_bgd, "LOAD MYSQL SERVERS TO RUNTIME" }; + + int rc = execute_all(admin, queries); + if (rc != EXIT_SUCCESS) { + diag("Error: failed to set active=0 and load the BGD configuration to runtime"); + return EXIT_FAILURE; + } + + // Wait until disabling restores the blue writer to its writer hostgroup. + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not return to the blue writer hostgroup"); + return EXIT_FAILURE; + } + + ok(true, "setting active=0 restores the blue writer from hostgroup 1341 to 1340"); + return EXIT_SUCCESS; +} + +int main() { + plan(4); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: set the blue/green writers to read_only=0 and publish AVAILABLE topology. + // ProxySQL: update mysql_servers and mysql_aws_rds_bgd_hostgroups with BGD configuration. + // Verify: runtime_mysql_aws_rds_bgd_hostgroups status reports AVAILABLE. + if (test_bgd_status_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS topology. + // Verify: runtime_mysql_aws_rds_bgd_hostgroups status reports WRITER_SWITCHOVER_IN_PROGRESS. + // Verify: runtime_mysql_servers moves the blue writer from writer hostgroup to reader hostgroup. + if (test_writer_switchover_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: set active=0 without changing mysql_servers or the configured BGD hostgroups. + // Verify: runtime_mysql_servers returns the blue writer from reader hostgroup to writer hostgroup. + if (test_disable_during_switchover(admin, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp b/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp new file mode 100644 index 0000000000..2ddc6ee15f --- /dev/null +++ b/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp @@ -0,0 +1,359 @@ +/** + * @file test_rds_bgd_explicit_startup-t.cpp + * @brief Starting an explicit BGD worker after both required inputs exist. + * + * Steps: + * + * 1. Load the BGD row for hostgroups 840-843 before loading its servers. + * 2. Verify no table-check occurs until an eligible blue server is loaded. + * 3. Load servers for hostgroups 850-853 before loading their BGD row. + * 4. Verify no table-check occurs until the explicit BGD row is loaded. + */ + +#include +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 800; + +struct TestState { + RDS_BGD_Cluster row_first { bgd_cluster_init() }; + RDS_BGD_Cluster servers_first { bgd_cluster_2_init() }; + BGD_Hostgroups row_first_hg { 840, 841, 842, 843 }; + BGD_Hostgroups servers_first_hg { 850, 851, 852, 853 }; + vector row_first_endpoints { row_first.get_endpoints() }; + vector servers_first_endpoints { servers_first.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int configure_monitor(MYSQL* admin, BGD_Hostgroups& hg) { + vector queries { + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + ")", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "SET mysql-monitor_read_only_interval=100", + "SET mysql-monitor_aws_rds_topology_discovery_interval=1", + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int insert_explicit_bgd_row(MYSQL* admin, BGD_Hostgroups& hg, string comment) { + string query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + "," + + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ",1,0,100,800," + bgd_sql_quote(comment) + ")"; + + int rc = mysql_query(admin, query.c_str()); + if (rc != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int add_cluster_servers(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + int blue_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (blue_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + int green_rc = bgd_admin_add_servers(admin, cluster, hg, green_servers, true, 0); + if (green_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, queries); + if (load_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +bool runtime_membership_matches(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector hostgroups { hg.blue_writer, hg.blue_reader, hg.green_writer, hg.green_reader }; + auto [rc, rows] = bgd_runtime_servers(admin, hostgroups); + if (rc != EXIT_SUCCESS || rows.size() != 6) { + return false; + } + + vector> expected { + { hg.blue_writer, cluster.blue_writer.hostname }, + { hg.blue_reader, cluster.blue_readers[0].hostname }, + { hg.blue_reader, cluster.blue_readers[1].hostname }, + { hg.green_writer, cluster.green_writer.hostname }, + { hg.green_reader, cluster.green_readers[0].hostname }, + { hg.green_reader, cluster.green_readers[1].hostname }, + }; + + for (pair& server : expected) { + bool found = false; + for (mysql_res_row& row : rows) { + if (row.size() == 5 && row[0] == to_string(server.first) && row[1] == server.second) { + found = true; + break; + } + } + if (!found) { + return false; + } + } + return true; +} + +/** + * Load the explicit BGD row before any eligible blue server. + * + * - Publish AVAILABLE topology for hostgroups 840-843. + * - Load the explicit BGD row without mysql_servers membership. + * - Verify no table-check probe starts. + * - Load all servers and verify AVAILABLE with explicit runtime membership. + */ +int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.row_first; + BGD_Hostgroups& hg = state.row_first_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure row-first simulated writers"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.row_first_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for hostgroups 840-843"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for hostgroups 840-843"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before loading wHG 840"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "BGD row before servers"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert the explicit BGD row for wHG 840"); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_row_rc = execute_all(admin, load_queries); + if (load_row_rc != EXIT_SUCCESS) { + diag("Error: failed to load wHG 840 before its servers"); + return EXIT_FAILURE; + } + + int no_probe_rc = bgd_expect_no_table_check(sim, seq, state.row_first_endpoints, kNegativeProbeTimeoutMs); + if (no_probe_rc != EXIT_SUCCESS) { + diag("Error: wHG 840 started before an eligible blue server existed"); + return EXIT_FAILURE; + } + + ok(true, "wHG 840 does not start before an eligible blue server exists"); + + int servers_rc = add_cluster_servers(admin, cluster, hg); + if (servers_rc != EXIT_SUCCESS) { + diag("Error: failed to load servers for hostgroups 840-843"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: loading the blue writer did not start the wHG 840 table check"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 840 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_membership_matches(admin, cluster, hg); + ok(membership_matches, "loading servers starts wHG 840 with explicit runtime membership"); + return EXIT_SUCCESS; +} + +/** + * Load all servers before their explicit BGD row. + * + * - Publish AVAILABLE topology for hostgroups 850-853. + * - Load mysql_servers membership without a BGD row. + * - Verify no table-check probe starts. + * - Load the explicit row and verify AVAILABLE with explicit membership. + */ +int test_servers_before_bgd_row(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.servers_first; + BGD_Hostgroups& hg = state.servers_first_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure servers-first simulated writers"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.servers_first_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for hostgroups 850-853"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for hostgroups 850-853"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before loading hostgroups 850-853"); + return EXIT_FAILURE; + } + + int servers_rc = add_cluster_servers(admin, cluster, hg); + if (servers_rc != EXIT_SUCCESS) { + diag("Error: failed to load servers before wHG 850"); + return EXIT_FAILURE; + } + + int no_probe_rc = bgd_expect_no_table_check(sim, seq, state.servers_first_endpoints, kNegativeProbeTimeoutMs); + if (no_probe_rc != EXIT_SUCCESS) { + diag("Error: servers in hostgroups 850-853 started without an explicit BGD row"); + return EXIT_FAILURE; + } + + ok(true, "servers in hostgroups 850-853 do not start without an explicit BGD row"); + + int row_rc = insert_explicit_bgd_row(admin, hg, "servers before BGD row"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert the explicit BGD row for wHG 850"); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_row_rc = execute_all(admin, load_queries); + if (load_row_rc != EXIT_SUCCESS) { + diag("Error: failed to load the explicit BGD row for wHG 850"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: loading wHG 850 did not start the blue table check"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 850 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_membership_matches(admin, cluster, hg); + ok(membership_matches, "loading wHG 850 starts the worker with explicit runtime membership"); + return EXIT_SUCCESS; +} + +int main() { + plan(4); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish AVAILABLE topology for hostgroups 840-843. + // ProxySQL: load the explicit BGD row before loading mysql_servers. + // Verify: no table-check starts until eligible blue membership exists. + if (test_bgd_row_before_servers(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish AVAILABLE topology for hostgroups 850-853. + // ProxySQL: load mysql_servers before loading the explicit BGD row. + // Verify: no table-check starts until wHG 850 is loaded. + if (test_servers_before_bgd_row(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp b/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp new file mode 100644 index 0000000000..08f1e2bfc4 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp @@ -0,0 +1,421 @@ +/** + * @file test_rds_bgd_green_membership_ordering-t.cpp + * @brief Loading configured green membership at three supported times. + * + * Steps: + * + * 1. Load green membership for hostgroups 862 and 863 before AVAILABLE. + * 2. Load green membership for hostgroups 872 and 873 after discovery. + * 3. Start wHG 880 against absent topology, then load green membership for + * hostgroups 882 and 883 before publishing AVAILABLE. + * 4. Verify all three orders produce complete runtime green membership. + */ + +#include +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster before_available { bgd_cluster_3_init() }; + RDS_BGD_Cluster after_discovery { bgd_cluster_1_deployment_b_init() }; + RDS_BGD_Cluster after_worker_start { bgd_cluster_init() }; + BGD_Hostgroups before_available_hg { 860, 861, 862, 863 }; + BGD_Hostgroups after_discovery_hg { 870, 871, 872, 873 }; + BGD_Hostgroups after_worker_start_hg { 880, 881, 882, 883 }; + vector before_available_endpoints { before_available.get_endpoints() }; + vector after_discovery_endpoints { after_discovery.get_endpoints() }; + vector after_worker_start_endpoints { after_worker_start.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int configure_monitor(MYSQL* admin, BGD_Hostgroups& hg) { + vector queries { + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + ")", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "SET mysql-monitor_read_only_interval=100", + "SET mysql-monitor_aws_rds_topology_discovery_interval=1", + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int insert_explicit_bgd_row(MYSQL* admin, BGD_Hostgroups& hg, string comment) { + string query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + "," + + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ",1,0,100,800," + bgd_sql_quote(comment) + ")"; + + int rc = mysql_query(admin, query.c_str()); + if (rc != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int add_blue_servers(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + int add_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (add_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, queries); + if (load_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int add_green_servers(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + int add_rc = bgd_admin_add_servers(admin, cluster, hg, green_servers, true, 0); + if (add_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, queries); + if (load_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +bool runtime_green_membership_matches(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector hostgroups { hg.green_writer, hg.green_reader }; + auto [rc, rows] = bgd_runtime_servers(admin, hostgroups); + if (rc != EXIT_SUCCESS || rows.size() != 3) { + return false; + } + + vector> expected { + { hg.green_writer, cluster.green_writer.hostname }, + { hg.green_reader, cluster.green_readers[0].hostname }, + { hg.green_reader, cluster.green_readers[1].hostname }, + }; + + for (pair& server : expected) { + bool found = false; + for (mysql_res_row& row : rows) { + if (row.size() == 5 && row[0] == to_string(server.first) && row[1] == server.second) { + found = true; + break; + } + } + if (!found) { + return false; + } + } + return true; +} + +/** + * Load complete green membership before the first AVAILABLE observation. + * + * - Configure wHG 860 and all blue/green mysql_servers rows. + * - Publish AVAILABLE topology after all membership exists. + * - Verify runtime hostgroups 862 and 863 contain the configured green set. + */ +int test_green_before_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.before_available; + BGD_Hostgroups& hg = state.before_available_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure green-before-AVAILABLE simulated writers"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for wHG 860"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "green membership before AVAILABLE"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert the explicit BGD row for wHG 860"); + return EXIT_FAILURE; + } + + int blue_rc = add_blue_servers(admin, cluster, hg); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to load blue membership for wHG 860"); + return EXIT_FAILURE; + } + + int green_rc = add_green_servers(admin, cluster, hg); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to load green membership for hostgroups 862 and 863"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.before_available_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 860"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 860 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_green_membership_matches(admin, cluster, hg); + ok(membership_matches, "green membership loaded before AVAILABLE appears in hostgroups 862 and 863"); + return EXIT_SUCCESS; +} + +/** + * Load green membership after the explicit worker discovers AVAILABLE. + * + * - Publish AVAILABLE and start wHG 870 with blue membership only. + * - Load the configured green writer/readers into hostgroups 872 and 873. + * - Verify runtime_mysql_servers contains the complete green membership after + * the worker observes the configuration change. + */ +int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.after_discovery; + BGD_Hostgroups& hg = state.after_discovery_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure green-after-discovery simulated writers"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.after_discovery_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 870"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for wHG 870"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "green membership after discovery"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert the explicit BGD row for wHG 870"); + return EXIT_FAILURE; + } + + int blue_rc = add_blue_servers(admin, cluster, hg); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to load blue membership for wHG 870"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 870 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before loading hostgroups 872 and 873"); + return EXIT_FAILURE; + } + + int green_rc = add_green_servers(admin, cluster, hg); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to load green membership for hostgroups 872 and 873"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: wHG 870 did not probe the green writer after membership load"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_green_membership_matches(admin, cluster, hg); + ok(membership_matches, "green membership loaded after discovery appears in hostgroups 872 and 873"); + return EXIT_SUCCESS; +} + +/** + * Start an explicit worker before topology and green membership exist. + * + * - Start wHG 880 with blue membership against absent topology. + * - Load green membership into hostgroups 882 and 883. + * - Publish AVAILABLE and verify complete runtime green membership. + */ +int test_green_after_worker_start(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.after_worker_start; + BGD_Hostgroups& hg = state.after_worker_start_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure green-after-worker-start simulated writers"); + return EXIT_FAILURE; + } + + int drop_rc = sim.topology_drop(state.after_worker_start_endpoints); + if (drop_rc != EXIT_SUCCESS) { + diag("Error: failed to publish absent topology for wHG 880"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for wHG 880"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "green membership after worker start"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert the explicit BGD row for wHG 880"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before starting wHG 880"); + return EXIT_FAILURE; + } + + int blue_rc = add_blue_servers(admin, cluster, hg); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to load blue membership for wHG 880"); + return EXIT_FAILURE; + } + + auto [start_rc, start_probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (start_rc != EXIT_SUCCESS) { + diag("Error: wHG 880 did not start the blue table-check probe"); + return EXIT_FAILURE; + } + + int green_rc = add_green_servers(admin, cluster, hg); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to load green membership for hostgroups 882 and 883"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.after_worker_start_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 880"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 880 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_green_membership_matches(admin, cluster, hg); + ok(membership_matches, "green membership loaded after worker start appears in hostgroups 882 and 883"); + return EXIT_SUCCESS; +} + +int main() { + plan(3); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // ProxySQL: load wHG 860 and complete blue/green membership before AVAILABLE. + // Simulator: publish AVAILABLE topology after membership exists. + // Verify: runtime_mysql_servers contains the configured green writer/readers in hostgroups 862 and 863. + if (test_green_before_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish AVAILABLE topology and start wHG 870 with blue membership. + // ProxySQL: load green writer/readers into hostgroups 872 and 873 after discovery. + // Verify: runtime_mysql_servers converges on complete green membership. + if (test_green_after_discovery(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: start wHG 880 against absent topology, then load green hostgroups 882 and 883. + // Simulator: publish AVAILABLE after the worker and green membership exist. + // Verify: runtime_mysql_servers converges on complete green membership. + if (test_green_after_worker_start(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp b/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp new file mode 100644 index 0000000000..181915b473 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp @@ -0,0 +1,552 @@ +/** + * @file test_rds_bgd_green_pool_cleanup-t.cpp + * @brief BGD rollback and successful cleanup for public green-server statuses. + * + * Steps: + * + * 1. Configure ONLINE, SHUNNED, OFFLINE_SOFT, and OFFLINE_HARD green rows and + * establish one causal connection pool for each hostname. + * 2. Roll back SWITCHOVER_IN_PROGRESS to AVAILABLE and verify that every + * green pool and configured status is preserved. + * 3. Complete writer and reader switchover, then publish empty topology. + * 4. Verify that cleanup drains ONLINE and SHUNNED pools, preserves + * OFFLINE_SOFT and OFFLINE_HARD pools, and retains all configured rows. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const int kRouterHostgroup = 1350; + +struct GreenServer { + int hostgroup; + RDS_BGD_Host host; + string status; +}; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_3_init() }; + RDS_BGD_Cluster extra { bgd_cluster_1_deployment_b_init() }; + BGD_Hostgroups hostgroups { 1300, 1301, 1302, 1303 }; + vector topology_endpoints { cluster.get_endpoints() }; + vector servers { + { hostgroups.green_writer, cluster.green_writer, "ONLINE" }, + { hostgroups.green_reader, cluster.green_readers[0], "SHUNNED" }, + { hostgroups.green_reader, cluster.green_readers[1], "OFFLINE_SOFT" }, + { hostgroups.green_reader, extra.green_readers[0], "OFFLINE_HARD" }, + }; + vector pool_before {}; + vector pool_after {}; + vector admin_snapshot {}; + vector runtime_snapshot {}; + + TestState() { + topology_endpoints.push_back(extra.green_readers[0].endpoint()); + } +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + for (RDS_BGD_Host& host : cluster.blue_readers) { + rows.push_back({ + host.hostname, + host.hostname, + host.port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + } + for (RDS_BGD_Host& host : cluster.green_readers) { + rows.push_back({ + host.hostname, + host.hostname, + host.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + } + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int add_server(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, string status) { + string query = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hostgroup) + "," + bgd_sql_quote(host.hostname) + "," + to_string(host.port) + + "," + bgd_sql_quote(status) + ",0," + bgd_sql_quote("BGD TAP pool " + host.ip) + ")"; + + int rc = mysql_query(admin, query.c_str()); + if (rc != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int set_server_status(MYSQL* admin, GreenServer& server) { + string query = + "UPDATE mysql_servers SET status=" + bgd_sql_quote(server.status) + + " WHERE hostgroup_id=" + to_string(server.hostgroup) + + " AND hostname=" + bgd_sql_quote(server.host.hostname) + + " AND port=" + to_string(server.host.port); + + int rc = mysql_query(admin, query.c_str()); + if (rc != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int create_pool(CommandLine& cl, MYSQL* admin, int hostgroup) { + int user_rc = set_default_hostgroup(admin, hostgroup); + if (user_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return EXIT_FAILURE; + } + + auto [echo_rc, echo] = bgd_backend_ip_echo(client); + mysql_close(client); + return echo_rc; +} + +rc_t pool_for_hostname(MYSQL* admin, string hostname) { + string query = + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool WHERE srv_host=" + + bgd_sql_quote(hostname); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + rc_t result { EXIT_FAILURE, 0 }; + return result; + } + + int64_t count = strtoll(rows[0][0].c_str(), nullptr, 10); + rc_t result { EXIT_SUCCESS, count }; + return result; +} + +rc_t> green_snapshot(MYSQL* admin, string table, BGD_Hostgroups& hg) { + string query = + "SELECT hostgroup_id,hostname,port,status,use_ssl,weight,max_connections FROM " + table + + " WHERE hostgroup_id IN (" + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ") ORDER BY hostgroup_id,hostname,port"; + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +int read_pools(MYSQL* admin, vector& servers, vector& pools) { + pools.clear(); + for (GreenServer& server : servers) { + auto [pool_rc, pool] = pool_for_hostname(admin, server.host.hostname); + if (pool_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + pools.push_back(pool); + } + return EXIT_SUCCESS; +} + +bool all_pools_nonzero(vector& pools) { + if (pools.size() != 4) { + return false; + } + + for (int64_t pool : pools) { + if (pool < 1) { + return false; + } + } + return true; +} + +int configure_status_matrix(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1300"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1300-1303"); + return EXIT_FAILURE; + } + + int extra_rc = add_server(admin, hg.green_reader, state.extra.green_readers[0], "ONLINE"); + if (extra_rc != EXIT_SUCCESS) { + diag("Error: failed to add the OFFLINE_HARD green reader to hostgroup 1303"); + return EXIT_FAILURE; + } + + for (size_t i = 0; i < state.servers.size(); ++i) { + int router_rc = add_server(admin, kRouterHostgroup + static_cast(i), state.servers[i].host, "ONLINE"); + if (router_rc != EXIT_SUCCESS) { + diag("Error: failed to add pool-router row for green status index %zu", i); + return EXIT_FAILURE; + } + } + + for (GreenServer& server : state.servers) { + int status_rc = set_server_status(admin, server); + if (status_rc != EXIT_SUCCESS) { + diag("Error: failed to set %s for green server %s", server.status.c_str(), server.host.hostname.c_str()); + return EXIT_FAILURE; + } + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load the green status matrix to runtime"); + return EXIT_FAILURE; + } + + auto [admin_snapshot_rc, admin_snapshot] = green_snapshot(admin, "mysql_servers", hg); + if (admin_snapshot_rc != EXIT_SUCCESS || admin_snapshot.size() != 4) { + diag("Error: failed to snapshot four persistent green status rows"); + return EXIT_FAILURE; + } + state.admin_snapshot = admin_snapshot; + + auto [runtime_snapshot_rc, runtime_snapshot] = green_snapshot(admin, "runtime_mysql_servers", hg); + if (runtime_snapshot_rc != EXIT_SUCCESS || runtime_snapshot.size() != 3) { + diag("Error: failed to snapshot ONLINE, SHUNNED, and OFFLINE_SOFT runtime rows"); + return EXIT_FAILURE; + } + state.runtime_snapshot = runtime_snapshot; + + for (size_t i = 0; i < state.servers.size(); ++i) { + int pool_rc = create_pool(cl, admin, kRouterHostgroup + static_cast(i)); + if (pool_rc != EXIT_SUCCESS) { + diag("Error: failed to create causal pool for green status index %zu", i); + return EXIT_FAILURE; + } + } + + int user_rc = set_default_hostgroup(admin, hg.blue_writer); + if (user_rc != EXIT_SUCCESS) { + diag("Error: failed to restore testuser to writer hostgroup 1300"); + return EXIT_FAILURE; + } + + int pools_rc = read_pools(admin, state.servers, state.pool_before); + if (pools_rc != EXIT_SUCCESS || !all_pools_nonzero(state.pool_before)) { + diag("Error: every green status must have a nonzero pool before lifecycle changes"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_topology(RDS_BGD_Simulator& sim, TestState& state, string status) { + vector topology = topology_with_readers(state.cluster, status); + + int rc = sim.topology_update(state.topology_endpoints, topology); + return rc; +} + +bool snapshots_match(MYSQL* admin, TestState& state) { + auto [admin_rc, admin_rows] = green_snapshot(admin, "mysql_servers", state.hostgroups); + auto [runtime_rc, runtime_rows] = green_snapshot(admin, "runtime_mysql_servers", state.hostgroups); + if (admin_rc != EXIT_SUCCESS || runtime_rc != EXIT_SUCCESS) { + return false; + } + + bool matches = admin_rows == state.admin_snapshot && runtime_rows == state.runtime_snapshot; + return matches; +} + +/** + * Roll back writer switchover with four green status pools. + * + * - Configure ONLINE, SHUNNED, OFFLINE_SOFT, and OFFLINE_HARD green rows. + * - Establish a nonzero causal pool for every green hostname. + * - Publish AVAILABLE, SWITCHOVER_IN_PROGRESS, then AVAILABLE. + * - Verify rollback preserves every green pool and exact configured row. + */ +int test_rollback_preserves_green_pools(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + int config_rc = configure_status_matrix(cl, admin, sim, state); + if (config_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int available_rc = publish_topology(sim, state, "AVAILABLE"); + if (available_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 1300"); + return EXIT_FAILURE; + } + + int available_status_rc = bgd_wait_for_status(admin, state.hostgroups, "AVAILABLE", kTimeoutSeconds); + if (available_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + int progress_rc = publish_topology(sim, state, "SWITCHOVER_IN_PROGRESS"); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG 1300"); + return EXIT_FAILURE; + } + + int progress_status_rc = + bgd_wait_for_status(admin, state.hostgroups, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int rollback_rc = publish_topology(sim, state, "AVAILABLE"); + if (rollback_rc != EXIT_SUCCESS) { + diag("Error: failed to publish rollback AVAILABLE topology for wHG 1300"); + return EXIT_FAILURE; + } + + int rollback_status_rc = bgd_wait_for_status(admin, state.hostgroups, "AVAILABLE", kTimeoutSeconds); + if (rollback_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not return to AVAILABLE"); + return EXIT_FAILURE; + } + + vector pools_after_rollback {}; + int pools_rc = read_pools(admin, state.servers, pools_after_rollback); + if (pools_rc != EXIT_SUCCESS || pools_after_rollback.size() != state.pool_before.size()) { + diag("Error: failed to read green pools after rollback"); + return EXIT_FAILURE; + } + + bool pools_preserved = true; + for (size_t i = 0; i < state.pool_before.size(); ++i) { + if (pools_after_rollback[i] < state.pool_before[i]) { + pools_preserved = false; + } + } + ok(pools_preserved, "AVAILABLE rollback preserves all four green status pools for wHG 1300"); + + bool rows_preserved = snapshots_match(admin, state); + ok(rows_preserved, "AVAILABLE rollback preserves the configured green rows and public statuses for wHG 1300"); + return EXIT_SUCCESS; +} + +/** + * Complete reader cleanup and drain eligible green pools. + * + * - Publish POST_PROCESSING and target-only SWITCHOVER_COMPLETED. + * - Require every green pool to remain nonzero immediately before cleanup. + * - Delete topology rows and verify that ONLINE and SHUNNED pools drain. + */ +int test_successful_cleanup_drains_non_offline(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + int post_rc = publish_topology(sim, state, "SWITCHOVER_IN_POST_PROCESSING"); + if (post_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for wHG 1300"); + return EXIT_FAILURE; + } + + int post_status_rc = + bgd_wait_for_status(admin, state.hostgroups, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (post_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + vector completed = target_only_completed(state.cluster); + int completed_rc = sim.topology_update(state.topology_endpoints, completed); + if (completed_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG 1300"); + return EXIT_FAILURE; + } + + int reader_status_rc = + bgd_wait_for_status(admin, state.hostgroups, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (reader_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not reach READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int baseline_rc = read_pools(admin, state.servers, state.pool_before); + if (baseline_rc != EXIT_SUCCESS || !all_pools_nonzero(state.pool_before)) { + diag("Error: every green status must have a nonzero pool immediately before reader cleanup"); + return EXIT_FAILURE; + } + + int empty_rc = sim.topology_delete(state.topology_endpoints); + if (empty_rc != EXIT_SUCCESS) { + diag("Error: failed to publish empty topology for wHG 1300"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, state.hostgroups, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not reach NONE during reader cleanup"); + return EXIT_FAILURE; + } + + int pools_rc = read_pools(admin, state.servers, state.pool_after); + if (pools_rc != EXIT_SUCCESS || state.pool_after.size() != 4) { + diag("Error: failed to read green pools after reader cleanup"); + return EXIT_FAILURE; + } + + bool non_offline_drained = state.pool_after[0] == 0 && state.pool_after[1] == 0; + ok(non_offline_drained, "reader cleanup drains ONLINE and SHUNNED green pools for wHG 1300"); + return EXIT_SUCCESS; +} + +/** + * Preserve offline pools and configured green rows during successful cleanup. + * + * - Compare OFFLINE_SOFT and OFFLINE_HARD pools with their causal baselines. + * - Verify persistent and runtime green rows still match their pre-lifecycle + * snapshots. + */ +int test_cleanup_preserves_offline_pools(MYSQL* admin, TestState& state) { + if (state.pool_before.size() != 4 || state.pool_after.size() != 4) { + diag("Error: green pool baselines are incomplete after reader cleanup"); + return EXIT_FAILURE; + } + + bool offline_pools_preserved = + state.pool_after[2] == state.pool_before[2] && + state.pool_after[3] == state.pool_before[3]; + ok(offline_pools_preserved, "reader cleanup preserves OFFLINE_SOFT and OFFLINE_HARD green pools for wHG 1300"); + + bool rows_preserved = snapshots_match(admin, state); + ok(rows_preserved, "reader cleanup retains all configured green rows and public statuses for wHG 1300"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // ProxySQL: configure four public green statuses and establish one causal pool for each hostname. + // Simulator: publish AVAILABLE, SWITCHOVER_IN_PROGRESS, then AVAILABLE. + // Verify: rollback preserves all four pools and exact configured green rows. + if (test_rollback_preserves_green_pools(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish POST_PROCESSING, target-only SWITCHOVER_COMPLETED, then empty topology. + // Verify: reader cleanup drains ONLINE and SHUNNED green pools. + if (test_successful_cleanup_drains_non_offline(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Verify: reader cleanup preserves OFFLINE_SOFT/OFFLINE_HARD pools and configured green rows. + if (test_cleanup_preserves_offline_pools(admin, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp b/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp new file mode 100644 index 0000000000..6e212f11b2 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp @@ -0,0 +1,349 @@ +/** + * @file test_rds_bgd_late_entry_completed-t.cpp + * @brief Starting a BGD worker from target-only SWITCHOVER_COMPLETED. + * + * Steps: + * + * 1. Publish target-only SWITCHOVER_COMPLETED before configuring wHG 1200. + * 2. Verify the first observation enters READER_SWITCHOVER_IN_PROGRESS + * without rebuilding writer-switchover routing or placement. + * 3. Establish green writer/reader pools. + * 4. Publish empty topology and verify status NONE and green-pool cleanup. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNoProbeTimeoutMs = 1200; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_1_deployment_b_init() }; + BGD_Hostgroups hostgroups { 1200, 1201, 1202, 1203 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int wait_for_blue_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { + auto [probe_rc, probe] = + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + return probe_rc; +} + +bool runtime_server_match(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, string status) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status=" + bgd_sql_quote(status); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +int wait_for_green_pool_drain(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "SELECT " + "(SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool WHERE hostgroup=" + + to_string(hg.green_writer) + ")=0 AND " + "(SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool WHERE hostgroup=" + + to_string(hg.green_reader) + ")=0"; + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +/** + * Start wHG 1200 from target-only SWITCHOVER_COMPLETED. + * + * - Publish SWITCHOVER_COMPLETED before configuring the BGD row. + * - Verify READER_SWITCHOVER_IN_PROGRESS and a blue metadata probe. + * - Verify no direct green metadata probe, blue routing remains active, and + * the blue writer is not demoted. + * - Establish green writer/reader pools for terminal cleanup. + */ +int test_first_completed(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1200"); + return EXIT_FAILURE; + } + + auto [publish_seq_rc, publish_seq] = sim.probe_log_last_sequence(); + if (publish_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the first COMPLETED probe sequence"); + return EXIT_FAILURE; + } + + vector topology = target_only_completed(cluster); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1200-1203"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1200 did not reach READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int blue_probe_rc = wait_for_blue_writer(sim, publish_seq, cluster); + if (blue_probe_rc != EXIT_SUCCESS) { + diag("Error: first COMPLETED observation did not probe the blue writer"); + return EXIT_FAILURE; + } + + ok(true, "first SWITCHOVER_COMPLETED observation reports READER_SWITCHOVER_IN_PROGRESS for wHG 1200"); + + int no_green_rc = bgd_expect_no_metadata_probe(sim, publish_seq, cluster.green_writer.endpoint(), kNoProbeTimeoutMs); + if (no_green_rc != EXIT_SUCCESS) { + diag("Error: first COMPLETED observation rebuilt a direct green metadata probe"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: first COMPLETED observation changed blue-writer placement"); + return EXIT_FAILURE; + } + + auto [blue_echo_rc, blue_echo] = connect_and_echo(cl); + bool blue_routing = blue_echo_rc == EXIT_SUCCESS && blue_echo.find(cluster.blue_writer.ip) != string::npos; + bool reader_online = runtime_server_match(admin, hg.blue_reader, cluster.blue_readers[1], "ONLINE"); + ok(blue_routing && reader_online, + "first SWITCHOVER_COMPLETED observation keeps blue routing without writer-phase pins or demotion"); + + int writer_hg_rc = set_default_hostgroup(admin, hg.green_writer); + if (writer_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green writer hostgroup 1202"); + return EXIT_FAILURE; + } + + int writer_echo_rc = connect_and_echo(cl).first; + if (writer_echo_rc != EXIT_SUCCESS) { + diag("Error: failed to establish a green-writer pool before terminal cleanup"); + return EXIT_FAILURE; + } + + int reader_hg_rc = set_default_hostgroup(admin, hg.green_reader); + if (reader_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green reader hostgroup 1203"); + return EXIT_FAILURE; + } + + int reader_echo_rc = connect_and_echo(cl).first; + if (reader_echo_rc != EXIT_SUCCESS) { + diag("Error: failed to establish a green-reader pool before terminal cleanup"); + return EXIT_FAILURE; + } + + int restore_hg_rc = set_default_hostgroup(admin, hg.blue_writer); + if (restore_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to restore the test user to blue writer hostgroup 1200"); + return EXIT_FAILURE; + } + + auto [writer_pool_rc, writer_pool] = bgd_connection_pool_count(admin, hg.green_writer); + auto [reader_pool_rc, reader_pool] = bgd_connection_pool_count(admin, hg.green_reader); + if (writer_pool_rc != EXIT_SUCCESS || writer_pool < 1 || reader_pool_rc != EXIT_SUCCESS || reader_pool < 1) { + diag("Error: green pools are empty before terminal completed cleanup"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +/** + * Publish empty topology during direct-entry reader switchover. + * + * - Remove the simulated topology after first-observation COMPLETED. + * - Verify BGD status NONE. + * - Verify eligible green writer/reader pools are drained. + * - Verify blue-writer and blue-reader placement remains available. + */ +int test_completed_empty_topology(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int topology_rc = sim.topology_delete(state.topology_endpoints); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish empty topology for wHG 1200"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1200 did not reach NONE after empty topology"); + return EXIT_FAILURE; + } + + int drain_rc = wait_for_green_pool_drain(admin, hg); + if (drain_rc != EXIT_SUCCESS) { + diag("Error: empty topology did not drain green pools for hostgroups 1202-1203"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: empty topology changed blue-writer placement for wHG 1200"); + return EXIT_FAILURE; + } + + bool reader_online = runtime_server_match(admin, hg.blue_reader, cluster.blue_readers[1], "ONLINE"); + ok(reader_online, "empty topology reaches NONE, drains green pools, and keeps blue hostgroups 1200-1201 available"); + return EXIT_SUCCESS; +} + +int main() { + plan(3); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish target-only SWITCHOVER_COMPLETED before wHG 1200 is configured. + // Verify: first observation enters READER_SWITCHOVER_IN_PROGRESS without green pins or writer demotion. + // Client: establish green writer/reader pools for terminal cleanup. + if (test_first_completed(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish empty topology during READER_SWITCHOVER_IN_PROGRESS. + // Verify: BGD status reaches NONE and green writer/reader pools are drained. + // Verify: blue writer and reader hostgroups remain available. + if (test_completed_empty_topology(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp b/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp new file mode 100644 index 0000000000..970378605e --- /dev/null +++ b/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp @@ -0,0 +1,624 @@ +/** + * @file test_rds_bgd_late_entry_writer_phases-t.cpp + * @brief Starting a BGD worker from each writer switchover phase. + * + * Steps: + * + * 1. Start wHG 1170 when topology already reports SWITCHOVER_INITIATED and + * verify read_only suppression without blue-writer demotion. + * 2. Start wHG 1180 when topology already reports SWITCHOVER_IN_PROGRESS and + * verify prerequisite construction before blue-writer demotion. + * 3. Create a blue-writer pool, then start wHG 1190 when topology already + * reports SWITCHOVER_IN_POST_PROCESSING. + * 4. Verify green routing, blue-pool drain, writer placement, and reader + * read_only suppression from the first POST_PROCESSING observation. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kReadOnlyObservationMs = 500; + +struct TestState { + RDS_BGD_Cluster initiated_cluster { bgd_cluster_init() }; + BGD_Hostgroups initiated_hg { 1170, 1171, 1172, 1173 }; + vector initiated_endpoints { initiated_cluster.get_endpoints() }; + + RDS_BGD_Cluster progress_cluster { bgd_cluster_2_init() }; + BGD_Hostgroups progress_hg { 1180, 1181, 1182, 1183 }; + vector progress_endpoints { progress_cluster.get_endpoints() }; + + RDS_BGD_Cluster post_cluster { bgd_cluster_3_init() }; + BGD_Hostgroups post_hg { 1190, 1191, 1192, 1193 }; + vector post_endpoints { post_cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); + + int rc = sim.topology_update(endpoints, topology); + return rc; +} + +int wait_for_green_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { + auto [probe_rc, probe] = + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + return probe_rc; +} + +bool runtime_server_match(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, string status) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status=" + bgd_sql_quote(status); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +int64_t last_read_only_log_time(MYSQL* admin, RDS_BGD_Host& host) { + string query = + "SELECT COALESCE(MAX(time_start_us),0) FROM mysql_server_read_only_log WHERE hostname=" + + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return -1; + } + + int64_t time = strtoll(rows[0][0].c_str(), nullptr, 10); + return time; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +int wait_for_blue_writer_pool_drain(MYSQL* admin, RDS_BGD_Cluster& cluster) { + string query = + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)=0 FROM stats_mysql_connection_pool WHERE srv_host=" + + bgd_sql_quote(cluster.blue_writer.hostname); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +int configure_servers_without_worker(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector config_queries { + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + ")", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "SET mysql-monitor_read_only_interval=100", + "SET mysql-monitor_aws_rds_topology_discovery_interval=1", + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "UPDATE mysql_users SET default_hostgroup=" + to_string(hg.blue_writer) + " WHERE username='testuser'", + }; + + int config_rc = execute_all(admin, config_queries); + if (config_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + int blue_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (blue_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int green_rc = bgd_admin_add_servers(admin, cluster, hg, green_servers, true, 0); + if (green_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector load_queries { + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL USERS TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int enable_bgd_worker(MYSQL* admin, BGD_Hostgroups& hg) { + string insert_bgd = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + "," + + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ",1,0,100,800,'BGD TAP late writer phase')"; + vector queries { + insert_bgd, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +/** + * Start wHG 1170 from SWITCHOVER_INITIATED. + * + * - Publish SWITCHOVER_INITIATED before configuring the BGD row. + * - Verify WRITER_SWITCHOVER_INITIATED without blue-writer demotion. + * - Change simulated writer/reader read_only values. + * - Verify read_only monitoring is suppressed for deployment members. + */ +int test_first_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.initiated_cluster; + BGD_Hostgroups& hg = state.initiated_hg; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1170"); + return EXIT_FAILURE; + } + + auto [publish_seq_rc, publish_seq] = sim.probe_log_last_sequence(); + if (publish_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the first INITIATED probe sequence"); + return EXIT_FAILURE; + } + + int topology_rc = publish_topology(sim, state.initiated_endpoints, cluster, "SWITCHOVER_INITIATED"); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED topology for wHG 1170"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1170-1173"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1170 did not reach WRITER_SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + int probe_rc = wait_for_green_writer(sim, publish_seq, cluster); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: first INITIATED observation did not probe the green writer"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: first INITIATED observation changed blue-writer placement"); + return EXIT_FAILURE; + } + + ok(true, "first SWITCHOVER_INITIATED observation keeps the blue writer in hostgroup 1170"); + + int64_t writer_log = last_read_only_log_time(admin, cluster.blue_writer); + int64_t reader_log = last_read_only_log_time(admin, cluster.blue_readers[0]); + if (writer_log < 0 || reader_log < 0) { + diag("Error: failed to read INITIATED read_only log baselines"); + return EXIT_FAILURE; + } + + auto [suppression_seq_rc, suppression_seq] = sim.probe_log_last_sequence(); + if (suppression_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the INITIATED suppression probe sequence"); + return EXIT_FAILURE; + } + + int writer_update_rc = bgd_set_host_read_only_1(sim, cluster.blue_writer); + if (writer_update_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=1 for the simulated blue writer"); + return EXIT_FAILURE; + } + + int reader_update_rc = bgd_set_host_read_only_0(sim, cluster.blue_readers[0]); + if (reader_update_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue reader"); + return EXIT_FAILURE; + } + + int suppression_probe_rc = wait_for_green_writer(sim, suppression_seq, cluster); + if (suppression_probe_rc != EXIT_SUCCESS) { + diag("Error: INITIATED suppression check did not observe the green writer"); + return EXIT_FAILURE; + } + + int writer_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_writer, writer_log, kReadOnlyObservationMs); + if (writer_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-writer read_only monitoring was not suppressed on first SWITCHOVER_INITIATED observation"); + return EXIT_FAILURE; + } + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], reader_log, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed on first SWITCHOVER_INITIATED observation"); + return EXIT_FAILURE; + } + + bool reader_online = runtime_server_match(admin, hg.blue_reader, cluster.blue_readers[0], "ONLINE"); + ok(reader_online, "first SWITCHOVER_INITIATED observation suppresses writer and reader read_only placement changes"); + return EXIT_SUCCESS; +} + +/** + * Start wHG 1180 from SWITCHOVER_IN_PROGRESS. + * + * - Publish SWITCHOVER_IN_PROGRESS before configuring the BGD row. + * - Verify WRITER_SWITCHOVER_IN_PROGRESS and blue-writer demotion. + * - Change the simulated blue-reader read_only value. + * - Verify read_only monitoring remains suppressed after demotion. + */ +int test_first_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.progress_cluster; + BGD_Hostgroups& hg = state.progress_hg; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1180"); + return EXIT_FAILURE; + } + + auto [publish_seq_rc, publish_seq] = sim.probe_log_last_sequence(); + if (publish_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the first IN_PROGRESS probe sequence"); + return EXIT_FAILURE; + } + + int topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "SWITCHOVER_IN_PROGRESS"); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG 1180"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1180-1183"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1180 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int probe_rc = wait_for_green_writer(sim, publish_seq, cluster); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: first IN_PROGRESS observation did not probe the green writer"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: first IN_PROGRESS observation did not demote the blue writer"); + return EXIT_FAILURE; + } + + ok(true, "first SWITCHOVER_IN_PROGRESS observation moves the blue writer from hostgroup 1180 to 1181"); + + int64_t reader_log = last_read_only_log_time(admin, cluster.blue_readers[0]); + if (reader_log < 0) { + diag("Error: failed to read the IN_PROGRESS blue-reader log baseline"); + return EXIT_FAILURE; + } + + auto [suppression_seq_rc, suppression_seq] = sim.probe_log_last_sequence(); + if (suppression_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the IN_PROGRESS suppression probe sequence"); + return EXIT_FAILURE; + } + + int reader_update_rc = bgd_set_host_read_only_0(sim, cluster.blue_readers[0]); + if (reader_update_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue reader"); + return EXIT_FAILURE; + } + + int suppression_probe_rc = wait_for_green_writer(sim, suppression_seq, cluster); + if (suppression_probe_rc != EXIT_SUCCESS) { + diag("Error: IN_PROGRESS suppression check did not observe the green writer"); + return EXIT_FAILURE; + } + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], reader_log, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed on first SWITCHOVER_IN_PROGRESS observation"); + return EXIT_FAILURE; + } + + bool reader_online = runtime_server_match(admin, hg.blue_reader, cluster.blue_readers[0], "ONLINE"); + ok(reader_online, "first SWITCHOVER_IN_PROGRESS observation suppresses blue-reader read_only placement changes"); + return EXIT_SUCCESS; +} + +/** + * Start wHG 1190 from SWITCHOVER_IN_POST_PROCESSING. + * + * - Publish POST_PROCESSING and create a blue-writer pool before enabling BGD. + * - Verify WRITER_SWITCHOVER_POST_PROCESSING and restored blue-writer placement. + * - Verify the old blue pool drains and new connections route to green. + * - Verify mapped blue readers remain under read_only suppression. + */ +int test_first_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.post_cluster; + BGD_Hostgroups& hg = state.post_hg; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1190"); + return EXIT_FAILURE; + } + + auto [publish_seq_rc, publish_seq] = sim.probe_log_last_sequence(); + if (publish_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the first POST_PROCESSING probe sequence"); + return EXIT_FAILURE; + } + + int topology_rc = publish_topology(sim, state.post_endpoints, cluster, "SWITCHOVER_IN_POST_PROCESSING"); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for wHG 1190"); + return EXIT_FAILURE; + } + + int servers_rc = configure_servers_without_worker(admin, cluster, hg); + if (servers_rc != EXIT_SUCCESS) { + diag("Error: failed to configure hostgroups 1190-1193 without a BGD worker"); + return EXIT_FAILURE; + } + + auto [blue_echo_rc, blue_echo] = connect_and_echo(cl); + if (blue_echo_rc != EXIT_SUCCESS || blue_echo.find(cluster.blue_writer.ip) == string::npos) { + diag("Error: failed to establish the pre-worker blue-writer pool"); + return EXIT_FAILURE; + } + + auto [pool_before_rc, pool_before] = bgd_connection_pool_count(admin, hg.blue_writer, cluster.blue_writer.hostname); + if (pool_before_rc != EXIT_SUCCESS || pool_before < 1) { + diag("Error: blue-writer pool is empty before enabling wHG 1190"); + return EXIT_FAILURE; + } + + int worker_rc = enable_bgd_worker(admin, hg); + if (worker_rc != EXIT_SUCCESS) { + diag("Error: failed to enable BGD worker for wHG 1190"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1190 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + int probe_rc = wait_for_green_writer(sim, publish_seq, cluster); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: first POST_PROCESSING observation did not probe the green writer"); + return EXIT_FAILURE; + } + + ok(true, "first POST_PROCESSING observation reports WRITER_SWITCHOVER_POST_PROCESSING for wHG 1190"); + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: first POST_PROCESSING observation did not keep the blue writer in hostgroup 1190"); + return EXIT_FAILURE; + } + + int drain_rc = wait_for_blue_writer_pool_drain(admin, cluster); + if (drain_rc != EXIT_SUCCESS) { + diag("Error: first POST_PROCESSING observation did not drain the old blue-writer pool"); + return EXIT_FAILURE; + } + + auto [green_echo_rc, green_echo] = connect_and_echo(cl); + bool green_routing = green_echo_rc == EXIT_SUCCESS && green_echo.find(cluster.green_writer.ip) != string::npos; + ok(green_routing, + "first POST_PROCESSING observation restores hostgroup 1190, drains its blue pool, and routes to green"); + + int64_t reader_log = last_read_only_log_time(admin, cluster.blue_readers[0]); + if (reader_log < 0) { + diag("Error: failed to read the POST_PROCESSING blue-reader log baseline"); + return EXIT_FAILURE; + } + + auto [suppression_seq_rc, suppression_seq] = sim.probe_log_last_sequence(); + if (suppression_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the POST_PROCESSING suppression probe sequence"); + return EXIT_FAILURE; + } + + int reader_update_rc = bgd_set_host_read_only_0(sim, cluster.blue_readers[0]); + if (reader_update_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue reader"); + return EXIT_FAILURE; + } + + int suppression_probe_rc = wait_for_green_writer(sim, suppression_seq, cluster); + if (suppression_probe_rc != EXIT_SUCCESS) { + diag("Error: POST_PROCESSING suppression check did not observe the green writer"); + return EXIT_FAILURE; + } + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], reader_log, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed on first POST_PROCESSING observation"); + return EXIT_FAILURE; + } + + bool reader_online = runtime_server_match(admin, hg.blue_reader, cluster.blue_readers[0], "ONLINE"); + ok(reader_online, "first POST_PROCESSING observation keeps the mapped blue reader ONLINE in hostgroup 1191"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish SWITCHOVER_INITIATED before wHG 1170 is configured. + // Verify: first observation reports WRITER_SWITCHOVER_INITIATED without writer demotion. + // Verify: writer and reader read_only placement changes are suppressed. + if (test_first_initiated(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS before wHG 1180 is configured. + // Verify: first observation reports WRITER_SWITCHOVER_IN_PROGRESS and moves the writer to hostgroup 1181. + // Verify: blue-reader read_only placement changes are suppressed. + if (test_first_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_POST_PROCESSING before wHG 1190 is configured. + // Client: create a blue-writer pool before enabling the BGD worker. + // Verify: first observation drains the pool, routes to green, restores writer placement, and retains reader placement. + if (test_first_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_probe_tls-t.cpp b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp new file mode 100644 index 0000000000..f6c616306b --- /dev/null +++ b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp @@ -0,0 +1,461 @@ +/** + * @file test_rds_bgd_probe_tls-t.cpp + * @brief Selecting BGD metadata targets and their TLS values. + * + * Steps: + * + * 1. Configure a plaintext blue reader before a TLS blue writer and verify + * automatic discovery uses the matched writer TLS. + * 2. Configure an exact TLS TARGET beside a plaintext distractor and verify + * the recorded TARGET is selected. + * 3. Leave green writer hostgroup 962 empty, set use_ssl=1 defaults, and + * verify the created TARGET row and metadata probe use TLS. + * 4. Verify table-check, blue metadata, and green metadata probe order. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +// Automatic discovery is asynchronous and starts after the monitor observes the +// runtime server. Allow the monitor and the BGD worker to become ready on slower CI runners. +const uint32_t kTimeoutSeconds = 15; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 1200; + +struct TestState { + RDS_BGD_Cluster automatic { bgd_cluster_init() }; + RDS_BGD_Cluster explicit_target { bgd_cluster_2_init() }; + RDS_BGD_Cluster distractor { bgd_cluster_1_deployment_b_init() }; + RDS_BGD_Cluster created_target { bgd_cluster_3_init() }; + BGD_Hostgroups automatic_hg { 940, 941, 942, 943 }; + BGD_Hostgroups explicit_target_hg { 950, 951, 952, 953 }; + BGD_Hostgroups created_target_hg { 960, 961, 962, 963 }; + vector automatic_endpoints { automatic.get_endpoints() }; + vector explicit_target_endpoints { explicit_target.get_endpoints() }; + vector created_target_endpoints { created_target.get_endpoints() }; +}; + +struct ProbeChain { + RDS_BGD_Probe_Log table; + RDS_BGD_Probe_Log blue; + RDS_BGD_Probe_Log green; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + vector attribute_queries { + "DELETE FROM mysql_hostgroup_attributes", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int attribute_rc = execute_all(admin, attribute_queries); + if (attribute_rc != EXIT_SUCCESS) { + diag("Error: failed to clean BGD TLS hostgroup attributes"); + } + + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (attribute_rc != EXIT_SUCCESS || admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int wait_for_probe_chain(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, + int blue_use_ssl, int green_use_ssl, ProbeChain& chain) +{ + auto [table_rc, table] = + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, blue_use_ssl); + if (table_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + auto [blue_rc, blue] = + sim.wait_for_probe_log(table.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, blue_use_ssl); + if (blue_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + auto [green_rc, green] = + sim.wait_for_probe_log(blue.sequence_id, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, green_use_ssl); + if (green_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + chain = { table, blue, green }; + return EXIT_SUCCESS; +} + +bool probe_chain_ordered(ProbeChain& chain) { + bool ordered = + chain.table.sequence_id < chain.blue.sequence_id && + chain.blue.sequence_id < chain.green.sequence_id && + chain.table.probe_kind == RDS_BGD_Probe_Kind::table_check && + chain.blue.probe_kind == RDS_BGD_Probe_Kind::metadata && + chain.green.probe_kind == RDS_BGD_Probe_Kind::metadata; + return ordered; +} + +bool runtime_server_tls_matches(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, int use_ssl) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND use_ssl=" + to_string(use_ssl); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +/** + * Select the automatic blue writer when a reader appears first. + * + * - Load a blue reader with use_ssl=0 before the blue writer with use_ssl=1. + * - Publish AVAILABLE topology. + * - Verify table-check and blue metadata use the writer with TLS. + * - Verify the mapped green writer metadata probe also uses TLS. + */ +int test_automatic_writer_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.automatic; + BGD_Hostgroups& hg = state.automatic_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic TLS simulated writers"); + return EXIT_FAILURE; + } + + vector no_servers {}; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::automatic, no_servers, no_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic TLS discovery for hostgroups 940-943"); + return EXIT_FAILURE; + } + + vector reader { cluster.blue_readers[0] }; + int reader_rc = bgd_admin_add_servers(admin, cluster, hg, reader, false, 0); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to load the plaintext blue reader in hostgroup 941"); + return EXIT_FAILURE; + } + + vector writer { cluster.blue_writer }; + int server_writer_rc = bgd_admin_add_servers(admin, cluster, hg, writer, false, 1); + if (server_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to load the TLS blue writer in hostgroup 940"); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load automatic TLS server rows"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the automatic TLS probe sequence"); + return EXIT_FAILURE; + } + + vector topology = cluster.get_topology("AVAILABLE"); + int topology_rc = sim.topology_update(state.automatic_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish automatic AVAILABLE topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 940 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ProbeChain chain {}; + int chain_rc = wait_for_probe_chain(sim, seq, cluster, 1, 1, chain); + if (chain_rc != EXIT_SUCCESS) { + diag("Error: automatic TLS probe chain did not complete"); + return EXIT_FAILURE; + } + + bool writer_tls = runtime_server_tls_matches(admin, hg.blue_writer, cluster.blue_writer, 1); + bool selected_writer = chain.table.backend.host == cluster.blue_writer.ip && chain.blue.backend.host == cluster.blue_writer.ip; + ok(writer_tls && selected_writer && chain.table.encrypted && chain.blue.encrypted, + "automatic discovery selects the TLS writer in hostgroup 940 instead of the plaintext reader"); + + bool green_target = chain.green.backend.host == cluster.green_writer.ip && chain.green.encrypted; + bool ordered = probe_chain_ordered(chain); + ok(green_target && ordered, "automatic probes run table-check, blue metadata, then TLS green metadata in order"); + return EXIT_SUCCESS; +} + +/** + * Select the exact explicit TARGET beside a valid-looking distractor. + * + * - Configure the exact green writer with use_ssl=1. + * - Configure a different green writer hostname with use_ssl=0 in hostgroup 952. + * - Publish AVAILABLE topology for the exact deployment. + * - Verify the exact TARGET and TLS value are used in probe order. + */ +int test_explicit_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.explicit_target; + RDS_BGD_Cluster& distractor = state.distractor; + BGD_Hostgroups& hg = state.explicit_target_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure explicit TLS simulated writers"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer }; + vector no_green_servers {}; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, + blue_servers, no_green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure explicit TLS hostgroups 950-953"); + return EXIT_FAILURE; + } + + string distractor_query = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hg.green_writer) + "," + bgd_sql_quote(distractor.green_writer.hostname) + + ",3306,'ONLINE',0,'BGD TAP TLS distractor')"; + string target_query = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hg.green_writer) + "," + bgd_sql_quote(cluster.green_writer.hostname) + + ",3306,'ONLINE',1,'BGD TAP exact TARGET')"; + vector target_queries { + distractor_query, + target_query, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int target_rc = execute_all(admin, target_queries); + if (target_rc != EXIT_SUCCESS) { + diag("Error: failed to load exact and distractor TARGET rows in hostgroup 952"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the explicit TLS probe sequence"); + return EXIT_FAILURE; + } + + vector topology_endpoints = state.explicit_target_endpoints; + vector distractor_endpoints = distractor.get_endpoints(); + topology_endpoints.insert(topology_endpoints.end(), distractor_endpoints.begin(), distractor_endpoints.end()); + vector topology = cluster.get_topology("AVAILABLE"); + int topology_rc = sim.topology_update(topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish explicit AVAILABLE topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 950 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ProbeChain chain {}; + int chain_rc = wait_for_probe_chain(sim, seq, cluster, 0, 1, chain); + if (chain_rc != EXIT_SUCCESS) { + diag("Error: explicit TLS probe chain did not complete"); + return EXIT_FAILURE; + } + + bool exact_target = + chain.green.backend.host == cluster.green_writer.ip && + chain.green.backend.host != distractor.green_writer.ip && + chain.green.encrypted; + int no_distractor_rc = bgd_expect_no_metadata_probe(sim, seq, distractor.green_writer.endpoint(), kNegativeProbeTimeoutMs); + if (no_distractor_rc != EXIT_SUCCESS) { + diag("Error: explicit discovery probed the plaintext green-writer distractor"); + return EXIT_FAILURE; + } + + ok(exact_target, "explicit discovery selects only the exact TLS TARGET and rejects the plaintext distractor"); + + bool ordered = probe_chain_ordered(chain); + bool blue_plaintext = !chain.table.encrypted && !chain.blue.encrypted; + ok(ordered && blue_plaintext, "explicit probes run plaintext table-check, blue metadata, then TLS TARGET metadata"); + return EXIT_SUCCESS; +} + +/** + * Apply green hostgroup TLS defaults when ProxySQL creates the TARGET row. + * + * - Leave green writer hostgroup 962 empty. + * - Configure servers_defaults use_ssl=1. + * - Publish AVAILABLE topology. + * - Verify the created TARGET runtime row and metadata probe use TLS. + */ +int test_created_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.created_target; + BGD_Hostgroups& hg = state.created_target_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure created-TARGET simulated writers"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer }; + vector no_green_servers {}; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, + blue_servers, no_green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure created-TARGET hostgroups 960-963"); + return EXIT_FAILURE; + } + + string defaults_query = + "INSERT INTO mysql_hostgroup_attributes(hostgroup_id,servers_defaults) VALUES (" + + to_string(hg.green_writer) + ",' {\"use_ssl\":1 }')"; + vector defaults_queries { + defaults_query, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int defaults_rc = execute_all(admin, defaults_queries); + if (defaults_rc != EXIT_SUCCESS) { + diag("Error: failed to configure TLS defaults for green writer hostgroup 962"); + return EXIT_FAILURE; + } + + string empty_query = + "SELECT COUNT(*)=0 FROM mysql_servers WHERE hostgroup_id=" + + to_string(hg.green_writer); + int empty_rc = bgd_wait_for_condition(admin, empty_query, kTimeoutSeconds); + if (empty_rc != EXIT_SUCCESS) { + diag("Error: green writer hostgroup 962 was not empty before discovery"); + return EXIT_FAILURE; + } + + ok(true, "green writer hostgroup 962 starts empty with use_ssl=1 defaults"); + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the created-TARGET probe sequence"); + return EXIT_FAILURE; + } + + vector topology = cluster.get_topology("AVAILABLE"); + int topology_rc = sim.topology_update(state.created_target_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish created-TARGET AVAILABLE topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 960 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + string created_query = + "SELECT COUNT(*)=1 FROM runtime_mysql_servers WHERE hostgroup_id=962 AND hostname=" + + bgd_sql_quote(cluster.green_writer.hostname) + " AND port=3306 AND use_ssl=1"; + int created_rc = bgd_wait_for_condition(admin, created_query, kTimeoutSeconds); + if (created_rc != EXIT_SUCCESS) { + diag("Error: discovery did not create the TLS TARGET row in hostgroup 962"); + return EXIT_FAILURE; + } + + bool runtime_tls = runtime_server_tls_matches(admin, hg.green_writer, cluster.green_writer, 1); + ok(runtime_tls, "AVAILABLE creates the TARGET runtime row with hostgroup 962 TLS defaults"); + + ProbeChain chain {}; + int chain_rc = wait_for_probe_chain(sim, seq, cluster, 0, 1, chain); + if (chain_rc != EXIT_SUCCESS) { + diag("Error: created-TARGET TLS probe chain did not complete"); + return EXIT_FAILURE; + } + + bool green_tls = chain.green.backend.host == cluster.green_writer.ip && chain.green.encrypted; + bool ordered = probe_chain_ordered(chain); + ok(green_tls && ordered, "created TARGET probes run table-check, blue metadata, then TLS green metadata"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // ProxySQL: load a plaintext reader before the TLS writer in automatic hostgroups 940 and 941. + // Simulator: publish AVAILABLE topology. + // Verify: table-check and metadata probes select the writer and use TLS. + if (test_automatic_writer_tls(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: load an exact TLS TARGET and plaintext distractor into green writer hostgroup 952. + // Simulator: publish AVAILABLE for the exact TARGET deployment. + // Verify: metadata probing selects the exact TARGET and preserves probe order. + if (test_explicit_target_tls(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: leave hostgroup 962 empty and configure use_ssl=1 server defaults. + // Simulator: publish AVAILABLE topology. + // Verify: ProxySQL creates and probes the TARGET row with TLS. + if (test_created_target_tls(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_reader_policy-t.cpp b/test/tap/tests/test_rds_bgd_reader_policy-t.cpp new file mode 100644 index 0000000000..b274b4e9c5 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_reader_policy-t.cpp @@ -0,0 +1,431 @@ +/** + * @file test_rds_bgd_reader_policy-t.cpp + * @brief BGD matched-reader, offline-reader, and writer-fallback routing. + * + * Steps: + * + * 1. Publish SWITCHOVER_IN_POST_PROCESSING with one mapped and one unmapped + * blue reader. + * 2. Verify that the mapped blue reader remains ONLINE and reader traffic + * reaches its green target instead of the unmapped blue reader. + * 3. Publish SWITCHOVER_IN_POST_PROCESSING without reader pairs while both + * blue readers are OFFLINE_SOFT or OFFLINE_HARD, then verify that they do + * not trigger writer fallback. + * 4. Make one blue reader ONLINE but omit it from topology and verify that + * writer fallback keeps the reader hostgroup routable. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; + +struct TestState { + RDS_BGD_Cluster matched { bgd_cluster_init() }; + BGD_Hostgroups matched_hg { 1280, 1281, 1282, 1283 }; + vector matched_endpoints { matched.get_endpoints() }; + + RDS_BGD_Cluster fallback { bgd_cluster_2_init() }; + BGD_Hostgroups fallback_hg { 1290, 1291, 1292, 1293 }; + vector fallback_endpoints { fallback.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pairs(RDS_BGD_Cluster& cluster, string status, size_t pairs) { + vector rows = cluster.get_topology(status); + for (size_t i = 0; i < pairs; ++i) { + rows.push_back({ + cluster.blue_readers[i].hostname, + cluster.blue_readers[i].hostname, + cluster.blue_readers[i].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[i].hostname, + cluster.green_readers[i].hostname, + cluster.green_readers[i].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + } + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int configure_bgd( + MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, size_t green_reader_count) +{ + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer }; + for (size_t i = 0; i < green_reader_count; ++i) { + green_servers.push_back(cluster.green_readers[i]); + } + + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_post_processing( + RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, size_t pairs) +{ + vector topology = + topology_with_reader_pairs(cluster, "SWITCHOVER_IN_POST_PROCESSING", pairs); + + int rc = sim.topology_update(endpoints, topology); + return rc; +} + +int wait_for_reader_online(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Host& host) { + string query = + "SELECT COUNT(*)=1 FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status='ONLINE'"; + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +int wait_for_writer_reader_membership(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster, int expected) { + string query = + "SELECT COUNT(*)=" + to_string(expected) + + " FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_writer.hostname) + + " AND port=" + to_string(cluster.blue_writer.port); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +int set_blue_reader_statuses(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster, + string first_status, string second_status) +{ + string first_query = + "UPDATE mysql_servers SET status=" + bgd_sql_quote(first_status) + + " WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_readers[0].hostname) + + " AND port=" + to_string(cluster.blue_readers[0].port); + string second_query = + "UPDATE mysql_servers SET status=" + bgd_sql_quote(second_status) + + " WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_readers[1].hostname) + + " AND port=" + to_string(cluster.blue_readers[1].port); + vector queries { + first_query, + second_query, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +/** + * Apply reader matching during SWITCHOVER_IN_POST_PROCESSING. + * + * - Configure blue readers 0 and 1 in hostgroup 1281. + * - Publish SWITCHOVER_IN_POST_PROCESSING topology with a pair only for blue + * reader 0. + * - Verify that the mapped reader remains ONLINE. + * - Route a client through hostgroup 1281 and verify that it reaches the + * mapped green reader instead of the unmapped blue reader. + */ +int test_matched_unmatched_readers(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.matched; + BGD_Hostgroups& hg = state.matched_hg; + + int config_rc = configure_bgd(admin, sim, cluster, hg, 1); + if (config_rc != EXIT_SUCCESS) { + diag("Error: failed to configure matched-reader scenario for wHG 1280"); + return EXIT_FAILURE; + } + + // Prefer the unmapped reader heavily so a routing check fails if BGD leaves it eligible. + string mapped_weight = + "UPDATE mysql_servers SET weight=1 WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_readers[0].hostname); + string unmapped_weight = + "UPDATE mysql_servers SET weight=1000000 WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_readers[1].hostname); + vector weight_queries { + mapped_weight, + unmapped_weight, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int weight_rc = execute_all(admin, weight_queries); + if (weight_rc != EXIT_SUCCESS) { + diag("Error: failed to configure deterministic reader weights in hostgroup 1281"); + return EXIT_FAILURE; + } + + int topology_rc = publish_post_processing(sim, state.matched_endpoints, cluster, 1); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish one-pair SWITCHOVER_IN_POST_PROCESSING topology for wHG 1280"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1280 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + int matched_rc = wait_for_reader_online(admin, hg, cluster.blue_readers[0]); + if (matched_rc != EXIT_SUCCESS) { + diag("Error: mapped blue reader did not remain ONLINE in hostgroup 1281"); + return EXIT_FAILURE; + } + + ok(true, "SWITCHOVER_IN_POST_PROCESSING keeps the mapped blue reader ONLINE in hostgroup 1281"); + + int user_rc = set_default_hostgroup(admin, hg.blue_reader); + if (user_rc != EXIT_SUCCESS) { + diag("Error: failed to route testuser through reader hostgroup 1281"); + return EXIT_FAILURE; + } + + auto [echo_rc, echo] = connect_and_echo(cl); + if (echo_rc != EXIT_SUCCESS) { + diag("Error: failed to connect through reader hostgroup 1281"); + return EXIT_FAILURE; + } + + bool mapped_reader_routing = echo.find(cluster.green_readers[0].ip) != string::npos; + ok(mapped_reader_routing, "SWITCHOVER_IN_POST_PROCESSING routes hostgroup 1281 through the mapped green reader"); + return EXIT_SUCCESS; +} + +/** + * Exclude offline blue readers from writer-fallback calculation. + * + * - Configure both blue readers in hostgroup 1291 as OFFLINE_SOFT and + * OFFLINE_HARD. + * - Publish SWITCHOVER_IN_POST_PROCESSING topology without reader pairs. + * - Verify that the blue writer is not added to reader hostgroup 1291. + */ +int test_offline_blue_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.fallback; + BGD_Hostgroups& hg = state.fallback_hg; + + int config_rc = configure_bgd(admin, sim, cluster, hg, 0); + if (config_rc != EXIT_SUCCESS) { + diag("Error: failed to configure offline-reader scenario for wHG 1290"); + return EXIT_FAILURE; + } + + int offline_rc = set_blue_reader_statuses(admin, hg, cluster, "OFFLINE_SOFT", "OFFLINE_HARD"); + if (offline_rc != EXIT_SUCCESS) { + diag("Error: failed to configure OFFLINE_SOFT and OFFLINE_HARD blue readers in hostgroup 1291"); + return EXIT_FAILURE; + } + + int topology_rc = publish_post_processing(sim, state.fallback_endpoints, cluster, 0); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish writer-only SWITCHOVER_IN_POST_PROCESSING topology for wHG 1290"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1290 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + int writer_absent_rc = wait_for_writer_reader_membership(admin, hg, cluster, 0); + if (writer_absent_rc != EXIT_SUCCESS) { + diag("Error: offline blue readers incorrectly triggered writer fallback in hostgroup 1291"); + return EXIT_FAILURE; + } + + ok(true, "OFFLINE_SOFT and OFFLINE_HARD blue readers do not trigger writer fallback in hostgroup 1291"); + return EXIT_SUCCESS; +} + +/** + * Route the reader hostgroup through writer fallback. + * + * - Make blue reader 0 ONLINE and publish SWITCHOVER_IN_POST_PROCESSING + * without reader pairs. + * - Verify that the blue writer is added to reader hostgroup 1291. + * - Connect through hostgroup 1291 and verify routing reaches the green writer + * IP pinned for the blue writer hostname. + */ +int test_writer_reader_fallback(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.fallback; + BGD_Hostgroups& hg = state.fallback_hg; + + int online_rc = set_blue_reader_statuses(admin, hg, cluster, "ONLINE", "OFFLINE_HARD"); + if (online_rc != EXIT_SUCCESS) { + diag("Error: failed to make one blue reader eligible for writer fallback"); + return EXIT_FAILURE; + } + + int topology_rc = publish_post_processing(sim, state.fallback_endpoints, cluster, 0); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish writer-only SWITCHOVER_IN_POST_PROCESSING topology for wHG 1290"); + return EXIT_FAILURE; + } + + int writer_present_rc = wait_for_writer_reader_membership(admin, hg, cluster, 1); + if (writer_present_rc != EXIT_SUCCESS) { + diag("Error: writer fallback did not add the blue writer to reader hostgroup 1291"); + return EXIT_FAILURE; + } + + int user_rc = set_default_hostgroup(admin, hg.blue_reader); + if (user_rc != EXIT_SUCCESS) { + diag("Error: failed to route testuser through reader hostgroup 1291"); + return EXIT_FAILURE; + } + + auto [echo_rc, echo] = connect_and_echo(cl); + if (echo_rc != EXIT_SUCCESS) { + diag("Error: failed to connect through writer fallback in reader hostgroup 1291"); + return EXIT_FAILURE; + } + + bool green_writer_routing = echo.find(cluster.green_writer.ip) != string::npos; + ok(green_writer_routing, "writer fallback keeps hostgroup 1291 routable through the green writer IP"); + return EXIT_SUCCESS; +} + +int main() { + plan(4); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish SWITCHOVER_IN_POST_PROCESSING with one reader pair for wHG 1280. + // Verify: only the mapped blue reader remains ONLINE in reader hostgroup 1281. + if (test_matched_unmatched_readers(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: configure both blue readers in hostgroup 1291 as OFFLINE_SOFT and OFFLINE_HARD. + // Simulator: publish SWITCHOVER_IN_POST_PROCESSING without reader pairs. + // Verify: offline blue readers do not add the blue writer to reader hostgroup 1291. + if (test_offline_blue_servers(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: make one blue reader ONLINE. + // Simulator: publish SWITCHOVER_IN_POST_PROCESSING without reader pairs. + // Verify: writer fallback keeps reader hostgroup 1291 routable through the green writer IP. + if (test_writer_reader_fallback(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp b/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp new file mode 100644 index 0000000000..3d10a06c50 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp @@ -0,0 +1,417 @@ +/** + * @file test_rds_bgd_reader_switchover_cleanup-t.cpp + * @brief BGD reader switchover and terminal empty-topology cleanup. + * + * Steps: + * + * 1. Configure hostgroups 980-983 and advance through writer post-processing. + * 2. Publish target-only SWITCHOVER_COMPLETED and verify + * READER_SWITCHOVER_IN_PROGRESS with green rows retained. + * 3. Repeat the completed observation and verify the reader phase is stable. + * 4. Publish empty topology and verify NONE, restored blue-reader routing, + * blue-IP probing, green-pool drain, and retained green rows. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_init() }; + BGD_Hostgroups hostgroups { 980, 981, 982, 983 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows { + { + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }, + }; + return rows; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int create_pool(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return EXIT_FAILURE; + } + + rc_t echo = bgd_backend_ip_echo(client); + mysql_close(client); + return echo.first; +} + +bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status='ONLINE'"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool online = rows[0][0] == "1"; + return online; +} + +bool green_rows_online(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + bool writer_online = runtime_server_online(admin, hg.green_writer, cluster.green_writer); + bool reader_online = runtime_server_online(admin, hg.green_reader, cluster.green_readers[0]); + bool rows_online = writer_online && reader_online; + return rows_online; +} + +int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + vector available = topology_with_reader_pair(cluster, "AVAILABLE"); + int available_rc = sim.topology_update(state.topology_endpoints, available); + if (available_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int available_status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_status_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector in_progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + int progress_rc = sim.topology_update(state.topology_endpoints, in_progress); + if (progress_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int progress_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_status_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector post_processing = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + int post_rc = sim.topology_update(state.topology_endpoints, post_processing); + if (post_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int post_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (post_status_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Enter reader switchover after writer post-processing. + * + * - Configure hostgroups 980-983 with one mapped reader pair. + * - Create pools in green writer and reader hostgroups. + * - Advance through writer post-processing. + * - Publish target-only SWITCHOVER_COMPLETED twice. + * - Verify READER_SWITCHOVER_IN_PROGRESS and retained green rows. + */ +int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int blue_writer_rc = bgd_set_host_read_only_0(sim, cluster.blue_writer); + if (blue_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue writer"); + return EXIT_FAILURE; + } + + int green_writer_rc = bgd_set_host_read_only_0(sim, cluster.green_writer); + if (green_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated green writer"); + return EXIT_FAILURE; + } + + int blue_reader_rc = bgd_set_host_read_only_1(sim, cluster.blue_readers[0]); + if (blue_reader_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=1 for the simulated blue reader"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 980-983"); + return EXIT_FAILURE; + } + + int green_writer_hg_rc = set_default_hostgroup(admin, hg.green_writer); + if (green_writer_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green writer hostgroup 982"); + return EXIT_FAILURE; + } + + int green_writer_pool_rc = create_pool(cl); + if (green_writer_pool_rc != EXIT_SUCCESS) { + diag("Error: failed to create a green-writer pool before reader cleanup"); + return EXIT_FAILURE; + } + + int green_reader_hg_rc = set_default_hostgroup(admin, hg.green_reader); + if (green_reader_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green reader hostgroup 983"); + return EXIT_FAILURE; + } + + int green_reader_pool_rc = create_pool(cl); + if (green_reader_pool_rc != EXIT_SUCCESS) { + diag("Error: failed to create a green-reader pool before reader cleanup"); + return EXIT_FAILURE; + } + + int restore_hg_rc = set_default_hostgroup(admin, hg.blue_writer); + if (restore_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to restore testuser to blue writer hostgroup 980"); + return EXIT_FAILURE; + } + + int post_rc = advance_to_post_processing(admin, sim, state); + if (post_rc != EXIT_SUCCESS) { + diag("Error: failed to reach WRITER_SWITCHOVER_POST_PROCESSING before reader switchover"); + return EXIT_FAILURE; + } + + vector completed = target_only_completed(cluster); + int completed_rc = sim.topology_update(state.topology_endpoints, completed); + if (completed_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology"); + return EXIT_FAILURE; + } + + int reader_status_rc = bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (reader_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 980 did not reach READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "target-only SWITCHOVER_COMPLETED sets BGD status for wHG 980 to READER_SWITCHOVER_IN_PROGRESS"); + + bool rows_online = green_rows_online(admin, cluster, hg); + ok(rows_online, "READER_SWITCHOVER_IN_PROGRESS retains configured green writer and reader rows"); + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the repeated reader-switchover probe sequence"); + return EXIT_FAILURE; + } + + vector repeated_completed = target_only_completed(cluster); + int repeat_rc = sim.topology_update(state.topology_endpoints, repeated_completed); + if (repeat_rc != EXIT_SUCCESS) { + diag("Error: failed to repeat target-only SWITCHOVER_COMPLETED topology"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: BGD did not observe repeated target-only SWITCHOVER_COMPLETED topology"); + return EXIT_FAILURE; + } + + int repeat_status_rc = bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (repeat_status_rc != EXIT_SUCCESS) { + diag("Error: repeated completion changed BGD status for wHG 980"); + return EXIT_FAILURE; + } + + bool repeated_rows_online = green_rows_online(admin, cluster, hg); + ok(repeated_rows_online, "repeated SWITCHOVER_COMPLETED preserves reader switchover and green rows"); + return EXIT_SUCCESS; +} + +/** + * Complete reader cleanup with present-but-empty topology. + * + * - Delete every topology row while the topology table remains present. + * - Verify BGD status NONE and restored blue-reader routing. + * - Verify metadata probing returns from the green pin to the blue writer. + * - Verify green pools drain while configured green rows remain ONLINE. + */ +int test_reader_switchover_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + auto [green_writer_pool_before_rc, green_writer_pool_before] = bgd_connection_pool_count(admin, hg.green_writer); + auto [green_reader_pool_before_rc, green_reader_pool_before] = bgd_connection_pool_count(admin, hg.green_reader); + if (green_writer_pool_before_rc != EXIT_SUCCESS || green_reader_pool_before_rc != EXIT_SUCCESS || + green_writer_pool_before < 1 || green_reader_pool_before < 1) { + diag("Error: green writer or reader pool is empty before reader cleanup"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before empty topology"); + return EXIT_FAILURE; + } + + int empty_rc = sim.topology_delete(state.topology_endpoints); + if (empty_rc != EXIT_SUCCESS) { + diag("Error: failed to publish present-but-empty topology"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 980 did not reach NONE"); + return EXIT_FAILURE; + } + + ok(true, "present-but-empty topology sets BGD status for wHG 980 to NONE"); + + bool unmatched_reader_online = runtime_server_online(admin, hg.blue_reader, cluster.blue_readers[1]); + ok(unmatched_reader_online, "reader cleanup restores the unmatched blue reader in hostgroup 981"); + + auto [green_probe_rc, green_probe] = + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (green_probe_rc != EXIT_SUCCESS) { + diag("Error: reader cleanup did not observe empty topology through the green pin"); + return EXIT_FAILURE; + } + + auto [blue_probe_rc, blue_probe] = + sim.wait_for_probe_log(green_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (blue_probe_rc != EXIT_SUCCESS) { + diag("Error: metadata probing did not return to the blue writer after reader cleanup"); + return EXIT_FAILURE; + } + + bool probe_order = green_probe.sequence_id < blue_probe.sequence_id; + ok(probe_order, "reader cleanup removes the green pin and resumes blue-writer metadata probing"); + + auto [green_writer_pool_rc, green_writer_pool] = bgd_connection_pool_count(admin, hg.green_writer); + auto [green_reader_pool_rc, green_reader_pool] = bgd_connection_pool_count(admin, hg.green_reader); + if (green_writer_pool_rc != EXIT_SUCCESS || green_reader_pool_rc != EXIT_SUCCESS) { + diag("Error: failed to read green pools after reader cleanup"); + return EXIT_FAILURE; + } + + ok(green_writer_pool == 0 && green_reader_pool == 0, "reader cleanup drains eligible green writer and reader pools"); + + bool rows_online = green_rows_online(admin, cluster, hg); + ok(rows_online, "reader cleanup retains configured green writer and reader rows as ONLINE"); + return EXIT_SUCCESS; +} + +int main() { + plan(8); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // ProxySQL: configure hostgroups 980-983 and establish green writer/reader pools. + // Simulator: advance through writer post-processing, then publish target-only SWITCHOVER_COMPLETED twice. + // Verify: BGD remains in READER_SWITCHOVER_IN_PROGRESS and green rows remain ONLINE. + if (test_reader_switchover_in_progress(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: delete all rows while keeping the topology table present. + // Verify: BGD reaches NONE, blue-reader routing and blue probing resume, green pools drain, and rows remain. + if (test_reader_switchover_cleanup(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp b/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp new file mode 100644 index 0000000000..dbfa042c38 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp @@ -0,0 +1,251 @@ +/** + * @file test_rds_bgd_remove_during_switchover-t.cpp + * @brief Removing BGD configuration during writer switchover. + * + * Steps: + * + * 1. Configure BGD hostgroups 1350-1353 and reach `AVAILABLE`. + * 2. Publish `SWITCHOVER_IN_PROGRESS` and verify that the blue writer moves + * from hostgroup 1350 to hostgroup 1351. + * 3. Delete writer hostgroup 1350 from `mysql_aws_rds_bgd_hostgroups`. + * 4. Verify that the blue writer returns to hostgroup 1350 and the runtime BGD + * row is removed. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_init() }; + BGD_Hostgroups hostgroups { 1350, 1351, 1352, 1353 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int wait_for_bgd_row_absent(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "SELECT COUNT(*)=0 FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hg.blue_writer); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +/** + * Configure BGD hostgroups 1350-1353. + * + * - Set `read_only=0` for the simulated blue and green writers. + * - Publish `AVAILABLE` topology. + * - Configure `mysql_servers` and `mysql_aws_rds_bgd_hostgroups`. + * - Verify that the runtime BGD row reaches `AVAILABLE`. + */ +int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Set read_only=0 for the simulated blue and green writers. + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated writer read_only values"); + return EXIT_FAILURE; + } + + // Publish AVAILABLE topology. + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology"); + return EXIT_FAILURE; + } + + // Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure mysql_servers and mysql_aws_rds_bgd_hostgroups"); + return EXIT_FAILURE; + } + + // Wait for the runtime BGD row to report AVAILABLE. + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime BGD status did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1350 reports AVAILABLE"); + return EXIT_SUCCESS; +} + +/** + * Move the BGD row for writer hostgroup 1350 into writer switchover. + * + * - Publish `SWITCHOVER_IN_PROGRESS`. + * - Verify `WRITER_SWITCHOVER_IN_PROGRESS`. + * - Verify that the blue writer moves to the blue reader hostgroup. + */ +int test_writer_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Publish SWITCHOVER_IN_PROGRESS and wait for the runtime BGD status. + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime BGD status did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1350 reports WRITER_SWITCHOVER_IN_PROGRESS"); + + // Verify the blue writer was moved from the writer to the reader hostgroup. + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not move to the blue reader hostgroup"); + return EXIT_FAILURE; + } + + ok(true, "SWITCHOVER_IN_PROGRESS moves the blue writer from hostgroup 1350 to 1351"); + return EXIT_SUCCESS; +} + +/** + * Remove BGD configuration during writer switchover. + * + * - Delete writer hostgroup 1350 from `mysql_aws_rds_bgd_hostgroups`. + * - Load the configuration to runtime without changing `mysql_servers`. + * - Verify that the blue writer returns to hostgroup 1350. + * - Verify that the runtime BGD row for writer hostgroup 1350 is removed. + */ +int test_remove_during_switchover(MYSQL* admin, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Delete the BGD row without changing mysql_servers or the configured hostgroups. + string delete_bgd = "DELETE FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hg.blue_writer); + vector queries { + delete_bgd, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + if (rc != EXIT_SUCCESS) { + diag("Error: failed to delete and load the BGD configuration"); + return EXIT_FAILURE; + } + + // Wait until deleting the row restores the blue writer to its writer hostgroup. + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not return to the blue writer hostgroup"); + return EXIT_FAILURE; + } + + // Wait until the deleted BGD row is absent from runtime. + int row_rc = wait_for_bgd_row_absent(admin, hg); + if (row_rc != EXIT_SUCCESS) { + diag("Error: deleted BGD configuration remains in the runtime table"); + return EXIT_FAILURE; + } + + ok(true, "deleting BGD configuration restores the blue writer from hostgroup 1351 to 1350"); + ok(true, "deleting wHG 1350 removes it from runtime_mysql_aws_rds_bgd_hostgroups"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: set the blue/green writers to read_only=0 and publish AVAILABLE topology. + // ProxySQL: update mysql_servers and mysql_aws_rds_bgd_hostgroups with BGD configuration. + // Verify: runtime_mysql_aws_rds_bgd_hostgroups status reports AVAILABLE. + if (test_bgd_status_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS topology. + // Verify: runtime_mysql_aws_rds_bgd_hostgroups status reports WRITER_SWITCHOVER_IN_PROGRESS. + // Verify: runtime_mysql_servers moves the blue writer from writer hostgroup to reader hostgroup. + if (test_writer_switchover_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: delete wHG 1350 from mysql_aws_rds_bgd_hostgroups without changing mysql_servers. + // Verify: the blue writer returns to hostgroup 1350 and the runtime BGD row is absent. + if (test_remove_during_switchover(admin, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp b/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp new file mode 100644 index 0000000000..b1852dedf8 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp @@ -0,0 +1,447 @@ +/** + * @file test_rds_bgd_repeated_deployment-t.cpp + * @brief Reusing BGD hostgroups 1400-1403 for a second deployment. + * + * Steps: + * + * 1. Configure deployment A, complete writer and reader switchover, and publish + * empty topology. + * 2. Replace the configured green servers with TLS-enabled deployment B. + * 3. Verify that only deployment B membership, probes, and routing are used + * during the second lifecycle. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 500; + +struct TestState { + RDS_BGD_Cluster deployment_a { bgd_cluster_init() }; + RDS_BGD_Cluster deployment_b { bgd_cluster_1_deployment_b_init() }; + BGD_Hostgroups hostgroups { 1400, 1401, 1402, 1403 }; + vector topology_endpoints { deployment_a.get_endpoints() }; + + TestState() { + vector deployment_b_green = deployment_b.get_green_endpoints(); + topology_endpoints.insert(topology_endpoints.end(), deployment_b_green.begin(), deployment_b_green.end()); + } +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + auto result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +bool runtime_green_membership_matches( + MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& present, RDS_BGD_Cluster& absent, int use_ssl) +{ + string query = "SELECT " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(present.green_writer.hostname) + " AND port=3306 AND use_ssl=" + + to_string(use_ssl) + ")=1 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(present.green_readers[0].hostname) + " AND port=3306 AND use_ssl=" + + to_string(use_ssl) + ")=1 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(absent.green_writer.hostname) + " AND port=3306)=0 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(absent.green_readers[0].hostname) + " AND port=3306)=0"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +int replace_green_membership( + MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& old_deployment, RDS_BGD_Cluster& new_deployment) +{ + vector delete_queries { + "DELETE FROM mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(old_deployment.green_writer.hostname) + " AND port=3306", + "DELETE FROM mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(old_deployment.green_readers[0].hostname) + " AND port=3306", + }; + if (execute_all(admin, delete_queries) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector green_servers { new_deployment.green_writer, new_deployment.green_readers[0] }; + if (bgd_admin_add_servers(admin, new_deployment, hg, green_servers, true, 1) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int rc = execute_all(admin, load_queries); + return rc; +} + +int publish_writer_lifecycle( + MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, string deployment) +{ + vector initiated = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + if (sim.topology_update(cluster.get_endpoints(), initiated) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED topology for deployment %s", deployment.c_str()); + return EXIT_FAILURE; + } + + if (bgd_wait_for_status(admin, state.hostgroups, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds) != EXIT_SUCCESS) { + diag("Error: BGD status for deployment %s did not reach WRITER_SWITCHOVER_INITIATED", deployment.c_str()); + return EXIT_FAILURE; + } + + vector progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + if (sim.topology_update(cluster.get_endpoints(), progress) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for deployment %s", deployment.c_str()); + return EXIT_FAILURE; + } + + if (bgd_wait_for_status(admin, state.hostgroups, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds) != EXIT_SUCCESS) { + diag("Error: BGD status for deployment %s did not reach WRITER_SWITCHOVER_IN_PROGRESS", deployment.c_str()); + return EXIT_FAILURE; + } + + vector post = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + if (sim.topology_update(cluster.get_endpoints(), post) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for deployment %s", deployment.c_str()); + return EXIT_FAILURE; + } + + int post_status_rc = + bgd_wait_for_status(admin, state.hostgroups, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (post_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for deployment %s did not reach WRITER_SWITCHOVER_POST_PROCESSING", deployment.c_str()); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int publish_reader_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, + string deployment) { + vector completed = target_only_completed(cluster); + if (sim.topology_update(cluster.get_endpoints(), completed) != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED for deployment %s", deployment.c_str()); + return EXIT_FAILURE; + } + + if (bgd_wait_for_status(admin, state.hostgroups, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds) != EXIT_SUCCESS) { + diag("Error: BGD status for deployment %s did not reach READER_SWITCHOVER_IN_PROGRESS", deployment.c_str()); + return EXIT_FAILURE; + } + + if (sim.topology_delete(state.topology_endpoints) != EXIT_SUCCESS) { + diag("Error: failed to publish empty topology for deployment %s", deployment.c_str()); + return EXIT_FAILURE; + } + + if (bgd_wait_for_status(admin, state.hostgroups, "NONE", kTimeoutSeconds) != EXIT_SUCCESS) { + diag("Error: BGD status for deployment %s did not reach NONE", deployment.c_str()); + return EXIT_FAILURE; + } + + if (bgd_wait_for_server_placement( + admin, state.hostgroups.blue_writer, state.hostgroups.blue_reader, + cluster.blue_writer, false, kTimeoutSeconds + ) != EXIT_SUCCESS) { + diag("Error: deployment %s did not restore the blue writer to hostgroup 1400", deployment.c_str()); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Complete deployment A before reusing its BGD hostgroups. + * + * - Configure BGD hostgroups 1400-1403 with deployment A. + * - Complete writer and reader switchover, then publish empty topology. + * - Verify NONE and baseline blue-writer placement. + */ +int test_deployment_a(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& deployment = state.deployment_a; + BGD_Hostgroups& hg = state.hostgroups; + + int read_only_rc = configure_read_only_values(sim, deployment); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for deployment A"); + return EXIT_FAILURE; + } + + vector available = topology_with_reader_pair(deployment, "AVAILABLE"); + int topology_rc = sim.topology_update(deployment.get_endpoints(), available); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for deployment A"); + return EXIT_FAILURE; + } + + vector blue_servers { deployment.blue_writer, deployment.blue_readers[0] }; + vector green_servers { deployment.green_writer, deployment.green_readers[0] }; + int admin_rc = bgd_admin_setup( + admin, deployment, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1400-1403 for deployment A"); + return EXIT_FAILURE; + } + + int available_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1400 did not reach AVAILABLE for deployment A"); + return EXIT_FAILURE; + } + + int writer_rc = publish_writer_lifecycle(admin, sim, state, deployment, "A"); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to complete writer switchover for deployment A"); + return EXIT_FAILURE; + } + + int reader_rc = publish_reader_cleanup(admin, sim, state, deployment, "A"); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to complete reader switchover cleanup for deployment A"); + return EXIT_FAILURE; + } + + ok(true, "deployment A cleanup sets BGD status for wHG 1400 to NONE and restores blue writer placement"); + return EXIT_SUCCESS; +} + +/** + * Reuse BGD hostgroups 1400-1403 for deployment B. + * + * - Replace deployment A green rows with TLS-enabled deployment B rows. + * - Verify that metadata probes and runtime rows use only deployment B. + * - Complete deployment B and verify that routing uses deployment B without + * recreating deployment A rows. + */ +int test_deployment_b(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& deployment_a = state.deployment_a; + RDS_BGD_Cluster& deployment_b = state.deployment_b; + BGD_Hostgroups& hg = state.hostgroups; + + int read_only_rc = configure_read_only_values(sim, deployment_b); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for deployment B"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before configuring deployment B"); + return EXIT_FAILURE; + } + + int replace_rc = replace_green_membership(admin, hg, deployment_a, deployment_b); + if (replace_rc != EXIT_SUCCESS) { + diag("Error: failed to replace deployment A green rows with deployment B rows"); + return EXIT_FAILURE; + } + + vector available = topology_with_reader_pair(deployment_b, "AVAILABLE"); + int topology_rc = sim.topology_update(deployment_b.get_endpoints(), available); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for deployment B"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1400 did not reach AVAILABLE for deployment B"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_green_membership_matches(admin, hg, deployment_b, deployment_a, 1); + ok(membership_matches, "runtime_mysql_servers contains only TLS-enabled deployment B green rows"); + + auto [probe_rc, probe] = sim.wait_for_probe_log( + seq, deployment_b.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: deployment B green writer did not receive a TLS metadata probe"); + return EXIT_FAILURE; + } + + int stale_probe_rc = + bgd_expect_no_metadata_probe(sim, probe.sequence_id, deployment_a.green_writer.endpoint(), kNegativeProbeTimeoutMs); + ok(stale_probe_rc == EXIT_SUCCESS, + "deployment B metadata probing does not return to the removed deployment A green writer"); + + int writer_rc = publish_writer_lifecycle(admin, sim, state, deployment_b, "B"); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to complete writer switchover for deployment B"); + return EXIT_FAILURE; + } + + int user_rc = set_default_hostgroup(admin, hg.blue_writer); + if (user_rc != EXIT_SUCCESS) { + diag("Error: failed to route testuser through writer hostgroup 1400"); + return EXIT_FAILURE; + } + + auto [route_rc, route] = connect_and_echo(cl); + bool route_matches = route_rc == EXIT_SUCCESS && route.find(deployment_b.green_writer.ip) != string::npos; + ok(route_matches, "deployment B post-processing routes new connections to deployment B"); + + int reader_rc = publish_reader_cleanup(admin, sim, state, deployment_b, "B"); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to complete reader switchover cleanup for deployment B"); + return EXIT_FAILURE; + } + + bool final_membership = runtime_green_membership_matches(admin, hg, deployment_b, deployment_a, 1); + ok(final_membership, "deployment B cleanup retains deployment B green rows without restoring deployment A rows"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish deployment A topology through writer and reader completion, then delete it. + // ProxySQL: configure BGD hostgroups 1400-1403 for deployment A. + // Verify: deployment A cleanup reaches NONE and restores blue-writer placement. + if (test_deployment_a(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish AVAILABLE through completion for deployment B on the same blue writer. + // ProxySQL: replace deployment A green rows with TLS-enabled deployment B rows. + // Verify: only deployment B membership, probes, and routing are used by the second lifecycle. + if (test_deployment_b(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_rollback-t.cpp b/test/tap/tests/test_rds_bgd_rollback-t.cpp new file mode 100644 index 0000000000..1e1bbe270f --- /dev/null +++ b/test/tap/tests/test_rds_bgd_rollback-t.cpp @@ -0,0 +1,631 @@ +/** + * @file test_rds_bgd_rollback-t.cpp + * @brief Returning from writer switchover to AVAILABLE. + * + * Steps: + * + * 1. Enter SWITCHOVER_INITIATED with a monitor-created green writer. + * 2. Return to AVAILABLE and verify blue-writer placement, read_only + * processing, and the monitor-created green writer. + * 3. Enter SWITCHOVER_IN_PROGRESS with explicit green servers and pools. + * 4. Return to AVAILABLE and verify blue routing without removing explicit + * green servers or draining their pools. + * 5. Repeat AVAILABLE and verify that rollback remains stable. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct GreenRows { + vector admin_writer {}; + vector runtime_writer {}; + vector admin_reader {}; + vector runtime_reader {}; +}; + +struct TestState { + RDS_BGD_Cluster initiated_cluster { bgd_cluster_init() }; + BGD_Hostgroups initiated_hg { 980, 981, 982, 983 }; + vector initiated_endpoints { initiated_cluster.get_endpoints() }; + + RDS_BGD_Cluster progress_cluster { bgd_cluster_2_init() }; + BGD_Hostgroups progress_hg { 990, 991, 992, 993 }; + vector progress_endpoints { progress_cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); + + int rc = sim.topology_update(endpoints, topology); + return rc; +} + +int wait_for_green_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { + auto [probe_rc, probe] = + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + return probe_rc; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +int64_t last_read_only_log_time(MYSQL* admin, RDS_BGD_Host& host) { + string query = + "SELECT COALESCE(MAX(time_start_us),0) FROM mysql_server_read_only_log WHERE hostname=" + + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return -1; + } + + int64_t time = strtoll(rows[0][0].c_str(), nullptr, 10); + return time; +} + +int wait_for_read_only_log(MYSQL* admin, RDS_BGD_Host& host, int64_t baseline) { + string query = + "SELECT COUNT(*)>0 FROM mysql_server_read_only_log WHERE hostname=" + + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND time_start_us>" + to_string(baseline); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +rc_t> server_row_snapshot(MYSQL* admin, string table, int hostgroup, RDS_BGD_Host& host) { + string query = + "SELECT hostgroup_id,hostname,port,status,use_ssl,weight,max_connections FROM " + table + + " WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port); + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +bool server_row_matches(MYSQL* admin, string table, int hostgroup, RDS_BGD_Host& host, vector expected) { + auto [rc, rows] = server_row_snapshot(admin, table, hostgroup, host); + if (rc != EXIT_SUCCESS) { + return false; + } + + bool matches = rows == expected; + return matches; +} + +rc_t green_rows_snapshot(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster, bool include_reader) { + GreenRows rows {}; + + auto [admin_writer_rc, admin_writer] = server_row_snapshot(admin, "mysql_servers", hg.green_writer, cluster.green_writer); + if (admin_writer_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + rows.admin_writer = admin_writer; + + auto [runtime_writer_rc, runtime_writer] = + server_row_snapshot(admin, "runtime_mysql_servers", hg.green_writer, cluster.green_writer); + if (runtime_writer_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + rows.runtime_writer = runtime_writer; + + if (include_reader) { + auto [admin_reader_rc, admin_reader] = + server_row_snapshot(admin, "mysql_servers", hg.green_reader, cluster.green_readers[0]); + if (admin_reader_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + rows.admin_reader = admin_reader; + + auto [runtime_reader_rc, runtime_reader] = + server_row_snapshot(admin, "runtime_mysql_servers", hg.green_reader, cluster.green_readers[0]); + if (runtime_reader_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + rows.runtime_reader = runtime_reader; + } + + return { EXIT_SUCCESS, rows }; +} + +bool green_rows_match(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster, GreenRows& expected, bool include_reader) { + bool admin_writer = server_row_matches(admin, "mysql_servers", hg.green_writer, cluster.green_writer, expected.admin_writer); + bool runtime_writer = + server_row_matches(admin, "runtime_mysql_servers", hg.green_writer, cluster.green_writer, expected.runtime_writer); + + bool admin_reader = true; + bool runtime_reader = true; + if (include_reader) { + admin_reader = + server_row_matches(admin, "mysql_servers", hg.green_reader, cluster.green_readers[0], expected.admin_reader); + runtime_reader = server_row_matches( + admin, "runtime_mysql_servers", hg.green_reader, cluster.green_readers[0], expected.runtime_reader + ); + } + + bool matches = admin_writer && runtime_writer && admin_reader && runtime_reader; + return matches; +} + +/** + * Return from SWITCHOVER_INITIATED to AVAILABLE. + * + * - Configure BGD without a green mysql_servers row and let the worker create + * the green writer in runtime. + * - Publish SWITCHOVER_INITIATED, then return to AVAILABLE. + * - Verify blue-writer placement and normal read_only processing are restored. + * - Repeat AVAILABLE and verify the monitor-created green writer remains. + */ +int test_initiated_rollback(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.initiated_cluster; + BGD_Hostgroups& hg = state.initiated_hg; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 980"); + return EXIT_FAILURE; + } + + int available_topology_rc = publish_topology(sim, state.initiated_endpoints, cluster, "AVAILABLE"); + if (available_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 980"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 980-983"); + return EXIT_FAILURE; + } + + int available_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 980 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + auto [created_rc, created_rows] = green_rows_snapshot(admin, hg, cluster, false); + if (created_rc != EXIT_SUCCESS || !created_rows.admin_writer.empty() || created_rows.runtime_writer.size() != 1) { + diag("Error: the green writer was not created only in runtime hostgroup 982"); + return EXIT_FAILURE; + } + + int initiated_topology_rc = publish_topology(sim, state.initiated_endpoints, cluster, "SWITCHOVER_INITIATED"); + if (initiated_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED topology for wHG 980"); + return EXIT_FAILURE; + } + + int initiated_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds); + if (initiated_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 980 did not reach WRITER_SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + int64_t read_only_baseline = last_read_only_log_time(admin, cluster.blue_readers[0]); + if (read_only_baseline < 0) { + diag("Error: failed to read the blue-reader read_only log baseline"); + return EXIT_FAILURE; + } + + auto [return_seq_rc, return_seq] = sim.probe_log_last_sequence(); + if (return_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the AVAILABLE rollback probe sequence"); + return EXIT_FAILURE; + } + + int return_topology_rc = publish_topology(sim, state.initiated_endpoints, cluster, "AVAILABLE"); + if (return_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to return wHG 980 topology to AVAILABLE"); + return EXIT_FAILURE; + } + + int returned_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (returned_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 980 did not return to AVAILABLE"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: initiated rollback did not restore the blue writer to hostgroup 980"); + return EXIT_FAILURE; + } + + int probe_rc = wait_for_green_writer(sim, return_seq, cluster); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: initiated rollback did not resume green-writer probing"); + return EXIT_FAILURE; + } + + ok(true, "returning from SWITCHOVER_INITIATED restores the blue writer to hostgroup 980"); + + int reader_update_rc = bgd_set_host_read_only_0(sim, cluster.blue_readers[0]); + if (reader_update_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue reader"); + return EXIT_FAILURE; + } + + int reader_log_rc = wait_for_read_only_log(admin, cluster.blue_readers[0], read_only_baseline); + if (reader_log_rc != EXIT_SUCCESS) { + diag("Error: read_only monitoring remained suppressed after initiated rollback"); + return EXIT_FAILURE; + } + + ok(true, "returning to AVAILABLE restores normal read_only monitoring"); + + auto [repeat_seq_rc, repeat_seq] = sim.probe_log_last_sequence(); + if (repeat_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the repeated AVAILABLE probe sequence"); + return EXIT_FAILURE; + } + + int repeat_topology_rc = publish_topology(sim, state.initiated_endpoints, cluster, "AVAILABLE"); + if (repeat_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to repeat AVAILABLE topology for wHG 980"); + return EXIT_FAILURE; + } + + int repeat_status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (repeat_status_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE did not keep BGD status for wHG 980"); + return EXIT_FAILURE; + } + + int repeat_placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (repeat_placement_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE changed blue-writer placement for wHG 980"); + return EXIT_FAILURE; + } + + int repeat_probe_rc = wait_for_green_writer(sim, repeat_seq, cluster); + if (repeat_probe_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE did not probe the green writer"); + return EXIT_FAILURE; + } + + bool created_rows_match = green_rows_match(admin, hg, cluster, created_rows, false); + ok(created_rows_match, "repeated AVAILABLE keeps the monitor-created green writer in runtime hostgroup 982"); + return EXIT_SUCCESS; +} + +/** + * Return from SWITCHOVER_IN_PROGRESS to AVAILABLE. + * + * - Configure explicit green writer/reader rows and establish their pools. + * - Enter SWITCHOVER_IN_PROGRESS and require blue-writer demotion. + * - Return to AVAILABLE and verify blue routing is restored. + * - Verify explicit green rows and pools remain unchanged. + * - Repeat AVAILABLE and verify rollback remains stable. + */ +int test_in_progress_rollback(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.progress_cluster; + BGD_Hostgroups& hg = state.progress_hg; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 990"); + return EXIT_FAILURE; + } + + int available_topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "AVAILABLE"); + if (available_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 990"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 990-993"); + return EXIT_FAILURE; + } + + int available_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 990 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + auto [green_rows_rc, green_rows] = green_rows_snapshot(admin, hg, cluster, true); + if (green_rows_rc != EXIT_SUCCESS || green_rows.admin_writer.size() != 1 || + green_rows.runtime_writer.size() != 1 || green_rows.admin_reader.size() != 1 || + green_rows.runtime_reader.size() != 1) { + diag("Error: failed to snapshot explicit green servers for wHG 990"); + return EXIT_FAILURE; + } + + int writer_hg_rc = set_default_hostgroup(admin, hg.green_writer); + if (writer_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green writer hostgroup 992"); + return EXIT_FAILURE; + } + + int writer_echo_rc = connect_and_echo(cl).first; + if (writer_echo_rc != EXIT_SUCCESS) { + diag("Error: failed to establish a green-writer connection pool"); + return EXIT_FAILURE; + } + + int reader_hg_rc = set_default_hostgroup(admin, hg.green_reader); + if (reader_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green reader hostgroup 993"); + return EXIT_FAILURE; + } + + int reader_echo_rc = connect_and_echo(cl).first; + if (reader_echo_rc != EXIT_SUCCESS) { + diag("Error: failed to establish a green-reader connection pool"); + return EXIT_FAILURE; + } + + int restore_hg_rc = set_default_hostgroup(admin, hg.blue_writer); + if (restore_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to restore the test user to blue writer hostgroup 990"); + return EXIT_FAILURE; + } + + auto [writer_pool_rc, writer_pool] = bgd_connection_pool_count(admin, hg.green_writer); + auto [reader_pool_rc, reader_pool] = bgd_connection_pool_count(admin, hg.green_reader); + if (writer_pool_rc != EXIT_SUCCESS || writer_pool < 1 || reader_pool_rc != EXIT_SUCCESS || reader_pool < 1) { + diag("Error: failed to establish green pools before in-progress rollback"); + return EXIT_FAILURE; + } + + int initiated_topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "SWITCHOVER_INITIATED"); + if (initiated_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED topology for wHG 990"); + return EXIT_FAILURE; + } + + int initiated_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds); + if (initiated_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 990 did not reach WRITER_SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + int progress_topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "SWITCHOVER_IN_PROGRESS"); + if (progress_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG 990"); + return EXIT_FAILURE; + } + + int progress_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 990 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int demotion_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (demotion_rc != EXIT_SUCCESS) { + diag("Error: the blue writer did not move to reader hostgroup 991"); + return EXIT_FAILURE; + } + + auto [return_seq_rc, return_seq] = sim.probe_log_last_sequence(); + if (return_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the in-progress rollback probe sequence"); + return EXIT_FAILURE; + } + + int return_topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "AVAILABLE"); + if (return_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to return wHG 990 topology to AVAILABLE"); + return EXIT_FAILURE; + } + + int returned_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (returned_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 990 did not return to AVAILABLE"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: in-progress rollback did not restore the blue writer to hostgroup 990"); + return EXIT_FAILURE; + } + + int probe_rc = wait_for_green_writer(sim, return_seq, cluster); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: in-progress rollback did not resume green-writer probing"); + return EXIT_FAILURE; + } + + auto [blue_echo_rc, blue_echo] = connect_and_echo(cl); + bool blue_routing = blue_echo_rc == EXIT_SUCCESS && blue_echo.find(cluster.blue_writer.ip) != string::npos; + ok(blue_routing, "returning from SWITCHOVER_IN_PROGRESS restores routing through blue writer hostgroup 990"); + + bool rows_match = green_rows_match(admin, hg, cluster, green_rows, true); + ok(rows_match, "in-progress rollback keeps explicit green servers in Admin and runtime hostgroups 992-993"); + + auto [post_writer_pool_rc, post_writer_pool] = bgd_connection_pool_count(admin, hg.green_writer); + auto [post_reader_pool_rc, post_reader_pool] = bgd_connection_pool_count(admin, hg.green_reader); + bool pools_match = post_writer_pool_rc == EXIT_SUCCESS && post_writer_pool >= writer_pool && + post_reader_pool_rc == EXIT_SUCCESS && post_reader_pool >= reader_pool; + ok(pools_match, "in-progress rollback does not drain green writer and reader pools"); + + auto [repeat_seq_rc, repeat_seq] = sim.probe_log_last_sequence(); + if (repeat_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the repeated AVAILABLE probe sequence for wHG 990"); + return EXIT_FAILURE; + } + + int repeat_topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "AVAILABLE"); + if (repeat_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to repeat AVAILABLE topology for wHG 990"); + return EXIT_FAILURE; + } + + int repeat_status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (repeat_status_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE did not keep BGD status for wHG 990"); + return EXIT_FAILURE; + } + + int repeat_placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (repeat_placement_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE changed blue-writer placement for wHG 990"); + return EXIT_FAILURE; + } + + int repeat_probe_rc = wait_for_green_writer(sim, repeat_seq, cluster); + if (repeat_probe_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE did not probe the green writer for wHG 990"); + return EXIT_FAILURE; + } + + auto [repeat_writer_pool_rc, repeat_writer_pool] = bgd_connection_pool_count(admin, hg.green_writer); + auto [repeat_reader_pool_rc, repeat_reader_pool] = bgd_connection_pool_count(admin, hg.green_reader); + bool repeat_rows = green_rows_match(admin, hg, cluster, green_rows, true); + bool repeat_pools = repeat_writer_pool_rc == EXIT_SUCCESS && repeat_writer_pool >= writer_pool && + repeat_reader_pool_rc == EXIT_SUCCESS && repeat_reader_pool >= reader_pool; + ok(repeat_rows && repeat_pools, "repeated AVAILABLE keeps blue placement, explicit green servers, and green pools"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish AVAILABLE, SWITCHOVER_INITIATED, then AVAILABLE for a monitor-created green writer. + // Verify: blue writer returns to hostgroup 980 and normal read_only monitoring resumes. + // Verify: repeated AVAILABLE keeps the monitor-created green writer in runtime hostgroup 982. + if (test_initiated_rollback(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish AVAILABLE, SWITCHOVER_INITIATED, SWITCHOVER_IN_PROGRESS, then AVAILABLE. + // ProxySQL: configure explicit green servers and establish green writer/reader pools. + // Verify: blue routing returns without removing green servers or draining their pools. + if (test_in_progress_rollback(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_smoke-t.cpp b/test/tap/tests/test_rds_bgd_smoke-t.cpp new file mode 100644 index 0000000000..8b72573e6b --- /dev/null +++ b/test/tap/tests/test_rds_bgd_smoke-t.cpp @@ -0,0 +1,218 @@ +/** + * @file test_rds_bgd_smoke-t.cpp + * @brief Explicitly configured BGD worker reaching AVAILABLE and probing the green writer. + * + * Steps: + * + * 1. Set read_only=0 for the blue and green writers and publish AVAILABLE topology. + * 2. Configure BGD hostgroups 10-40 with the blue writer in hostgroup 10. + * 3. Verify BGD status AVAILABLE and a plaintext metadata probe to the green writer. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_init() }; + BGD_Hostgroups hostgroups { 10, 20, 30, 40 }; + vector topology_endpoints { cluster.get_writers() }; + uint64_t probe_sequence { 0 }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int configure_explicit_bgd(MYSQL* admin, TestState& state) { + RDS_BGD_Host& writer = state.cluster.blue_writer; + BGD_Hostgroups& hg = state.hostgroups; + + string add_replication_hostgroups = + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + ")"; + string add_bgd_hostgroups = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + "," + + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ",1,0,100,800,'BGD simulator smoke test')"; + string add_blue_writer = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,use_ssl,comment) VALUES (" + + to_string(hg.blue_writer) + "," + bgd_sql_quote(writer.hostname) + "," + + to_string(writer.port) + ",0,'blue writer')"; + vector queries { + add_replication_hostgroups, + add_bgd_hostgroups, + add_blue_writer, + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +/** + * Publish AVAILABLE topology for writable blue and green writers. + * + * - Set read_only=0 for both simulated writers. + * - Record the probe sequence before publishing topology. + * - Publish AVAILABLE topology to the blue and green writer endpoints. + */ +int publish_available_topology(RDS_BGD_Simulator& sim, TestState& state) { + int writer_rc = bgd_set_writer_read_only_0(sim, state.cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated writers"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before publishing AVAILABLE topology"); + return EXIT_FAILURE; + } + state.probe_sequence = seq; + + vector topology = state.cluster.get_topology("AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology"); + return EXIT_FAILURE; + } + + ok(true, "simulator publishes AVAILABLE topology to the blue and green writers"); + return EXIT_SUCCESS; +} + +/** + * Configure an explicit BGD worker for writer hostgroup 10. + * + * - Insert mysql_replication_hostgroups and mysql_aws_rds_bgd_hostgroups rows. + * - Insert the blue writer in mysql_servers hostgroup 10. + * - Verify that the runtime BGD status reaches AVAILABLE. + */ +int configure_bgd_available(MYSQL* admin, TestState& state) { + int config_rc = configure_explicit_bgd(admin, state); + if (config_rc != EXIT_SUCCESS) { + diag("Error: failed to configure mysql_servers and mysql_aws_rds_bgd_hostgroups"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, state.hostgroups, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 10 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 10 reports AVAILABLE"); + return EXIT_SUCCESS; +} + +/** + * Verify the AVAILABLE worker probes the green writer. + * + * - Wait for a metadata probe after the topology publication sequence. + * - Require the probe on the green writer IP without TLS. + */ +int test_plaintext_green_writer_probe(RDS_BGD_Simulator& sim, TestState& state) { + auto [probe_rc, probe] = sim.wait_for_probe_log( + state.probe_sequence, state.cluster.green_writer.endpoint(), + RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: green writer did not receive a plaintext metadata probe"); + return EXIT_FAILURE; + } + + ok(true, "BGD worker probes the green writer IP over plaintext"); + return EXIT_SUCCESS; +} + +int main() { + plan(3); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: set blue/green writer read_only=0 and publish AVAILABLE topology. + // Verify: topology publication succeeds for both writer endpoints. + if (publish_available_topology(sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: configure mysql_servers and mysql_aws_rds_bgd_hostgroups for wHG 10. + // Verify: BGD status for wHG 10 reports AVAILABLE. + if (configure_bgd_available(admin, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: run the explicitly configured BGD worker without TLS. + // Verify: the green writer IP receives a plaintext metadata probe. + if (test_plaintext_green_writer_probe(sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp b/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp new file mode 100644 index 0000000000..1c938b306f --- /dev/null +++ b/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp @@ -0,0 +1,560 @@ +/** + * @file test_rds_bgd_topology_empty_absent-t.cpp + * @brief Empty and absent BGD topology before and after writer completion. + * + * Steps: + * + * 1. Delete topology rows during WRITER_SWITCHOVER_IN_PROGRESS and verify + * rollback through a successful metadata probe. + * 2. Drop the topology table during WRITER_SWITCHOVER_IN_PROGRESS and verify + * rollback followed by a blue-writer table check. + * 3. Delete topology rows during READER_SWITCHOVER_IN_PROGRESS and verify + * reader cleanup through a successful metadata probe. + * 4. Drop the topology table during READER_SWITCHOVER_IN_PROGRESS and verify + * reader cleanup followed by a blue-writer table check. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster empty_before { bgd_cluster_init() }; + BGD_Hostgroups empty_before_hg { 1100, 1101, 1102, 1103 }; + vector empty_before_endpoints { empty_before.get_endpoints() }; + + RDS_BGD_Cluster absent_before { bgd_cluster_2_init() }; + BGD_Hostgroups absent_before_hg { 1110, 1111, 1112, 1113 }; + vector absent_before_endpoints { absent_before.get_endpoints() }; + + RDS_BGD_Cluster empty_reader { bgd_cluster_3_init() }; + BGD_Hostgroups empty_reader_hg { 1120, 1121, 1122, 1123 }; + vector empty_reader_endpoints { empty_reader.get_endpoints() }; + + RDS_BGD_Cluster absent_reader { bgd_cluster_1_deployment_b_init() }; + BGD_Hostgroups absent_reader_hg { 1130, 1131, 1132, 1133 }; + vector absent_reader_endpoints { absent_reader.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status='ONLINE'"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool online = rows[0][0] == "1"; + return online; +} + +int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); + + int rc = sim.topology_update(endpoints, topology); + return rc; +} + +int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, + BGD_Hostgroups& hg, vector endpoints) +{ + int config_rc = configure_bgd(admin, sim, cluster, hg); + if (config_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int available_rc = publish_topology(sim, endpoints, cluster, "AVAILABLE"); + if (available_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int available_status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach AVAILABLE", hg.blue_writer); + return EXIT_FAILURE; + } + + int progress_rc = publish_topology(sim, endpoints, cluster, "SWITCHOVER_IN_PROGRESS"); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int progress_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach WRITER_SWITCHOVER_IN_PROGRESS", hg.blue_writer); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer for wHG %d did not move to its reader hostgroup", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, + BGD_Hostgroups& hg, vector endpoints) +{ + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, endpoints); + if (progress_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int post_rc = publish_topology(sim, endpoints, cluster, "SWITCHOVER_IN_POST_PROCESSING"); + if (post_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int post_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (post_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach WRITER_SWITCHOVER_POST_PROCESSING", hg.blue_writer); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer for wHG %d did not return to its writer hostgroup", hg.blue_writer); + return EXIT_FAILURE; + } + + vector completed = target_only_completed(cluster); + int completed_rc = sim.topology_update(endpoints, completed); + if (completed_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int reader_status_rc = bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (reader_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach READER_SWITCHOVER_IN_PROGRESS", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int disable_bgd(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "UPDATE mysql_aws_rds_bgd_hostgroups SET active=0 WHERE writer_hostgroup=" + + to_string(hg.blue_writer); + vector queries { + query, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +/** + * Delete topology rows during writer switchover. + * + * - Reach WRITER_SWITCHOVER_IN_PROGRESS for wHG 1100. + * - Delete every topology row while the topology table remains present. + * - Verify BGD status NONE, restored blue-writer placement, and metadata + * telemetry from the pinned green writer. + */ +int test_empty_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.empty_before; + BGD_Hostgroups& hg = state.empty_before_hg; + + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, state.empty_before_endpoints); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to reach writer switchover for wHG 1100"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before empty topology for wHG 1100"); + return EXIT_FAILURE; + } + + int empty_rc = sim.topology_delete(state.empty_before_endpoints); + if (empty_rc != EXIT_SUCCESS) { + diag("Error: failed to delete topology rows for wHG 1100"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1100 did not reach NONE after empty topology"); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: empty topology did not restore the blue writer for wHG 1100"); + return EXIT_FAILURE; + } + + ok(true, "empty topology restores the blue writer and sets BGD status for wHG 1100 to NONE"); + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: empty topology for wHG 1100 was not observed through green-writer metadata"); + return EXIT_FAILURE; + } + + ok(true, "empty topology for wHG 1100 is observed through a successful green-writer metadata probe"); + + int disable_rc = disable_bgd(admin, hg); + if (disable_rc != EXIT_SUCCESS) { + diag("Error: failed to stop wHG 1100 before the next topology scenario"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Drop the topology table during writer switchover. + * + * - Reach WRITER_SWITCHOVER_IN_PROGRESS for wHG 1110. + * - Drop the topology table on the simulated blue and green endpoints. + * - Verify BGD status NONE, restored blue-writer placement, and a new + * blue-writer table-check probe. + */ +int test_absent_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.absent_before; + BGD_Hostgroups& hg = state.absent_before_hg; + + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, state.absent_before_endpoints); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to reach writer switchover for wHG 1110"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before absent topology for wHG 1110"); + return EXIT_FAILURE; + } + + int absent_rc = sim.topology_drop(state.absent_before_endpoints); + if (absent_rc != EXIT_SUCCESS) { + diag("Error: failed to drop the topology table for wHG 1110"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1110 did not reach NONE after absent topology"); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: absent topology did not restore the blue writer for wHG 1110"); + return EXIT_FAILURE; + } + + ok(true, "absent topology restores the blue writer and sets BGD status for wHG 1110 to NONE"); + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: absent topology for wHG 1110 did not return to blue-writer table checks"); + return EXIT_FAILURE; + } + + ok(true, "absent topology for wHG 1110 returns probing to the blue-writer table check"); + + int disable_rc = disable_bgd(admin, hg); + if (disable_rc != EXIT_SUCCESS) { + diag("Error: failed to stop wHG 1110 before the next topology scenario"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Delete topology rows during reader switchover. + * + * - Reach READER_SWITCHOVER_IN_PROGRESS for wHG 1120. + * - Delete every topology row while the topology table remains present. + * - Verify BGD status NONE, restored blue-reader routing, retained green + * rows, and metadata telemetry from the pinned green writer. + */ +int test_empty_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.empty_reader; + BGD_Hostgroups& hg = state.empty_reader_hg; + + int reader_rc = enter_reader_switchover(admin, sim, cluster, hg, state.empty_reader_endpoints); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to reach reader switchover for wHG 1120"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before empty topology for wHG 1120"); + return EXIT_FAILURE; + } + + int empty_rc = sim.topology_delete(state.empty_reader_endpoints); + if (empty_rc != EXIT_SUCCESS) { + diag("Error: failed to delete topology rows for wHG 1120"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1120 did not reach NONE after empty topology"); + return EXIT_FAILURE; + } + + bool blue_reader_online = runtime_server_online(admin, hg.blue_reader, cluster.blue_readers[1]); + bool green_writer_online = runtime_server_online(admin, hg.green_writer, cluster.green_writer); + bool green_reader_online = runtime_server_online(admin, hg.green_reader, cluster.green_readers[0]); + ok(blue_reader_online && green_writer_online && green_reader_online, + "empty topology completes reader cleanup for wHG 1120 and retains configured green rows"); + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: empty topology for wHG 1120 was not observed through green-writer metadata"); + return EXIT_FAILURE; + } + + ok(true, "reader cleanup for wHG 1120 starts from a successful green-writer metadata probe"); + + int disable_rc = disable_bgd(admin, hg); + if (disable_rc != EXIT_SUCCESS) { + diag("Error: failed to stop wHG 1120 before the next topology scenario"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Drop the topology table during reader switchover. + * + * - Reach READER_SWITCHOVER_IN_PROGRESS for wHG 1130. + * - Drop the topology table on the simulated blue and green endpoints. + * - Verify BGD status NONE, restored blue-reader routing, retained green + * rows, and a new blue-writer table-check probe. + */ +int test_absent_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.absent_reader; + BGD_Hostgroups& hg = state.absent_reader_hg; + + int reader_rc = enter_reader_switchover(admin, sim, cluster, hg, state.absent_reader_endpoints); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to reach reader switchover for wHG 1130"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before absent topology for wHG 1130"); + return EXIT_FAILURE; + } + + int absent_rc = sim.topology_drop(state.absent_reader_endpoints); + if (absent_rc != EXIT_SUCCESS) { + diag("Error: failed to drop the topology table for wHG 1130"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1130 did not reach NONE after absent topology"); + return EXIT_FAILURE; + } + + bool blue_reader_online = runtime_server_online(admin, hg.blue_reader, cluster.blue_readers[1]); + bool green_writer_online = runtime_server_online(admin, hg.green_writer, cluster.green_writer); + bool green_reader_online = runtime_server_online(admin, hg.green_reader, cluster.green_readers[0]); + ok(blue_reader_online && green_writer_online && green_reader_online, + "absent topology completes reader cleanup for wHG 1130 and retains configured green rows"); + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: absent topology for wHG 1130 did not return to blue-writer table checks"); + return EXIT_FAILURE; + } + + ok(true, "reader cleanup for wHG 1130 returns probing to the blue-writer table check"); + return EXIT_SUCCESS; +} + +int main() { + plan(8); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish SWITCHOVER_IN_PROGRESS, then delete all topology rows. + // Verify: wHG 1100 reaches NONE, restores its blue writer, and records green-writer metadata. + if (test_empty_before_completion(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS, then drop the topology table. + // Verify: wHG 1110 reaches NONE, restores its blue writer, and returns to blue-writer table checks. + if (test_absent_before_completion(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish target-only SWITCHOVER_COMPLETED, then delete all topology rows. + // Verify: wHG 1120 completes reader cleanup through successful green-writer metadata. + if (test_empty_during_reader_switchover(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish target-only SWITCHOVER_COMPLETED, then drop the topology table. + // Verify: wHG 1130 completes reader cleanup and returns to blue-writer table checks. + if (test_absent_during_reader_switchover(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_topology_errors-t.cpp b/test/tap/tests/test_rds_bgd_topology_errors-t.cpp new file mode 100644 index 0000000000..06bc414ffe --- /dev/null +++ b/test/tap/tests/test_rds_bgd_topology_errors-t.cpp @@ -0,0 +1,497 @@ +/** + * @file test_rds_bgd_topology_errors-t.cpp + * @brief BGD metadata error 1146 and generic metadata-error handling. + * + * Steps: + * + * 1. Return metadata error 1146 during WRITER_SWITCHOVER_IN_PROGRESS and + * verify rollback followed by blue-writer table checks. + * 2. Return metadata error 1146 during READER_SWITCHOVER_IN_PROGRESS and + * verify reader cleanup followed by blue-writer table checks. + * 3. Return a generic metadata error during WRITER_SWITCHOVER_IN_PROGRESS and + * verify that the active status and blue-writer demotion remain unchanged. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 800; + +struct TestState { + RDS_BGD_Cluster before_completion { bgd_cluster_init() }; + BGD_Hostgroups before_completion_hg { 1140, 1141, 1142, 1143 }; + vector before_completion_endpoints { before_completion.get_endpoints() }; + + RDS_BGD_Cluster reader_switchover { bgd_cluster_2_init() }; + BGD_Hostgroups reader_switchover_hg { 1150, 1151, 1152, 1153 }; + vector reader_switchover_endpoints { reader_switchover.get_endpoints() }; + + RDS_BGD_Cluster generic_error { bgd_cluster_3_init() }; + BGD_Hostgroups generic_error_hg { 1160, 1161, 1162, 1163 }; + vector generic_error_endpoints { generic_error.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status='ONLINE'"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool online = rows[0][0] == "1"; + return online; +} + +int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); + + int rc = sim.topology_update(endpoints, topology); + return rc; +} + +int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, + BGD_Hostgroups& hg, vector endpoints) +{ + int config_rc = configure_bgd(admin, sim, cluster, hg); + if (config_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int available_rc = publish_topology(sim, endpoints, cluster, "AVAILABLE"); + if (available_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int available_status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach AVAILABLE", hg.blue_writer); + return EXIT_FAILURE; + } + + int progress_rc = publish_topology(sim, endpoints, cluster, "SWITCHOVER_IN_PROGRESS"); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int progress_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach WRITER_SWITCHOVER_IN_PROGRESS", hg.blue_writer); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer for wHG %d did not move to its reader hostgroup", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, + BGD_Hostgroups& hg, vector endpoints) +{ + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, endpoints); + if (progress_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int post_rc = publish_topology(sim, endpoints, cluster, "SWITCHOVER_IN_POST_PROCESSING"); + if (post_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int post_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (post_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach WRITER_SWITCHOVER_POST_PROCESSING", hg.blue_writer); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer for wHG %d did not return to its writer hostgroup", hg.blue_writer); + return EXIT_FAILURE; + } + + vector completed = target_only_completed(cluster); + int completed_rc = sim.topology_update(endpoints, completed); + if (completed_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int reader_status_rc = bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (reader_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach READER_SWITCHOVER_IN_PROGRESS", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int wait_for_metadata_error(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, + int error_number, string error_message, RDS_BGD_Probe_Log& probe) +{ + vector green_endpoint { cluster.green_writer.endpoint() }; + int error_rc = sim.topology_error(green_endpoint, error_number, error_message); + if (error_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + auto [probe_rc, metadata_probe] = + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + probe = metadata_probe; + return EXIT_SUCCESS; +} + +int wait_for_blue_table_check(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { + vector blue_endpoint { cluster.blue_writer.endpoint() }; + int drop_rc = sim.topology_drop(blue_endpoint); + if (drop_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + return probe_rc; +} + +/** + * Return metadata error 1146 before writer completion. + * + * - Reach WRITER_SWITCHOVER_IN_PROGRESS for wHG 1140. + * - Return error 1146 from the pinned green-writer metadata probe. + * - Verify BGD status NONE, restored blue-writer placement, and a subsequent + * blue-writer table check. + */ +int test_error_1146_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.before_completion; + BGD_Hostgroups& hg = state.before_completion_hg; + + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, state.before_completion_endpoints); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to reach writer switchover for wHG 1140"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before metadata error 1146 for wHG 1140"); + return EXIT_FAILURE; + } + + RDS_BGD_Probe_Log metadata {}; + int metadata_rc = + wait_for_metadata_error(sim, seq, cluster, 1146, "Table 'mysql.rds_topology' doesn't exist", metadata); + if (metadata_rc != EXIT_SUCCESS) { + diag("Error: wHG 1140 did not observe metadata error 1146 on the green writer"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1140 did not reach NONE after metadata error 1146"); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: metadata error 1146 did not restore the blue writer for wHG 1140"); + return EXIT_FAILURE; + } + + ok(true, "metadata error 1146 restores the blue writer and sets BGD status for wHG 1140 to NONE"); + + int table_rc = wait_for_blue_table_check(sim, metadata.sequence_id, cluster); + if (table_rc != EXIT_SUCCESS) { + diag("Error: wHG 1140 did not return to blue-writer table checks after metadata error 1146"); + return EXIT_FAILURE; + } + + ok(true, "metadata error 1146 returns wHG 1140 from green metadata to blue-writer table checks"); + return EXIT_SUCCESS; +} + +/** + * Return metadata error 1146 during reader switchover. + * + * - Reach READER_SWITCHOVER_IN_PROGRESS for wHG 1150. + * - Return error 1146 from the pinned green-writer metadata probe. + * - Verify BGD status NONE, restored blue-reader routing, retained green rows, + * and a subsequent blue-writer table check. + */ +int test_error_1146_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.reader_switchover; + BGD_Hostgroups& hg = state.reader_switchover_hg; + + int reader_rc = enter_reader_switchover(admin, sim, cluster, hg, state.reader_switchover_endpoints); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to reach reader switchover for wHG 1150"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before metadata error 1146 for wHG 1150"); + return EXIT_FAILURE; + } + + RDS_BGD_Probe_Log metadata {}; + int metadata_rc = + wait_for_metadata_error(sim, seq, cluster, 1146, "Table 'mysql.rds_topology' doesn't exist", metadata); + if (metadata_rc != EXIT_SUCCESS) { + diag("Error: wHG 1150 did not observe metadata error 1146 on the green writer"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1150 did not reach NONE after metadata error 1146"); + return EXIT_FAILURE; + } + + bool blue_reader_online = runtime_server_online(admin, hg.blue_reader, cluster.blue_readers[1]); + bool green_writer_online = runtime_server_online(admin, hg.green_writer, cluster.green_writer); + bool green_reader_online = runtime_server_online(admin, hg.green_reader, cluster.green_readers[0]); + ok(blue_reader_online && green_writer_online && green_reader_online, + "metadata error 1146 completes reader cleanup for wHG 1150 and retains configured green rows"); + + int table_rc = wait_for_blue_table_check(sim, metadata.sequence_id, cluster); + if (table_rc != EXIT_SUCCESS) { + diag("Error: wHG 1150 did not return to blue-writer table checks after metadata error 1146"); + return EXIT_FAILURE; + } + + ok(true, "metadata error 1146 returns wHG 1150 from green metadata to blue-writer table checks"); + return EXIT_SUCCESS; +} + +/** + * Return a generic metadata error before writer completion. + * + * - Reach WRITER_SWITCHOVER_IN_PROGRESS for wHG 1160. + * - Return error 1105 from the pinned green-writer metadata probe. + * - Verify that table checking does not restart and that the in-progress + * status and blue-writer demotion remain unchanged. + */ +int test_generic_metadata_error(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.generic_error; + BGD_Hostgroups& hg = state.generic_error_hg; + + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, state.generic_error_endpoints); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to reach writer switchover for wHG 1160"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before generic metadata error for wHG 1160"); + return EXIT_FAILURE; + } + + RDS_BGD_Probe_Log metadata {}; + int metadata_rc = wait_for_metadata_error(sim, seq, cluster, 1105, "simulated generic metadata failure", metadata); + if (metadata_rc != EXIT_SUCCESS) { + diag("Error: wHG 1160 did not observe the generic metadata error on the green writer"); + return EXIT_FAILURE; + } + + int no_table_rc = + bgd_expect_no_table_check(sim, metadata.sequence_id, state.generic_error_endpoints, kNegativeProbeTimeoutMs); + if (no_table_rc != EXIT_SUCCESS) { + diag("Error: generic metadata error restarted table checking for wHG 1160"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: generic metadata error changed BGD status for wHG 1160"); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: generic metadata error changed blue-writer placement for wHG 1160"); + return EXIT_FAILURE; + } + + ok(true, "generic metadata error keeps wHG 1160 in progress with the blue writer in hostgroup 1161"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: return metadata error 1146 during SWITCHOVER_IN_PROGRESS. + // Verify: wHG 1140 reaches NONE, restores its blue writer, and returns to blue-writer table checks. + if (test_error_1146_before_completion(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: return metadata error 1146 during READER_SWITCHOVER_IN_PROGRESS. + // Verify: wHG 1150 completes reader cleanup and returns to blue-writer table checks. + if (test_error_1146_during_reader_switchover(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: return generic metadata error 1105 during SWITCHOVER_IN_PROGRESS. + // Verify: wHG 1160 remains in progress with its blue writer in reader hostgroup 1161. + if (test_generic_metadata_error(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp b/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp new file mode 100644 index 0000000000..1643200102 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp @@ -0,0 +1,589 @@ +/** + * @file test_rds_bgd_worker_config_refresh-t.cpp + * @brief Refreshing an active BGD worker after server and scalar configuration changes. + * + * Steps: + * + * 1. Configure BGD hostgroups 1370-1373 and reach `AVAILABLE`. + * 2. Change `weight` and `comment` and verify that discovery does not restart. + * 3. Change green-writer TLS and verify that metadata probes use the new value. + * 4. Replace green-reader membership and verify that discovery does not restart. + * 5. Move the green writer offline and online and verify that direct metadata + * probing stops and resumes without a table-check restart. + * 6. Change `check_interval_ms` and verify the metadata probe cadence. + */ + +#include +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 800; +const uint32_t kRefreshedCheckIntervalMs = 1000; +const uint32_t kMinimumProbeIntervalMs = 500; +const uint32_t kMaximumProbeIntervalMs = 1500; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_2_init() }; + BGD_Hostgroups hostgroups { 1370, 1371, 1372, 1373 }; + vector topology_endpoints { cluster.get_endpoints() }; + uint64_t available_probe_sequence { 0 }; + string topology_discovery_interval {}; + bool topology_discovery_interval_saved { false }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int restore_topology_discovery_interval(MYSQL* admin, TestState& state) { + if (!state.topology_discovery_interval_saved) { + return EXIT_SUCCESS; + } + + vector queries { + "SET mysql-monitor_aws_rds_topology_discovery_interval=" + state.topology_discovery_interval, + "LOAD MYSQL VARIABLES TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + + int discovery_rc = restore_topology_discovery_interval(admin, state); + if (discovery_rc != EXIT_SUCCESS) { + diag("Error: failed to restore mysql-monitor_aws_rds_topology_discovery_interval"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || discovery_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int disable_topology_discovery_probes(MYSQL* admin, TestState& state) { + string query = + "SELECT variable_value FROM runtime_global_variables " + "WHERE variable_name='mysql-monitor_aws_rds_topology_discovery_interval'"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + diag("Error: failed to read mysql-monitor_aws_rds_topology_discovery_interval"); + return EXIT_FAILURE; + } + + state.topology_discovery_interval = rows[0][0]; + state.topology_discovery_interval_saved = true; + + vector queries { + "SET mysql-monitor_aws_rds_topology_discovery_interval=0", + "LOAD MYSQL VARIABLES TO RUNTIME", + }; + + int disable_rc = execute_all(admin, queries); + if (disable_rc != EXIT_SUCCESS) { + diag("Error: failed to disable automatic AWS topology discovery"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int wait_for_server_status(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, string status) { + string query = + "SELECT COUNT(*)=1 FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status=" + bgd_sql_quote(status); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +/** + * Configure BGD hostgroups 1370-1373 and change ignored server fields. + * + * - Set `read_only=0` for the simulated blue and green writers. + * - Publish `AVAILABLE` topology and configure the explicit BGD row. + * - Verify that the runtime BGD row reaches `AVAILABLE`. + * - Change blue-writer `weight` and `comment`. + * - Verify that the active worker does not restart with a table check. + */ +int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Set read_only=0 for the simulated blue and green writers. + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated writer read_only values"); + return EXIT_FAILURE; + } + + // Publish AVAILABLE topology. + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology"); + return EXIT_FAILURE; + } + + // Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure mysql_servers and mysql_aws_rds_bgd_hostgroups"); + return EXIT_FAILURE; + } + + // Disable automatic AWS topology discovery so its metadata queries are not + // mistaken for probes from the explicitly configured BGD worker. + int discovery_rc = disable_topology_discovery_probes(admin, state); + if (discovery_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + // Wait for the runtime BGD row to report AVAILABLE. + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime BGD status did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1370 reports AVAILABLE"); + + // Record a green metadata probe before changing ignored server fields. + auto [probe_seq_rc, probe_seq] = sim.probe_log_last_sequence(); + if (probe_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the AVAILABLE metadata probe"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = sim.wait_for_probe_log( + probe_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: green writer did not receive the AVAILABLE metadata probe"); + return EXIT_FAILURE; + } + state.available_probe_sequence = probe.sequence_id; + + // Change weight and comment, which are not BGD worker inputs. + string update_server = + "UPDATE mysql_servers SET weight=weight+7,comment='BGD TAP ignored fields' WHERE hostgroup_id=" + + to_string(hg.blue_writer) + " AND hostname=" + bgd_sql_quote(cluster.blue_writer.hostname) + + " AND port=" + to_string(cluster.blue_writer.port); + vector queries { + update_server, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int update_rc = execute_all(admin, queries); + if (update_rc != EXIT_SUCCESS) { + diag("Error: failed to update blue-writer weight and comment"); + return EXIT_FAILURE; + } + + int no_table_rc = bgd_expect_no_table_check(sim, state.available_probe_sequence, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (no_table_rc != EXIT_SUCCESS) { + diag("Error: weight or comment change restarted BGD discovery"); + return EXIT_FAILURE; + } + + ok(true, "weight and comment changes do not restart BGD discovery for wHG 1370"); + return EXIT_SUCCESS; +} + +/** + * Refresh green-writer TLS for writer hostgroup 1370. + * + * - Set `use_ssl=1` on the configured green writer. + * - Load `mysql_servers` to runtime. + * - Verify that the next green-writer metadata probe uses TLS. + * - Verify that discovery does not restart. + */ +int test_tls_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Record the probe sequence before changing green-writer TLS. + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the TLS refresh"); + return EXIT_FAILURE; + } + + // Set use_ssl=1 on the green writer and load mysql_servers to runtime. + string update_tls = + "UPDATE mysql_servers SET use_ssl=1 WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(cluster.green_writer.hostname) + + " AND port=" + to_string(cluster.green_writer.port); + vector queries { + update_tls, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int update_rc = execute_all(admin, queries); + if (update_rc != EXIT_SUCCESS) { + diag("Error: failed to set use_ssl=1 for the green writer"); + return EXIT_FAILURE; + } + + // Wait for the active worker to use TLS without starting a table check. + auto [probe_rc, probe] = sim.wait_for_probe_log( + seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: green-writer metadata probe did not use TLS after refresh"); + return EXIT_FAILURE; + } + + int no_table_rc = bgd_expect_no_table_check(sim, seq, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (no_table_rc != EXIT_SUCCESS) { + diag("Error: TLS refresh restarted BGD discovery"); + return EXIT_FAILURE; + } + + ok(true, "use_ssl=1 refreshes green-writer metadata probes without a table-check restart"); + return EXIT_SUCCESS; +} + +/** + * Replace green-reader membership for writer hostgroup 1370. + * + * - Add the second green reader to hostgroup 1373. + * - Delete the first green reader from hostgroup 1373. + * - Verify that discovery does not restart. + */ +int test_green_membership_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Record the probe sequence before changing green-reader membership. + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the membership refresh"); + return EXIT_FAILURE; + } + + // Replace the first green reader with the second green reader. + string add_reader = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hg.green_reader) + "," + bgd_sql_quote(cluster.green_readers[1].hostname) + + "," + to_string(cluster.green_readers[1].port) + ",'ONLINE',0,'BGD TAP green reader')"; + string delete_reader = + "DELETE FROM mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(cluster.green_readers[0].hostname) + + " AND port=" + to_string(cluster.green_readers[0].port); + vector queries { + add_reader, + delete_reader, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int update_rc = execute_all(admin, queries); + if (update_rc != EXIT_SUCCESS) { + diag("Error: failed to replace green-reader membership"); + return EXIT_FAILURE; + } + + int no_table_rc = bgd_expect_no_table_check(sim, seq, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (no_table_rc != EXIT_SUCCESS) { + diag("Error: green-reader membership refresh restarted BGD discovery"); + return EXIT_FAILURE; + } + + ok(true, "green-reader membership refresh does not restart BGD discovery for wHG 1370"); + return EXIT_SUCCESS; +} + +/** + * Refresh green-writer eligibility for writer hostgroup 1370. + * + * - Move the green writer to `OFFLINE_SOFT`. + * - Verify that direct metadata probing stops without a table-check restart. + * - Return the green writer to `ONLINE`. + * - Verify that TLS metadata probing resumes without a table-check restart. + */ +int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Record the probe sequence before changing green-writer eligibility. + auto [refresh_seq_rc, refresh_seq] = sim.probe_log_last_sequence(); + if (refresh_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the OFFLINE_SOFT refresh"); + return EXIT_FAILURE; + } + + // Move the green writer to OFFLINE_SOFT. + string set_offline = + "UPDATE mysql_servers SET status='OFFLINE_SOFT' WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(cluster.green_writer.hostname) + + " AND port=" + to_string(cluster.green_writer.port); + vector offline_queries { + set_offline, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int offline_rc = execute_all(admin, offline_queries); + if (offline_rc != EXIT_SUCCESS) { + diag("Error: failed to set the green writer OFFLINE_SOFT"); + return EXIT_FAILURE; + } + + int status_rc = wait_for_server_status(admin, hg.green_writer, cluster.green_writer, "OFFLINE_SOFT"); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime green writer did not reach OFFLINE_SOFT"); + return EXIT_FAILURE; + } + + // Wait for the worker to apply the refreshed server list and return to its eligible blue writer. + auto [blue_probe_rc, blue_probe] = sim.wait_for_probe_log( + refresh_seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (blue_probe_rc != EXIT_SUCCESS) { + diag("Error: OFFLINE_SOFT refresh did not return metadata probing to the blue writer"); + return EXIT_FAILURE; + } + + int no_metadata_rc = + bgd_expect_no_metadata_probe(sim, blue_probe.sequence_id, cluster.green_writer.endpoint(), kNegativeProbeTimeoutMs); + if (no_metadata_rc != EXIT_SUCCESS) { + diag("Error: OFFLINE_SOFT green writer continued receiving metadata probes"); + return EXIT_FAILURE; + } + + int offline_no_table_rc = + bgd_expect_no_table_check(sim, refresh_seq, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (offline_no_table_rc != EXIT_SUCCESS) { + diag("Error: OFFLINE_SOFT refresh restarted BGD discovery"); + return EXIT_FAILURE; + } + + ok(true, "OFFLINE_SOFT stops green-writer metadata probes without a table-check restart"); + + // Return the green writer to ONLINE. + auto [online_seq_rc, online_seq] = sim.probe_log_last_sequence(); + if (online_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the ONLINE refresh"); + return EXIT_FAILURE; + } + + string set_online = + "UPDATE mysql_servers SET status='ONLINE' WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(cluster.green_writer.hostname) + + " AND port=" + to_string(cluster.green_writer.port); + vector online_queries { + set_online, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int online_rc = execute_all(admin, online_queries); + if (online_rc != EXIT_SUCCESS) { + diag("Error: failed to return the green writer to ONLINE"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = sim.wait_for_probe_log( + online_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: ONLINE green writer did not resume TLS metadata probes"); + return EXIT_FAILURE; + } + + int online_no_table_rc = bgd_expect_no_table_check(sim, online_seq, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (online_no_table_rc != EXIT_SUCCESS) { + diag("Error: ONLINE refresh restarted BGD discovery"); + return EXIT_FAILURE; + } + + ok(true, "ONLINE resumes green-writer TLS metadata probes without a table-check restart"); + return EXIT_SUCCESS; +} + +/** + * Refresh the configured polling interval for writer hostgroup 1370. + * + * - Set `check_interval_ms=1000`. + * - Load the BGD configuration to runtime. + * - Publish empty topology so the worker uses its configured baseline interval. + * - Verify that the next metadata probe occurs between 500 and 1500 milliseconds. + * - Verify that the configuration refresh does not restart with a table check. + */ +int test_check_interval_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Record the probe sequence before changing check_interval_ms. + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the check_interval_ms refresh"); + return EXIT_FAILURE; + } + + // Set check_interval_ms=1000 and load the BGD configuration to runtime. + string update_bgd = + "UPDATE mysql_aws_rds_bgd_hostgroups SET check_interval_ms=" + + to_string(kRefreshedCheckIntervalMs) + " WHERE writer_hostgroup=" + + to_string(hg.blue_writer); + vector queries { + update_bgd, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int update_rc = execute_all(admin, queries); + if (update_rc != EXIT_SUCCESS) { + diag("Error: failed to update check_interval_ms"); + return EXIT_FAILURE; + } + + // Publish empty topology so the worker leaves AVAILABLE and uses check_interval_ms. + int topology_rc = sim.topology_delete(state.topology_endpoints); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish empty topology before checking the probe interval"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status did not reach NONE before checking the probe interval"); + return EXIT_FAILURE; + } + + auto [baseline_rc, baseline] = sim.probe_log_last_sequence(); + if (baseline_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence after BGD reached NONE"); + return EXIT_FAILURE; + } + + // Consume the immediate refresh probe and the first blue probe after the worker reaches NONE. + auto [first_rc, first_probe] = + sim.wait_for_probe_log(baseline, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); + if (first_rc != EXIT_SUCCESS) { + diag("Error: failed to observe the first blue metadata probe after the check_interval_ms refresh"); + return EXIT_FAILURE; + } + + auto [settled_rc, settled_probe] = + sim.wait_for_probe_log(first_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); + if (settled_rc != EXIT_SUCCESS) { + diag("Error: failed to observe the settled blue metadata probe after the check_interval_ms refresh"); + return EXIT_FAILURE; + } + + // Measure the steady-state interval between consecutive blue metadata probes. + unsigned long long interval_start = monotonic_time(); + auto [next_rc, next_probe] = sim.wait_for_probe_log( + settled_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kMaximumProbeIntervalMs, -1 + ); + if (next_rc != EXIT_SUCCESS) { + diag("Error: metadata probing did not occur within 1.5 times check_interval_ms"); + return EXIT_FAILURE; + } + + unsigned long long elapsed_ms = (monotonic_time() - interval_start) / 1000; + if (elapsed_ms < kMinimumProbeIntervalMs) { + diag("Error: consecutive metadata probes occurred before half of check_interval_ms elapsed"); + return EXIT_FAILURE; + } + + int no_table_rc = bgd_expect_no_table_check(sim, seq, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (no_table_rc != EXIT_SUCCESS) { + diag("Error: check_interval_ms refresh restarted discovery"); + return EXIT_FAILURE; + } + + ok(true, "check_interval_ms=1000 schedules the next metadata probe between 500 and 1500 milliseconds"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: set blue/green writers to read_only=0 and publish AVAILABLE topology. + // ProxySQL: configure BGD hostgroups 1370-1373, then change blue-writer weight and comment. + // Verify: BGD reaches AVAILABLE and ignored fields do not start a new table check. + if (test_irrelevant_server_fields(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: set use_ssl=1 for the green writer in hostgroup 1372. + // Verify: the next green-writer metadata probe uses TLS without restarting discovery. + if (test_tls_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: replace the green reader in hostgroup 1373 and load mysql_servers to runtime. + // Verify: green-reader membership refresh does not restart BGD discovery. + if (test_green_membership_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: move the green writer OFFLINE_SOFT and then ONLINE. + // Verify: direct metadata probes stop and resume without a table-check restart. + if (test_server_eligibility_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: set check_interval_ms=1000 for wHG 1370 and publish empty topology. + // Verify: metadata probing follows the configured interval without restarting discovery. + if (test_check_interval_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim, state) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp b/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp new file mode 100644 index 0000000000..6bd3d7ac64 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp @@ -0,0 +1,359 @@ +/** + * @file test_rds_bgd_worker_hostgroup_refresh-t.cpp + * @brief Refreshing hostgroups and the mapped blue writer during writer switchover. + * + * Steps: + * + * 1. Configure BGD hostgroups 1380-1383 and reach + * `WRITER_SWITCHOVER_IN_PROGRESS`. + * 2. Change the reader and green hostgroups to 1384-1386. + * 3. Verify that the BGD status remains in progress and runtime placement + * moves to the configured reader hostgroup. + * 4. Verify metadata probes use TLS configured only in the refreshed green + * hostgroups. + * 5. Move a blue reader into writer hostgroup 1380 and publish topology that + * maps it to a different green target. + * 6. Verify that the previous writer returns to hostgroup 1380, the newly + * mapped writer moves to hostgroup 1384, uses TLS from green writer + * hostgroup 1385, and stops probing the stale target. + */ + +#include +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 800; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_3_init() }; + BGD_Hostgroups hostgroups { 1380, 1381, 1382, 1383 }; + BGD_Hostgroups refreshed_hostgroups { 1380, 1384, 1385, 1386 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_as_writer(RDS_BGD_Cluster& cluster) { + RDS_BGD_Host& blue_writer = cluster.blue_readers[0]; + RDS_BGD_Host& green_writer = cluster.green_readers[0]; + + vector rows { + { blue_writer.hostname, blue_writer.hostname, blue_writer.port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", "SWITCHOVER_IN_PROGRESS" }, + { green_writer.hostname, green_writer.hostname, green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", "SWITCHOVER_IN_PROGRESS" }, + }; + return rows; +} + +/** + * Refresh the reader and green hostgroups during writer switchover. + * + * - Configure BGD hostgroups 1380-1383. + * - Publish `SWITCHOVER_IN_PROGRESS` and require the blue writer in hostgroup + * 1381. + * - Change the reader and green hostgroups to 1384-1386. + * - Verify that `WRITER_SWITCHOVER_IN_PROGRESS` is preserved. + * - Verify writer placement in hostgroup 1384. + * - Verify metadata probes use TLS from green hostgroups 1385 and 1386. + */ +int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + BGD_Hostgroups& refreshed_hg = state.refreshed_hostgroups; + + // Set read_only=0 for the simulated blue and green writers. + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated writer read_only values"); + return EXIT_FAILURE; + } + + // Publish SWITCHOVER_IN_PROGRESS topology. + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); + return EXIT_FAILURE; + } + + // Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1380-1383"); + return EXIT_FAILURE; + } + + // Require the in-progress status and blue-writer demotion before changing hostgroups. + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1380 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not move from hostgroup 1380 to 1381"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before refreshing hostgroups"); + return EXIT_FAILURE; + } + + // Move blue readers and TLS-enabled green servers into the refreshed hostgroups. + string update_replication = + "UPDATE mysql_replication_hostgroups SET reader_hostgroup=" + to_string(refreshed_hg.blue_reader) + + " WHERE writer_hostgroup=" + to_string(refreshed_hg.blue_writer); + string move_blue = + "UPDATE mysql_servers SET hostgroup_id=" + to_string(refreshed_hg.blue_reader) + + " WHERE hostgroup_id=" + to_string(hg.blue_reader); + string move_green_writer = + "UPDATE mysql_servers SET hostgroup_id=" + to_string(refreshed_hg.green_writer) + ",use_ssl=1" + + " WHERE hostgroup_id=" + to_string(hg.green_writer); + string move_green_readers = + "UPDATE mysql_servers SET hostgroup_id=" + to_string(refreshed_hg.green_reader) + ",use_ssl=1" + + " WHERE hostgroup_id=" + to_string(hg.green_reader); + string update_bgd = + "UPDATE mysql_aws_rds_bgd_hostgroups SET reader_hostgroup=" + to_string(refreshed_hg.blue_reader) + + ",green_writer_hostgroup=" + to_string(refreshed_hg.green_writer) + + ",green_reader_hostgroup=" + to_string(refreshed_hg.green_reader) + + " WHERE writer_hostgroup=" + to_string(refreshed_hg.blue_writer); + vector queries { + update_replication, + move_blue, + move_green_writer, + move_green_readers, + update_bgd, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int refresh_rc = execute_all(admin, queries); + if (refresh_rc != EXIT_SUCCESS) { + diag("Error: failed to refresh BGD hostgroups from 1381-1383 to 1384-1386"); + return EXIT_FAILURE; + } + + // Verify that the runtime BGD status is preserved. + int refreshed_status_rc = bgd_wait_for_status(admin, refreshed_hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (refreshed_status_rc != EXIT_SUCCESS) { + diag("Error: hostgroup refresh did not preserve WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1380 remains WRITER_SWITCHOVER_IN_PROGRESS after hostgroup refresh"); + + // Verify that the blue writer uses the refreshed reader hostgroup. + int refreshed_placement_rc = bgd_wait_for_server_placement( + admin, refreshed_hg.blue_writer, refreshed_hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds + ); + if (refreshed_placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not move from reader hostgroup 1381 to 1384"); + return EXIT_FAILURE; + } + + ok(true, "hostgroup refresh moves the demoted blue writer from hostgroup 1381 to 1384"); + + // Require TLS from the green writer row in refreshed green writer hostgroup 1385. + auto [probe_rc, probe] = sim.wait_for_probe_log( + seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: metadata probe did not use TLS from green writer hostgroup 1385"); + return EXIT_FAILURE; + } + + ok(true, "hostgroup refresh uses TLS from the green writer in hostgroup 1385"); + return EXIT_SUCCESS; +} + +/** + * Refresh the mapped blue writer during writer switchover. + * + * - Move the first blue reader from hostgroup 1384 to writer hostgroup 1380. + * - Publish `SWITCHOVER_IN_PROGRESS` topology that maps it to the first green + * reader. + * - Move that green target from reader hostgroup 1386 to writer hostgroup + * 1385. + * - Verify that the previous writer returns to hostgroup 1380. + * - Verify that the newly mapped writer moves to hostgroup 1384. + * - Verify that metadata probing uses TLS from the new green writer target. + */ +int test_mapped_writer_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.refreshed_hostgroups; + RDS_BGD_Host& previous_writer = cluster.blue_writer; + RDS_BGD_Host& mapped_writer = cluster.blue_readers[0]; + RDS_BGD_Host& mapped_target = cluster.green_readers[0]; + + // Publish topology that maps the first blue reader to the first green reader. + vector topology = topology_with_reader_as_writer(cluster); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for the new mapped writer"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before changing the mapped writer"); + return EXIT_FAILURE; + } + + // Move the new blue/green writer pair into writer hostgroups 1380 and 1385. + string move_writer = + "UPDATE mysql_servers SET hostgroup_id=" + to_string(hg.blue_writer) + + " WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(mapped_writer.hostname) + + " AND port=" + to_string(mapped_writer.port); + string move_target = + "UPDATE mysql_servers SET hostgroup_id=" + to_string(hg.green_writer) + + " WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(mapped_target.hostname) + + " AND port=" + to_string(mapped_target.port); + vector queries { + move_writer, + move_target, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int refresh_rc = execute_all(admin, queries); + if (refresh_rc != EXIT_SUCCESS) { + diag("Error: failed to move the new mapped writer pair into hostgroups 1380 and 1385"); + return EXIT_FAILURE; + } + + // Require the preserved BGD status and the new target metadata probe. + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: mapped-writer refresh did not preserve WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = sim.wait_for_probe_log( + seq, mapped_target.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: metadata probing did not use TLS from green writer hostgroup 1385"); + return EXIT_FAILURE; + } + + // Verify that the previous writer was restored to writer hostgroup 1380. + int previous_writer_rc = bgd_wait_for_server_placement( + admin, hg.blue_writer, hg.blue_reader, previous_writer, false, kTimeoutSeconds + ); + if (previous_writer_rc != EXIT_SUCCESS) { + diag("Error: previous blue writer was not restored to hostgroup 1380"); + return EXIT_FAILURE; + } + + ok(true, "mapped-writer refresh restores the previous blue writer from hostgroup 1384 to 1380"); + + // Verify that the newly mapped writer was demoted to reader hostgroup 1384. + int mapped_writer_rc = bgd_wait_for_server_placement( + admin, hg.blue_writer, hg.blue_reader, mapped_writer, true, kTimeoutSeconds + ); + if (mapped_writer_rc != EXIT_SUCCESS) { + diag("Error: newly mapped writer did not move to reader hostgroup 1384"); + return EXIT_FAILURE; + } + + ok(true, "mapped-writer refresh moves the new blue writer from hostgroup 1380 to 1384"); + + // Verify that the previous green target receives no metadata probes after the new target. + int stale_probe_rc = + bgd_expect_no_metadata_probe(sim, probe.sequence_id, cluster.green_writer.endpoint(), kNegativeProbeTimeoutMs); + if (stale_probe_rc != EXIT_SUCCESS) { + diag("Error: previous green target continued receiving metadata probes"); + return EXIT_FAILURE; + } + + ok(true, "mapped-writer refresh uses TLS from hostgroup 1385 and stops probing the previous target"); + return EXIT_SUCCESS; +} + +int main() { + plan(6); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish SWITCHOVER_IN_PROGRESS for the blue/green writers. + // ProxySQL: configure BGD hostgroups 1380-1383, then change reader/green hostgroups to 1384-1386. + // Verify: BGD status stays in progress, writer placement uses 1384, and target probing uses TLS from 1385. + if (test_hostgroup_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS with the first reader pair as the writer pair. + // ProxySQL: move the first blue reader from hostgroup 1384 to writer hostgroup 1380. + // Verify: the previous writer is restored, the new writer is demoted, and probing uses TLS from hostgroup 1385. + if (test_mapped_writer_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp b/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp new file mode 100644 index 0000000000..9bf8b8ee94 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp @@ -0,0 +1,558 @@ +/** + * @file test_rds_bgd_writer_switchover-t.cpp + * @brief BGD writer switchover from AVAILABLE through POST_PROCESSING. + * + * Steps: + * + * 1. Configure BGD hostgroups 970-973 and reach AVAILABLE. + * 2. Publish SWITCHOVER_INITIATED and verify read-only placement suppression. + * 3. Publish SWITCHOVER_IN_PROGRESS and verify blue-writer demotion. + * 4. Create a blue-writer pool through normal routing hostgroup 974. + * 5. Publish SWITCHOVER_IN_POST_PROCESSING and verify writer restoration, + * blue-pool drain, and green backend routing. + * 6. Create a post-cutover pool and verify repeated POST_PROCESSING does not + * drain it again. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kReadOnlyObservationMs = 500; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_init() }; + BGD_Hostgroups hostgroups { 970, 971, 972, 973 }; + int pool_hostgroup { 974 }; + vector topology_endpoints { cluster.get_endpoints() }; + int64_t reader_log_baseline { -1 }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +int wait_for_green_observation(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { + auto [probe_rc, probe] = + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + return probe_rc; +} + +int64_t last_read_only_log_time(MYSQL* admin, RDS_BGD_Host& host) { + string query = + "SELECT COALESCE(MAX(time_start_us),0) FROM mysql_server_read_only_log WHERE hostname=" + + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return -1; + } + + int64_t time = strtoll(rows[0][0].c_str(), nullptr, 10); + return time; +} + +bool server_match_count(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, int expected_count) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == to_string(expected_count); + return matches; +} + +int wait_for_blue_writer_pool_drain(MYSQL* admin, RDS_BGD_Cluster& cluster) { + string query = + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)=0 FROM stats_mysql_connection_pool WHERE srv_host=" + + bgd_sql_quote(cluster.blue_writer.hostname); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int create_blue_writer_pool(CommandLine& cl, MYSQL* admin, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + + string add_server = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(state.pool_hostgroup) + "," + bgd_sql_quote(cluster.blue_writer.hostname) + + "," + to_string(cluster.blue_writer.port) + ",'ONLINE','BGD TAP blue pool router')"; + vector queries { + add_server, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int server_rc = execute_all(admin, queries); + if (server_rc != EXIT_SUCCESS) { + diag("Error: failed to configure blue-pool routing hostgroup 974"); + return EXIT_FAILURE; + } + + int user_rc = set_default_hostgroup(admin, state.pool_hostgroup); + if (user_rc != EXIT_SUCCESS) { + diag("Error: failed to route testuser through blue-pool hostgroup 974"); + return EXIT_FAILURE; + } + + auto [echo_rc, echo] = connect_and_echo(cl); + if (echo_rc != EXIT_SUCCESS || echo.find(cluster.blue_writer.ip) == string::npos) { + diag("Error: failed to create a blue-writer connection through hostgroup 974"); + return EXIT_FAILURE; + } + + int restore_rc = set_default_hostgroup(admin, state.hostgroups.blue_writer); + if (restore_rc != EXIT_SUCCESS) { + diag("Error: failed to restore testuser to writer hostgroup 970"); + return EXIT_FAILURE; + } + + string query = + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)>=1 FROM stats_mysql_connection_pool WHERE hostgroup=" + + to_string(state.pool_hostgroup) + " AND srv_host=" + bgd_sql_quote(cluster.blue_writer.hostname); + + int pool_rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return pool_rc; +} + +/** + * Configure wHG 970 and reach AVAILABLE. + * + * - Set writer read_only=0 and reader read_only=1 values. + * - Publish AVAILABLE topology with one reader pair. + * - Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. + * - Verify BGD status AVAILABLE. + */ +int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int blue_writer_rc = bgd_set_host_read_only_0(sim, cluster.blue_writer); + if (blue_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue writer"); + return EXIT_FAILURE; + } + + int green_writer_rc = bgd_set_host_read_only_0(sim, cluster.green_writer); + if (green_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated green writer"); + return EXIT_FAILURE; + } + + int blue_reader_0_rc = bgd_set_host_read_only_1(sim, cluster.blue_readers[0]); + if (blue_reader_0_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=1 for the first simulated blue reader"); + return EXIT_FAILURE; + } + + int blue_reader_1_rc = bgd_set_host_read_only_1(sim, cluster.blue_readers[1]); + if (blue_reader_1_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=1 for the second simulated blue reader"); + return EXIT_FAILURE; + } + + vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 970"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 970-973"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 970 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 970 reports AVAILABLE"); + return EXIT_SUCCESS; +} + +/** + * Enter writer switchover initiated. + * + * - Publish SWITCHOVER_INITIATED. + * - Verify WRITER_SWITCHOVER_INITIATED. + * - Change simulated blue writer/reader read_only values. + * - Verify BGD suppresses their normal placement changes. + */ +int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the SWITCHOVER_INITIATED probe sequence"); + return EXIT_FAILURE; + } + + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 970 did not reach WRITER_SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 970 reports WRITER_SWITCHOVER_INITIATED"); + + int64_t writer_baseline = last_read_only_log_time(admin, cluster.blue_writer); + state.reader_log_baseline = last_read_only_log_time(admin, cluster.blue_readers[0]); + + int writer_ro_rc = bgd_set_host_read_only_1(sim, cluster.blue_writer); + if (writer_ro_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=1 for the simulated blue writer"); + return EXIT_FAILURE; + } + + int reader_ro_rc = bgd_set_host_read_only_0(sim, cluster.blue_readers[0]); + if (reader_ro_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue reader"); + return EXIT_FAILURE; + } + + auto [suppression_seq_rc, suppression_seq] = sim.probe_log_last_sequence(); + if (suppression_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the initiated suppression probe sequence"); + return EXIT_FAILURE; + } + + vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + int repeat_rc = sim.topology_update(state.topology_endpoints, repeat_topology); + if (repeat_rc != EXIT_SUCCESS) { + diag("Error: failed to repeat SWITCHOVER_INITIATED topology"); + return EXIT_FAILURE; + } + + int observation_rc = wait_for_green_observation(sim, suppression_seq, cluster); + if (observation_rc != EXIT_SUCCESS) { + diag("Error: BGD did not observe repeated SWITCHOVER_INITIATED topology"); + return EXIT_FAILURE; + } + + int writer_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_writer, writer_baseline, kReadOnlyObservationMs); + if (writer_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-writer read_only monitoring was not suppressed during SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], state.reader_log_baseline, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed during SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + bool writer_in_writer_hg = server_match_count(admin, hg.blue_writer, cluster.blue_writer, 1); + bool writer_absent_reader_hg = server_match_count(admin, hg.blue_reader, cluster.blue_writer, 0); + bool reader_in_reader_hg = server_match_count(admin, hg.blue_reader, cluster.blue_readers[0], 1); + bool reader_absent_writer_hg = server_match_count(admin, hg.blue_writer, cluster.blue_readers[0], 0); + ok(writer_in_writer_hg && writer_absent_reader_hg && reader_in_reader_hg && reader_absent_writer_hg, + "SWITCHOVER_INITIATED suppresses blue writer and reader placement changes"); + return EXIT_SUCCESS; +} + +/** + * Enter writer switchover in progress. + * + * - Publish SWITCHOVER_IN_PROGRESS. + * - Verify WRITER_SWITCHOVER_IN_PROGRESS. + * - Verify the blue writer moves from hostgroup 970 to 971. + * - Verify the mapped blue reader remains suppressed in hostgroup 971. + */ +int test_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 970 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 970 reports WRITER_SWITCHOVER_IN_PROGRESS"); + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not move from hostgroup 970 to 971"); + return EXIT_FAILURE; + } + + ok(true, "SWITCHOVER_IN_PROGRESS moves the blue writer from hostgroup 970 to 971"); + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], state.reader_log_baseline, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed during SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + bool reader_in_reader_hg = server_match_count(admin, hg.blue_reader, cluster.blue_readers[0], 1); + bool reader_absent_writer_hg = server_match_count(admin, hg.blue_writer, cluster.blue_readers[0], 0); + ok(reader_in_reader_hg && reader_absent_writer_hg, + "SWITCHOVER_IN_PROGRESS keeps the mapped blue reader suppressed in hostgroup 971"); + return EXIT_SUCCESS; +} + +/** + * Enter writer switchover post-processing. + * + * - Create a blue-writer pool through normal routing hostgroup 974. + * - Publish SWITCHOVER_IN_POST_PROCESSING. + * - Verify WRITER_SWITCHOVER_POST_PROCESSING. + * - Verify writer restoration, blue-pool drain, and green backend routing. + * - Repeat POST_PROCESSING and verify the post-cutover pool is not drained. + */ +int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int blue_pool_rc = create_blue_writer_pool(cl, admin, state); + if (blue_pool_rc != EXIT_SUCCESS) { + diag("Error: failed to establish the blue-writer pool before SWITCHOVER_IN_POST_PROCESSING"); + return EXIT_FAILURE; + } + + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 970 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 970 reports WRITER_SWITCHOVER_POST_PROCESSING"); + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not return from hostgroup 971 to 970"); + return EXIT_FAILURE; + } + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], state.reader_log_baseline, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed during SWITCHOVER_IN_POST_PROCESSING"); + return EXIT_FAILURE; + } + + bool reader_in_reader_hg = server_match_count(admin, hg.blue_reader, cluster.blue_readers[0], 1); + bool reader_absent_writer_hg = server_match_count(admin, hg.blue_writer, cluster.blue_readers[0], 0); + ok(reader_in_reader_hg && reader_absent_writer_hg, + "POST_PROCESSING restores the blue writer to hostgroup 970 and keeps the reader in 971"); + + int pool_drain_rc = wait_for_blue_writer_pool_drain(admin, cluster); + if (pool_drain_rc != EXIT_SUCCESS) { + diag("Error: POST_PROCESSING did not drain the old blue-writer pool"); + return EXIT_FAILURE; + } + + ok(true, "POST_PROCESSING drains the old blue-writer connection pool"); + + auto [echo_rc, echo] = connect_and_echo(cl); + if (echo_rc != EXIT_SUCCESS) { + diag("Error: failed to connect through wHG 970 after POST_PROCESSING"); + return EXIT_FAILURE; + } + + bool green_routing = echo.find(cluster.green_writer.ip) != string::npos; + ok(green_routing, "POST_PROCESSING routes the blue writer hostname to the green backend IP"); + + auto [pool_before_rc, pool_before] = bgd_connection_pool_count(admin, hg.blue_writer, cluster.blue_writer.hostname); + if (pool_before_rc != EXIT_SUCCESS || pool_before < 1) { + diag("Error: failed to establish the post-cutover pool before repeated POST_PROCESSING"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the repeated POST_PROCESSING probe sequence"); + return EXIT_FAILURE; + } + + vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + int repeat_rc = sim.topology_update(state.topology_endpoints, repeat_topology); + if (repeat_rc != EXIT_SUCCESS) { + diag("Error: failed to repeat SWITCHOVER_IN_POST_PROCESSING topology"); + return EXIT_FAILURE; + } + + int observation_rc = wait_for_green_observation(sim, seq, cluster); + if (observation_rc != EXIT_SUCCESS) { + diag("Error: BGD did not observe repeated POST_PROCESSING topology"); + return EXIT_FAILURE; + } + + auto [pool_after_rc, pool_after] = bgd_connection_pool_count(admin, hg.blue_writer, cluster.blue_writer.hostname); + if (pool_after_rc != EXIT_SUCCESS) { + diag("Error: failed to read the pool after repeated POST_PROCESSING"); + return EXIT_FAILURE; + } + + ok(pool_after >= pool_before, "repeated POST_PROCESSING does not drain the post-cutover connection pool"); + return EXIT_SUCCESS; +} + +int main() { + plan(11); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: set writer/reader read_only values and publish AVAILABLE topology. + // ProxySQL: configure BGD hostgroups 970-973. + // Verify: BGD status for wHG 970 reports AVAILABLE. + if (test_bgd_status_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_INITIATED and reverse one blue writer/reader read_only pair. + // Verify: BGD status is WRITER_SWITCHOVER_INITIATED and placement changes remain suppressed. + if (test_switchover_initiated(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS topology. + // Verify: BGD status for wHG 970 reports WRITER_SWITCHOVER_IN_PROGRESS. + // Verify: the blue writer moves from hostgroup 970 to 971. + // Verify: the mapped blue reader remains suppressed in hostgroup 971. + if (test_switchover_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: create a blue-writer pool through normal routing hostgroup 974. + // Simulator: publish SWITCHOVER_IN_POST_PROCESSING twice. + // Verify: writer placement is restored, the old pool drains, and routing reaches the green IP. + // Verify: repeated POST_PROCESSING preserves a connection created after cutover. + if (test_switchover_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/unit/Makefile b/test/tap/tests/unit/Makefile index e584fcb07e..09e852c175 100644 --- a/test/tap/tests/unit/Makefile +++ b/test/tap/tests/unit/Makefile @@ -397,6 +397,7 @@ UNIT_TESTS := smoke_test-t query_cache_unit-t query_processor_unit-t \ mysql_error_classifier_unit-t \ backend_sync_unit-t \ mysql_encode_unit-t \ + mysql_variables_unit-t \ mysql_decompress_payload_unit-t \ mysql_resolution_unit-t \ pgsql_variables_validator_unit-t \ @@ -421,7 +422,9 @@ UNIT_TESTS := smoke_test-t query_cache_unit-t query_processor_unit-t \ gtid_server_data_unit-t \ admin_disk_upgrade_unit-t \ glovars_unit-t \ - pgsql_servers_ssl_params_unit-t + pgsql_servers_ssl_params_unit-t \ + connection_unhealthy_unit-t \ + cluster_sync_unit-t # Plugin-chassis + mysqlx-plugin unit tests — built only when # libproxysql.a was compiled with -DPROXYSQL40 (autodetected higher up diff --git a/test/tap/tests/unit/cluster_sync_unit-t.cpp b/test/tap/tests/unit/cluster_sync_unit-t.cpp new file mode 100644 index 0000000000..a6bc86ca7b --- /dev/null +++ b/test/tap/tests/unit/cluster_sync_unit-t.cpp @@ -0,0 +1,116 @@ +#include "tap.h" +#include "test_globals.h" +#include "test_init.h" + +#include "proxysql.h" + +#include +#include +#include + +// The incoming_servers_t struct is defined in proxysql_admin.h, which cannot +// be included in this test due to circular include dependencies. Instead we +// re-declare the struct here to verify its layout. The canonical definition +// is the source of truth; this copy must match it exactly. +// +// WARNING: If the struct in proxysql_admin.h changes, this test will +// silently drift. Keep in sync. +struct incoming_servers_t { + void* incoming_mysql_servers_v2 = nullptr; + void* incoming_replication_hostgroups = nullptr; + void* incoming_group_replication_hostgroups = nullptr; + void* incoming_galera_hostgroups = nullptr; + void* incoming_aurora_hostgroups = nullptr; + void* incoming_hostgroup_attributes = nullptr; + void* incoming_mysql_servers_ssl_params = nullptr; + void* incoming_aws_rds_bgd_hostgroups = nullptr; + void* runtime_mysql_servers = nullptr; +}; + +static_assert(sizeof(incoming_servers_t) == 9 * sizeof(void*), + "incoming_servers_t must have exactly 9 pointer-sized fields"); + +static void test_incoming_servers_t_size() { + size_t expected = 9; + size_t actual = sizeof(incoming_servers_t) / sizeof(void*); + ok(actual == expected, + "sizeof(incoming_servers_t)/sizeof(void*) == %zu (expected %zu)", + actual, expected); +} + +static void test_incoming_servers_t_field_positions() { + incoming_servers_t s; + s.incoming_mysql_servers_v2 = (void*)1; + s.incoming_replication_hostgroups = (void*)2; + s.incoming_group_replication_hostgroups = (void*)3; + s.incoming_galera_hostgroups = (void*)4; + s.incoming_aurora_hostgroups = (void*)5; + s.incoming_hostgroup_attributes = (void*)6; + s.incoming_mysql_servers_ssl_params = (void*)7; + s.incoming_aws_rds_bgd_hostgroups = (void*)8; + s.runtime_mysql_servers = (void*)9; + + ok(s.incoming_mysql_servers_v2 == (void*)1, "field 0 set correctly"); + ok(s.incoming_replication_hostgroups == (void*)2, "field 1 set correctly"); + ok(s.incoming_group_replication_hostgroups == (void*)3, "field 2 set correctly"); + ok(s.incoming_galera_hostgroups == (void*)4, "field 3 set correctly"); + ok(s.incoming_aurora_hostgroups == (void*)5, "field 4 set correctly"); + ok(s.incoming_hostgroup_attributes == (void*)6, "field 5 set correctly"); + ok(s.incoming_mysql_servers_ssl_params == (void*)7, "field 6 set correctly"); + ok(s.incoming_aws_rds_bgd_hostgroups == (void*)8, "field 7 (BGD) set correctly"); + ok(s.runtime_mysql_servers == (void*)9, "field 8 set correctly"); +} + +// CLUSTER_QUERY_MYSQL_AWS_RDS_BGD is defined in ProxySQL_Cluster.hpp which +// has the same include dependency issue. Define it locally instead. +#define CLUSTER_QUERY_MYSQL_AWS_RDS_BGD \ + "PROXY_SELECT writer_hostgroup, reader_hostgroup, green_writer_hostgroup, " \ + "green_reader_hostgroup, active, writer_is_also_reader, check_interval_ms, " \ + "check_timeout_ms, comment, auto_generated, status " \ + "FROM runtime_mysql_aws_rds_bgd_hostgroups " \ + "WHERE auto_generated=0 ORDER BY writer_hostgroup" + +static void test_cluster_query_rds_bgd() { + const char* query = CLUSTER_QUERY_MYSQL_AWS_RDS_BGD; + ok(strncmp(query, "PROXY_SELECT", 12) == 0, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD starts with PROXY_SELECT"); + ok(strstr(query, "auto_generated=0") != nullptr, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD filters auto_generated=0"); + ok(strstr(query, "green_writer_hostgroup") != nullptr, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD includes green_writer_hostgroup"); + ok(strstr(query, "green_reader_hostgroup") != nullptr, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD includes green_reader_hostgroup"); + ok(strstr(query, "runtime_mysql_aws_rds_bgd_hostgroups") != nullptr, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD queries runtime_mysql_aws_rds_bgd_hostgroups"); + ok(strstr(query, "ORDER BY writer_hostgroup") != nullptr, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD has ORDER BY writer_hostgroup"); +} + +static void test_convert_size_check() { + std::vector v8(8, nullptr); + std::vector v9(9, nullptr); + std::vector v10(10, nullptr); + + size_t expected_struct_ptrs = sizeof(incoming_servers_t) / sizeof(void*); + + ok(v9.size() == expected_struct_ptrs, + "9-element vector matches sizeof(incoming_servers_t)/sizeof(void*) (%zu)", + expected_struct_ptrs); + ok(v8.size() != expected_struct_ptrs, + "8-element vector does NOT match (%zu vs %zu)", + v8.size(), expected_struct_ptrs); + ok(v10.size() != expected_struct_ptrs, + "10-element vector does NOT match (%zu vs %zu)", + v10.size(), expected_struct_ptrs); +} + +int main() { + plan(19); + + test_incoming_servers_t_size(); + test_incoming_servers_t_field_positions(); + test_cluster_query_rds_bgd(); + test_convert_size_check(); + + return exit_status(); +} diff --git a/test/tap/tests/unit/config_write_unit-t.cpp b/test/tap/tests/unit/config_write_unit-t.cpp index 656404156f..cda1fac9a0 100644 --- a/test/tap/tests/unit/config_write_unit-t.cpp +++ b/test/tap/tests/unit/config_write_unit-t.cpp @@ -356,6 +356,7 @@ static void test_write_mysql_servers_empty() { db->execute(ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); db->execute(ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); db->execute(ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); @@ -391,6 +392,7 @@ static void test_write_mysql_servers_with_data() { db->execute(ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); db->execute(ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); db->execute(ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); // Populate sub-tables with data (not empty!) @@ -398,6 +400,7 @@ static void test_write_mysql_servers_with_data() { db->execute("INSERT INTO mysql_group_replication_hostgroups (writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment) VALUES (30,31,32,33,1,2,0,100,'gr')"); db->execute("INSERT INTO mysql_galera_hostgroups (writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment) VALUES (40,41,42,43,1,3,1,50,'galera')"); db->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,comment) VALUES (50,51,1,3306,'.aurora.example',100,1000,800,0,1,0,0,1,'aurora')"); + db->execute("INSERT INTO mysql_aws_rds_bgd_hostgroups (writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (70,71,72,73,1,0,1000,800,'rds')"); db->execute("INSERT INTO mysql_hostgroup_attributes (hostgroup_id,max_num_online_servers,autocommit,free_connections_pct,init_connect,multiplex,connection_warming,throttle_connections_per_sec,ignore_session_variables,hostgroup_settings,servers_defaults,comment) VALUES (60,100,-1,50,'SET autocommit=1',1,0,100,'','{}','{}','hg60')"); db->execute("INSERT INTO mysql_servers_ssl_params (hostname,port,username,ssl_ca,ssl_cert,ssl_key,ssl_capath,ssl_crl,ssl_crlpath,ssl_cipher,tls_version,comment) VALUES ('h1',3306,'u1','/ca','/cert','/key','','','','','TLSv1.2','ssl1')"); @@ -432,6 +435,7 @@ static void test_write_mysql_servers_replication_hostgroups() { db->execute(ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); db->execute(ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); db->execute(ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); // Also populate one more sub for good measure diff --git a/test/tap/tests/unit/connection_unhealthy_unit-t.cpp b/test/tap/tests/unit/connection_unhealthy_unit-t.cpp new file mode 100644 index 0000000000..599adcbb81 --- /dev/null +++ b/test/tap/tests/unit/connection_unhealthy_unit-t.cpp @@ -0,0 +1,147 @@ +/** + * @file connection_unhealthy_unit-t.cpp + * @brief Verify unhealthy MySQL connections cannot re-enter connection pools. + * + * Exercises the real MySQL connection, thread-local cache, and HostGroups + * Manager boundaries without opening a backend network connection. + */ + +#include "tap.h" +#include "test_globals.h" +#include "test_init.h" + +#include "proxysql.h" +#include "cpp.h" +#include "MySQL_Logger.hpp" + +extern MySQL_HostGroups_Manager *MyHGM; +extern MySQL_Threads_Handler *GloMTH; +extern MySQL_Logger *GloMyLogger; + +static MySrvC *create_server(unsigned int hostgroup_id, const char *address) { + srv_info_t info; + info.addr = address; + info.port = 3306; + info.kind = "connection-unhealthy-unit"; + + srv_opts_t opts; + opts.weigth = 1; + opts.max_conns = 100; + opts.use_ssl = 0; + + MyHGM->wrlock(); + int rc = MyHGM->create_new_server_in_hg(hostgroup_id, info, opts); + MyHGC *hostgroup = MyHGM->MyHGC_find(hostgroup_id); + MyHGM->wrunlock(); + + if (rc != 0 || hostgroup == nullptr || hostgroup->mysrvs->cnt() != 1) { + BAIL_OUT("failed to create server for hostgroup %u", hostgroup_id); + } + + return hostgroup->mysrvs->idx(0); +} + +static MySQL_Connection *create_used_connection(MySrvC *server, bool healthy) { + MySQL_Connection *connection = new MySQL_Connection(); + connection->mysql = mysql_init(nullptr); + if (connection->mysql == nullptr) { + delete connection; + BAIL_OUT("mysql_init() failed for unit-test connection"); + } + + connection->parent = server; + connection->healthy = healthy; + connection->reusable = healthy; + connection->async_state_machine = ASYNC_IDLE; + connection->largest_query_length = 0; + server->ConnectionsUsed->add(connection); + return connection; +} + +static void check_pool_state(MySrvC *server, unsigned int exp_used, unsigned int exp_free, const char *msg) { + unsigned int used = server->ConnectionsUsed->conns_length(); + unsigned int free = server->ConnectionsFree->conns_length(); + ok(used == exp_used && free == exp_free, "%s (used=%u, free=%u)", msg, used, free); +} + +static void test_unhealthy_global_pool() { + MySrvC *server = create_server(101, "unhealthy-global"); + MySQL_Connection *connection = create_used_connection(server, false); + + connection->reset(); + MyHGM->push_MyConn_to_pool(connection); + + check_pool_state(server, 0, 0, "reset unhealthy connection is destroyed at the global pool boundary"); +} + +static void test_unhealthy_local_pool(MySQL_Thread &worker) { + MySrvC *server = create_server(102, "unhealthy-local"); + MySQL_Connection *connection = create_used_connection(server, false); + + connection->reset(); + worker.push_MyConn_local(connection); + + check_pool_state(server, 0, 0, "reset unhealthy connection is destroyed at the local pool boundary"); + + // If the assertion failed because the connection entered the local cache, + // return it before continuing so later cases remain isolated. + worker.return_local_connections(); +} + +static void test_healthy_global_pool() { + MySrvC *server = create_server(103, "healthy-global"); + MySQL_Connection *connection = create_used_connection(server, true); + + MyHGM->push_MyConn_to_pool(connection); + + check_pool_state(server, 0, 1, "healthy connection enters the global free pool"); +} + +static void test_healthy_local_pool(MySQL_Thread &worker) { + MySrvC *server = create_server(104, "healthy-local"); + MySQL_Connection *connection = create_used_connection(server, true); + + worker.push_MyConn_local(connection); + check_pool_state(server, 1, 0, "healthy connection remains used while cached locally"); + + worker.return_local_connections(); + check_pool_state(server, 0, 1, "healthy local connection enters the global free pool when returned"); +} + +int main() { + plan(5); + + if (test_init_minimal() != 0) { + BAIL_OUT("test_init_minimal() failed"); + } + if (test_init_query_processor() != 0) { + BAIL_OUT("test_init_query_processor() failed"); + } + GloMyLogger = new MySQL_Logger(); + if (test_init_hostgroups() != 0) { + BAIL_OUT("test_init_hostgroups() failed"); + } + + // Make the local-cache decision deterministic: with one worker, every + // otherwise eligible connection is cached locally. + GloMTH->num_threads = 1; + { + MySQL_Thread worker; + if (!worker.init()) { + BAIL_OUT("MySQL_Thread::init() failed"); + } + + test_unhealthy_global_pool(); + test_unhealthy_local_pool(worker); + test_healthy_global_pool(); + test_healthy_local_pool(worker); + } + + test_cleanup_hostgroups(); + delete GloMyLogger; + GloMyLogger = nullptr; + test_cleanup_query_processor(); + test_cleanup_minimal(); + + return exit_status(); +} diff --git a/test/tap/tests/unit/mysql_variables_unit-t.cpp b/test/tap/tests/unit/mysql_variables_unit-t.cpp new file mode 100644 index 0000000000..fcf42acfcb --- /dev/null +++ b/test/tap/tests/unit/mysql_variables_unit-t.cpp @@ -0,0 +1,52 @@ +#include "tap.h" +#include "test_globals.h" + +#include "MySQL_Thread.h" + +static void test_mysql_integer_variables_are_registered() { + test_globals_init(); + MySQL_Threads_Handler handler; + char **variables = handler.get_variables_list(); + + ok(handler.get_variable_int("aws_blue_green_deployment_auto_discovery") == 1, + "aws_blue_green_deployment_auto_discovery is registered as an integer variable"); + ok(handler.get_variable_int("session_track_variables") == 0, + "session_track_variables is registered as an integer variable"); + + if (variables) { + for (char **p = variables; *p != nullptr; ++p) { + free(*p); + } + free(reinterpret_cast(variables)); + } + test_globals_cleanup(); +} + +static void test_mysql_integer_boolean_aliases() { + test_globals_init(); + MySQL_Threads_Handler handler; + char **variables = handler.get_variables_list(); + char variable_name[] = "aws_blue_green_deployment_auto_discovery"; + + ok(handler.set_variable(variable_name, "true") && + handler.get_variable_int(variable_name) == 1, + "aws_blue_green_deployment_auto_discovery accepts true"); + ok(handler.set_variable(variable_name, "false") && + handler.get_variable_int(variable_name) == 0, + "aws_blue_green_deployment_auto_discovery accepts false"); + + if (variables) { + for (char **p = variables; *p != nullptr; ++p) { + free(*p); + } + free(reinterpret_cast(variables)); + } + test_globals_cleanup(); +} + +int main() { + plan(4); + test_mysql_integer_variables_are_registered(); + test_mysql_integer_boolean_aliases(); + return exit_status(); +}