diff --git a/.github/workflows/CI-package-arm64-almalinux10-genai.yml b/.github/workflows/CI-package-arm64-almalinux10-genai.yml index e70f2ed082..ec0f056375 100644 --- a/.github/workflows/CI-package-arm64-almalinux10-genai.yml +++ b/.github/workflows/CI-package-arm64-almalinux10-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-almalinux10.yml b/.github/workflows/CI-package-arm64-almalinux10.yml index 0633cb10c5..c132e931a2 100644 --- a/.github/workflows/CI-package-arm64-almalinux10.yml +++ b/.github/workflows/CI-package-arm64-almalinux10.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-almalinux8-genai.yml b/.github/workflows/CI-package-arm64-almalinux8-genai.yml index 794abc7c8c..ac4aec1661 100644 --- a/.github/workflows/CI-package-arm64-almalinux8-genai.yml +++ b/.github/workflows/CI-package-arm64-almalinux8-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-almalinux8.yml b/.github/workflows/CI-package-arm64-almalinux8.yml index e6ea361c46..8e294cfd3c 100644 --- a/.github/workflows/CI-package-arm64-almalinux8.yml +++ b/.github/workflows/CI-package-arm64-almalinux8.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-almalinux9-genai.yml b/.github/workflows/CI-package-arm64-almalinux9-genai.yml index c19e7e2eeb..a289a5b834 100644 --- a/.github/workflows/CI-package-arm64-almalinux9-genai.yml +++ b/.github/workflows/CI-package-arm64-almalinux9-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-almalinux9.yml b/.github/workflows/CI-package-arm64-almalinux9.yml index 5a394fa87b..968c1f3aa4 100644 --- a/.github/workflows/CI-package-arm64-almalinux9.yml +++ b/.github/workflows/CI-package-arm64-almalinux9.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-centos10-genai.yml b/.github/workflows/CI-package-arm64-centos10-genai.yml index 567aa4c232..c7b0dbe659 100644 --- a/.github/workflows/CI-package-arm64-centos10-genai.yml +++ b/.github/workflows/CI-package-arm64-centos10-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-centos10.yml b/.github/workflows/CI-package-arm64-centos10.yml index 5bf8109af5..73c73358f8 100644 --- a/.github/workflows/CI-package-arm64-centos10.yml +++ b/.github/workflows/CI-package-arm64-centos10.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-centos9-genai.yml b/.github/workflows/CI-package-arm64-centos9-genai.yml index a9451dedcb..1eef443f48 100644 --- a/.github/workflows/CI-package-arm64-centos9-genai.yml +++ b/.github/workflows/CI-package-arm64-centos9-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-centos9.yml b/.github/workflows/CI-package-arm64-centos9.yml index 56fe9eaf1e..68f044a309 100644 --- a/.github/workflows/CI-package-arm64-centos9.yml +++ b/.github/workflows/CI-package-arm64-centos9.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-debian12-genai.yml b/.github/workflows/CI-package-arm64-debian12-genai.yml index 51775fd72a..1f49f99293 100644 --- a/.github/workflows/CI-package-arm64-debian12-genai.yml +++ b/.github/workflows/CI-package-arm64-debian12-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-debian12.yml b/.github/workflows/CI-package-arm64-debian12.yml index ec558df27e..a7d3183833 100644 --- a/.github/workflows/CI-package-arm64-debian12.yml +++ b/.github/workflows/CI-package-arm64-debian12.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-debian13-genai.yml b/.github/workflows/CI-package-arm64-debian13-genai.yml index ff2e039d9f..49d8755423 100644 --- a/.github/workflows/CI-package-arm64-debian13-genai.yml +++ b/.github/workflows/CI-package-arm64-debian13-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-debian13.yml b/.github/workflows/CI-package-arm64-debian13.yml index 6b9b9378cc..4a5ab6abec 100644 --- a/.github/workflows/CI-package-arm64-debian13.yml +++ b/.github/workflows/CI-package-arm64-debian13.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-fedora42-genai.yml b/.github/workflows/CI-package-arm64-fedora42-genai.yml index 84961c04e8..f6f3b8a0aa 100644 --- a/.github/workflows/CI-package-arm64-fedora42-genai.yml +++ b/.github/workflows/CI-package-arm64-fedora42-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-fedora42.yml b/.github/workflows/CI-package-arm64-fedora42.yml index d2af9f4384..8414ff778a 100644 --- a/.github/workflows/CI-package-arm64-fedora42.yml +++ b/.github/workflows/CI-package-arm64-fedora42.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-fedora43-genai.yml b/.github/workflows/CI-package-arm64-fedora43-genai.yml index c71f2d3a86..bdce6db3dc 100644 --- a/.github/workflows/CI-package-arm64-fedora43-genai.yml +++ b/.github/workflows/CI-package-arm64-fedora43-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-fedora43.yml b/.github/workflows/CI-package-arm64-fedora43.yml index b39b83bb93..811e733626 100644 --- a/.github/workflows/CI-package-arm64-fedora43.yml +++ b/.github/workflows/CI-package-arm64-fedora43.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-fedora44-genai.yml b/.github/workflows/CI-package-arm64-fedora44-genai.yml index fa3da17159..e439126c21 100644 --- a/.github/workflows/CI-package-arm64-fedora44-genai.yml +++ b/.github/workflows/CI-package-arm64-fedora44-genai.yml @@ -111,6 +111,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-fedora44.yml b/.github/workflows/CI-package-arm64-fedora44.yml index 19d2d76cc2..253e267292 100644 --- a/.github/workflows/CI-package-arm64-fedora44.yml +++ b/.github/workflows/CI-package-arm64-fedora44.yml @@ -111,6 +111,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-opensuse15-genai.yml b/.github/workflows/CI-package-arm64-opensuse15-genai.yml index 701460b0e5..ae8c97bfa1 100644 --- a/.github/workflows/CI-package-arm64-opensuse15-genai.yml +++ b/.github/workflows/CI-package-arm64-opensuse15-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-opensuse15.yml b/.github/workflows/CI-package-arm64-opensuse15.yml index 708f25113e..a529c35165 100644 --- a/.github/workflows/CI-package-arm64-opensuse15.yml +++ b/.github/workflows/CI-package-arm64-opensuse15.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-opensuse16-genai.yml b/.github/workflows/CI-package-arm64-opensuse16-genai.yml index 8f4e573ec0..d7704a4c70 100644 --- a/.github/workflows/CI-package-arm64-opensuse16-genai.yml +++ b/.github/workflows/CI-package-arm64-opensuse16-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-opensuse16.yml b/.github/workflows/CI-package-arm64-opensuse16.yml index c308e0d682..af92653a4c 100644 --- a/.github/workflows/CI-package-arm64-opensuse16.yml +++ b/.github/workflows/CI-package-arm64-opensuse16.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-ubuntu22-genai.yml b/.github/workflows/CI-package-arm64-ubuntu22-genai.yml index e91114e4eb..b18e8425af 100644 --- a/.github/workflows/CI-package-arm64-ubuntu22-genai.yml +++ b/.github/workflows/CI-package-arm64-ubuntu22-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-ubuntu22.yml b/.github/workflows/CI-package-arm64-ubuntu22.yml index ab958b8e24..2db6efb050 100644 --- a/.github/workflows/CI-package-arm64-ubuntu22.yml +++ b/.github/workflows/CI-package-arm64-ubuntu22.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-ubuntu24-genai.yml b/.github/workflows/CI-package-arm64-ubuntu24-genai.yml index 7e5a1305e8..1c5706710e 100644 --- a/.github/workflows/CI-package-arm64-ubuntu24-genai.yml +++ b/.github/workflows/CI-package-arm64-ubuntu24-genai.yml @@ -109,6 +109,16 @@ jobs: run: | PROXYSQL40=1 make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-package-arm64-ubuntu24.yml b/.github/workflows/CI-package-arm64-ubuntu24.yml index eb852aa671..2806a12101 100644 --- a/.github/workflows/CI-package-arm64-ubuntu24.yml +++ b/.github/workflows/CI-package-arm64-ubuntu24.yml @@ -109,6 +109,16 @@ jobs: run: | make ${{ env.MAKE_TARGET }} + - name: Verify package installs on clean ${{ env.DISTRO }} + run: | + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + test/infra/control/verify-package-install.bash "$PKG" + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI-unit-tests-asan-coverage.yml b/.github/workflows/CI-unit-tests-asan-coverage.yml index 71c66cecb2..2f5589d675 100644 --- a/.github/workflows/CI-unit-tests-asan-coverage.yml +++ b/.github/workflows/CI-unit-tests-asan-coverage.yml @@ -64,17 +64,6 @@ jobs: steps: - - uses: LouisBrunner/checks-action@v2.0.0 - id: checks - if: always() - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: '${{ github.workflow }} / ${{ github.job }}' - repo: ${{ github.repository }} - sha: ${{ env.SHA }} - status: 'in_progress' - details_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' - - name: Checkout repository uses: actions/checkout@v4 with: @@ -172,7 +161,7 @@ jobs: if: always() with: token: ${{ secrets.GITHUB_TOKEN }} - check_id: ${{ steps.checks.outputs.check_id }} + name: '${{ github.workflow }} / ${{ github.job }}' repo: ${{ github.repository }} sha: ${{ env.SHA }} conclusion: ${{ job.status }} diff --git a/.github/workflows/gh-actions-reusable/verify-package-install.yml b/.github/workflows/gh-actions-reusable/verify-package-install.yml new file mode 100644 index 0000000000..acb44e6e41 --- /dev/null +++ b/.github/workflows/gh-actions-reusable/verify-package-install.yml @@ -0,0 +1,38 @@ +name: verify-package-install +run-name: 'Verify ${{ inputs.distro }} package install' + +on: + workflow_call: + inputs: + distro: + type: string + required: true + description: 'Distro name (ubuntu24, centos9, etc.) — used for the verify log' + secrets: + GITHUB_TOKEN: + required: false + +jobs: + verify: + runs-on: ubuntu-24.04-arm + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Locate package + id: locate + run: | + set -euo pipefail + PKG=$(ls binaries/*.deb binaries/*.rpm 2>/dev/null | head -1) + if [[ -z "$PKG" ]]; then + echo "ERROR: no package file found in binaries/" >&2 + ls -la binaries/ + exit 1 + fi + echo "package=$PKG" >> "$GITHUB_OUTPUT" + + - name: Verify package installs on clean ${{ inputs.distro }} + run: | + test/infra/control/verify-package-install.bash "${{ steps.locate.outputs.package }}" diff --git a/CLAUDE.md b/CLAUDE.md index 07f7406e82..ec05901074 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,22 +40,10 @@ The same codebase produces three product tiers via feature flags: |------|------|---------|------| | Stable | (default) | v3.0.x | Core proxy | | Innovative | `PROXYSQL31=1` | v3.1.x | FFTO, TSDB | -| Plugin Chassis | `PROXYSQL40=1` | v4.0.x | Plugin loader + ABI (4-phase lifecycle, query-hook, shared Prometheus) | -| AI/MCP | `PROXYSQLGENAI=1` | v4.0.x | Builds the genai plugin (`plugins/genai/`) — GenAI, MCP, Anomaly Detection | - -`PROXYSQLGENAI=1` implies `PROXYSQL40=1` implies `PROXYSQL31=1`, which -implies `PROXYSQLFFTO=1` and `PROXYSQLTSDB=1`. - -**As of the GenAI plugin carve-out (Step 7), `PROXYSQLGENAI` no longer -gates any code in core with the exception of the sqlite-vec hook -(`proxy_sqlite3_vec_init` in `lib/proxy_sqlite3_symbols.cpp` and the -`vec.o` link in `lib/Makefile` / `src/Makefile`), which is only needed -when the genai plugin is loaded.** All other AI/MCP/RAG/LLM features -live in `plugins/genai/` and load as a `.so` at runtime. The -user-facing `PROXYSQLGENAI=1` flag still exists, and now means "build -the genai plugin alongside core proxysql" — it propagates through the -plugin chassis (`PROXYSQL40=1`) and the sqlite-vec link, but core -compiles identically with or without it otherwise. +| Plugin Chassis | `PROXYSQL40=1` | v4.0.x | Plugin loader + ABI (4-phase lifecycle, query-hook, shared Prometheus); builds and packages all v4.0 plugins including mysqlx and genai/MCP | + +**`PROXYSQL40=1` implies `PROXYSQL31=1` which implies `PROXYSQLFFTO=1` and `PROXYSQLTSDB=1`.** +There is no separate `PROXYSQLGENAI` flag — `PROXYSQL40=1` builds and packages all v4.0 plugins (mysqlx, genai/MCP, anomaly detection). All AI/MCP/RAG/LLM features live in `plugins/genai/` and load as a `.so` at runtime. ### Build Flags diff --git a/deps/parsersql/parsersql b/deps/parsersql/parsersql index 782f86ce9d..2e6b83db0b 120000 --- a/deps/parsersql/parsersql +++ b/deps/parsersql/parsersql @@ -1 +1 @@ -parsersql-1.0.2 \ No newline at end of file +parsersql-1.0.3 \ No newline at end of file diff --git a/deps/parsersql/parsersql-1.0.2.tar.gz b/deps/parsersql/parsersql-1.0.2.tar.gz deleted file mode 100644 index 7335dca916..0000000000 Binary files a/deps/parsersql/parsersql-1.0.2.tar.gz and /dev/null differ diff --git a/deps/parsersql/parsersql-1.0.3.tar.gz b/deps/parsersql/parsersql-1.0.3.tar.gz new file mode 100644 index 0000000000..66ab6d4a88 Binary files /dev/null and b/deps/parsersql/parsersql-1.0.3.tar.gz differ diff --git a/docker/images/proxysql/deb-compliant/entrypoint/entrypoint.bash b/docker/images/proxysql/deb-compliant/entrypoint/entrypoint.bash index 763a6dd0a8..acaf1bc368 100755 --- a/docker/images/proxysql/deb-compliant/entrypoint/entrypoint.bash +++ b/docker/images/proxysql/deb-compliant/entrypoint/entrypoint.bash @@ -41,19 +41,17 @@ else build_target="$PROXYSQL_BUILD_TYPE" fi -# When the chassis tier is enabled (PROXYSQL40=1, or PROXYSQLGENAI=1 -# which implies PROXYSQL40 via the cascade in include/makefiles_vars.mk), -# the build recurses into plugins/mysqlx/ which dynamically links -# against the system libprotobuf (3.x). Some of the v4.0.0 packaging -# images were built before plugins/mysqlx existed and do not yet ship -# libprotobuf-dev. Install it on demand here so the plugin's pkg-config -# check at plugins/mysqlx/Makefile:47 succeeds. The install is -# idempotent — apt-get returns 0 if the package is already present. -# Skip silently for v3.x builds where neither flag is set and the plugin -# path is not exercised. -if [[ "${PROXYSQLGENAI:-}" == "1" || "${PROXYSQL40:-}" == "1" ]]; then +# The v4.0 chassis tier (PROXYSQL40=1) builds plugins/mysqlx/ which +# dynamically links against the system libprotobuf (3.x). Some of the +# v4.0.0 packaging images were built before plugins/mysqlx existed and +# do not yet ship libprotobuf-dev. Install it on demand here so the +# plugin's pkg-config check succeeds. The install is idempotent — +# apt-get returns 0 if the package is already present. PROXYSQL40=1 +# builds and packages all v4.0 plugins — there is no separate +# PROXYSQLGENAI flag. +if [[ "${PROXYSQL40:-}" == "1" ]]; then if ! pkg-config --exists protobuf 2>/dev/null; then - echo "==> Installing libprotobuf-dev (required for the mysqlx plugin build under PROXYSQL40 / PROXYSQLGENAI)" + echo "==> Installing libprotobuf-dev (required for the mysqlx plugin build under PROXYSQL40)" apt-get update -qq apt-get install -y --no-install-recommends libprotobuf-dev fi @@ -65,13 +63,10 @@ fi # Pass through the chassis tier flag explicitly to make. WITHTSAN / # WITHASAN / WITHGCOV come through via the docker-compose.yml env # passthrough (the Makefile reads them via $(WITHTSAN) etc.), so they -# don't need to be replicated here. PROXYSQLGENAI=1 implies -# PROXYSQL40=1 via the Makefile cascade, so passing one is sufficient -# in either case — but we mirror whichever the operator set so the -# matrix stays explicit for downstream commands that match on env. +# don't need to be replicated here. PROXYSQL40=1 builds and packages +# all v4.0 plugins — PROXYSQLGENAI is no longer a separate flag. EXTRA="" [[ "${PROXYSQL40:-}" == "1" ]] && EXTRA="$EXTRA PROXYSQL40=1" -[[ "${PROXYSQLGENAI:-}" == "1" ]] && EXTRA="$EXTRA PROXYSQLGENAI=1" ${MAKE} ${MAKEOPT} ${EXTRA} ${deps_target} ${MAKE} ${MAKEOPT} ${EXTRA} ${build_target} @@ -92,10 +87,10 @@ cp -r ../tools ./tools cp -r ../systemd ./systemd # Plugin .so artefacts (v4.0+ chassis): the proxysql binary is the -# loader; runtime features (mysqlx, genai/MCP) ship as separate .so -# files installed to /usr/lib/proxysql/ and named in proxysql.cnf -# `plugins=("...")` to be loaded. Conditional on the same flags that -# gated the build above so v3.x deb packaging stays unchanged. +# loader; runtime features (mysqlx, genai/MCP, etc.) ship as separate +# .so files installed to /usr/lib/proxysql/ and named in proxysql.cnf +# `plugins=("...")` to be loaded. Conditional on PROXYSQL40=1 so +# v3.x deb packaging stays unchanged. mkdir -p ./plugins PLUGIN_FILES_BLOCK="" add_plugin_to_pkg() { @@ -111,10 +106,8 @@ add_plugin_to_pkg() { PLUGIN_FILES_BLOCK+=$' plugins/'"${basename}"$' /usr/lib/proxysql/\n' fi } -if [[ "${PROXYSQL40:-}" == "1" || "${PROXYSQLGENAI:-}" == "1" ]]; then +if [[ "${PROXYSQL40:-}" == "1" ]]; then add_plugin_to_pkg "../plugins/mysqlx/ProxySQL_MySQLX_Plugin.so" "ProxySQL_MySQLX_Plugin.so" -fi -if [[ "${PROXYSQLGENAI:-}" == "1" ]]; then add_plugin_to_pkg "../plugins/genai/ProxySQL_GenAI_Plugin.so" "ProxySQL_GenAI_Plugin.so" fi @@ -164,6 +157,53 @@ cp "./${PKG}" "../binaries/proxysql_${CURVER}-${PKG_RELEASE}_${ARCH}.deb" ar -p "${PKG}" $(ar t "${PKG}" | grep '^data\.tar') | unxz -c - | tar xvf - ./usr/bin/proxysql -O > tmp/proxysql sha1sum tmp/proxysql | sed 's|tmp/||' | tee tmp/proxysql.sha1 cp tmp/proxysql.sha1 ../binaries/proxysql_${CURVER}-${PKG_RELEASE}_${ARCH}.id-hash +# Verify plugin .so files are present in the package +if [[ "${PROXYSQL40:-}" == "1" ]]; then + echo "==> Verifying plugin .so files in package" + for plugin in ProxySQL_MySQLX_Plugin.so ProxySQL_GenAI_Plugin.so; do + if dpkg -c "${PKG}" 2>/dev/null | grep -q "/usr/lib/proxysql/${plugin}"; then + echo " OK ${plugin}" + else + echo " FAIL ${plugin} not found in package" >&2 + exit 1 + fi + done + echo "==> Plugin packaging verification PASSED" +fi +# Plugin smoke test: verify .so files are valid and export the expected +# descriptor symbol. +if [[ "${PROXYSQL40:-}" == "1" ]]; then + echo "==> Running plugin smoke test" + SMOKE_DIR=$(mktemp -d) + dpkg-deb -R "${PKG}" "${SMOKE_DIR}" + ALL_OK=0 + for plugin in ProxySQL_MySQLX_Plugin.so ProxySQL_GenAI_Plugin.so; do + plugin_path="${SMOKE_DIR}/usr/lib/proxysql/${plugin}" + if [[ -f "${plugin_path}" ]]; then + if file "${plugin_path}" | grep -q 'ELF 64-bit.*shared object'; then + echo " OK ${plugin} (valid ELF shared library)" + if nm -D "${plugin_path}" 2>/dev/null | grep -q 'proxysql_plugin_descriptor_v1'; then + echo " OK ${plugin} (exports proxysql_plugin_descriptor_v1)" + else + echo " WARN ${plugin} (no proxysql_plugin_descriptor_v1 symbol)" >&2 + ALL_OK=1 + fi + else + echo " FAIL ${plugin} (not a valid ELF shared object)" >&2 + ALL_OK=1 + fi + else + echo " FAIL ${plugin} (not found in package)" >&2 + ALL_OK=1 + fi + done + rm -rf "${SMOKE_DIR}" + if [[ "${ALL_OK}" != "0" ]]; then + echo "==> Plugin smoke test FAILED" >&2 + exit 1 + fi + echo "==> Plugin smoke test PASSED" +fi popd # Cleanup current build rm -rf /opt/proxysql/pkgroot diff --git a/docker/images/proxysql/rhel-compliant/entrypoint/entrypoint.bash b/docker/images/proxysql/rhel-compliant/entrypoint/entrypoint.bash index 852c3a3d52..56c075d30d 100755 --- a/docker/images/proxysql/rhel-compliant/entrypoint/entrypoint.bash +++ b/docker/images/proxysql/rhel-compliant/entrypoint/entrypoint.bash @@ -41,14 +41,15 @@ else build_target="$PROXYSQL_BUILD_TYPE" fi -# See deb-compliant entrypoint for the rationale: PROXYSQLGENAI=1 -# triggers a build of plugins/mysqlx/ which dynamically links against -# the system libprotobuf (3.x). Some of the v4.0.0 packaging images -# were built before plugins/mysqlx existed and do not yet ship -# protobuf-devel. Install it on demand for RHEL-family images. -if [[ "${PROXYSQLGENAI:-}" == "1" ]]; then +# The v4.0 chassis tier (PROXYSQL40=1) builds plugins/mysqlx/ which +# dynamically links against the system libprotobuf (3.x). Some of the +# v4.0.0 packaging images were built before plugins/mysqlx existed and +# do not yet ship protobuf-devel. Install it on demand for RHEL-family +# images. PROXYSQL40=1 builds and packages all v4.0 plugins — there +# is no separate PROXYSQLGENAI flag. +if [[ "${PROXYSQL40:-}" == "1" ]]; then if ! pkg-config --exists protobuf 2>/dev/null; then - echo "==> Installing protobuf-devel (required for PROXYSQLGENAI=1 mysqlx plugin build)" + echo "==> Installing protobuf-devel (required for PROXYSQL40=1 mysqlx plugin build)" if command -v dnf >/dev/null 2>&1; then dnf install -y protobuf-devel elif command -v yum >/dev/null 2>&1; then @@ -62,13 +63,13 @@ fi # clean is expensive, do it before, outside of container #${MAKE} cleanbuild -if [[ "${PROXYSQLGENAI:-}" == "1" ]]; then - ${MAKE} ${MAKEOPT} PROXYSQLGENAI=1 ${deps_target} - ${MAKE} ${MAKEOPT} PROXYSQLGENAI=1 ${build_target} -else - ${MAKE} ${MAKEOPT} ${deps_target} - ${MAKE} ${MAKEOPT} ${build_target} -fi +# PROXYSQL40=1 enables the plugin chassis tier; all v4.0 plugins +# (mysqlx, genai, etc.) are built and packaged automatically. +# PROXYSQLGENAI is no longer a separate flag. +EXTRA="" +[[ "${PROXYSQL40:-}" == "1" ]] && EXTRA="$EXTRA PROXYSQL40=1" +${MAKE} ${MAKEOPT} ${EXTRA} ${deps_target} +${MAKE} ${MAKEOPT} ${EXTRA} ${build_target} touch /opt/proxysql/src/proxysql @@ -83,20 +84,19 @@ mkdir -p proxysql/usr/share/proxysql/tools cp -a tools/proxysql_galera_checker.sh tools/proxysql_galera_writer.pl proxysql/usr/share/proxysql/tools # Plugin .so artefacts (v4.0+ chassis): proxysql becomes the loader; -# runtime features (mysqlx, genai/MCP) ship as separate .so files +# runtime features (mysqlx, genai/MCP, etc.) ship as separate .so files # installed to /usr/lib/proxysql/ and named in proxysql.cnf -# `plugins=("...")` to be loaded. Conditional on the same flags that -# gated the build above so v3.x rpm packaging stays unchanged. The -# proxysql.spec %files section already lists /usr/lib/proxysql/* so -# anything dropped in this directory ends up packaged automatically. -if [[ "${PROXYSQL40:-}" == "1" || "${PROXYSQLGENAI:-}" == "1" ]]; then +# `plugins=("...")` to be loaded. Conditional on PROXYSQL40=1 so +# v3.x rpm packaging stays unchanged. The proxysql.spec %files section +# already lists /usr/lib/proxysql/* so anything dropped in this +# directory ends up packaged automatically. +if [[ "${PROXYSQL40:-}" == "1" ]]; then mkdir -p proxysql/usr/lib/proxysql + # mysqlx protocol plugin if [[ -f plugins/mysqlx/ProxySQL_MySQLX_Plugin.so ]]; then cp plugins/mysqlx/ProxySQL_MySQLX_Plugin.so proxysql/usr/lib/proxysql/ fi -fi -if [[ "${PROXYSQLGENAI:-}" == "1" ]]; then - mkdir -p proxysql/usr/lib/proxysql + # genai/MCP plugin if [[ -f plugins/genai/ProxySQL_GenAI_Plugin.so ]]; then cp plugins/genai/ProxySQL_GenAI_Plugin.so proxysql/usr/lib/proxysql/ fi @@ -133,5 +133,54 @@ rpm2cpio /root/rpmbuild/RPMS/${ARCH}/proxysql-${CURVER}-1.${ARCH}.rpm | cpio -iu sha1sum tmp/proxysql | sed 's|tmp/||' | tee tmp/proxysql.sha1 cp tmp/proxysql.sha1 ../binaries/proxysql-${CURVER}-1-${PKG_RELEASE}.${ARCH}.id-hash popd +# Verify plugin .so files are present in the package +if [[ "${PROXYSQL40:-}" == "1" ]]; then + echo "==> Verifying plugin .so files in package" + PKG_PATH="/root/rpmbuild/RPMS/${ARCH}/proxysql-${CURVER}-1.${ARCH}.rpm" + for plugin in ProxySQL_MySQLX_Plugin.so ProxySQL_GenAI_Plugin.so; do + if rpm -qpl "${PKG_PATH}" 2>/dev/null | grep -q "/usr/lib/proxysql/${plugin}"; then + echo " OK ${plugin}" + else + echo " FAIL ${plugin} not found in package" >&2 + exit 1 + fi + done + echo "==> Plugin packaging verification PASSED" +fi +# Plugin smoke test: verify .so files are valid and export the expected +# descriptor symbol, then attempt a quick start to confirm plugin loading. +if [[ "${PROXYSQL40:-}" == "1" ]]; then + echo "==> Running plugin smoke test" + SMOKE_DIR=$(mktemp -d) + pushd "${SMOKE_DIR}" >/dev/null + rpm2cpio /root/rpmbuild/RPMS/${ARCH}/proxysql-${CURVER}-1.${ARCH}.rpm | cpio -idm 2>/dev/null + ALL_OK=0 + for plugin in usr/lib/proxysql/ProxySQL_MySQLX_Plugin.so usr/lib/proxysql/ProxySQL_GenAI_Plugin.so; do + if [[ -f "${plugin}" ]]; then + if file "${plugin}" | grep -q 'ELF 64-bit.*shared object'; then + echo " OK ${plugin} (valid ELF shared library)" + if nm -D "${plugin}" 2>/dev/null | grep -q 'proxysql_plugin_descriptor_v1'; then + echo " OK ${plugin} (exports proxysql_plugin_descriptor_v1)" + else + echo " WARN ${plugin} (no proxysql_plugin_descriptor_v1 symbol)" >&2 + ALL_OK=1 + fi + else + echo " FAIL ${plugin} (not a valid ELF shared object)" >&2 + ALL_OK=1 + fi + else + echo " FAIL ${plugin} (not found in package)" >&2 + ALL_OK=1 + fi + done + popd >/dev/null + rm -rf "${SMOKE_DIR}" + if [[ "${ALL_OK}" != "0" ]]; then + echo "==> Plugin smoke test FAILED" >&2 + exit 1 + fi + echo "==> Plugin smoke test PASSED" +fi # cleanup rm -fr /root/.pki /root/rpmbuild/{BUILDROOT,RPMS,SRPMS,BUILD,SOURCES,tmp} /opt/proxysql/proxysql /opt/proxysql/proxysql-${CURVER} ./pkgroot diff --git a/docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/SPECS/proxysql.spec b/docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/SPECS/proxysql.spec index 56ad6ab7f0..c80629d43e 100644 --- a/docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/SPECS/proxysql.spec +++ b/docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/SPECS/proxysql.spec @@ -78,9 +78,10 @@ rm -rf %{buildroot} /usr/share/proxysql/tools/proxysql_galera_writer.pl # Plugin .so artefacts for v4.0+ chassis builds (mysqlx, genai/MCP). # Gated on the `with_plugins` define passed by the rhel-compliant -# entrypoint when PROXYSQL40=1 or PROXYSQLGENAI=1. Without the gate -# the wildcard match would fail on v3.x release builds where the -# directory is empty and rpmbuild aborts on no-files-match. +# entrypoint when PROXYSQL40=1. Without the gate the wildcard match +# would fail on v3.x release builds where the directory is empty and +# rpmbuild aborts on no-files-match. PROXYSQL40=1 builds and packages +# all v4.0 plugins; PROXYSQLGENAI is no longer a separate flag. %if 0%{?with_plugins} %dir /usr/lib/proxysql /usr/lib/proxysql/*.so diff --git a/docker/images/proxysql/suse-compliant/entrypoint/entrypoint.bash b/docker/images/proxysql/suse-compliant/entrypoint/entrypoint.bash index c20760195a..f5b335d54f 100755 --- a/docker/images/proxysql/suse-compliant/entrypoint/entrypoint.bash +++ b/docker/images/proxysql/suse-compliant/entrypoint/entrypoint.bash @@ -41,14 +41,15 @@ else build_target="$PROXYSQL_BUILD_TYPE" fi -# See deb-compliant entrypoint for the rationale: PROXYSQLGENAI=1 -# triggers a build of plugins/mysqlx/ which dynamically links against -# the system libprotobuf (3.x). Some of the v4.0.0 packaging images -# were built before plugins/mysqlx existed and do not yet ship -# protobuf-devel. Install it on demand for SUSE-family images. -if [[ "${PROXYSQLGENAI:-}" == "1" ]]; then +# The v4.0 chassis tier (PROXYSQL40=1) builds plugins/mysqlx/ which +# dynamically links against the system libprotobuf (3.x). Some of the +# v4.0.0 packaging images were built before plugins/mysqlx existed and +# do not yet ship protobuf-devel. Install it on demand for SUSE-family +# images. PROXYSQL40=1 builds and packages all v4.0 plugins — there +# is no separate PROXYSQLGENAI flag. +if [[ "${PROXYSQL40:-}" == "1" ]]; then if ! pkg-config --exists protobuf 2>/dev/null; then - echo "==> Installing protobuf-devel (required for PROXYSQLGENAI=1 mysqlx plugin build)" + echo "==> Installing protobuf-devel (required for PROXYSQL40=1 mysqlx plugin build)" if command -v zypper >/dev/null 2>&1; then zypper install -y libprotobuf-c-devel || zypper install -y protobuf-devel else @@ -60,13 +61,13 @@ fi # clean is expensive, do it before, outside of container #${MAKE} cleanbuild -if [[ "${PROXYSQLGENAI:-}" == "1" ]]; then - ${MAKE} ${MAKEOPT} PROXYSQLGENAI=1 ${deps_target} - ${MAKE} ${MAKEOPT} PROXYSQLGENAI=1 ${build_target} -else - ${MAKE} ${MAKEOPT} ${deps_target} - ${MAKE} ${MAKEOPT} ${build_target} -fi +# PROXYSQL40=1 enables the plugin chassis tier; all v4.0 plugins +# (mysqlx, genai, etc.) are built and packaged automatically. +# PROXYSQLGENAI is no longer a separate flag. +EXTRA="" +[[ "${PROXYSQL40:-}" == "1" ]] && EXTRA="$EXTRA PROXYSQL40=1" +${MAKE} ${MAKEOPT} ${EXTRA} ${deps_target} +${MAKE} ${MAKEOPT} ${EXTRA} ${build_target} touch /opt/proxysql/src/proxysql @@ -85,17 +86,16 @@ cp -a etc/proxysql.cnf proxysql-${CURVER}/etc/ cp -a etc/logrotate.d proxysql-${CURVER}/etc/ cp -a tools/proxysql_galera_checker.sh tools/proxysql_galera_writer.pl proxysql-${CURVER}/usr/share/proxysql/tools -# Plugin .so artefacts (v4.0+ chassis); see rhel-compliant entrypoint -# for the full rationale. Gated on the same build flags so v3.x -# packaging is unchanged. -if [[ "${PROXYSQL40:-}" == "1" || "${PROXYSQLGENAI:-}" == "1" ]]; then +# Plugin .so artefacts (v4.0+ chassis): see rhel-compliant entrypoint +# for the full rationale. Gated on PROXYSQL40=1 so v3.x packaging is +# unchanged. PROXYSQL40=1 builds and packages all v4.0 plugins. +if [[ "${PROXYSQL40:-}" == "1" ]]; then mkdir -p proxysql-${CURVER}/usr/lib/proxysql + # mysqlx protocol plugin if [[ -f plugins/mysqlx/ProxySQL_MySQLX_Plugin.so ]]; then cp plugins/mysqlx/ProxySQL_MySQLX_Plugin.so proxysql-${CURVER}/usr/lib/proxysql/ fi -fi -if [[ "${PROXYSQLGENAI:-}" == "1" ]]; then - mkdir -p proxysql-${CURVER}/usr/lib/proxysql + # genai/MCP plugin if [[ -f plugins/genai/ProxySQL_GenAI_Plugin.so ]]; then cp plugins/genai/ProxySQL_GenAI_Plugin.so proxysql-${CURVER}/usr/lib/proxysql/ fi @@ -127,5 +127,54 @@ rpm2cpio /root/rpmbuild/RPMS/${ARCH}/proxysql-${CURVER}-1.${ARCH}.rpm | cpio -iu sha1sum tmp/proxysql | sed 's|tmp/||' | tee tmp/proxysql.sha1 cp tmp/proxysql.sha1 ../binaries/proxysql-${CURVER}-1-${PKG_RELEASE}.${ARCH}.id-hash popd +# Verify plugin .so files are present in the package +if [[ "${PROXYSQL40:-}" == "1" ]]; then + echo "==> Verifying plugin .so files in package" + PKG_PATH="/root/rpmbuild/RPMS/${ARCH}/proxysql-${CURVER}-1.${ARCH}.rpm" + for plugin in ProxySQL_MySQLX_Plugin.so ProxySQL_GenAI_Plugin.so; do + if rpm -qpl "${PKG_PATH}" 2>/dev/null | grep -q "/usr/lib/proxysql/${plugin}"; then + echo " OK ${plugin}" + else + echo " FAIL ${plugin} not found in package" >&2 + exit 1 + fi + done + echo "==> Plugin packaging verification PASSED" +fi +# Plugin smoke test: verify .so files are valid and export the expected +# descriptor symbol, then attempt a quick start to confirm plugin loading. +if [[ "${PROXYSQL40:-}" == "1" ]]; then + echo "==> Running plugin smoke test" + SMOKE_DIR=$(mktemp -d) + pushd "${SMOKE_DIR}" >/dev/null + rpm2cpio /root/rpmbuild/RPMS/${ARCH}/proxysql-${CURVER}-1.${ARCH}.rpm | cpio -idm 2>/dev/null + ALL_OK=0 + for plugin in usr/lib/proxysql/ProxySQL_MySQLX_Plugin.so usr/lib/proxysql/ProxySQL_GenAI_Plugin.so; do + if [[ -f "${plugin}" ]]; then + if file "${plugin}" | grep -q 'ELF 64-bit.*shared object'; then + echo " OK ${plugin} (valid ELF shared library)" + if nm -D "${plugin}" 2>/dev/null | grep -q 'proxysql_plugin_descriptor_v1'; then + echo " OK ${plugin} (exports proxysql_plugin_descriptor_v1)" + else + echo " WARN ${plugin} (no proxysql_plugin_descriptor_v1 symbol)" >&2 + ALL_OK=1 + fi + else + echo " FAIL ${plugin} (not a valid ELF shared object)" >&2 + ALL_OK=1 + fi + else + echo " FAIL ${plugin} (not found in package)" >&2 + ALL_OK=1 + fi + done + popd >/dev/null + rm -rf "${SMOKE_DIR}" + if [[ "${ALL_OK}" != "0" ]]; then + echo "==> Plugin smoke test FAILED" >&2 + exit 1 + fi + echo "==> Plugin smoke test PASSED" +fi # cleanup rm -fr /root/.pki /root/rpmbuild/{BUILDROOT,RPMS,SRPMS,BUILD,SOURCES,tmp} ./proxysql-${CURVER} ./pkgroot diff --git a/include/DNS_Cache.hpp b/include/DNS_Cache.hpp new file mode 100644 index 0000000000..a994a23b7f --- /dev/null +++ b/include/DNS_Cache.hpp @@ -0,0 +1,167 @@ +#ifndef __CLASS_DNS_CACHE_H +#define __CLASS_DNS_CACHE_H + +// Shared DNS cache infrastructure used by both MySQL_Monitor and PgSQL_Monitor. +// The cache itself is just a hostname -> [IP] map with TTL; the resolver thread +// does getaddrinfo and feeds the cache. Each monitor instantiates its own +// DNS_Cache and supplies its own counters, so the two protocols share code but +// have independent cache state. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "thread.h" +#include "wqueue.h" + +struct DNS_Cache_Record { + DNS_Cache_Record() = default; + DNS_Cache_Record(DNS_Cache_Record&&) = default; + DNS_Cache_Record(const DNS_Cache_Record&) = default; + DNS_Cache_Record& operator=(DNS_Cache_Record&&) = default; + DNS_Cache_Record& operator=(const DNS_Cache_Record&) = default; + DNS_Cache_Record(const std::string& hostname, const std::vector& ips, unsigned long long ttl = 0) + : hostname_(hostname), ttl_(ttl) { + std::copy(ips.begin(), ips.end(), std::inserter(ips_, ips_.end())); + } + DNS_Cache_Record(const std::string& hostname, std::set&& ips, unsigned long long ttl = 0) + : hostname_(hostname), ips_(std::move(ips)), ttl_(ttl) { } + + ~DNS_Cache_Record() = default; + + std::string hostname_; + std::set ips_; + unsigned long long ttl_ = 0; +}; + +class DNS_Cache { + +public: + DNS_Cache() : enabled(true) { + int rc = pthread_rwlock_init(&rwlock_, nullptr); + assert(rc == 0); + } + + ~DNS_Cache() { + pthread_rwlock_destroy(&rwlock_); + } + + inline + void set_enabled_flag(bool value) { + enabled = value; + } + + // Wire the cache up to per-instance counters maintained by the owning + // monitor. Pass nullptr to leave any counter unincremented; this is what + // keeps MySQL_Monitor and PgSQL_Monitor cache state independent. The + // counters are atomic because resolver workers increment them while + // p_update_metrics() reads them from another thread. + inline + void set_counters(std::atomic* queried, + std::atomic* lookup_success, + std::atomic* record_updated) { + counter_queried_ = queried; + counter_lookup_success_ = lookup_success; + counter_record_updated_ = record_updated; + } + + bool add(const std::string& hostname, std::vector&& ips); + bool add_if_not_exist(const std::string& hostname, std::vector&& ips); + void remove(const std::string& hostname); + void clear(); + bool empty() const; + std::string lookup(const std::string& hostname, size_t* ip_count) const; + +private: + struct IP_ADDR { + std::vector ips; + // 'counter' is bumped by get_next_ip() (a const method) for + // round-robin selection; the logical state of the cache record is + // unchanged, so mutable is the right tool here and lets us drop a + // const_cast at the call site. + mutable unsigned long counter = 0; + }; + + std::string get_next_ip(const IP_ADDR& ip_addr) const; + std::unordered_map records; + std::atomic_bool enabled; + mutable pthread_rwlock_t rwlock_; + + std::atomic* counter_queried_ { nullptr }; + std::atomic* counter_lookup_success_ { nullptr }; + std::atomic* counter_record_updated_ { nullptr }; +}; + +struct DNS_Resolve_Data { + std::promise> result; + std::shared_ptr dns_cache; + std::string hostname; + std::set cached_ips; + unsigned int ttl = 0; + unsigned int refresh_intv = 0; + // Address family for getaddrinfo, set by the caller from its protocol's + // resolution_family setting. AF_UNSPEC keeps the OS default behavior. + int ai_family = AF_UNSPEC; +}; + +// Worker function for the DNS resolver thread pool. Performs getaddrinfo +// on dns_resolve_data->hostname, populates the cache, and fulfils the +// future on dns_resolve_data->result. +void* monitor_dns_resolver_thread(const std::vector& dns_resolve_data_list); + +// Minimal worker thread that drains a wqueue and calls +// monitor_dns_resolver_thread() on each item. Independent of any monitor +// singleton so it can back PgSQL_Monitor's DNS cache loop alongside (or +// instead of) MySQL_Monitor's existing ConsumerThread-based pool. +// +// Each item ownership is transferred to this worker: the worker frees it +// after running monitor_dns_resolver_thread. A NULL dequeue is the exit +// signal — the loop terminates cleanly so callers can shut the pool down by +// pushing one NULL per worker. +class DNSResolverWorker : public Thread { +public: + DNSResolverWorker(wqueue& q, const char* name = nullptr) + : queue_(q) { + if (name && name[0]) + snprintf(thr_name_, sizeof(thr_name_), "%.15s", name); + else + snprintf(thr_name_, sizeof(thr_name_), "DNSResolver"); + } + + void* run() override; + +private: + wqueue& queue_; + char thr_name_[16] {}; +}; + +// Return true if 'ip' is a valid IPv4 or IPv6 address string. +bool validate_ip(const std::string& ip); + +// Return the IP address of the peer connected to 'socket_fd', or "" on +// failure / non-IP families. +std::string get_connected_peer_ip_from_socket(int socket_fd); + +// Helper: stringify a list of IPs for debug logging. Defined inline because +// it's templated over the iterable type used by the various call sites. +template +std::string debug_iplisttostring(const T& ips) { + std::stringstream sstr; + for (const std::string& ip : ips) + sstr << ip << " "; + return sstr.str(); +} + +#endif // __CLASS_DNS_CACHE_H diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index c217950c3c..ed1020fe39 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -4,6 +4,7 @@ #include "prometheus/counter.h" #include "prometheus/gauge.h" +#include "DNS_Cache.hpp" #include "MySQL_HostGroups_Manager.h" #include "proxysql.h" #include "cpp.h" @@ -385,71 +386,9 @@ struct mon_metrics_map_idx { }; }; -struct DNS_Cache_Record { - DNS_Cache_Record() = default; - DNS_Cache_Record(DNS_Cache_Record&&) = default; - DNS_Cache_Record(const DNS_Cache_Record&) = default; - DNS_Cache_Record& operator=(DNS_Cache_Record&&) = default; - DNS_Cache_Record& operator=(const DNS_Cache_Record&) = default; - DNS_Cache_Record(const std::string& hostname, const std::vector& ips, unsigned long long ttl = 0) : hostname_(hostname), - ttl_(ttl) { - std::copy(ips.begin(), ips.end(), std::inserter(ips_, ips_.end())); - } - DNS_Cache_Record(const std::string& hostname, std::set&& ips, unsigned long long ttl = 0) : hostname_(hostname), - ips_(std::move(ips)), ttl_(ttl) - { } - - ~DNS_Cache_Record() = default; - - std::string hostname_; - std::set ips_; - unsigned long long ttl_ = 0; -}; - -class DNS_Cache { - -public: - DNS_Cache() : enabled(true) { - int rc = pthread_rwlock_init(&rwlock_, nullptr); - assert(rc == 0); - } - - ~DNS_Cache() { - pthread_rwlock_destroy(&rwlock_); - } - - inline - void set_enabled_flag(bool value) { - enabled = value; - } - - bool add(const std::string& hostname, std::vector&& ips); - bool add_if_not_exist(const std::string& hostname, std::vector&& ips); - void remove(const std::string& hostname); - void clear(); - bool empty() const; - std::string lookup(const std::string& hostname, size_t* ip_count) const; - -private: - struct IP_ADDR { - std::vector ips; - unsigned long counter = 0; - }; - - std::string get_next_ip(const IP_ADDR& ip_addr) const; - std::unordered_map records; - std::atomic_bool enabled; - mutable pthread_rwlock_t rwlock_; -}; - -struct DNS_Resolve_Data { - std::promise> result; - std::shared_ptr dns_cache; - std::string hostname; - std::set cached_ips; - unsigned int ttl = 0; - unsigned int refresh_intv = 0; -}; +// DNS_Cache, DNS_Cache_Record, DNS_Resolve_Data and the resolver helpers now +// live in DNS_Cache.hpp (included above) so the same machinery can back the +// independent PgSQL_Monitor DNS cache. class MySQL_Monitor { @@ -495,9 +434,11 @@ class MySQL_Monitor { unsigned long long read_only_check_ERR; unsigned long long replication_lag_check_OK; unsigned long long replication_lag_check_ERR; - unsigned long long dns_cache_queried; - unsigned long long dns_cache_lookup_success; //cache hit - unsigned long long dns_cache_record_updated; + // DNS cache counters are incremented by resolver workers and read by + // p_update_metrics() from another thread; atomic to avoid the race. + std::atomic dns_cache_queried; + std::atomic dns_cache_lookup_success; //cache hit + std::atomic dns_cache_record_updated; std::atomic_bool force_dns_cache_update; struct { /// Prometheus metrics arrays diff --git a/include/PgSQL_Connection.h b/include/PgSQL_Connection.h index 77c1133a50..a20e35fe48 100644 --- a/include/PgSQL_Connection.h +++ b/include/PgSQL_Connection.h @@ -253,6 +253,11 @@ class PgSQL_Connection { PG_ASYNC_ST handler(short event); void connect_start(); void connect_cont(short event); + // Consults PgSQL_Monitor::dns_lookup; returns the cached IP on a hit, + // empty std::string on a miss. Used by connect_start() to set + // `hostaddr=` in the libpq conninfo so PQconnectStart skips + // getaddrinfo and doesn't block the worker thread on DNS. + std::string connect_start_DNS_lookup(); void query_start(); void query_cont(short event); void fetch_result_start(); diff --git a/include/PgSQL_Monitor.hpp b/include/PgSQL_Monitor.hpp index 45661cc58c..77f03b2142 100644 --- a/include/PgSQL_Monitor.hpp +++ b/include/PgSQL_Monitor.hpp @@ -3,10 +3,13 @@ #include "libpq-fe.h" +#include "DNS_Cache.hpp" #include "sqlite3db.h" #include "proxysql_structs.h" +#include #include +#include #include #include @@ -23,8 +26,10 @@ #define MONITOR_SQLITE_TABLE_PROXYSQL_SERVERS "CREATE TABLE proxysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port) )" struct PgSQL_Monitor { - // @brief Flags if monitoring threads should be shutdown. - bool shutdown = false; + // @brief Flags if monitoring threads should be shutdown. Atomic because + // the resolver / monitor loops read it from worker threads while + // lifecycle code writes it from the main thread. + std::atomic shutdown { false }; // @brief Mutex to hold to update `monitor_internal.pgsql_servers` std::mutex pgsql_srvs_mutex {}; // @brief Mutex to hold to update/read `pgsql_servers` to monitor @@ -45,8 +50,22 @@ struct PgSQL_Monitor { uint64_t repl_lag_check_OK { 0 }; uint64_t ssl_connections_OK { 0 }; uint64_t non_ssl_connections_OK { 0 }; + // DNS cache counters (independent of MySQL_Monitor's counters). Atomic + // because the resolver workers increment them while readers (stats, + // p_update_metrics) load them from other threads. + std::atomic dns_cache_queried { 0 }; + std::atomic dns_cache_lookup_success { 0 }; + std::atomic dns_cache_record_updated { 0 }; /////////////////////////////////////////////////////////////////////////// + // PgSQL-side DNS cache, independent of MySQL_Monitor's cache. Lifetime is + // the same as the PgSQL_Monitor singleton; the resolver loop and the + // PgSQL_Connection lookup path both go through this shared_ptr. + std::shared_ptr dns_cache; + // Set to true by trigger_dns_cache_update() to short-circuit the + // refresh_interval sleep on the resolver loop. + std::atomic_bool force_dns_cache_update { false }; + std::vector tables_defs_monitor { { const_cast("pgsql_server_connect_log"), @@ -74,6 +93,29 @@ struct PgSQL_Monitor { }; PgSQL_Monitor(); + + // DNS cache facade. Mirrors MySQL_Monitor's surface so PgSQL_Connection + // can call PgSQL_Monitor::dns_lookup(hostname) and get a cached IP without + // blocking on getaddrinfo. + static std::string dns_lookup(const std::string& hostname, + bool return_hostname_if_lookup_fails = true, size_t* ip_count = nullptr); + static std::string dns_lookup(const char* hostname, + bool return_hostname_if_lookup_fails = true, size_t* ip_count = nullptr); + // Seed the cache from an already-connected PGconn. Called on the + // SUCCESSFUL branch of PgSQL_Connection's handler so the next + // connect avoids getaddrinfo even before the resolver loop has run. + static bool update_dns_cache_from_pgsql_conn(PGconn* pgsql_conn); + // Wake the resolver loop early (e.g. after a pgsql_servers update). + static void trigger_dns_cache_update(); + + // Background loop that drives the resolver pool. Mirrors + // MySQL_Monitor::monitor_dns_cache but walks PgHGM->pgsql_servers_to_monitor + // and reads pgsql-monitor_local_dns_* settings. + void* monitor_dns_cache(); + +private: + static bool _dns_cache_update(const std::string& hostname, + std::vector&& ip_address); }; struct pgsql_conn_t { @@ -86,4 +128,10 @@ struct pgsql_conn_t { void* PgSQL_monitor_scheduler_thread(); +// pthread entry-point that drives PgSQL_Monitor::monitor_dns_cache() until +// shutdown. Launched independently of the monitor scheduler so the DNS +// cache stays warm even when pgsql-monitor_enabled=false (same pattern as +// MySQL_Monitor's resolver thread, which is unconditional). +void* PgSQL_monitor_dns_cache_pthread(void* arg); + #endif diff --git a/include/cpp.h b/include/cpp.h index beb79afaaf..10db44990f 100644 --- a/include/cpp.h +++ b/include/cpp.h @@ -40,7 +40,9 @@ // AI_Features_Manager, AI_Tool_Handler, RAG_Tool_Handler // Step 6: AI_Vector_Storage, MySQL_Catalog, Discovery_Schema, // Static_Harvester, PgSQL_Static_Harvester -// Step 7 deletes the PROXYSQLGENAI macro (and this comment block). +// PROXYSQLGENAI is no longer a build flag — PROXYSQL40=1 builds and +// packages all v4.0 plugins (mysqlx, genai/MCP, ...). All +// genai/MCP/AI/RAG code lives in plugins/genai/. #include "MySQL_HostGroups_Manager.h" #include "PgSQL_HostGroups_Manager.h" diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index da7ac6000a..b381fb0468 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -1221,6 +1221,9 @@ __thread int pgsql_thread___monitor_replication_lag_count; __thread char* pgsql_thread___monitor_replication_lag_use_percona_heartbeat; __thread bool pgsql_thread___monitor_writer_is_also_reader; __thread int pgsql_thread___monitor_threads; +__thread int pgsql_thread___monitor_local_dns_cache_ttl; +__thread int pgsql_thread___monitor_local_dns_cache_refresh_interval; +__thread int pgsql_thread___monitor_local_dns_resolver_queue_maxsize; __thread char* pgsql_thread___monitor_username; __thread char* pgsql_thread___monitor_password; __thread char* pgsql_thread___monitor_dbname; @@ -1559,6 +1562,9 @@ extern __thread char* pgsql_thread___monitor_replication_lag_use_percona_heartbe extern __thread int pgsql_thread___monitor_read_only_max_timeout_count; extern __thread bool pgsql_thread___monitor_writer_is_also_reader; extern __thread int pgsql_thread___monitor_threads; +extern __thread int pgsql_thread___monitor_local_dns_cache_ttl; +extern __thread int pgsql_thread___monitor_local_dns_cache_refresh_interval; +extern __thread int pgsql_thread___monitor_local_dns_resolver_queue_maxsize; extern __thread char* pgsql_thread___monitor_username; extern __thread char* pgsql_thread___monitor_password; extern __thread char* pgsql_thread___monitor_dbname; diff --git a/lib/DNS_Cache.cpp b/lib/DNS_Cache.cpp new file mode 100644 index 0000000000..e584c16809 --- /dev/null +++ b/lib/DNS_Cache.cpp @@ -0,0 +1,345 @@ +#include "DNS_Cache.hpp" + +#include +#include +#include +#include + +#include +#include + +#include "proxysql.h" +#include "proxysql_debug.h" +#include "proxysql_utils.h" +#include "proxysql_glovars.hpp" +#include "gen_utils.h" + +#include + + +bool validate_ip(const std::string& ip) { + // inet_pton returns 1 on success, 0 if input is not a valid address for + // the family, -1 on error (e.g. unsupported family). Treating != 0 as + // success would misclassify -1 errors as valid IPs. + struct sockaddr_in sa4; + if (inet_pton(AF_INET, ip.c_str(), &(sa4.sin_addr)) == 1) + return true; + + struct sockaddr_in6 sa6; + if (inet_pton(AF_INET6, ip.c_str(), &(sa6.sin6_addr)) == 1) + return true; + + return false; +} + +std::string get_connected_peer_ip_from_socket(int socket_fd) { + std::string result; + char ip_addr[INET6_ADDRSTRLEN]; + + struct sockaddr_storage custom_sockaddr; + socklen_t addrlen = sizeof(custom_sockaddr); + memset(&custom_sockaddr, 0, sizeof(custom_sockaddr)); + + if (getpeername(socket_fd, (struct sockaddr*)&custom_sockaddr, &addrlen) != 0) + return result; + + // Only assign to result when sa_family is one we know how to format. + // Other families (AF_UNIX, AF_NETLINK, ...) shouldn't happen for a TCP + // peer fd, but if they ever do we'd previously emit uninitialized memory + // from ip_addr. + if (custom_sockaddr.ss_family == AF_INET) { + const struct sockaddr_in* ipv4 = (const struct sockaddr_in*)&custom_sockaddr; + if (inet_ntop(AF_INET, &ipv4->sin_addr, ip_addr, INET_ADDRSTRLEN)) + result = ip_addr; + } + else if (custom_sockaddr.ss_family == AF_INET6) { + const struct sockaddr_in6* ipv6 = (const struct sockaddr_in6*)&custom_sockaddr; + if (inet_ntop(AF_INET6, &ipv6->sin6_addr, ip_addr, INET6_ADDRSTRLEN)) + result = ip_addr; + } + + return result; +} + +void* monitor_dns_resolver_thread(const std::vector& dns_resolve_data_list) { + assert(!dns_resolve_data_list.empty()); + DNS_Resolve_Data* dns_resolve_data = dns_resolve_data_list.front(); + + struct addrinfo hints, *res = NULL; + + memset(&hints, 0, sizeof(hints)); + hints.ai_protocol = IPPROTO_TCP; + hints.ai_socktype = SOCK_STREAM; + // AI_ADDRCONFIG: only return AF_INET addrs if the local system has at least + // one IPv4 address configured, and AF_INET6 only if it has at least one + // IPv6 address configured. Loopback is not considered configured for this + // purpose. Useful on IPv4-only hosts so getaddrinfo() doesn't return IPv6 + // addresses that connect/bind would always fail on. + hints.ai_flags = AI_ADDRCONFIG; + hints.ai_family = dns_resolve_data->ai_family; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Resolving hostname:[%s] to its mapped IP address.\n", + dns_resolve_data->hostname.c_str()); + int gai_rc = getaddrinfo(dns_resolve_data->hostname.c_str(), NULL, &hints, &res); + + if (gai_rc != 0 || !res) { + proxy_error("An error occurred while resolving hostname: %s [%d]\n", + dns_resolve_data->hostname.c_str(), gai_rc); + goto __error; + } + + try { + std::vector ips; + ips.reserve(64); + + char ip_addr[INET6_ADDRSTRLEN]; + + for (auto p = res; p != NULL; p = p->ai_next) { + if (p->ai_family == AF_INET) { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)p->ai_addr; + inet_ntop(p->ai_addr->sa_family, &ipv4->sin_addr, ip_addr, INET_ADDRSTRLEN); + ips.push_back(ip_addr); + } + else { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)p->ai_addr; + inet_ntop(p->ai_addr->sa_family, &ipv6->sin6_addr, ip_addr, INET6_ADDRSTRLEN); + ips.push_back(ip_addr); + } + } + + freeaddrinfo(res); + + if (!ips.empty()) { + + bool to_update_cache = false; + int cache_ttl = dns_resolve_data->ttl; + if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) { + thread_local std::mt19937 gen(std::random_device{}()); + const int jitter = static_cast(dns_resolve_data->ttl * 0.025); + std::uniform_int_distribution dis(-jitter, jitter); + cache_ttl += dis(gen); + } + + if (!dns_resolve_data->cached_ips.empty()) { + + if (dns_resolve_data->cached_ips.size() == ips.size()) { + for (const std::string& ip : ips) { + if (dns_resolve_data->cached_ips.find(ip) == dns_resolve_data->cached_ips.end()) { + to_update_cache = true; + break; + } + } + } + else + to_update_cache = true; + + if (!to_update_cache) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "DNS cache record already up-to-date. (Hostname:[%s] IP:[%s])\n", + dns_resolve_data->hostname.c_str(), + debug_iplisttostring(ips).c_str()); + dns_resolve_data->result.set_value(std::make_tuple<>(true, + DNS_Cache_Record(dns_resolve_data->hostname, + std::move(dns_resolve_data->cached_ips), + monotonic_time() + (1000ULL * static_cast(cache_ttl))))); + } + } + else + to_update_cache = true; + + if (to_update_cache) { + dns_resolve_data->result.set_value(std::make_tuple<>(true, + DNS_Cache_Record(dns_resolve_data->hostname, ips, + monotonic_time() + (1000ULL * static_cast(cache_ttl))))); + dns_resolve_data->dns_cache->add(dns_resolve_data->hostname, std::move(ips)); + } + + return NULL; + } + } + catch (std::exception& ex) { + proxy_error("An exception occurred while resolving hostname: %s [%s]\n", + dns_resolve_data->hostname.c_str(), ex.what()); + } + catch (...) { + proxy_error("An unknown exception has occurred while resolving hostname: %s\n", + dns_resolve_data->hostname.c_str()); + } + +__error: + dns_resolve_data->result.set_value(std::make_tuple<>(false, DNS_Cache_Record())); + + return NULL; +} + + +void* DNSResolverWorker::run() { + set_thread_name(thr_name_, GloVars.set_thread_name); + + while (true) { + DNS_Resolve_Data* item = static_cast(queue_.remove()); + if (item == nullptr) { + // Sentinel: caller pushed NULL to ask the worker to exit. + break; + } + std::vector list { item }; + try { + monitor_dns_resolver_thread(list); + } catch (...) { + // monitor_dns_resolver_thread() can in principle throw + // (allocations inside the try/catch in there are guarded, but + // other paths — set_thread_name, vector growth, etc. — aren't). + // If it did and the promise was not satisfied, future::get() on + // the producer side would block forever and hang shutdown. + // Forcibly satisfy the promise with a failure result. set_value + // throws if the promise is already satisfied, which we also + // swallow here. + try { + item->result.set_value(std::make_tuple<>(false, DNS_Cache_Record())); + } catch (...) { } + } + delete item; + } + return nullptr; +} + + +bool DNS_Cache::add(const std::string& hostname, std::vector&& ips) { + + if (!enabled) return false; + + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Updating DNS cache. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), debug_iplisttostring(ips).c_str()); + int rc = pthread_rwlock_wrlock(&rwlock_); + assert(rc == 0); + auto& ip_addr = records[hostname]; + ip_addr.ips = std::move(ips); + __sync_fetch_and_and(&ip_addr.counter, 0); + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + + if (counter_record_updated_) + counter_record_updated_->fetch_add(1, std::memory_order_relaxed); + + return true; +} + +bool DNS_Cache::add_if_not_exist(const std::string& hostname, std::vector&& ips) { + if (!enabled) return false; + + bool inserted = false; + int rc = pthread_rwlock_wrlock(&rwlock_); + assert(rc == 0); + if (records.find(hostname) == records.end()) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Updating DNS cache. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), debug_iplisttostring(ips).c_str()); + auto& ip_addr = records[hostname]; + ip_addr.ips = std::move(ips); + __sync_fetch_and_and(&ip_addr.counter, 0); + inserted = true; + } + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + + if (inserted && counter_record_updated_) + counter_record_updated_->fetch_add(1, std::memory_order_relaxed); + + return inserted; +} + +std::string DNS_Cache::get_next_ip(const IP_ADDR& ip_addr) const { + + if (ip_addr.ips.empty()) + return ""; + + const auto counter_val = __sync_fetch_and_add(&ip_addr.counter, 1); + + return ip_addr.ips[counter_val % ip_addr.ips.size()]; +} + +std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) const { + if (!enabled) { + if (ip_count) + *ip_count = 0; + return ""; + } + + std::string ip; + + if (counter_queried_) + counter_queried_->fetch_add(1, std::memory_order_relaxed); + + int rc = pthread_rwlock_rdlock(&rwlock_); + assert(rc == 0); + auto itr = records.find(hostname); + + if (itr != records.end()) { + ip = get_next_ip(itr->second); + + if (ip_count) + *ip_count = itr->second.ips.size(); + + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "DNS cache lookup success. (Hostname:[%s] IP returned:[%s])\n", + hostname.c_str(), ip.c_str()); + } + else { + if (ip_count) + *ip_count = 0; + } + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + + if (!ip.empty() && counter_lookup_success_) + counter_lookup_success_->fetch_add(1, std::memory_order_relaxed); + + return ip; +} + +void DNS_Cache::remove(const std::string& hostname) { + bool item_removed = false; + + int rc = pthread_rwlock_wrlock(&rwlock_); + assert(rc == 0); + auto itr = records.find(hostname); + if (itr != records.end()) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Removing DNS cache record. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), debug_iplisttostring(itr->second.ips).c_str()); + records.erase(itr); + item_removed = true; + } + rc = pthread_rwlock_unlock(&rwlock_); + + if (item_removed && counter_record_updated_) + counter_record_updated_->fetch_add(1, std::memory_order_relaxed); + + assert(rc == 0); +} + +void DNS_Cache::clear() { + size_t records_removed = 0; + int rc = pthread_rwlock_wrlock(&rwlock_); + assert(rc == 0); + records_removed = records.size(); + records.clear(); + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + if (records_removed && counter_record_updated_) + counter_record_updated_->fetch_add(records_removed, std::memory_order_relaxed); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache was cleared.\n"); +} + +bool DNS_Cache::empty() const { + bool result = true; + + int rc = pthread_rwlock_rdlock(&rwlock_); + assert(rc == 0); + result = records.empty(); + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + + return result; +} diff --git a/lib/Makefile b/lib/Makefile index ed6378081b..03a6027881 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -102,7 +102,7 @@ _OBJ_CXX := ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo proxy_protocol_info.oo \ proxysql_find_charset.oo ProxySQL_Poll.oo \ PgSQL_Protocol.oo PgSQL_Thread.oo PgSQL_Data_Stream.oo PgSQL_Session.oo PgSQL_Variables.oo PgSQL_HostGroups_Manager.oo PgSQL_Connection.oo PgSQL_Backend.oo PgSQL_Logger.oo PgSQL_Authentication.oo PgSQL_Error_Helper.oo \ - MySQL_Query_Cache.oo PgSQL_Query_Cache.oo PgSQL_Monitor.oo \ + MySQL_Query_Cache.oo PgSQL_Query_Cache.oo PgSQL_Monitor.oo DNS_Cache.oo \ MySQL_Set_Stmt_Parser.oo PgSQL_Set_Stmt_Parser.oo \ PgSQL_Variables_Validator.oo PgSQL_ExplicitTxnStateMgr.oo \ PgSQL_PreparedStatement.oo PgSQL_Extended_Query_Message.oo \ diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 2bc251ffc1..1ea5077175 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -746,19 +746,37 @@ void MySQL_Monitor_State_Data::init_async() { " wsrep_cluster_status, pxc_maint_mode FROM HOST_STATUS_GALERA WHERE hostgroup_id="; query_ += std::to_string(writer_hostgroup) + " AND hostname='" + std::string(hostname) + "' AND port=" + std::to_string(port); #else - if (strncmp(mysql->server_version, (char*)"5.7", 3) == 0 || strncmp(mysql->server_version, (char*)"8", 1) == 0) { - // the backend is either MySQL 5.7 or MySQL 8 : INFORMATION_SCHEMA.GLOBAL_STATUS is deprecated - query_ = "SELECT (SELECT VARIABLE_VALUE FROM performance_schema.global_status WHERE VARIABLE_NAME='WSREP_LOCAL_STATE') " - "wsrep_local_state, @@read_only read_only, (SELECT VARIABLE_VALUE FROM performance_schema.global_status WHERE VARIABLE_NAME='WSREP_LOCAL_RECV_QUEUE') wsrep_local_recv_queue , " - "@@wsrep_desync wsrep_desync, @@wsrep_reject_queries wsrep_reject_queries, @@wsrep_sst_donor_rejects_queries wsrep_sst_donor_rejects_queries, " - "(SELECT VARIABLE_VALUE FROM performance_schema.global_status WHERE VARIABLE_NAME='WSREP_CLUSTER_STATUS') wsrep_cluster_status , " - "(SELECT COALESCE(MAX(VARIABLE_VALUE),'DISABLED') FROM performance_schema.global_variables WHERE variable_name='pxc_maint_mode') pxc_maint_mode "; - } else { - // any other version - query_ = "SELECT (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_STATE') " - "wsrep_local_state, @@read_only read_only, (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_RECV_QUEUE') wsrep_local_recv_queue , " - "@@wsrep_desync wsrep_desync, @@wsrep_reject_queries wsrep_reject_queries, @@wsrep_sst_donor_rejects_queries wsrep_sst_donor_rejects_queries, " - "(SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_CLUSTER_STATUS') wsrep_cluster_status , (SELECT 'DISABLED') pxc_maint_mode"; + { + // performance_schema.global_status is a MySQL-only table (added in 5.7). + // INFORMATION_SCHEMA.GLOBAL_STATUS was deprecated in MySQL 8.0 and + // removed in MySQL 8.4, so MySQL >= 5.7 must use performance_schema + // (otherwise the monitor query fails on MySQL 8.4+/9.x). MariaDB never + // implemented the performance_schema status tables and keeps + // GLOBAL_STATUS in information_schema across all versions, so it must + // be excluded explicitly (atoi("10.x-MariaDB") >= 8 would otherwise + // route it to the performance_schema branch). + // (The enclosing braces are required: this is a `switch` case body + // and the variable declarations below are jumped over by sibling + // `case` labels — without a scope, GCC errors with "jump to case + // label" / "crosses initialization".) + const char *sv = mysql->server_version; + bool is_mariadb = (sv != NULL && strstr(sv, "MariaDB") != NULL); + int sv_major = (sv != NULL) ? atoi(sv) : 0; + bool use_perf_schema = !is_mariadb && + (sv_major >= 8 || (sv_major == 5 && sv != NULL && strncmp(sv, "5.7", 3) == 0)); + if (use_perf_schema) { + query_ = "SELECT (SELECT VARIABLE_VALUE FROM performance_schema.global_status WHERE VARIABLE_NAME='WSREP_LOCAL_STATE') " + "wsrep_local_state, @@read_only read_only, (SELECT VARIABLE_VALUE FROM performance_schema.global_status WHERE VARIABLE_NAME='WSREP_LOCAL_RECV_QUEUE') wsrep_local_recv_queue , " + "@@wsrep_desync wsrep_desync, @@wsrep_reject_queries wsrep_reject_queries, @@wsrep_sst_donor_rejects_queries wsrep_sst_donor_rejects_queries, " + "(SELECT VARIABLE_VALUE FROM performance_schema.global_status WHERE VARIABLE_NAME='WSREP_CLUSTER_STATUS') wsrep_cluster_status , " + "(SELECT COALESCE(MAX(VARIABLE_VALUE),'DISABLED') FROM performance_schema.global_variables WHERE variable_name='pxc_maint_mode') pxc_maint_mode "; + } else { + // MariaDB Galera (any version) and legacy MySQL/PXC < 5.7 + query_ = "SELECT (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_STATE') " + "wsrep_local_state, @@read_only read_only, (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_RECV_QUEUE') wsrep_local_recv_queue , " + "@@wsrep_desync wsrep_desync, @@wsrep_reject_queries wsrep_reject_queries, @@wsrep_sst_donor_rejects_queries wsrep_sst_donor_rejects_queries, " + "(SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_CLUSTER_STATUS') wsrep_cluster_status , (SELECT 'DISABLED') pxc_maint_mode"; + } } #endif // TEST_GALERA task_timeout_ = mysql_thread___monitor_galera_healthcheck_timeout; @@ -1045,6 +1063,7 @@ mon_metrics_map = std::make_tuple( MySQL_Monitor::MySQL_Monitor() { dns_cache = std::make_shared(); + dns_cache->set_counters(&dns_cache_queried, &dns_cache_lookup_success, &dns_cache_record_updated); GloMyMon = this; My_Conn_Pool=new MySQL_Monitor_Connection_Pool(); @@ -1185,9 +1204,9 @@ void MySQL_Monitor::p_update_metrics() { p_update_counter(this->metrics.p_counter_array[p_mon_counter::mysql_monitor_read_only_check_err], GloMyMon->read_only_check_ERR); p_update_counter(this->metrics.p_counter_array[p_mon_counter::mysql_monitor_replication_lag_check_ok], GloMyMon->replication_lag_check_OK); p_update_counter(this->metrics.p_counter_array[p_mon_counter::mysql_monitor_replication_lag_check_err], GloMyMon->replication_lag_check_ERR); - p_update_counter(this->metrics.p_counter_array[p_mon_counter::mysql_monitor_dns_cache_queried], GloMyMon->dns_cache_queried); - p_update_counter(this->metrics.p_counter_array[p_mon_counter::mysql_monitor_dns_cache_lookup_success], GloMyMon->dns_cache_lookup_success); - p_update_counter(this->metrics.p_counter_array[p_mon_counter::mysql_monitor_dns_cache_record_updated], GloMyMon->dns_cache_record_updated); + p_update_counter(this->metrics.p_counter_array[p_mon_counter::mysql_monitor_dns_cache_queried], GloMyMon->dns_cache_queried.load()); + p_update_counter(this->metrics.p_counter_array[p_mon_counter::mysql_monitor_dns_cache_lookup_success], GloMyMon->dns_cache_lookup_success.load()); + p_update_counter(this->metrics.p_counter_array[p_mon_counter::mysql_monitor_dns_cache_record_updated], GloMyMon->dns_cache_record_updated.load()); } } @@ -2293,16 +2312,23 @@ void * monitor_galera_thread(const std::vector& data) mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, q2); free(q2); #else - char *sv = mmsd->mysql->server_version; - if (strncmp(sv,(char *)"5.7",3)==0 || strncmp(sv,(char *)"8",1)==0) { - // the backend is either MySQL 5.7 or MySQL 8 : INFORMATION_SCHEMA.GLOBAL_STATUS is deprecated + // Same MariaDB / MySQL 8.4+ consideration as the synchronous Galera + // healthcheck above (lib/MySQL_Monitor.cpp:749): MariaDB always uses + // INFORMATION_SCHEMA; non-MariaDB MySQL >= 5.7 must use + // performance_schema (INFORMATION_SCHEMA.GLOBAL_STATUS removed in 8.4). + const char *sv = mmsd->mysql->server_version; + bool is_mariadb = (sv != NULL && strstr(sv, "MariaDB") != NULL); + int sv_major = (sv != NULL) ? atoi(sv) : 0; + bool use_perf_schema = !is_mariadb && + (sv_major >= 8 || (sv_major == 5 && sv != NULL && strncmp(sv, "5.7", 3) == 0)); + if (use_perf_schema) { mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT (SELECT VARIABLE_VALUE FROM performance_schema.global_status WHERE VARIABLE_NAME='WSREP_LOCAL_STATE') " "wsrep_local_state, @@read_only read_only, (SELECT VARIABLE_VALUE FROM performance_schema.global_status WHERE VARIABLE_NAME='WSREP_LOCAL_RECV_QUEUE') wsrep_local_recv_queue , " "@@wsrep_desync wsrep_desync, @@wsrep_reject_queries wsrep_reject_queries, @@wsrep_sst_donor_rejects_queries wsrep_sst_donor_rejects_queries, " "(SELECT VARIABLE_VALUE FROM performance_schema.global_status WHERE VARIABLE_NAME='WSREP_CLUSTER_STATUS') wsrep_cluster_status , " "(SELECT COALESCE(MAX(VARIABLE_VALUE),'DISABLED') FROM performance_schema.global_variables WHERE variable_name='pxc_maint_mode') pxc_maint_mode "); } else { - // any other version + // MariaDB Galera (any version) and legacy MySQL/PXC < 5.7 mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_STATE') " "wsrep_local_state, @@read_only read_only, (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_RECV_QUEUE') wsrep_local_recv_queue , " "@@wsrep_desync wsrep_desync, @@wsrep_reject_queries wsrep_reject_queries, @@wsrep_sst_donor_rejects_queries wsrep_sst_donor_rejects_queries, " @@ -4622,169 +4648,9 @@ void * MySQL_Monitor::monitor_replication_lag() { return NULL; } -bool validate_ip(const std::string& ip) { - - // check if ip is vaild IPV4 ip address - struct sockaddr_in sa4; - if (inet_pton(AF_INET, ip.c_str(), &(sa4.sin_addr)) != 0) - return true; - - // check if ip is vaild IPV6 ip address - struct sockaddr_in6 sa6; - if (inet_pton(AF_INET6, ip.c_str(), &(sa6.sin6_addr)) != 0) - return true; - - return false; -} - -std::string get_connected_peer_ip_from_socket(int socket_fd) { - std::string result; - char ip_addr[INET6_ADDRSTRLEN]; - - union { - struct sockaddr_in in; - struct sockaddr_in6 in6; - } custom_sockaddr; - - struct sockaddr* addr = (struct sockaddr*)malloc(sizeof(custom_sockaddr)); - socklen_t addrlen = sizeof(custom_sockaddr); - memset(addr, 0, sizeof(custom_sockaddr)); - - int rc = getpeername(socket_fd, addr, &addrlen); - - if (rc == 0) { - if (addr->sa_family == AF_INET) { - struct sockaddr_in* ipv4 = (struct sockaddr_in*)addr; - inet_ntop(addr->sa_family, &ipv4->sin_addr, ip_addr, INET_ADDRSTRLEN); - } - else if (addr->sa_family == AF_INET6) { - struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)addr; - inet_ntop(addr->sa_family, &ipv6->sin6_addr, ip_addr, INET6_ADDRSTRLEN); - } - - result = ip_addr; - } - - free(addr); - - return result; -} - -template -std::string debug_iplisttostring(const T& ips) { - std::stringstream sstr; - - for (const std::string& ip : ips) - sstr << ip << " "; - - return sstr.str(); -} - -void* monitor_dns_resolver_thread(const std::vector& dns_resolve_data_list) { - assert(!dns_resolve_data_list.empty()); - DNS_Resolve_Data* dns_resolve_data = dns_resolve_data_list.front(); - - struct addrinfo hints, *res = NULL; - - /* set hints for getaddrinfo */ - memset(&hints, 0, sizeof(hints)); - hints.ai_protocol = IPPROTO_TCP; - hints.ai_socktype = SOCK_STREAM; - /* AI_ADDRCONFIG: IPv4 addresses are returned in the list pointed to by res only if the - local system has at least one IPv4 address configured, and IPv6 - addresses are returned only if the local system has at least one - IPv6 address configured. The loopback address is not considered - for this case as valid as a configured address. This flag is - useful on, for example, IPv4-only systems, to ensure that - getaddrinfo() does not return IPv6 socket addresses that would - always fail in connect or bind. */ - hints.ai_flags = AI_ADDRCONFIG; - hints.ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Resolving hostname:[%s] to its mapped IP address.\n", dns_resolve_data->hostname.c_str()); - int gai_rc = getaddrinfo(dns_resolve_data->hostname.c_str(), NULL, &hints, &res); - - if (gai_rc != 0 || !res) - { - proxy_error("An error occurred while resolving hostname: %s [%d]\n", dns_resolve_data->hostname.c_str(), gai_rc); - goto __error; - } - - try { - std::vector ips; - ips.reserve(64); - - char ip_addr[INET6_ADDRSTRLEN]; - - for (auto p = res; p != NULL; p = p->ai_next) { - - if (p->ai_family == AF_INET) { - struct sockaddr_in* ipv4 = (struct sockaddr_in*)p->ai_addr; - inet_ntop(p->ai_addr->sa_family, &ipv4->sin_addr, ip_addr, INET_ADDRSTRLEN); - ips.push_back(ip_addr); - } - else { - struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)p->ai_addr; - inet_ntop(p->ai_addr->sa_family, &ipv6->sin6_addr, ip_addr, INET6_ADDRSTRLEN); - ips.push_back(ip_addr); - } - } - - freeaddrinfo(res); - - if (!ips.empty()) { - - bool to_update_cache = false; - int cache_ttl = dns_resolve_data->ttl; - if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) { - thread_local std::mt19937 gen(std::random_device{}()); - const int jitter = static_cast(dns_resolve_data->ttl * 0.025); - std::uniform_int_distribution dis(-jitter, jitter); - cache_ttl += dis(gen); - } - - if (!dns_resolve_data->cached_ips.empty()) { - - if (dns_resolve_data->cached_ips.size() == ips.size()) { - for (const std::string& ip : ips) { - - if (dns_resolve_data->cached_ips.find(ip) == dns_resolve_data->cached_ips.end()) { - to_update_cache = true; - break; - } - } - } - else - to_update_cache = true; - - // only update dns_records_bookkeeping - if (!to_update_cache) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache record already up-to-date. (Hostname:[%s] IP:[%s])\n", dns_resolve_data->hostname.c_str(), debug_iplisttostring(ips).c_str()); - dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * cache_ttl)))); - } - } - else - to_update_cache = true; - - if (to_update_cache) { - dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * cache_ttl)))); - dns_resolve_data->dns_cache->add(dns_resolve_data->hostname, std::move(ips)); - } - - return NULL; - } - } - catch (std::exception& ex) { - proxy_error("An exception occurred while resolving hostname: %s [%s]\n", dns_resolve_data->hostname.c_str(), ex.what()); - } - catch (...) { - proxy_error("An unknown exception has occurred while resolving hostname: %s\n", dns_resolve_data->hostname.c_str()); - } - -__error: - dns_resolve_data->result.set_value(std::make_tuple<>(false, DNS_Cache_Record())); - - return NULL; -} +// validate_ip(), get_connected_peer_ip_from_socket(), debug_iplisttostring() +// and monitor_dns_resolver_thread() moved to lib/DNS_Cache.cpp so they can +// back both MySQL_Monitor and PgSQL_Monitor. void* MySQL_Monitor::monitor_dns_cache() { // initialize the MySQL Thread (note: this is not a real thread, just the structures associated with it) @@ -4946,6 +4812,7 @@ void* MySQL_Monitor::monitor_dns_cache() { dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl; dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval; dns_resolve_data->dns_cache = dns_cache; + dns_resolve_data->ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Removing expired DNS record from bookkeeper. (Hostname:[%s] IP:[%s])\n", itr->hostname_.c_str(), debug_iplisttostring(dns_resolve_data->cached_ips).c_str()); @@ -4998,6 +4865,7 @@ void* MySQL_Monitor::monitor_dns_cache() { dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl; dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval; dns_resolve_data->dns_cache = dns_cache; + dns_resolve_data->ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); dns_resolver_queue.add(new WorkItem(dns_resolve_data.release(), monitor_dns_resolver_thread)); usleep(delay_us); @@ -6832,8 +6700,12 @@ bool MySQL_Monitor::_dns_cache_update(const std::string &hostname, std::vectordns_cache; if (dns_cache_thread) { + // Render the IP list for the debug log BEFORE moving the vector + // into add_if_not_exist; reading the moved-from vector would yield + // "valid but unspecified" content (typically empty). + const std::string ip_list_for_log = debug_iplisttostring(ip_address); if (dns_cache_thread->add_if_not_exist(trim(hostname), std::move(ip_address))) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Direct DNS cache update. (Hostname:[%s] IP:[%s])\n", hostname.c_str(), debug_iplisttostring(ip_address).c_str()); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Direct DNS cache update. (Hostname:[%s] IP:[%s])\n", hostname.c_str(), ip_list_for_log.c_str()); return true; } } @@ -6848,133 +6720,8 @@ void MySQL_Monitor::trigger_dns_cache_update() { } } -bool DNS_Cache::add(const std::string& hostname, std::vector&& ips) { - - if (!enabled) return false; - - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Updating DNS cache. (Hostname:[%s] IP:[%s])\n", hostname.c_str(), debug_iplisttostring(ips).c_str()); - int rc = pthread_rwlock_wrlock(&rwlock_); - assert(rc == 0); - auto& ip_addr = records[hostname]; - ip_addr.ips = std::move(ips); - __sync_fetch_and_and(&ip_addr.counter, 0); - rc = pthread_rwlock_unlock(&rwlock_); - assert(rc == 0); - - if (GloMyMon) - __sync_fetch_and_add(&GloMyMon->dns_cache_record_updated, 1); - - return true; -} - -bool DNS_Cache::add_if_not_exist(const std::string& hostname, std::vector&& ips) { - if (!enabled) return false; - - int rc = pthread_rwlock_wrlock(&rwlock_); - assert(rc == 0); - if (records.find(hostname) == records.end()) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Updating DNS cache. (Hostname:[%s] IP:[%s])\n", hostname.c_str(), debug_iplisttostring(ips).c_str()); - auto& ip_addr = records[hostname]; - ip_addr.ips = std::move(ips); - __sync_fetch_and_and(&ip_addr.counter, 0); - } - rc = pthread_rwlock_unlock(&rwlock_); - assert(rc == 0); - - if (GloMyMon) - __sync_fetch_and_add(&GloMyMon->dns_cache_record_updated, 1); - - return true; -} - -std::string DNS_Cache::get_next_ip(const IP_ADDR& ip_addr) const { - - if (ip_addr.ips.empty()) - return ""; - - const auto counter_val = __sync_fetch_and_add(const_cast(&ip_addr.counter), 1); - - return ip_addr.ips[counter_val%ip_addr.ips.size()]; -} - -std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) const { - if (!enabled) return ""; - - std::string ip; - - __sync_fetch_and_add(&GloMyMon->dns_cache_queried, 1); - - int rc = pthread_rwlock_rdlock(&rwlock_); - assert(rc == 0); - auto itr = records.find(hostname); - - if (itr != records.end()) { - ip = get_next_ip(itr->second); - - if (ip_count) - *ip_count = itr->second.ips.size(); - - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache lookup success. (Hostname:[%s] IP returned:[%s])\n", hostname.c_str(), ip.c_str()); - } - else { - if (ip_count) - *ip_count = 0; - } - rc = pthread_rwlock_unlock(&rwlock_); - assert(rc == 0); - - if (!ip.empty() && GloMyMon) { - __sync_fetch_and_add(&GloMyMon->dns_cache_lookup_success, 1); - } - - return ip; -} - -void DNS_Cache::remove(const std::string& hostname) { - bool item_removed = false; - - int rc = pthread_rwlock_wrlock(&rwlock_); - assert(rc == 0); - auto itr = records.find(hostname); - if (itr != records.end()) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Removing DNS cache record. (Hostname:[%s] IP:[%s])\n", hostname.c_str(), debug_iplisttostring(itr->second.ips).c_str()); - records.erase(itr); - item_removed = true; - } - rc = pthread_rwlock_unlock(&rwlock_); - - if (item_removed && GloMyMon) - __sync_fetch_and_add(&GloMyMon->dns_cache_record_updated, 1); - - assert(rc == 0); - - -} - -void DNS_Cache::clear() { - size_t records_removed = 0; - int rc = pthread_rwlock_wrlock(&rwlock_); - assert(rc == 0); - records_removed = records.size(); - records.clear(); - rc = pthread_rwlock_unlock(&rwlock_); - assert(rc == 0); - if (records_removed) - __sync_fetch_and_add(&GloMyMon->dns_cache_record_updated, records_removed); - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache was cleared.\n"); -} - -bool DNS_Cache::empty() const { - bool result = true; - - int rc = pthread_rwlock_rdlock(&rwlock_); - assert(rc == 0); - result = records.empty(); - rc = pthread_rwlock_unlock(&rwlock_); - assert(rc == 0); - - return result; -} +// DNS_Cache::add(), add_if_not_exist(), get_next_ip(), lookup(), remove(), +// clear(), and empty() moved to lib/DNS_Cache.cpp. #define NEXT_IMMEDIATE(new_st) do { async_state_machine_=new_st; goto __again; } while (0) diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 957c1c25a0..c8e69dd2b5 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -2604,15 +2604,25 @@ bool MySQL_Session::handler_again___status_SETTING_GENERIC_VARIABLE(int *_rc, co } query=(char *)malloc(strlen(q)+strlen(var_name)+strlen(var_value)); if (strncasecmp("tx_isolation", var_name, 12) == 0) { - char *sv = mybe->server_myds->myconn->mysql->server_version; - if (strncmp(sv,(char *)"8",1)==0) { + // MySQL 8.0+ uses `transaction_isolation`; `tx_isolation` was + // deprecated in 8.0 and removed in 8.4 (so MySQL 9.x also + // requires the modern name). MariaDB (any version, including + // 11.x where atoi(server_version) >= 11) keeps `tx_isolation`, + // so the major-version check must exclude MariaDB explicitly. + const char *sv = mybe->server_myds->myconn->mysql->server_version; + bool is_mariadb = (sv != NULL && strstr(sv, "MariaDB") != NULL); + int sv_major = (sv != NULL) ? atoi(sv) : 0; + if (!is_mariadb && sv_major >= 8) { sprintf(query,q,"transaction_isolation", var_value); } else { sprintf(query,q,"tx_isolation", var_value); } } else if (strncasecmp("tx_read_only", var_name, 12) == 0) { - char* sv = mybe->server_myds->myconn->mysql->server_version; - if (strncmp(sv, (char *)"8", 1) == 0) { + // Same MariaDB / MySQL 9.x consideration as `tx_isolation` above. + const char *sv = mybe->server_myds->myconn->mysql->server_version; + bool is_mariadb = (sv != NULL && strstr(sv, "MariaDB") != NULL); + int sv_major = (sv != NULL) ? atoi(sv) : 0; + if (!is_mariadb && sv_major >= 8) { sprintf(query,q,"transaction_read_only", var_value); } else { sprintf(query,q,"tx_read_only", var_value); @@ -6373,8 +6383,18 @@ void MySQL_Session::handler_WCD_SS_MCQ_qpo_LargePacket(PtrSize_t *pkt) { string errmsg = "Got a packet bigger than 'max_allowed_packet' bytes"; client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,client_myds->pkt_sid+1,1153,(char *)"08S01", errmsg.c_str(), true); MyHGM->add_mysql_errors(current_hostgroup, (char *)"", 0, client_myds->myconn->userinfo->username, (client_myds->addr.addr ? client_myds->addr.addr : (char *)"unknown" ), client_myds->myconn->userinfo->schemaname, 1153, (char *)errmsg.c_str()); + // Issue #5639: when called from the COM_STMT_EXECUTE path, + // MySQL_Protocol::get_binds_from_pkt() has set stmt_meta->pkt to alias + // pkt->ptr. RequestEnd() -> Query_Info::end() then free()s stmt_meta->pkt + // (the fix for bug #796), so the l_free() below would be a double-free of + // the same buffer. Capture the alias before RequestEnd() (which may reset + // stmt_meta) and skip the second free when the alias was present. + const bool stmt_meta_owns_pkt = + (CurrentQuery.stmt_meta != NULL && CurrentQuery.stmt_meta->pkt == pkt->ptr); RequestEnd(NULL, 1153, errmsg.c_str()); - l_free(pkt->size,pkt->ptr); + if (!stmt_meta_owns_pkt) { + l_free(pkt->size,pkt->ptr); + } } bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo(PtrSize_t *pkt, bool *lock_hostgroup, ps_type prepare_stmt_type) { @@ -6522,8 +6542,19 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C nq.erase(pos + 1); // remove trailing spaces and semicolumns } // detect MariaDB SET STATEMENT ... FOR syntax - // pass through to backend without hostgroup locking - if (strncasecmp(nq.c_str(), (char *)"SET STATEMENT ", 14) == 0 && strcasestr(nq.c_str(), (char *)" FOR ")) { + // pass through to backend without hostgroup locking. + // + // Use 'dig' (the digest text) for the keyword check rather than + // 'nq' (the raw query). 'nq' preserves the client's original + // whitespace, so a multi-line "SET STATEMENT max_statement_time + // = 300 FOR\n SELECT ..." has '\n' immediately after 'FOR' and + // strcasestr(nq, " FOR ") never matches — falling through to + // unable_to_parse_set_statement() and locking the hostgroup. + // 'dig' is already whitespace-normalised by the digest builder, + // so " FOR " always has a space on both sides regardless of how + // the client formatted the query. See issue #5686 follow-up and + // the customer report on PR #5708. + if (strncasecmp(dig, (char *)"SET STATEMENT ", 14) == 0 && strcasestr(dig, (char *)" FOR ")) { return false; } if ( diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 1e48476639..dac2df0518 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -5336,19 +5336,19 @@ SQLite3_result * MySQL_Threads_Handler::SQL3_GlobalStatus(bool _memory) { } { pta[0] = (char*)"MySQL_Monitor_dns_cache_queried"; - sprintf(buf, "%llu", GloMyMon->dns_cache_queried); + sprintf(buf, "%llu", GloMyMon->dns_cache_queried.load()); pta[1] = buf; result->add_row(pta); } { pta[0] = (char*)"MySQL_Monitor_dns_cache_lookup_success"; - sprintf(buf, "%llu", GloMyMon->dns_cache_lookup_success); + sprintf(buf, "%llu", GloMyMon->dns_cache_lookup_success.load()); pta[1] = buf; result->add_row(pta); } { pta[0] = (char*)"MySQL_Monitor_dns_cache_record_updated"; - sprintf(buf, "%llu", GloMyMon->dns_cache_record_updated); + sprintf(buf, "%llu", GloMyMon->dns_cache_record_updated.load()); pta[1] = buf; result->add_row(pta); } diff --git a/lib/MySQL_Variables.cpp b/lib/MySQL_Variables.cpp index a370c297cb..669ce7ce45 100644 --- a/lib/MySQL_Variables.cpp +++ b/lib/MySQL_Variables.cpp @@ -337,9 +337,19 @@ bool validate_charset(MySQL_Session* session, int idx, int &_rc) { unsigned int replace_collation_nr = 0; std::stringstream ss; int charset = atoi(mysql_variables.client_get_value(session, idx)); - // Pre-8.0 MySQL cannot handle collations with id >= 255. - // MySQL 8.x, 9.x and later can — match by numeric major version, not first character. - if (charset >= 255 && atoi(myconn->mysql->server_version) < 8) { + // Collations with id >= 255 (e.g. utf8mb4_0900_ai_ci) are a MySQL 8.0+ + // feature. They are not understood by: + // - MySQL < 8.0 (any major version below 8) + // - MariaDB (any version): MariaDB uses different collation IDs and + // does not implement utf8mb4_0900_ai_ci, regardless of its numeric + // major version (10.x, 11.x would otherwise pass an atoi() < 8 check). + const char *sv = myconn->mysql->server_version; + bool is_mariadb = (sv != NULL && strstr(sv, "MariaDB") != NULL); + // `atoi(NULL)` is undefined behavior; in practice `sv` should not be + // NULL here (validate_charset runs after a backend handshake has + // populated server_version), but the guard is essentially free. + int sv_major = (sv != NULL) ? atoi(sv) : 0; + if (charset >= 255 && (is_mariadb || sv_major < 8)) { switch(mysql_thread___handle_unknown_charset) { case HANDLE_UNKNOWN_CHARSET__DISCONNECT_CLIENT: snprintf(msg,sizeof(msg),"Can't initialize character set %s", mysql_variables.client_get_value(session, idx)); diff --git a/lib/PgSQL_Connection.cpp b/lib/PgSQL_Connection.cpp index dab5ed3d48..6168171902 100644 --- a/lib/PgSQL_Connection.cpp +++ b/lib/PgSQL_Connection.cpp @@ -8,6 +8,7 @@ using json = nlohmann::json; #define PROXYJSON #include "PgSQL_HostGroups_Manager.h" +#include "PgSQL_Monitor.hpp" #include "proxysql.h" #include "cpp.h" #include "PgSQL_PreparedStatement.h" @@ -373,7 +374,10 @@ PG_ASYNC_ST PgSQL_Connection::handler(short event) { } __sync_fetch_and_add(&PgHGM->status.server_connections_connected, 1); __sync_fetch_and_add(&parent->connect_OK, 1); - //MySQL_Monitor::update_dns_cache_from_mysql_conn(pgsql); + // Seed the PgSQL DNS cache from the just-established connection so + // the next connect for this hostname can skip getaddrinfo even if + // the background resolver loop hasn't visited it yet. + PgSQL_Monitor::update_dns_cache_from_pgsql_conn(pgsql_conn); break; case ASYNC_CONNECT_FAILED: //PQfinish(pgsql_conn);//release connection even on error @@ -948,6 +952,16 @@ static void append_conninfo_param(std::ostringstream& conninfo, const char* key, } } +std::string PgSQL_Connection::connect_start_DNS_lookup() { + // PgSQL_Monitor::dns_lookup() returns an IP on cache hit, or empty + // on miss / when 'parent->address' is itself an IP / when the cache is + // disabled. Empty result means "don't pass hostaddr to libpq" so the + // existing behavior (libpq does getaddrinfo) is preserved. + const std::string ip = PgSQL_Monitor::dns_lookup(parent->address, + /*return_hostname_if_lookup_fails=*/false); + return ip; +} + void PgSQL_Connection::connect_start() { PROXY_TRACE(); assert(pgsql_conn == NULL); // already there is a connection @@ -959,6 +973,16 @@ void PgSQL_Connection::connect_start() { append_conninfo_param(conninfo, "password", userinfo->password); // password append_conninfo_param(conninfo, "dbname", userinfo->dbname); // dbname append_conninfo_param(conninfo, "host", parent->address); // backend address + // If the DNS cache has resolved this hostname already, also pass + // hostaddr=. libpq documents this combo specifically to skip name + // resolution while keeping the hostname for TLS verification and error + // messages. Empty IP -> cache miss / IP literal / disabled, leave as-is. + { + const std::string ip = connect_start_DNS_lookup(); + if (!ip.empty() && ip != std::string(parent->address)) { + append_conninfo_param(conninfo, "hostaddr", const_cast(ip.c_str())); + } + } conninfo << "port=" << parent->port << " "; // backend port conninfo << "application_name=proxysql "; // application name //conninfo << "require_auth=" << AUTHENTICATION_METHOD_STR[pgsql_thread___authentication_method]; // authentication method diff --git a/lib/PgSQL_HostGroups_Manager.cpp b/lib/PgSQL_HostGroups_Manager.cpp index 237d5f7451..628a1e86b4 100644 --- a/lib/PgSQL_HostGroups_Manager.cpp +++ b/lib/PgSQL_HostGroups_Manager.cpp @@ -1834,7 +1834,11 @@ void PgSQL_HostGroups_Manager::update_table_pgsql_servers_for_monitor(bool lock) wrunlock(); } - MySQL_Monitor::trigger_dns_cache_update(); + // Wake the *PgSQL* resolver loop, not MySQL's. Pre-#5806 this was a + // MySQL_Monitor:: call inherited from a copy of the MySQL HGM; that + // kicked the wrong cache and pgsql hostnames had to wait for the next + // refresh_interval (default 60 s) before becoming resolvable. + PgSQL_Monitor::trigger_dns_cache_update(); } SQLite3_result * PgSQL_HostGroups_Manager::dump_table_pgsql(const string& name) { diff --git a/lib/PgSQL_Monitor.cpp b/lib/PgSQL_Monitor.cpp index 7f499d265e..01dd4dcc94 100644 --- a/lib/PgSQL_Monitor.cpp +++ b/lib/PgSQL_Monitor.cpp @@ -118,6 +118,9 @@ void check_and_build_standard_tables(SQLite3DB& db, const vector& t } PgSQL_Monitor::PgSQL_Monitor() { + dns_cache = std::make_shared(); + dns_cache->set_counters(&dns_cache_queried, &dns_cache_lookup_success, &dns_cache_record_updated); + int rc = monitordb.open( const_cast("file:mem_monitordb?mode=memory&cache=shared"), SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX @@ -2505,7 +2508,7 @@ void* PgSQL_monitor_scheduler_thread() { tasks_intvs_t next_intvs {}; vector tasks_batches {}; - while (GloPgMon->shutdown == false && pgsql_thread___monitor_enabled == true) { + while (GloPgMon->shutdown.load(std::memory_order_acquire) == false && pgsql_thread___monitor_enabled == true) { cur_intv_start = monotonic_time(); uint64_t closest_intv { @@ -2718,3 +2721,364 @@ void* PgSQL_monitor_scheduler_thread() { return nullptr; } + + +// ============================================================================ +// PgSQL_Monitor DNS cache +// ---------------------------------------------------------------------------- +// Mirrors MySQL_Monitor's DNS cache: a hostname -> [IP] map populated by a +// background resolver loop and consulted on the PgSQL connect path so the +// worker thread doesn't block on getaddrinfo inside libpq's PQconnectStart. +// +// Cache state is INDEPENDENT of MySQL_Monitor's cache. The two share only +// the DNS_Cache class itself (see DNS_Cache.hpp); each monitor owns its own +// DNS_Cache instance, its own counters, and its own resolver loop. This way +// pgsql-monitor_local_dns_cache_* settings on one side don't affect the +// other. +// ============================================================================ + +std::string PgSQL_Monitor::dns_lookup(const std::string& hostname, + bool return_hostname_if_lookup_fails, size_t* ip_count) { + + static thread_local std::shared_ptr dns_cache_thread; + + // If hostname is already an IP, return as-is (cache miss avoided). + if (hostname.empty() || validate_ip(hostname)) + return hostname; + + if (!dns_cache_thread && GloPgMon) + dns_cache_thread = GloPgMon->dns_cache; + + std::string ip; + + if (dns_cache_thread) { + ip = dns_cache_thread->lookup(trim(hostname), ip_count); + } + + // Apply the hostname fallback even when the cache facade is unavailable + // (e.g. early startup or shutdown), so callers consistently get the + // hostname back as documented when return_hostname_if_lookup_fails=true. + if (ip.empty() && return_hostname_if_lookup_fails) { + ip = hostname; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "DNS cache lookup was a miss. (Hostname:[%s])\n", hostname.c_str()); + } + + return ip; +} + +std::string PgSQL_Monitor::dns_lookup(const char* hostname, + bool return_hostname_if_lookup_fails, size_t* ip_count) { + return PgSQL_Monitor::dns_lookup(std::string(hostname), + return_hostname_if_lookup_fails, ip_count); +} + +bool PgSQL_Monitor::update_dns_cache_from_pgsql_conn(PGconn* pgsql_conn) { + if (!pgsql_conn) return false; + + // PQhost returns "host" parameter value (hostname or IP); PQsocket gives + // the actual fd we can call getpeername on. + const char* host_param = PQhost(pgsql_conn); + if (!host_param || !host_param[0]) + return false; + + const std::string hostname { host_param }; + + // If user configured an IP directly, no cache update needed. + if (validate_ip(hostname)) + return false; + + const int sock = PQsocket(pgsql_conn); + if (sock < 0) + return false; + + const std::string& ip_addr = get_connected_peer_ip_from_socket(sock); + if (ip_addr.empty()) + return false; + + return _dns_cache_update(hostname, { ip_addr }); +} + +bool PgSQL_Monitor::_dns_cache_update(const std::string& hostname, + std::vector&& ip_address) { + + static thread_local std::shared_ptr dns_cache_thread; + + if (!dns_cache_thread && GloPgMon) + dns_cache_thread = GloPgMon->dns_cache; + + if (dns_cache_thread) { + if (dns_cache_thread->add_if_not_exist(trim(hostname), std::move(ip_address))) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Direct DNS cache update. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), debug_iplisttostring(ip_address).c_str()); + return true; + } + } + + return false; +} + +void PgSQL_Monitor::trigger_dns_cache_update() { + if (GloPgMon) { + GloPgMon->force_dns_cache_update = true; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Triggering PgSQL DNS cache update sequence.\n"); + } +} + +// Background loop that drives the pgsql DNS cache. Mirrors +// MySQL_Monitor::monitor_dns_cache structurally but pulls hostnames from +// PgHGM->pgsql_servers_to_monitor instead of MyHGM, and reads the +// pgsql-monitor_local_dns_* settings. +void* PgSQL_Monitor::monitor_dns_cache() { + constexpr unsigned int num_dns_resolver_threads = 1; + constexpr unsigned int num_dns_resolver_max_threads = 32; + unsigned long long t1 = 0; + unsigned long long t2 = 0; + unsigned long long next_loop_at = 0; + bool dns_cache_enable = true; + + // Per-instance Thread variable refresher. Without this the pgsql_thread___ + // globals stay at their startup defaults inside this worker. Refresh + // once up-front so the first loop iteration sees the configured TTL / + // refresh interval rather than zero-initialized values — the + // version-bump check below then only fires on later config changes. + std::unique_ptr pgsql_thr { new PgSQL_Thread() }; + pgsql_thr->curtime = monotonic_time(); + pgsql_thr->refresh_variables(); + unsigned int local_thread_vars_version = GloPTH ? GloPTH->get_global_version() : 0; + if (pgsql_thread___monitor_local_dns_cache_ttl == 0 || + pgsql_thread___monitor_local_dns_cache_refresh_interval == 0) { + dns_cache_enable = false; + dns_cache->set_enabled_flag(false); + } else { + dns_cache->set_enabled_flag(true); + } + + std::list dns_records_bookkeeping; + wqueue dns_resolver_queue; + + while (GloPgMon->shutdown.load(std::memory_order_acquire) == false) { + if (!GloPTH) return nullptr; + + const unsigned int glover = GloPTH->get_global_version(); + if (local_thread_vars_version < glover) { + local_thread_vars_version = glover; + pgsql_thr->refresh_variables(); + next_loop_at = 0; + + if (pgsql_thread___monitor_local_dns_cache_ttl == 0 || + pgsql_thread___monitor_local_dns_cache_refresh_interval == 0) { + dns_cache_enable = false; + dns_cache->set_enabled_flag(false); + dns_cache->clear(); + dns_records_bookkeeping.clear(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "PgSQL DNS cache is disabled.\n"); + } else { + dns_cache_enable = true; + dns_cache->set_enabled_flag(true); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "PgSQL DNS cache is enabled.\n"); + } + } + + if (!dns_cache_enable) { + usleep(200000); + continue; + } + + t1 = monotonic_time(); + if (t1 >= next_loop_at || force_dns_cache_update) { + force_dns_cache_update = false; + next_loop_at = t1 + (1000ULL * pgsql_thread___monitor_local_dns_cache_refresh_interval); + + // Collect the set of distinct hostnames currently configured. Done + // under the pgsql_servers_to_monitor mutex because the resultset + // pointer can be swapped by an admin LOAD. + std::set hostnames; + { + std::lock_guard guard(PgHGM->pgsql_servers_to_monitor_mutex); + SQLite3_result* rs = PgHGM->pgsql_servers_to_monitor; + if (rs != nullptr) { + int hostname_col = -1; + for (int i = 0; i < rs->columns; i++) { + if (rs->column_definition[i] && + rs->column_definition[i]->name && + strcmp(rs->column_definition[i]->name, "hostname") == 0) { + hostname_col = static_cast(i); + break; + } + } + if (hostname_col >= 0) { + for (const auto row : rs->rows) { + if (!row || !row->fields[hostname_col]) continue; + // Trim before validate_ip / insert so the lookup + // path (which also trims) and the bookkeeper + // agree on the hostname key. Matches the + // SELECT trim(hostname) the MySQL resolver does + // in SQL. + const std::string hostname { trim(row->fields[hostname_col]) }; + if (hostname.empty()) continue; + if (!validate_ip(hostname)) + hostnames.insert(hostname); + } + } + } + } + + if (hostnames.empty()) { + if (!dns_cache->empty()) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Clearing all orphaned PgSQL DNS records from cache.\n"); + dns_cache->clear(); + } + if (!dns_records_bookkeeping.empty()) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Clearing all orphaned PgSQL DNS records from bookkeeper.\n"); + dns_records_bookkeeping.clear(); + } + } else { + std::vector dns_resolver_threads(num_dns_resolver_threads); + for (unsigned int i = 0; i < num_dns_resolver_threads; i++) { + dns_resolver_threads[i] = new DNSResolverWorker(dns_resolver_queue, "pgDnsResolver"); + dns_resolver_threads[i]->start(2048, false); + } + + std::list>> dns_resolve_result; + + // Stagger enqueueing so the resolver pool doesn't see a burst. + int delay_us = 100; + if (!hostnames.empty()) { + delay_us = pgsql_thread___monitor_local_dns_cache_refresh_interval / 2 / hostnames.size(); + delay_us *= 40; + if (delay_us > 1000000 || delay_us <= 0) + delay_us = 10000; + delay_us = delay_us + rand() % delay_us; + } + + // Walk the bookkeeper: drop orphans, requeue expired ones, keep + // the rest. Items that survive are also removed from the + // "hostnames" set so we don't enqueue them twice below. + if (!dns_records_bookkeeping.empty()) { + const unsigned long long current_time = monotonic_time(); + for (auto itr = dns_records_bookkeeping.begin(); + itr != dns_records_bookkeeping.end(); ) { + if (hostnames.find(itr->hostname_) == hostnames.end()) { + dns_cache->remove(itr->hostname_); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Removing orphaned PgSQL DNS record from bookkeeper. (Hostname:[%s] IP:[%s])\n", + itr->hostname_.c_str(), debug_iplisttostring(itr->ips_).c_str()); + itr = dns_records_bookkeeping.erase(itr); + } else { + hostnames.erase(itr->hostname_); + if (current_time > itr->ttl_) { + std::unique_ptr dns_resolve_data(new DNS_Resolve_Data()); + dns_resolve_data->hostname = std::move(itr->hostname_); + dns_resolve_data->cached_ips = std::move(itr->ips_); + dns_resolve_data->ttl = pgsql_thread___monitor_local_dns_cache_ttl; + dns_resolve_data->refresh_intv = pgsql_thread___monitor_local_dns_cache_refresh_interval; + dns_resolve_data->dns_cache = dns_cache; + // PgSQL has no separate resolution-family setting yet; + // default to AF_UNSPEC so we honor the OS resolver. + dns_resolve_data->ai_family = AF_UNSPEC; + dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); + + // Capture the IP list for the debug log + // BEFORE std::move'ing dns_resolve_data away. + const std::string ip_log = debug_iplisttostring(dns_resolve_data->cached_ips); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Removing expired PgSQL DNS record from bookkeeper. (Hostname:[%s] IP:[%s])\n", + dns_resolve_data->hostname.c_str(), + ip_log.c_str()); + dns_resolver_queue.add(dns_resolve_data.release()); + itr = dns_records_bookkeeping.erase(itr); + usleep(delay_us); + continue; + } + itr++; + } + } + } + + // Scale the resolver pool up if the queue is filling. + auto maybe_scale_pool = [&](unsigned int divisor) { + unsigned int qsize = dns_resolver_queue.size(); + unsigned int num_threads = dns_resolver_threads.size(); + if (qsize > (static_cast(pgsql_thread___monitor_local_dns_resolver_queue_maxsize) / divisor)) { + proxy_warning("PgSQL DNS resolver queue too big: %d.\n", qsize); + unsigned int threads_max = num_dns_resolver_max_threads; + if (threads_max > num_threads) { + unsigned int new_threads = threads_max - num_threads; + if ((qsize / divisor) < new_threads) + new_threads = qsize / divisor; + if (new_threads) { + unsigned int old_num_threads = num_threads; + num_threads += new_threads; + dns_resolver_threads.resize(num_threads); + for (unsigned int i = old_num_threads; i < num_threads; i++) { + dns_resolver_threads[i] = new DNSResolverWorker(dns_resolver_queue, "pgDnsResolver"); + dns_resolver_threads[i]->start(2048, false); + } + } + } + } + }; + maybe_scale_pool(8); + + // Enqueue the remaining (new) hostnames. + for (const std::string& hostname : hostnames) { + std::unique_ptr dns_resolve_data(new DNS_Resolve_Data()); + dns_resolve_data->hostname = hostname; + dns_resolve_data->ttl = pgsql_thread___monitor_local_dns_cache_ttl; + dns_resolve_data->refresh_intv = pgsql_thread___monitor_local_dns_cache_refresh_interval; + dns_resolve_data->dns_cache = dns_cache; + dns_resolve_data->ai_family = AF_UNSPEC; + dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); + dns_resolver_queue.add(dns_resolve_data.release()); + usleep(delay_us); + } + + maybe_scale_pool(4); + + // Push one NULL per worker so each loop's `remove()` returns + // the sentinel and the worker exits cleanly. + for (size_t i = 0; i < dns_resolver_threads.size(); i++) + dns_resolver_queue.add(nullptr); + + // Collect results. Successful resolutions feed the bookkeeper. + for (auto& dns_result : dns_resolve_result) { + auto ret_value = dns_result.get(); + if (std::get<0>(ret_value)) { + DNS_Cache_Record dns_record = std::get<1>(ret_value); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Adding PgSQL DNS record to bookkeeper. (Hostname:[%s] IP:[%s])\n", + dns_record.hostname_.c_str(), debug_iplisttostring(dns_record.ips_).c_str()); + dns_records_bookkeeping.emplace_back(std::move(dns_record)); + } + } + + for (DNSResolverWorker* const w : dns_resolver_threads) { + w->join(); + delete w; + } + + if (GloPgMon->shutdown.load(std::memory_order_acquire)) return nullptr; + } + } + + t2 = monotonic_time(); + if (t2 < next_loop_at) { + unsigned long long st = next_loop_at - t2; + if (st > 500000) st = 500000; + usleep(st); + } + } + + return nullptr; +} + +void* PgSQL_monitor_dns_cache_pthread(void* /*arg*/) { + if (GloPgMon) + GloPgMon->monitor_dns_cache(); + return nullptr; +} diff --git a/lib/PgSQL_Session.cpp b/lib/PgSQL_Session.cpp index c63ac7c432..1d64da390f 100644 --- a/lib/PgSQL_Session.cpp +++ b/lib/PgSQL_Session.cpp @@ -4513,7 +4513,7 @@ bool PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___handle_ for (auto it = std::begin(set); it != std::end(set); ++it) { std::string var = it->first; proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET variable %s\n", var.c_str()); - if (it->second.size() < 1 || it->second.size() > 2) { + if (it->second.size() < 1) { // error not enough arguments string query_str = string((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength); string digest_str = string(CurrentQuery.get_digest_text()); @@ -4532,7 +4532,18 @@ bool PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___handle_ return false; } + // PostgreSQL allows multi-value lists for some variables, notably + // search_path and datestyle (e.g. `SET search_path TO "$user", public`). + // ParserSQL v1.0.3 captures every value as a separate sibling; here + // we collapse them into the comma-separated form PG itself uses for + // these parameters before handing to the per-variable validator and + // tracker. Single-value SETs hit this loop with size()==1 and take + // just it->second.front() (unchanged behaviour). std::string value1 = it->second.front(); + for (size_t vi = 1; vi < it->second.size(); ++vi) { + value1 += ", "; + value1 += it->second[vi]; + } if (std::find(pgsql_critical_variables.begin(), pgsql_critical_variables.end(), var) != pgsql_critical_variables.end() || pgsql_other_variables.find(var) != pgsql_other_variables.end()) { diff --git a/lib/PgSQL_Thread.cpp b/lib/PgSQL_Thread.cpp index e99e3f8c5e..51a3730152 100644 --- a/lib/PgSQL_Thread.cpp +++ b/lib/PgSQL_Thread.cpp @@ -356,13 +356,13 @@ static char* pgsql_thread_variables_names[] = { (char*)"monitor_slave_lag_when_null", */ (char*)"monitor_threads", + (char*)"monitor_local_dns_cache_ttl", + (char*)"monitor_local_dns_cache_refresh_interval", + (char*)"monitor_local_dns_resolver_queue_maxsize", /* (char*)"monitor_threads_min", (char*)"monitor_threads_max", (char*)"monitor_threads_queue_maxsize", - (char*)"monitor_local_dns_cache_ttl", - (char*)"monitor_local_dns_cache_refresh_interval", - (char*)"monitor_local_dns_resolver_queue_maxsize", (char*)"monitor_wait_timeout", */ (char*)"monitor_writer_is_also_reader", @@ -4056,6 +4056,9 @@ void PgSQL_Thread::refresh_variables() { pgsql_thread___monitor_read_only_timeout = GloPTH->get_variable_int((char*)"monitor_read_only_timeout"); pgsql_thread___monitor_read_only_max_timeout_count = GloPTH->get_variable_int((char*)"monitor_read_only_max_timeout_count"); pgsql_thread___monitor_threads = GloPTH->get_variable_int((char*)"monitor_threads"); + pgsql_thread___monitor_local_dns_cache_ttl = GloPTH->get_variable_int((char*)"monitor_local_dns_cache_ttl"); + pgsql_thread___monitor_local_dns_cache_refresh_interval = GloPTH->get_variable_int((char*)"monitor_local_dns_cache_refresh_interval"); + pgsql_thread___monitor_local_dns_resolver_queue_maxsize = GloPTH->get_variable_int((char*)"monitor_local_dns_resolver_queue_maxsize"); /* NOTE: Disabled until 'pt-heartbeat' supports PostgreSQL is fixed: https://perconadev.atlassian.net/browse/PT-2030 if (pgsql_thread___monitor_replication_lag_use_percona_heartbeat) free(pgsql_thread___monitor_replication_lag_use_percona_heartbeat); pgsql_thread___monitor_replication_lag_use_percona_heartbeat = GloPTH->get_variable_string((char*)"monitor_replication_lag_use_percona_heartbeat"); @@ -4755,6 +4758,24 @@ SQLite3_result* PgSQL_Threads_Handler::SQL3_GlobalStatus(bool _memory) { pta[1] = buf; result->add_row(pta); } + { + pta[0] = (char*)"PgSQL_Monitor_dns_cache_queried"; + sprintf(buf, "%llu", GloPgMon->dns_cache_queried.load()); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"PgSQL_Monitor_dns_cache_lookup_success"; + sprintf(buf, "%llu", GloPgMon->dns_cache_lookup_success.load()); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"PgSQL_Monitor_dns_cache_record_updated"; + sprintf(buf, "%llu", GloPgMon->dns_cache_record_updated.load()); + pta[1] = buf; + result->add_row(pta); + } /* { pta[0] = (char*)"MySQL_Monitor_replication_lag_check_OK"; diff --git a/lib/Query_Processor_ParserSQL.cpp b/lib/Query_Processor_ParserSQL.cpp index 1843e69ac3..67eb5a369c 100644 --- a/lib/Query_Processor_ParserSQL.cpp +++ b/lib/Query_Processor_ParserSQL.cpp @@ -439,15 +439,24 @@ static std::map> walk_set_stmt( } case NodeType::NODE_VAR_ASSIGNMENT: { const AstNode* target = child->first_child; - const AstNode* rhs = target ? target->next_sibling : nullptr; if (!target || target->type != NodeType::NODE_VAR_TARGET) break; std::string var_name = normalize_set_var_name( emit_node_text(target, arena)); - std::string val = finalize_var_value( - resolve_var_value(target, rhs, query, query_len, arena)); - result[var_name] = {val}; + // Collect every RHS sibling of the target. For MySQL there is + // always exactly one. For PostgreSQL, multi-value lists such + // as `SET search_path TO 'a', 'b', 'c'` produce one VAR_TARGET + // followed by N value-expression siblings (see set_parser.h). + std::vector vals; + for (const AstNode* rhs = target->next_sibling; + rhs; rhs = rhs->next_sibling) { + vals.push_back(finalize_var_value( + resolve_var_value(target, rhs, query, query_len, arena))); + } + if (vals.empty()) vals.push_back(""); + + result[var_name] = std::move(vals); break; } // SET TRANSACTION is handled separately by MySQL_Session::parse2() diff --git a/lib/proxy_sqlite3_symbols.cpp b/lib/proxy_sqlite3_symbols.cpp index 70c231d35f..4358323261 100644 --- a/lib/proxy_sqlite3_symbols.cpp +++ b/lib/proxy_sqlite3_symbols.cpp @@ -50,14 +50,11 @@ int (*proxy_sqlite3_prepare_v2)(sqlite3*, const char*, int, sqlite3_stmt**, cons int (*proxy_sqlite3_open_v2)(const char*, sqlite3**, int, const char*) = sqlite3_open_v2; int (*proxy_sqlite3_exec)(sqlite3*, const char*, int (*)(void*,int,char**,char**), void*, char**) = sqlite3_exec; -// Hook for sqlite-vec. Gated on PROXYSQLGENAI: vec.o + sqlite-vec.h +// Hook for sqlite-vec. Gated on PROXYSQL40: vec.o + sqlite-vec.h // only get built (deps/Makefile) and linked (src/Makefile) when the -// genai plugin is part of this build. Non-genai v3.0/v3.1 release +// v4.0 plugin chassis is enabled. Non-genai v3.0/v3.1 release // binaries don't carry the vector-search extension; the genai plugin's -// AI_Vector_Storage is the only consumer of the hook anyway. An -// earlier Step-7 attempt to make this unconditional was wrong — it -// left non-genai dbg builds failing at `#include "sqlite-vec.h"` -// because deps still skipped sqlite-vec when PROXYSQLGENAI was unset. +// AI_Vector_Storage is the only consumer of the hook anyway. #ifdef PROXYSQL40 #include "sqlite-vec.h" int (*proxy_sqlite3_vec_init)(sqlite3*, char**, const sqlite3_api_routines*) = sqlite3_vec_init; diff --git a/src/main.cpp b/src/main.cpp index 3019823cfa..e7ae264560 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -101,6 +101,8 @@ static std::unique_ptr GloPluginManager; #endif /* PROXYSQL40 */ std::thread* pgsql_monitor_thread = nullptr; +pthread_t pgsql_monitor_dns_cache_thread {}; +bool pgsql_monitor_dns_cache_thread_started = false; extern int ProxySQL_create_or_load_TLS(bool bootstrap, std::string& msg); @@ -925,7 +927,7 @@ void ProxySQL_Main_init_main_modules() { GloPTH=NULL; // MCP_Threads_Handler / GenAI_Threads_Handler / AI_Features_Manager // are all constructed by the genai plugin's init() callback now -// (Steps 4.C and 5). Core has no PROXYSQLGENAI initializers here. +// (Steps 4.C and 5). Core has no PROXYSQL40 plugin-dedicated initializers here. #ifdef PROXYSQLCLICKHOUSE GloClickHouseAuth=NULL; #endif /* PROXYSQLCLICKHOUSE */ @@ -1126,7 +1128,7 @@ void ProxySQL_Main_join_all_threads() { GloMyMon->shutdown=true; } if (GloPgMon) { - GloPgMon->shutdown=true; + GloPgMon->shutdown.store(true, std::memory_order_release); } // join GloMyMon thread if (GloMyMon && MyMon_thread) { @@ -1147,6 +1149,10 @@ void ProxySQL_Main_join_all_threads() { std::cerr << "GloPgMon joined in "; #endif } + if (pgsql_monitor_dns_cache_thread_started) { + pthread_join(pgsql_monitor_dns_cache_thread, NULL); + pgsql_monitor_dns_cache_thread_started = false; + } /* Unified QC Purge Thread for both MySQL and PgSQL query cache // join GloMyQC thread if (GloMyQC) { @@ -1291,7 +1297,7 @@ void ProxySQL_Main_shutdown_all_modules() { #endif } // MCP / GenAI / AI shutdown is performed entirely by the genai plugin's -// stop() callback (Steps 4.C and 5). Core has no PROXYSQLGENAI +// stop() callback (Steps 4.C and 5). Core has no PROXYSQL40 plugin-dedicated // teardown here. if (GloMyLogger) { cpu_timer t; @@ -1518,7 +1524,7 @@ void ProxySQL_Main_init_phase2___not_started(const bootstrap_info_t& boostrap_in ProxySQL_Main_init_main_modules(); -// PROXYSQLGENAI init moved entirely to the genai plugin's init/start +// GenAI init moved entirely to the genai plugin's init/start // callbacks (Steps 4.C and 5). No core invocation needed. #ifdef PROXYSQL40 @@ -1723,6 +1729,17 @@ bool ProxySQL_Main_init_phase3___start_all() { { cpu_timer t; pgsql_monitor_thread = new std::thread(&PgSQL_monitor_scheduler_thread); + // DNS cache lives independently of the monitor scheduler so it + // stays warm even when pgsql-monitor_enabled is later toggled off. + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 2048 * 1024); + if (pthread_create(&pgsql_monitor_dns_cache_thread, &attr, &PgSQL_monitor_dns_cache_pthread, NULL) == 0) { + pgsql_monitor_dns_cache_thread_started = true; + } else { + proxy_error("Thread creation: PgSQL DNS cache\n"); + } + pthread_attr_destroy(&attr); #ifdef DEBUG std::cerr << "Main phase3 : PgSQL Monitor initialized in "; #endif diff --git a/test/deps/cluster_simulator/Makefile b/test/deps/cluster_simulator/Makefile index 3c61c6f8c1..eabc559248 100644 --- a/test/deps/cluster_simulator/Makefile +++ b/test/deps/cluster_simulator/Makefile @@ -68,7 +68,7 @@ ifeq ($(UNAME_S),Linux) STATIC_LIBS += $(COREDUMPER_LDIR)/libcoredumper.a endif -ifeq ($(PROXYSQLGENAI),1) +ifeq ($(PROXYSQL40),1) STATIC_LIBS += $(SQLITE3_LDIR)/vec.o endif diff --git a/test/infra/control/run-unit-tests-asan-coverage.bash b/test/infra/control/run-unit-tests-asan-coverage.bash index 443b22429a..b9d4bade20 100755 --- a/test/infra/control/run-unit-tests-asan-coverage.bash +++ b/test/infra/control/run-unit-tests-asan-coverage.bash @@ -19,7 +19,7 @@ # Preconditions: # * cwd is the repository root (the script cd's there from $0 if not). # * lib/, src/, and test/tap/tests/unit/ have been built with -# WITHASAN=1 WITHGCOV=1 NOJEMALLOC=1 PROXYSQLGENAI=1 (the make +# WITHASAN=1 WITHGCOV=1 NOJEMALLOC=1 PROXYSQL40=1 (the make # target `ubuntu24-tap` does this when invoked with those flags). # * The container has apt — we install lcov + libprotobuf-dev on # demand if missing (the build image is package-build-focused diff --git a/test/infra/control/start-proxysql-isolated.bash b/test/infra/control/start-proxysql-isolated.bash index f0418deede..315c3302dc 100755 --- a/test/infra/control/start-proxysql-isolated.bash +++ b/test/infra/control/start-proxysql-isolated.bash @@ -144,7 +144,7 @@ GENAI_PLUGIN_MOUNT="" if [ "${PROXYSQL_LOAD_GENAI_PLUGIN:-0}" = "1" ]; then if [ ! -f "${GENAI_PLUGIN_SRC}" ]; then echo "ERROR: PROXYSQL_LOAD_GENAI_PLUGIN=1 but plugin .so missing at ${GENAI_PLUGIN_SRC}" >&2 - echo " Build it first: PROXYSQLGENAI=1 make (or cd plugins/genai && make with the right flags)" >&2 + echo " Build it first: PROXYSQL40=1 make (or cd plugins/genai && make with the right flags)" >&2 exit 1 fi GENAI_PLUGIN_MOUNT="-v ${GENAI_PLUGIN_SRC}:/usr/lib/proxysql/ProxySQL_GenAI_Plugin.so:ro" diff --git a/test/infra/control/verify-package-install.bash b/test/infra/control/verify-package-install.bash new file mode 100755 index 0000000000..dec6d10bc7 --- /dev/null +++ b/test/infra/control/verify-package-install.bash @@ -0,0 +1,198 @@ +#!/bin/bash +# verify-package-install.bash +# +# Install a just-built ProxySQL package on a clean Docker image of the target +# distro and run smoke tests. Catches: +# - Missing runtime dependencies (the binary or plugins can't load) +# - File conflicts / incorrect paths +# - Binary startup failures +# - Missing plugin .so files (belt-and-braces beyond the entrypoint check) +# +# Usage: +# verify-package-install.bash +# +# The distro is extracted from the package filename (e.g. +# "proxysql_2.6.0-ubuntu24_arm64.deb" => ubuntu24 => ubuntu:24.04). +# +# The script auto-detects whether the package contains plugins (mysqlx, genai) +# by inspecting the package metadata, and adjusts the verification accordingly. + +set -euo pipefail + +PKG_PATH="${1:-}" +if [[ -z "$PKG_PATH" || ! -f "$PKG_PATH" ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi +PKG_PATH="$(realpath "$PKG_PATH")" +PKG_FILE="$(basename "$PKG_PATH")" + +# ---- Detect package type and extract distro ---- +if [[ "$PKG_FILE" == *.deb ]]; then + PKG_TYPE=deb +elif [[ "$PKG_FILE" == *.rpm ]]; then + PKG_TYPE=rpm +else + echo "ERROR: unknown package type (not .deb or .rpm): $PKG_FILE" >&2 + exit 1 +fi + +# Extract distro from filename. +# DEB: proxysql_-_.deb => ubuntu24 +# RPM: proxysql--1-..rpm => centos9 +DISTRO="$(echo "$PKG_FILE" | sed -n 's/.*\-\([a-z0-9]\+\)[-_.].*\.\(deb\|rpm\)$/\1/p')" +if [[ -z "$DISTRO" ]]; then + echo "ERROR: could not extract distro from filename: $PKG_FILE" >&2 + echo " Expected: proxysql_-_.deb or proxysql--1-..rpm" >&2 + exit 1 +fi + +# ---- Detect whether this package ships plugins ---- +# Check the package metadata for .so files under /usr/lib/proxysql/. +HAS_PLUGINS=false +if [[ "$PKG_TYPE" == deb ]]; then + if dpkg -c "$PKG_PATH" 2>/dev/null | grep -q 'usr/lib/proxysql/.*\.so'; then + HAS_PLUGINS=true + fi +else + if rpm -qpl "$PKG_PATH" 2>/dev/null | grep -q '/usr/lib/proxysql/.*\.so'; then + HAS_PLUGINS=true + fi +fi + +# ---- Map distro name to a clean Docker image ---- +declare -A IMAGE_MAP=( + [ubuntu22]="ubuntu:22.04" + [ubuntu24]="ubuntu:24.04" + [debian12]="debian:bookworm-slim" + [debian13]="debian:trixie-slim" + [centos9]="quay.io/centos/centos:stream9" + [centos10]="quay.io/centos/centos:stream10" + [almalinux8]="almalinux:8" + [almalinux9]="almalinux:9" + [almalinux10]="almalinux:10" + [fedora42]="fedora:42" + [fedora43]="fedora:43" + [fedora44]="fedora:44" + [opensuse15]="opensuse/leap:15.6" + [opensuse16]="opensuse/leap:16.0" +) +IMAGE="${IMAGE_MAP[$DISTRO]:-}" +if [[ -z "$IMAGE" ]]; then + echo "ERROR: unknown distro '$DISTRO' — add it to IMAGE_MAP in $0" >&2 + exit 1 +fi + +echo "==> Verifying package: $PKG_FILE" +echo "==> Distro: $DISTRO => Image: $IMAGE" +echo "==> Plugins detected: $HAS_PLUGINS" +echo "" + +# ---- Pull the clean distro image ---- +echo "==> Pulling $IMAGE ..." +if ! docker pull "$IMAGE" >/dev/null 2>&1; then + echo "WARNING: could not pull $IMAGE — skipping verification" >&2 + exit 0 +fi +echo "" + +# ---- Prepare the test script that runs inside the container ---- +TEST_SCRIPT=$(cat << 'SCRIPT_BODY' +#!/bin/bash +set -euo pipefail +PKG_TYPE="$1" +HAS_PLUGINS="$2" + +echo "==> Installing package: /tmp/pkg.$PKG_TYPE" +case "$PKG_TYPE" in + deb) + apt-get update -qq 2>/dev/null || true + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq /tmp/pkg.deb 2>&1 + ;; + rpm) + if command -v dnf &>/dev/null; then + dnf install -y -q /tmp/pkg.rpm 2>&1 + elif command -v yum &>/dev/null; then + yum install -y -q /tmp/pkg.rpm 2>&1 + elif command -v zypper &>/dev/null; then + zypper --non-interactive install -y /tmp/pkg.rpm 2>&1 + else + echo "ERROR: no supported package manager (dnf/yum/zypper)" >&2 + exit 1 + fi + ;; +esac +echo "" + +echo "==> Binary smoke test (proxysql --version)" +proxysql --version 2>&1 | head -5 +echo "" + +echo "==> Plugin .so presence check" +ALL_OK=0 +if [[ "$HAS_PLUGINS" == "true" ]]; then + for plugin in ProxySQL_MySQLX_Plugin.so ProxySQL_GenAI_Plugin.so; do + path="/usr/lib/proxysql/${plugin}" + if [[ -f "$path" ]]; then + echo " OK ${plugin} (exists)" + if file "$path" | grep -q 'ELF 64-bit'; then + echo " OK ${plugin} (valid ELF)" + else + echo " FAIL ${plugin} (not a valid ELF file)" >&2 + ALL_OK=1 + fi + else + echo " FAIL ${plugin} (not found at ${path})" >&2 + ALL_OK=1 + fi + done +else + echo " SKIP (no plugins in this package)" +fi + +# Binary sanity check +if file "$(which proxysql)" | grep -q 'ELF 64-bit'; then + echo " OK proxysql binary (valid ELF)" +else + echo " FAIL proxysql binary (not a valid ELF file)" >&2 + ALL_OK=1 +fi + +if [[ "$ALL_OK" != "0" ]]; then + echo "" >&2 + echo "==> Package verification FAILED" >&2 + exit 1 +fi +echo "" +echo "==> Package verification PASSED" +SCRIPT_BODY +) + +# ---- Run verification in clean distro container ---- +CID="proxysql-verify-$$" +cleanup() { + docker kill "$CID" >/dev/null 2>&1 || true + docker rm "$CID" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "==> Starting clean container from $IMAGE ..." +if ! docker run -d --name "$CID" "$IMAGE" sleep 120 >/dev/null 2>&1; then + echo "WARNING: could not start container from $IMAGE — skipping" >&2 + exit 0 +fi + +# Copy package into container +docker cp "$PKG_PATH" "$CID:/tmp/pkg.$PKG_TYPE" + +# Run the test script inside the container +if docker exec -i "$CID" bash -s "$PKG_TYPE" "$HAS_PLUGINS" <<< "$TEST_SCRIPT"; then + echo "" + echo "==> Package verification PASSED for $PKG_FILE" + exit 0 +else + EXIT_CODE=$? + echo "" + echo "==> Package verification FAILED for $PKG_FILE (exit code $EXIT_CODE)" >&2 + exit $EXIT_CODE +fi diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index f055580348..96932f96c0 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -178,6 +178,7 @@ "pgsql-set_parameter_validation_test-t" : [ "legacy-g6","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","set_parser_algorithm_3-g1" ], "pgsql-set_statement_test-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","set_parser_algorithm_3-g1" ], "pgsql-ssl_keylog-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4" ], + "pgsql-test_dns_cache-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4" ], "pgsql-test_malformed_packet-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4" ], "pgsql-transaction_state_comprehensive-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4" ], "pgsql-transaction_variable_state_tracking-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4" ], @@ -261,7 +262,9 @@ "reg_test_5306-show_warnings_with_comment-t" : [ "legacy-g2","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql84-g2","mysql90-g2","mysql95-g2" ], "reg_test_5389-flush_logs_no_drop-t" : [ "legacy-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], "reg_test_5620_fast_routing_qr_leak-t" : [ "legacy-g6" ], + "reg_test_5639_stmt_execute_max_allowed_packet-t" : [ "mysql84-g6","mysql95-g1" ], "reg_test_5766_libconfig_escape_passthrough-t" : [ "mysql95-g4" ], + "reg_test_5790-mariadb_collation_255-t" : [ "mariadb10-galera-g1" ], "reg_test__ssl_client_busy_wait-t" : [ "legacy-g2","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g2","mysql90-g2","mysql95-g2" ], "reg_test_com_change_user_malformed_packet-t" : [ "mysql84-g6","mysql95-g1" ], "reg_test_compression_split_packets-t" : [ "legacy-g2","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g2","mysql90-g2","mysql95-g2" ], @@ -384,7 +387,6 @@ "test_mysqlx_e2e_routing-t" : [ "mysqlx-e2e-g1","@proxysql_min_version:4.0" ], "test_mysqlx_plugin_load-t" : [ "unit-tests-g1","@proxysql_min_version:4.0" ], "test_mysqlx_route_drop_inflight-t" : [ "mysqlx-soak-g1","@proxysql_min_version:4.0" ], - "test_mysqlx_sigterm_inflight-t" : [ "mysqlx-soak-g1","@proxysql_min_version:4.0" ], "test_mysqlx_soak_behavioral-t" : [ "mysqlx-soak-g1","@proxysql_min_version:4.0" ], "test_mysqlx_soak_stress-t" : [ "mysqlx-soak-g1","@proxysql_min_version:4.0" ], "test_noise_injection-t" : [ "legacy-g8","mysql84-g8","mysql90-g3","mysql95-g3" ], @@ -409,7 +411,7 @@ "test_session_status_flags-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], "test_set_character_results-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4","set_parser_algorithm_3-g1" ], "test_set_collation-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4","set_parser_algorithm_3-g1" ], - "test_set_statement_for-t" : [ "legacy-g2","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g2","mysql90-g2","mysql95-g2" ], + "test_set_statement_for-t" : [ "mariadb10-galera-g2" ], "test_simple_embedded_HTTP_server-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], "test_sqlite3_from_unixtime-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], "test_sqlite3_pass_exts-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], diff --git a/test/tap/tap/tap.cpp b/test/tap/tap/tap.cpp index 5298ced21c..c5df26bf22 100644 --- a/test/tap/tap/tap.cpp +++ b/test/tap/tap/tap.cpp @@ -275,17 +275,7 @@ plan(int count) } -void -skip_all(char const *reason, ...) -{ - va_list ap; - va_start(ap, reason); - fprintf(tapout, "1..0 # skip "); - vfprintf(tapout, reason, ap); - fflush(tapout); - va_end(ap); - exit(0); -} +// skip_all() removed — see comment in tap.h. Use BAIL_OUT() instead. void ok(int pass, char const *fmt, ...) @@ -771,11 +761,5 @@ int tests_last() { to use anything than a block. */ -/** - @example skip_all.t.c - - Sometimes, you skip an entire test because it's testing a feature - that doesn't exist on the system that you're testing. To skip an - entire test, use the skip_all() function according to - this example. - */ +// (The @example skip_all.t.c doc reference was removed alongside the +// skip_all() function itself — see comment in tap.h.) diff --git a/test/tap/tap/tap.h b/test/tap/tap/tap.h index 635a0c14dd..d577b6bd3c 100644 --- a/test/tap/tap/tap.h +++ b/test/tap/tap/tap.h @@ -275,16 +275,12 @@ void BAIL_OUT(char const *fmt, ...) int exit_status(void); -/** - Skip entire test suite. - - To skip the entire test suite, use this function. It will - automatically call exit(), so there is no need to have checks - around it. - */ - -void skip_all(char const *reason, ...) - __attribute__((noreturn, format(printf, 1, 2))); +// skip_all() was removed: silently skipping a whole test suite when a +// runtime precondition is missing hides setup bugs and erodes coverage. +// Use BAIL_OUT() to fail loudly instead, and fix the misregistration in +// test/tap/groups/groups.json or the missing env in the group's env.sh. +// See the PR that removed skip_all for the rationale and the conversion +// pattern for the four former callers. /** diff --git a/test/tap/tests/Makefile b/test/tap/tests/Makefile index e62aff8b0f..545a9d3939 100644 --- a/test/tap/tests/Makefile +++ b/test/tap/tests/Makefile @@ -390,16 +390,14 @@ test_mysqlx_route_drop_inflight-t: test_mysqlx_route_drop_inflight-t.cpp $(PROXY -I$(PROXYSQL_PATH)/plugins/mysqlx/include -I$(MYSQLX_PROTO_DIR) \ $(IDIRS) $(LDIRS) $(OPT) -lprotobuf -lssl -lcrypto -ltap -lcpp_dotenv -lcurl -lmariadbclient -lpthread -lz -ldl -o $@ -# Behavioural validation TAP scaffolding for issue #5678 part (a): runtime -# is skip_all() unless MYSQLX_SIGTERM_INFLIGHT_OPT_IN=1 is set. The body -# would mirror behavioral_validation.py --scenario sigterm; today it -# refuses to run automated and points operators at that script. Link -# line keeps -lssl/-lcrypto and -lmariadbclient because libtap.so pulls -# in those symbols (admin connection helpers); we don't use protobuf -# here so we omit -lprotobuf and the .pb.o objects. -test_mysqlx_sigterm_inflight-t: test_mysqlx_sigterm_inflight-t.cpp $(TAP_LDIR)/libtap.so - $(CXX) $< \ - $(IDIRS) $(LDIRS) $(OPT) -lssl -lcrypto -ltap -lcpp_dotenv -lcurl -lmariadbclient -lpthread -lz -ldl -o $@ +# test_mysqlx_sigterm_inflight-t (the C++ stub binary) was deleted in the +# same PR that removed skip_all() from the framework. The stub had never +# been more than a skip_all() guard pointing operators at the real +# implementation, which lives in +# test/scripts/mysqlx/behavioral_validation.py --scenario sigterm. Once +# skip_all was removed, the stub had no purpose other than to BAIL_OUT +# on invocation, so the source and this Makefile target were removed +# together. The python scenario is the canonical sigterm-inflight test. ### clean targets @@ -417,14 +415,14 @@ setparser_parsersql_test-t: setparser_parsersql_test ln -fs setparser_parsersql_test setparser_parsersql_test-t setparser_parsersql_test: setparser_parsersql_test.cpp $(TAP_LDIR)/libtap.so setparser_test_common.h $(LIBPROXYSQLAR) $(LIBCOREDUMPERAR) $(PARSERSQL_LDIR)/libsqlparser.a -ifeq ($(PROXYSQLGENAI),1) +ifeq ($(PROXYSQL40),1) $(CXX) $< -DEXCLUDE_TRACKING_VARIABLES $(IDIRS) $(LDIRS) $(OPT) $(MYLIBS) $(LIBCOREDUMPERAR) $(SQLITE3_LDIR)/vec.o $(PARSERSQL_LDIR)/libsqlparser.a -o $@ else $(CXX) $< -DEXCLUDE_TRACKING_VARIABLES $(IDIRS) $(LDIRS) $(OPT) $(MYLIBS) $(LIBCOREDUMPERAR) $(PARSERSQL_LDIR)/libsqlparser.a -o $@ endif parsersql_digest_test: parsersql_digest_test.cpp $(TAP_LDIR)/libtap.so $(LIBPROXYSQLAR) $(LIBCOREDUMPERAR) $(PARSERSQL_LDIR)/libsqlparser.a -ifeq ($(PROXYSQLGENAI),1) +ifeq ($(PROXYSQL40),1) $(CXX) $< -DEXCLUDE_TRACKING_VARIABLES $(IDIRS) $(LDIRS) $(OPT) $(MYLIBS) $(LIBCOREDUMPERAR) $(SQLITE3_LDIR)/vec.o $(PARSERSQL_LDIR)/libsqlparser.a -o $@ else $(CXX) $< -DEXCLUDE_TRACKING_VARIABLES $(IDIRS) $(LDIRS) $(OPT) $(MYLIBS) $(LIBCOREDUMPERAR) $(PARSERSQL_LDIR)/libsqlparser.a -o $@ diff --git a/test/tap/tests/pgsql-test_dns_cache-t.cpp b/test/tap/tests/pgsql-test_dns_cache-t.cpp new file mode 100644 index 0000000000..c7beef380a --- /dev/null +++ b/test/tap/tests/pgsql-test_dns_cache-t.cpp @@ -0,0 +1,436 @@ +/** + * @file pgsql-test_dns_cache-t.cpp + * @brief Tests the PgSQL-side DNS cache added for issue #5768. + * + * Covers: + * - Cache is on by default and a resolvable hostname populates it + * (PgSQL_Monitor_dns_cache_record_updated increments). + * - IP literals bypass the cache (record_updated stays flat). + * - Client connect through ProxySQL goes through the cache + * (PgSQL_Monitor_dns_cache_queried + _lookup_success increment). + * - Hostnames with leading/trailing whitespace are trimmed before + * resolution. + * - An unresolvable hostname is queried but never produces a record. + * - Removing a pgsql_servers row drops the corresponding cache record. + * - Disabling the cache (refresh_interval=0) flatlines all counters. + * - PgSQL cache state is independent of MySQL cache state: tweaking + * pgsql-monitor_local_dns_* and inserting pgsql hostnames must not + * move the MySQL_Monitor_dns_cache_* counters, and vice versa. + * + * Reads counters from stats_pgsql_global / stats_mysql_global rather + * than Prometheus because the PgSQL DNS counters are only exposed + * through stats_pgsql_global today. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libpq-fe.h" + +#include "command_line.h" +#include "tap.h" + +CommandLine cl; + +using PGConnPtr = std::unique_ptr; + +static PGConnPtr admin_connect() { + std::stringstream ss; + ss << "host=" << cl.pgsql_admin_host + << " port=" << cl.pgsql_admin_port + << " user=" << cl.admin_username + << " password=" << cl.admin_password + << " sslmode=disable"; + PGconn* c = PQconnectdb(ss.str().c_str()); + if (PQstatus(c) != CONNECTION_OK) { + diag("ADMIN connect failed: %s", PQerrorMessage(c)); + PQfinish(c); + return PGConnPtr(nullptr, &PQfinish); + } + return PGConnPtr(c, &PQfinish); +} + +static PGConnPtr backend_connect() { + std::stringstream ss; + ss << "host=" << cl.pgsql_host + << " port=" << cl.pgsql_port + << " user=" << cl.pgsql_username + << " password=" << cl.pgsql_password + << " sslmode=disable"; + PGconn* c = PQconnectdb(ss.str().c_str()); + // Don't fail the test on connection error; the test exercises the DNS + // cache via connection attempts that may legitimately fail (bad host, + // bad port). Caller checks PQstatus. + return PGConnPtr(c, &PQfinish); +} + +// Run a one-off statement on the admin connection. Returns true on success. +// Statements where we don't care about the row count use this. +static bool admin_exec(PGConnPtr& a, const std::string& sql) { + PGresult* r = PQexec(a.get(), sql.c_str()); + const ExecStatusType st = PQresultStatus(r); + bool ok = (st == PGRES_COMMAND_OK || st == PGRES_TUPLES_OK); + if (!ok) { + diag("admin_exec('%s') failed: %s", sql.c_str(), PQerrorMessage(a.get())); + } + PQclear(r); + return ok; +} + +// Read a single counter from stats_pgsql_global (or stats_mysql_global). +// Returns -1 on failure. +static long admin_counter(PGConnPtr& a, const char* table, const char* name) { + std::stringstream q; + q << "SELECT Variable_Value FROM " << table + << " WHERE Variable_Name='" << name << "'"; + PGresult* r = PQexec(a.get(), q.str().c_str()); + if (PQresultStatus(r) != PGRES_TUPLES_OK || PQntuples(r) == 0) { + PQclear(r); + return -1; + } + long v = atol(PQgetvalue(r, 0, 0)); + PQclear(r); + return v; +} + +struct DnsCounters { + long queried { -1 }; + long lookup_success { -1 }; + long record_updated { -1 }; +}; + +static DnsCounters read_pg_counters(PGConnPtr& a) { + DnsCounters c; + c.queried = admin_counter(a, "stats_pgsql_global", "PgSQL_Monitor_dns_cache_queried"); + c.lookup_success = admin_counter(a, "stats_pgsql_global", "PgSQL_Monitor_dns_cache_lookup_success"); + c.record_updated = admin_counter(a, "stats_pgsql_global", "PgSQL_Monitor_dns_cache_record_updated"); + return c; +} + +// Force a `SELECT 1` round-trip through the proxy a few times so the worker +// thread opens a backend connection and the DNS cache lookup path runs. +// Returns the number of successful attempts; we don't require all of them +// because a hostname pointing at a dummy port is expected to fail at TCP +// connect — that's fine, the DNS lookup happened first. +static int hammer_proxy(int n) { + int ok_count = 0; + for (int i = 0; i < n; i++) { + PGConnPtr c = backend_connect(); + if (PQstatus(c.get()) == CONNECTION_OK) { + PGresult* r = PQexec(c.get(), "SELECT 1"); + if (PQresultStatus(r) == PGRES_TUPLES_OK) ok_count++; + PQclear(r); + } + // Brief pacing so we don't outpace the resolver loop. + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + return ok_count; +} + +static void sleep_seconds(int s) { + std::this_thread::sleep_for(std::chrono::seconds(s)); +} + +// Wait up to deadline_secs for the resolver loop to populate the cache; poll +// every poll_ms. Returns true if record_updated grew past `baseline`. +static bool wait_for_record_growth(PGConnPtr& a, long baseline, int deadline_secs) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(deadline_secs); + while (std::chrono::steady_clock::now() < deadline) { + const long now_val = admin_counter(a, "stats_pgsql_global", "PgSQL_Monitor_dns_cache_record_updated"); + if (now_val > baseline) return true; + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + } + return false; +} + +int main(int /*argc*/, char** /*argv*/) { + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return -1; + } + + plan(17); + + PGConnPtr admin = admin_connect(); + if (!admin) { + BAIL_OUT("Could not connect to ProxySQL Admin via libpq"); + return exit_status(); + } + + // Make the cache tunable and fast, so test cycles complete in seconds. + // 500 ms refresh interval, 5 s TTL. Save the originals so we restore + // them on the way out. + const long orig_refresh = admin_counter(admin, "global_variables", "pgsql-monitor_local_dns_cache_refresh_interval"); + const long orig_ttl = admin_counter(admin, "global_variables", "pgsql-monitor_local_dns_cache_ttl"); + const long orig_qsize = admin_counter(admin, "global_variables", "pgsql-monitor_local_dns_resolver_queue_maxsize"); + + if (!admin_exec(admin, "SET pgsql-monitor_local_dns_cache_refresh_interval=500")) { + BAIL_OUT("Could not set pgsql-monitor_local_dns_cache_refresh_interval"); + return exit_status(); + } + if (!admin_exec(admin, "SET pgsql-monitor_local_dns_cache_ttl=5000") || + !admin_exec(admin, "LOAD PGSQL VARIABLES TO RUNTIME")) { + BAIL_OUT("Could not apply pgsql DNS cache runtime settings"); + return exit_status(); + } + + // Confirm the new values actually landed in runtime (regression for the + // "names commented out in pgsql_thread_variables_names" bug fixed during + // initial bring-up). + const long rt_refresh = admin_counter(admin, "runtime_global_variables", "pgsql-monitor_local_dns_cache_refresh_interval"); + const long rt_ttl = admin_counter(admin, "runtime_global_variables", "pgsql-monitor_local_dns_cache_ttl"); + ok(rt_refresh == 500, "pgsql-monitor_local_dns_cache_refresh_interval propagated to runtime (got %ld, expected 500)", rt_refresh); + ok(rt_ttl == 5000, "pgsql-monitor_local_dns_cache_ttl propagated to runtime (got %ld, expected 5000)", rt_ttl); + + // Wipe the pgsql_servers table down to a known state. Tests below add + // servers in a high hostgroup that real test infra doesn't use. + admin_exec(admin, "DELETE FROM pgsql_servers WHERE hostgroup_id=999"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + + // Install a routing rule so client queries hitting the proxy land on + // hostgroup 999 (the test's own hostgroup) instead of whatever the + // infra's default backend is. Without this, hammer_proxy() routes + // through the default hostgroup — typically configured with an IP + // literal in the test infrastructure (see e.g. + // test/infra/docker-pgsql16-single/conf/proxysql/config.sql) which + // bypasses DNS entirely via validate_ip() and never exercises + // PgSQL_Connection::connect_start_DNS_lookup. Step 3 below would + // then assert on counters bumped by the resolver loop alone, masking + // a broken connect-path. Rule id 99999 is well outside ranges used + // by the test infra; we delete it on the way out. + admin_exec(admin, "DELETE FROM pgsql_query_rules WHERE rule_id=99999"); + { + // Use the runtime test username so the rule actually matches the + // connections from backend_connect() / hammer_proxy(); a hardcoded + // 'testuser' would silently bypass hostgroup 999 whenever the infra + // uses a different user. + std::stringstream q; + q << "INSERT INTO pgsql_query_rules " + << "(rule_id,active,username,destination_hostgroup,apply) " + << "VALUES (99999,1,'" << cl.pgsql_username << "',999,1)"; + admin_exec(admin, q.str()); + } + admin_exec(admin, "LOAD PGSQL QUERY RULES TO RUNTIME"); + // Give the resolver loop time to drop any orphaned bookkeeping entry. + sleep_seconds(2); + + // ===================================================================== + // Step 1: IP literal must NOT cause DNS work + // ===================================================================== + diag("---- Step 1: IP-literal server should not populate the cache"); + DnsCounters before = read_pg_counters(admin); + admin_exec(admin, + "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,max_connections,comment) " + "VALUES (999,'0.0.0.0',7861,10,'pgsql-dns-test ip-literal')"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + sleep_seconds(2); + DnsCounters after = read_pg_counters(admin); + ok(after.record_updated == before.record_updated, + "IP literal does not touch cache (record_updated %ld -> %ld)", + before.record_updated, after.record_updated); + + // ===================================================================== + // Step 2: Resolvable hostname populates record_updated + // ===================================================================== + // We use example.com — IANA-reserved and stable. Public DNS hostnames + // are also what the upstream MySQL test uses. If the test runner has + // no outbound DNS, this step (and steps 3 and 4) will be skipped via + // the wait_for_record_growth helper, which fails the ok(). CI runners + // have network access; the failure surface from a missing resolver is + // noisy on purpose so we notice. + diag("---- Step 2: Resolvable hostname populates the cache"); + before = read_pg_counters(admin); + admin_exec(admin, + "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,max_connections,comment) " + "VALUES (999,'example.com',7861,10,'pgsql-dns-test resolvable')"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + // trigger_dns_cache_update fires on LOAD so refresh shouldn't take a + // full interval, but give it a generous window. + bool grew = wait_for_record_growth(admin, before.record_updated, 10); + ok(grew, "resolver populated cache for example.com (record_updated %ld -> %ld)", + before.record_updated, admin_counter(admin, "stats_pgsql_global", "PgSQL_Monitor_dns_cache_record_updated")); + + // ===================================================================== + // Step 3: Client connect through proxy hits the cache + // ===================================================================== + diag("---- Step 3: Client connect attempt drives lookup counters"); + before = read_pg_counters(admin); + // Drive several attempts. These will fail at the TCP layer (port 7861 + // is not listening) but the DNS lookup path runs first and that's what + // we're counting. + hammer_proxy(3); + sleep_seconds(1); + after = read_pg_counters(admin); + ok(after.queried > before.queried, + "dns_cache_queried bumped after client connect attempts (%ld -> %ld)", + before.queried, after.queried); + ok(after.lookup_success > before.lookup_success, + "dns_cache_lookup_success bumped (cache hit for example.com) (%ld -> %ld)", + before.lookup_success, after.lookup_success); + + // ===================================================================== + // Step 4: Whitespace around hostname is trimmed before resolution + // ===================================================================== + diag("---- Step 4: Hostname whitespace is trimmed"); + before = read_pg_counters(admin); + admin_exec(admin, + "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,max_connections,comment) " + "VALUES (999,' example.org ',7861,10,'pgsql-dns-test whitespace')"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + grew = wait_for_record_growth(admin, before.record_updated, 10); + ok(grew, "trimmed hostname ' example.org ' was resolved (record_updated %ld -> %ld)", + before.record_updated, admin_counter(admin, "stats_pgsql_global", "PgSQL_Monitor_dns_cache_record_updated")); + + // ===================================================================== + // Step 5: Unresolvable hostname — lookup is queried but produces nothing + // ===================================================================== + diag("---- Step 5: Unresolvable hostname"); + // Drop everything except an unresolvable host so the cache has a clean + // shape; then drive a lookup attempt. + admin_exec(admin, "DELETE FROM pgsql_servers WHERE hostgroup_id=999"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + sleep_seconds(2); + before = read_pg_counters(admin); + admin_exec(admin, + "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,max_connections,comment) " + "VALUES (999,'this-host-should-not-resolve.invalid',7861,10,'pgsql-dns-test nxdomain')"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + sleep_seconds(3); + after = read_pg_counters(admin); + ok(after.record_updated == before.record_updated, + "unresolvable host did not produce a record (record_updated %ld -> %ld)", + before.record_updated, after.record_updated); + + // Now drive a client connect and confirm dns_cache_queried bumps but + // lookup_success stays flat (cache miss, falls back to hostname). + before = after; + hammer_proxy(3); + sleep_seconds(1); + after = read_pg_counters(admin); + ok(after.queried > before.queried, + "dns_cache_queried bumped on unresolved hostname attempts (%ld -> %ld)", + before.queried, after.queried); + ok(after.lookup_success == before.lookup_success, + "dns_cache_lookup_success stayed flat (cache miss) (%ld -> %ld)", + before.lookup_success, after.lookup_success); + + // ===================================================================== + // Step 6: Removing servers drops orphaned cache records + // ===================================================================== + diag("---- Step 6: Removing pgsql_servers clears orphaned cache records"); + // Add a resolvable host, wait for the cache to populate, then drop and + // confirm record_updated grew (the bookkeeper signals a remove via the + // same counter). + admin_exec(admin, "DELETE FROM pgsql_servers WHERE hostgroup_id=999"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + sleep_seconds(2); + before = read_pg_counters(admin); + admin_exec(admin, + "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,max_connections,comment) " + "VALUES (999,'example.net',7861,10,'pgsql-dns-test orphan-cleanup')"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + bool added = wait_for_record_growth(admin, before.record_updated, 10); + long after_add = admin_counter(admin, "stats_pgsql_global", "PgSQL_Monitor_dns_cache_record_updated"); + ok(added, "added example.net (record_updated %ld -> %ld)", before.record_updated, after_add); + + // Now drop the row and wait for the orphan-cleanup pass. + admin_exec(admin, "DELETE FROM pgsql_servers WHERE hostgroup_id=999"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + bool removed = wait_for_record_growth(admin, after_add, 10); + ok(removed, + "orphan removal bumped record_updated (was %ld, now %ld)", + after_add, admin_counter(admin, "stats_pgsql_global", "PgSQL_Monitor_dns_cache_record_updated")); + + // ===================================================================== + // Step 7: Disabled cache (refresh_interval=0) flatlines counters + // ===================================================================== + diag("---- Step 7: Cache disabled by refresh_interval=0"); + admin_exec(admin, "DELETE FROM pgsql_servers WHERE hostgroup_id=999"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + sleep_seconds(1); + admin_exec(admin, "SET pgsql-monitor_local_dns_cache_refresh_interval=0"); + admin_exec(admin, "LOAD PGSQL VARIABLES TO RUNTIME"); + // Use a resolvable hostname so the assertion exercises the actual cache + // path — if disabling the cache regressed (resolver still ran, or the + // connect path still called into the cache), record_updated and queried + // would move and the test would fail. An IP literal would bypass DNS + // regardless of refresh_interval and miss that regression. + admin_exec(admin, + "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,max_connections,comment) " + "VALUES (999,'example.com',7861,10,'pgsql-dns-test cache-off')"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + sleep_seconds(3); + before = read_pg_counters(admin); + hammer_proxy(3); + sleep_seconds(2); + after = read_pg_counters(admin); + ok(after.queried == before.queried, + "cache off: dns_cache_queried unchanged (%ld -> %ld)", + before.queried, after.queried); + ok(after.lookup_success == before.lookup_success, + "cache off: dns_cache_lookup_success unchanged (%ld -> %ld)", + before.lookup_success, after.lookup_success); + ok(after.record_updated == before.record_updated, + "cache off: dns_cache_record_updated unchanged (%ld -> %ld)", + before.record_updated, after.record_updated); + + // ===================================================================== + // Step 8: PgSQL cache is independent of MySQL cache + // ===================================================================== + diag("---- Step 8: PgSQL cache state independent of MySQL cache"); + // Re-enable the cache so the next inserts populate it, then make sure + // the MySQL counters don't budge in response to pgsql-side activity. + admin_exec(admin, "SET pgsql-monitor_local_dns_cache_refresh_interval=500"); + admin_exec(admin, "LOAD PGSQL VARIABLES TO RUNTIME"); + const long my_before_q = admin_counter(admin, "stats_mysql_global", "MySQL_Monitor_dns_cache_queried"); + const long my_before_u = admin_counter(admin, "stats_mysql_global", "MySQL_Monitor_dns_cache_record_updated"); + admin_exec(admin, "DELETE FROM pgsql_servers WHERE hostgroup_id=999"); + admin_exec(admin, + "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,max_connections,comment) " + "VALUES (999,'example.com',7861,10,'pgsql-dns-test independence')"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + sleep_seconds(3); + hammer_proxy(2); + sleep_seconds(1); + const long my_after_q = admin_counter(admin, "stats_mysql_global", "MySQL_Monitor_dns_cache_queried"); + const long my_after_u = admin_counter(admin, "stats_mysql_global", "MySQL_Monitor_dns_cache_record_updated"); + ok(my_after_q == my_before_q, + "pgsql activity did not bump MySQL_Monitor_dns_cache_queried (%ld -> %ld)", + my_before_q, my_after_q); + ok(my_after_u == my_before_u, + "pgsql activity did not bump MySQL_Monitor_dns_cache_record_updated (%ld -> %ld)", + my_before_u, my_after_u); + + // ===================================================================== + // Cleanup + // ===================================================================== + admin_exec(admin, "DELETE FROM pgsql_servers WHERE hostgroup_id=999"); + admin_exec(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + admin_exec(admin, "DELETE FROM pgsql_query_rules WHERE rule_id=99999"); + admin_exec(admin, "LOAD PGSQL QUERY RULES TO RUNTIME"); + { + std::stringstream q; + q << "SET pgsql-monitor_local_dns_cache_refresh_interval=" << (orig_refresh > 0 ? orig_refresh : 60000); + admin_exec(admin, q.str()); + } + { + std::stringstream q; + q << "SET pgsql-monitor_local_dns_cache_ttl=" << (orig_ttl > 0 ? orig_ttl : 300000); + admin_exec(admin, q.str()); + } + { + std::stringstream q; + q << "SET pgsql-monitor_local_dns_resolver_queue_maxsize=" << (orig_qsize > 0 ? orig_qsize : 128); + admin_exec(admin, q.str()); + } + admin_exec(admin, "LOAD PGSQL VARIABLES TO RUNTIME"); + + return exit_status(); +} diff --git a/test/tap/tests/reg_test_5639_stmt_execute_max_allowed_packet-t.cpp b/test/tap/tests/reg_test_5639_stmt_execute_max_allowed_packet-t.cpp new file mode 100644 index 0000000000..239290fad6 --- /dev/null +++ b/test/tap/tests/reg_test_5639_stmt_execute_max_allowed_packet-t.cpp @@ -0,0 +1,195 @@ +#include +#include +#include +#include + +#include "mysql.h" + +#include "tap.h" +#include "utils.h" +#include "command_line.h" + +// Regression test for https://github.com/sysown/proxysql/issues/5639 +// +// A COM_STMT_EXECUTE whose binary packet exceeds mysql-max_allowed_packet +// reaches MySQL_Session::handler_WCD_SS_MCQ_qpo_LargePacket +// (lib/MySQL_Session.cpp:6370) via +// handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_EXECUTE +// -> handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo +// at lib/MySQL_Session.cpp:6405-6406. The reporter observes a SIGSEGV in +// that frame on v3.0.1/v3.0.6/v3.0.7. This test exercises that path and +// asserts ProxySQL rejects the packet cleanly with ER_NET_PACKET_TOO_LARGE +// (1153), keeps the offending connection usable, and stays alive for new +// connections. + +namespace { + +constexpr unsigned int kMaxAllowedPacket = 8192; // minimum allowed by ProxySQL +constexpr size_t kBoundStringSize = 12 * 1024; // guarantees pkt > kMaxAllowedPacket +constexpr unsigned int kRestoreMaxAllowedPacket = 67108864; // ProxySQL default + +MYSQL* connect_admin(const CommandLine& cl) { + MYSQL* admin = mysql_init(nullptr); + if (admin == nullptr) { + return nullptr; + } + if (!mysql_real_connect(admin, cl.admin_host, cl.admin_username, cl.admin_password, + nullptr, cl.admin_port, nullptr, 0)) { + diag("admin mysql_real_connect failed: %s", mysql_error(admin)); + mysql_close(admin); + return nullptr; + } + return admin; +} + +MYSQL* connect_proxy(const CommandLine& cl) { + MYSQL* mysql = mysql_init(nullptr); + if (mysql == nullptr) { + return nullptr; + } + if (!mysql_real_connect(mysql, cl.host, cl.username, cl.password, + nullptr, cl.port, nullptr, 0)) { + diag("proxy mysql_real_connect failed: %s", mysql_error(mysql)); + mysql_close(mysql); + return nullptr; + } + return mysql; +} + +bool simple_select_works(MYSQL* mysql) { + if (mysql_query(mysql, "SELECT 1")) { + return false; + } + MYSQL_RES* res = mysql_store_result(mysql); + if (res == nullptr) { + return false; + } + MYSQL_ROW row = mysql_fetch_row(res); + const bool ok_ret = row != nullptr && row[0] != nullptr && strcmp(row[0], "1") == 0; + mysql_free_result(res); + return ok_ret; +} + +void restore_max_allowed_packet(MYSQL* admin) { + char buf[128]; + snprintf(buf, sizeof(buf), "SET mysql-max_allowed_packet=%u", kRestoreMaxAllowedPacket); + mysql_query(admin, buf); + mysql_query(admin, "LOAD MYSQL VARIABLES TO RUNTIME"); +} + +} // namespace + +int main(int /*argc*/, char** /*argv*/) { + CommandLine cl; + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return EXIT_FAILURE; + } + + plan(8); + + // --- 1. Lower mysql-max_allowed_packet to the minimum (8192) --- + MYSQL* admin = connect_admin(cl); + const bool admin_ok = admin != nullptr; + ok(admin_ok, "Connect to ProxySQL admin"); + if (!admin_ok) { + return exit_status(); + } + + { + char buf[128]; + snprintf(buf, sizeof(buf), "SET mysql-max_allowed_packet=%u", kMaxAllowedPacket); + MYSQL_QUERY(admin, buf); + } + MYSQL_QUERY(admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + ok(true, "Lowered mysql-max_allowed_packet to %u", kMaxAllowedPacket); + + // --- 2. Frontend connection used to send the oversized STMT_EXECUTE --- + MYSQL* sess = connect_proxy(cl); + const bool sess_ok = sess != nullptr; + ok(sess_ok, "Connect to ProxySQL frontend"); + if (!sess_ok) { + restore_max_allowed_packet(admin); + mysql_close(admin); + return exit_status(); + } + + // --- 3. Prepare a trivial stmt with one VARCHAR parameter --- + MYSQL_STMT* stmt = mysql_stmt_init(sess); + const char prep[] = "SELECT ? AS x"; + const bool prepared = stmt != nullptr && + mysql_stmt_prepare(stmt, prep, sizeof(prep) - 1) == 0; + ok(prepared, "Prepared 'SELECT ? AS x'"); + if (!prepared) { + if (stmt) { + diag("mysql_stmt_prepare failed: %s", mysql_stmt_error(stmt)); + mysql_stmt_close(stmt); + } + mysql_close(sess); + restore_max_allowed_packet(admin); + mysql_close(admin); + return exit_status(); + } + + // --- 4. Bind a parameter large enough that the binary STMT_EXECUTE + // packet exceeds kMaxAllowedPacket. The total wire packet for a single + // VARCHAR parameter is roughly payload_size + ~16 bytes of stmt header; + // kBoundStringSize > kMaxAllowedPacket comfortably crosses the limit. --- + std::string big(kBoundStringSize, 'A'); + unsigned long big_len = static_cast(big.size()); + + MYSQL_BIND bind {}; + bind.buffer_type = MYSQL_TYPE_STRING; + bind.buffer = const_cast(big.data()); + bind.buffer_length = static_cast(big.size()); + bind.length = &big_len; + bind.is_null = nullptr; + + if (mysql_stmt_bind_param(stmt, &bind)) { + diag("mysql_stmt_bind_param failed: %s", mysql_stmt_error(stmt)); + mysql_stmt_close(stmt); + mysql_close(sess); + restore_max_allowed_packet(admin); + mysql_close(admin); + ok(false, "mysql_stmt_bind_param"); + return exit_status(); + } + + // --- 5. Execute. ProxySQL must reject with ER_NET_PACKET_TOO_LARGE + // instead of crashing in handler_WCD_SS_MCQ_qpo_LargePacket. --- + const int rc = mysql_stmt_execute(stmt); + const unsigned int err = mysql_stmt_errno(stmt); + const char* errstr = mysql_stmt_error(stmt); + + ok(rc != 0, "mysql_stmt_execute is rejected for oversized binary packet (rc=%d)", rc); + ok( + err == 1153, + "Error is ER_NET_PACKET_TOO_LARGE (1153); got %u (%s)", + err, errstr ? errstr : "(no message)" + ); + + mysql_stmt_close(stmt); + + // --- 6. The connection that sent the oversized packet must remain usable. --- + ok(simple_select_works(sess), "Same frontend connection still serves queries"); + + // --- 7. ProxySQL still accepts new connections (process alive). --- + MYSQL* fresh = connect_proxy(cl); + ok( + fresh != nullptr && simple_select_works(fresh), + "Fresh frontend connection works after oversized STMT_EXECUTE" + ); + if (fresh != nullptr) { + mysql_close(fresh); + } + + mysql_close(sess); + + // --- 8. Restore default max_allowed_packet so the run does not bleed + // state into other tests in the same group. --- + restore_max_allowed_packet(admin); + mysql_close(admin); + + return exit_status(); +} diff --git a/test/tap/tests/reg_test_5790-mariadb_collation_255-t.cpp b/test/tap/tests/reg_test_5790-mariadb_collation_255-t.cpp new file mode 100644 index 0000000000..c9d7cefc8e --- /dev/null +++ b/test/tap/tests/reg_test_5790-mariadb_collation_255-t.cpp @@ -0,0 +1,154 @@ +/** + * @file reg_test_5790-mariadb_collation_255-t.cpp + * @brief Regression for issue #5790: SET NAMES utf8mb4 COLLATE utf8mb4_0900_ai_ci + * against a MariaDB backend must NOT propagate collation 255 to the backend. + * + * @details Collation id 255 (utf8mb4_0900_ai_ci) is a MySQL 8.0+ feature. + * MariaDB does not implement it, so ProxySQL must replace it with the configured + * default (per mysql-handle_unknown_charset) before opening / changing the + * backend session, otherwise the backend returns: + * ERROR 1273 (HY000): Unknown collation: 'utf8mb4_0900_ai_ci' + * + * A regression introduced after v3.0.8 changed the version guard in + * validate_charset() from `server_version[0] != '8'` to + * `atoi(server_version) < 8`, which inadvertently treats MariaDB 10.x / 11.x + * (atoi() = 10 / 11) the same as MySQL >= 8 and skips the replacement. + * + * The test: + * 1. Skips unless the backend reports as MariaDB. + * 2. Configures handle_unknown_charset = REPLACE_WITH_DEFAULT_VERBOSE (1) + * and default_collation_connection = utf8mb4_general_ci. + * 3. Opens a frontend connection, issues SET NAMES utf8mb4 COLLATE + * utf8mb4_0900_ai_ci, then runs a query that forces a backend + * SET NAMES via handler_again___status_CHANGING_CHARSET. + * 4. Verifies the query succeeds and that collation_connection on the + * backend was replaced (i.e. it is NOT utf8mb4_0900_ai_ci). + */ + +#include +#include +#include +#include + +#include +#include "mysql.h" + +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +using std::string; + +int main(int argc, char** argv) { + CommandLine cl; + + if (cl.getEnv()) + return exit_status(); + + plan(3); + + // ---- 1. Detect MariaDB backend --------------------------------------- + MYSQL* mysql_probe = mysql_init(NULL); + if (!mysql_probe) return exit_status(); + if (!mysql_real_connect(mysql_probe, cl.host, cl.username, cl.password, + NULL, cl.port, NULL, 0)) { + diag("Frontend connect for probe failed: %s", mysql_error(mysql_probe)); + return exit_status(); + } + string backend_version; + get_server_version(mysql_probe, backend_version); + mysql_close(mysql_probe); + diag("Backend server_version reported through ProxySQL: '%s'", + backend_version.c_str()); + + if (backend_version.find("MariaDB") == string::npos) { + diag("Backend is not MariaDB; this regression only applies to MariaDB " + "backends. Skipping."); + skip(3, "Backend is not MariaDB"); + return exit_status(); + } + + // ---- 2. Configure ProxySQL ------------------------------------------ + MYSQL* admin = mysql_init(NULL); + if (!admin) return exit_status(); + if (!mysql_real_connect(admin, cl.admin_host, cl.admin_username, cl.admin_password, + NULL, cl.admin_port, NULL, 0)) { + diag("Admin connect failed: %s", mysql_error(admin)); + return exit_status(); + } + + // REPLACE_WITH_DEFAULT_VERBOSE so we also log the replacement in the + // proxysql error log (useful for forensics). + set_admin_global_variable(admin, "mysql-handle_unknown_charset", "1"); + set_admin_global_variable(admin, "mysql-default_charset", "utf8mb4"); + set_admin_global_variable(admin, "mysql-default_collation_connection", + "utf8mb4_general_ci"); + if (mysql_query(admin, "LOAD MYSQL VARIABLES TO RUNTIME")) { + diag("LOAD failed: %s", mysql_error(admin)); + return exit_status(); + } + mysql_close(admin); + + // ---- 3. Frontend connection + reproducer ----------------------------- + MYSQL* mysql = mysql_init(NULL); + if (!mysql) return exit_status(); + if (!mysql_real_connect(mysql, cl.host, cl.username, cl.password, + NULL, cl.port, NULL, 0)) { + diag("Frontend connect failed: %s", mysql_error(mysql)); + return exit_status(); + } + + // Force a fresh backend connection so the SET NAMES propagates via + // handler_again___status_CHANGING_CHARSET on the first backend round-trip. + const char* set_names_q = + "SET NAMES utf8mb4 COLLATE utf8mb4_0900_ai_ci"; + int set_rc = mysql_query(mysql, set_names_q); + ok(set_rc == 0, + "Client `%s` accepted by ProxySQL (no client-side error). " + "errno=%u, err=`%s`", + set_names_q, mysql_errno(mysql), mysql_error(mysql)); + + // This SELECT requires a backend connection. With the bug, the SET NAMES + // against MariaDB fails with errno 1273 and the connection is destroyed; + // the client sees the propagated error. With the fix, validate_charset() + // has already swapped 255 for the configured default (45), so SET NAMES + // utf8mb4_general_ci succeeds on MariaDB and the SELECT returns 1. + const char* select_q = + "SELECT /* create_new_connection=1 */ 1"; + int sel_rc = mysql_query(mysql, select_q); + if (sel_rc != 0) { + diag("Backend query failed: errno=%u err=`%s`", + mysql_errno(mysql), mysql_error(mysql)); + } + ok(sel_rc == 0, + "Backend query after unsupported COLLATE must succeed. " + "errno=%u, err=`%s`", + mysql_errno(mysql), mysql_error(mysql)); + + // Drain the result if any. + if (sel_rc == 0) { + MYSQL_RES* r = mysql_store_result(mysql); + if (r) mysql_free_result(r); + } + + // Inspect what the backend actually ended up with. With the fix, the + // backend session's collation_connection should be the configured default + // (utf8mb4_general_ci), never utf8mb4_0900_ai_ci. + const string collation_query { + "SELECT /* create_new_connection=1 */ @@collation_connection" + }; + ext_val_t col_ext { mysql_query_ext_val(mysql, collation_query, string{}) }; + if (col_ext.err) { + diag("collation_connection query failed: %s", + get_ext_val_err(mysql, col_ext).c_str()); + } + diag("Backend collation_connection after replacement: '%s'", + col_ext.val.c_str()); + ok(col_ext.val != "utf8mb4_0900_ai_ci" && !col_ext.val.empty(), + "Backend collation_connection must not be utf8mb4_0900_ai_ci. " + "Actual: '%s'", + col_ext.val.c_str()); + + mysql_close(mysql); + return exit_status(); +} diff --git a/test/tap/tests/setparser_parsersql_test.cpp b/test/tap/tests/setparser_parsersql_test.cpp index 336070718f..548f08c377 100644 --- a/test/tap/tests/setparser_parsersql_test.cpp +++ b/test/tap/tests/setparser_parsersql_test.cpp @@ -1,14 +1,18 @@ /** * @file setparser_parsersql_test.cpp - * @brief Validates that parsersql_parse_set_mysql() produces the same output - * as the existing MySQL_Set_Stmt_Parser for all SET statement test cases - * defined in setparser_test_common.h. + * @brief Validates that parsersql_parse_set_{mysql,pgsql}() produces the same + * output as the existing MySQL_Set_Stmt_Parser for all SET statement test + * cases defined in setparser_test_common.h. * - * Controlled by: mysql-set_parser_algorithm = 3 + * Controlled by: mysql-set_parser_algorithm = 3 / pgsql-set_parser_algorithm = 3 * * Note: The AST-based parser normalizes quoting (double quotes to single quotes) * and whitespace, while the regex parser preserves raw text. The comparison * normalizes these cosmetic differences before checking equality. + * + * The PostgreSQL test groups (search_path multi-value, TIME ZONE alias) are + * the regression net for ParserSQL v1.0.3's PG SET fixes — they exercise the + * library + adapter end-to-end without needing a live backend. */ #include "setparser_test_common.h" @@ -23,6 +27,78 @@ static Test parsersql_syntax_errors[] = { { Expected("sql_mode", { "SELCT" } ) } }, }; +// ---------------------------------------------------------------------------- +// MySQL queries from test_filtered_set_statements-t (variables that ProxySQL +// is supposed to filter out — should still parse cleanly via ParserSQL). +// ---------------------------------------------------------------------------- +static Test parsersql_mysql_filtered_set[] = { + { "SET wait_timeout=28801", { Expected("wait_timeout", {"28801"}) } }, + { "SET @@wait_timeout = 28801", { Expected("wait_timeout", {"28801"}) } }, + { "SET SESSION wait_timeout = 28801", { Expected("wait_timeout", {"28801"}) } }, + { "SET `wait_timeout` = 28801", { Expected("wait_timeout", {"28801"}) } }, + { "SET character_set_results=latin1", { Expected("character_set_results", {"latin1"}) } }, + { "SET autocommit=1", { Expected("autocommit", {"1"}) } }, + { "SET max_join_size=18446744073709551615", { Expected("max_join_size", {"18446744073709551615"}) } }, +}; + +// MySQL multi-variable SET cases sampled from set_testing-240.csv (the fixture +// driving set_testing-t). Exercises comma-separated multi-variable parsing. +static Test parsersql_mysql_set_testing[] = { + { "SET aurora_read_replica_read_committed=Off, auto_increment_increment=320, sql_select_limit=3656, sql_quote_show_create=\"OFF\"", + { Expected("aurora_read_replica_read_committed", {"Off"}), + Expected("auto_increment_increment", {"320"}), + Expected("sql_quote_show_create", {"OFF"}), + Expected("sql_select_limit", {"3656"}) } }, + { "SET max_heap_table_size=19456, log_slow_filter=`not_using_index`", + { Expected("log_slow_filter", {"not_using_index"}), + Expected("max_heap_table_size", {"19456"}) } }, + { "SET lock_wait_timeout=431, sql_safe_updates=1, aurora_read_replica_read_committed=\"ON\", max_execution_time=13940", + { Expected("aurora_read_replica_read_committed", {"ON"}), + Expected("lock_wait_timeout", {"431"}), + Expected("max_execution_time", {"13940"}), + Expected("sql_safe_updates", {"1"}) } }, + { "SET session_track_gtids=OWN_GTID, optimizer_switch=\"index_merge_union=off\", foreign_key_checks=`OFF`, aurora_read_replica_read_committed=OFF", + { Expected("aurora_read_replica_read_committed", {"OFF"}), + Expected("foreign_key_checks", {"OFF"}), + Expected("optimizer_switch", {"index_merge_union=off"}), + Expected("session_track_gtids", {"OWN_GTID"}) } }, +}; + +// ---------------------------------------------------------------------------- +// PostgreSQL search_path tests — pgsql-set_parameter_validation_test-t shapes. +// Pre-ParserSQL-1.0.3 the multi-value cases silently dropped every value past +// the first; the v1.0.3 fix retains them under the same VAR_ASSIGNMENT node +// and the ProxySQL adapter walks every RHS sibling. +// ---------------------------------------------------------------------------- +static Test parsersql_pgsql_search_path[] = { + { "SET search_path TO \"$user\", public", { Expected("search_path", {"$user", "public"}) } }, + { "SET search_path TO \"$user\",public", { Expected("search_path", {"$user", "public"}) } }, + { "SET search_path = '\"$user\" , public'", { Expected("search_path", {"\"$user\" , public"}) } }, + { "SET search_path = 'public '", { Expected("search_path", {"public "}) } }, + { "SET search_path = \"$user\"", { Expected("search_path", {"$user"}) } }, + { "SET search_path = '$user'", { Expected("search_path", {"$user"}) } }, + { "SET search_path = ''", { Expected("search_path", {""}) } }, + { "SET search_path = public", { Expected("search_path", {"public"}) } }, +}; + +// PostgreSQL TIME ZONE tests — pgsql-set_statement_test-t shapes. +// Pre-ParserSQL-1.0.3 these were parsed as `time = ZONE` and the rest of the +// statement was dropped. The v1.0.3 fix recognizes "TIME ZONE" as the PG +// alias for `SET TimeZone = ...` and walks the trailing expression. +// +// NOTE: `SET TIME ZONE INTERVAL '7' HOUR` is intentionally *not* covered +// here. ParserSQL's expression parser does not yet consume the full +// INTERVAL ... modifier chain — it currently captures just the +// `INTERVAL` token. Asserting that as the expected output would lock in +// incomplete-but-current behaviour and would flip the test red when the +// parser is later fixed to capture the full interval expression. Add a +// case here once ParserSQL grows full INTERVAL modifier support. +static Test parsersql_pgsql_time_zone[] = { + { "SET TIME ZONE 'UTC'", { Expected("timezone", {"UTC"}) } }, + { "SET TIME ZONE DEFAULT", { Expected("timezone", {"DEFAULT"}) } }, + { "SET TIME ZONE '+05:30'", { Expected("timezone", {"+05:30"}) } }, +}; + static std::string normalize_value(const std::string& s) { std::string r; r.reserve(s.size()); @@ -56,6 +132,17 @@ static bool maps_match( return true; } +// Join a vector of values into a single " | "-separated string for diag output. +// Used by TestParse / TestParsePgsql when reporting expected-vs-actual mismatches. +static std::string join_values_for_diag(const std::vector& vals) { + std::string joined; + for (size_t j = 0; j < vals.size(); ++j) { + if (j) joined += " | "; + joined += vals[j]; + } + return joined; +} + void TestParse(const Test* tests, int ntests, const std::string& title) { for (int i = 0; i < ntests; i++) { std::map> data; @@ -66,7 +153,38 @@ void TestParse(const Test* tests, int ntests, const std::string& title) { std::map> result = parsersql_parse_set_mysql(tests[i].query); bool size_ok = (result.size() == data.size()); - ok(size_ok, "[%s %d] Sizes match: %lu, %lu", title.c_str(), i, result.size(), data.size()); + ok(size_ok, "[%s %d] Sizes match: %zu, %zu", title.c_str(), i, result.size(), data.size()); + if (!size_ok) { + diag(" FAIL: sizes differ for query: %s", tests[i].query); + } + + bool elem_ok = maps_match(result, data); + ok(elem_ok, "[%s %d] Elements match", title.c_str(), i); + if (!elem_ok) { + diag(" FAIL: elements differ for query: %s", tests[i].query); + for (auto& kv : result) { + diag(" result[%s] = [%s]", kv.first.c_str(), join_values_for_diag(kv.second).c_str()); + } + for (auto& kv : data) { + diag(" expected[%s] = [%s]", kv.first.c_str(), join_values_for_diag(kv.second).c_str()); + } + } + } +} + + +// Parallel TestParse for PostgreSQL — same shape, dispatches to parsersql_parse_set_pgsql. +void TestParsePgsql(const Test* tests, int ntests, const std::string& title) { + for (int i = 0; i < ntests; i++) { + std::map> data; + for (auto it = std::begin(tests[i].results); it != std::end(tests[i].results); ++it) { + data[it->var] = it->values; + } + + std::map> result = parsersql_parse_set_pgsql(tests[i].query); + + bool size_ok = (result.size() == data.size()); + ok(size_ok, "[%s %d] Sizes match: %zu, %zu", title.c_str(), i, result.size(), data.size()); if (!size_ok) { diag(" FAIL: sizes differ for query: %s", tests[i].query); } @@ -76,10 +194,10 @@ void TestParse(const Test* tests, int ntests, const std::string& title) { if (!elem_ok) { diag(" FAIL: elements differ for query: %s", tests[i].query); for (auto& kv : result) { - diag(" result[%s] = %s", kv.first.c_str(), normalize_value(kv.second.empty() ? "" : kv.second[0]).c_str()); + diag(" result[%s] = [%s]", kv.first.c_str(), join_values_for_diag(kv.second).c_str()); } for (auto& kv : data) { - diag(" expected[%s] = %s", kv.first.c_str(), normalize_value(kv.second.empty() ? "" : kv.second[0]).c_str()); + diag(" expected[%s] = [%s]", kv.first.c_str(), join_values_for_diag(kv.second).c_str()); } } } @@ -97,6 +215,10 @@ int main(int argc, char** argv) { p += arraysize(multiple); p += arraysize(Set1_v2); p += arraysize(parsersql_syntax_errors); + p += arraysize(parsersql_mysql_filtered_set); + p += arraysize(parsersql_mysql_set_testing); + p += arraysize(parsersql_pgsql_search_path); + p += arraysize(parsersql_pgsql_time_zone); p *= 2; plan(p); TestParse(sql_mode, arraysize(sql_mode), "sql_mode"); @@ -108,5 +230,10 @@ int main(int argc, char** argv) { TestParse(multiple, arraysize(multiple), "multiple"); TestParse(Set1_v2, arraysize(Set1_v2), "Set1_v2"); TestParse(parsersql_syntax_errors, arraysize(parsersql_syntax_errors), "parsersql_syntax_errors"); + TestParse(parsersql_mysql_filtered_set, arraysize(parsersql_mysql_filtered_set), "mysql_filtered_set"); + TestParse(parsersql_mysql_set_testing, arraysize(parsersql_mysql_set_testing), "mysql_set_testing"); + TestParsePgsql(parsersql_pgsql_search_path, arraysize(parsersql_pgsql_search_path), "pgsql_search_path"); + TestParsePgsql(parsersql_pgsql_time_zone, arraysize(parsersql_pgsql_time_zone), "pgsql_time_zone"); + return exit_status(); } diff --git a/test/tap/tests/test_mysqlx_e2e_handshake-t.cpp b/test/tap/tests/test_mysqlx_e2e_handshake-t.cpp index d73cddcb91..eaa67ee143 100644 --- a/test/tap/tests/test_mysqlx_e2e_handshake-t.cpp +++ b/test/tap/tests/test_mysqlx_e2e_handshake-t.cpp @@ -223,8 +223,12 @@ int main() { E2EConfig cfg; const char* host_env = std::getenv("MYSQLX_E2E_HOST"); if (!host_env) { - skip_all("MYSQLX_E2E_HOST not set; skipping E2E handshake test"); - return exit_status(); + // Group setup bug: test/tap/groups/mysqlx-e2e/env.sh defines + // MYSQLX_E2E_HOST=127.0.0.1 and CI-mysqlx.yml's e2e-tests job + // sources it. If the var is missing at runtime the harness did + // not source the env file — fail loud so the gap is fixed, not + // silently hide the regression as the previous skip_all did. + BAIL_OUT("MYSQLX_E2E_HOST not set — group env (test/tap/groups/mysqlx-e2e/env.sh) was not sourced before invoking this test"); } cfg.host = host_env; cfg.port = static_cast( diff --git a/test/tap/tests/test_mysqlx_e2e_routing-t.cpp b/test/tap/tests/test_mysqlx_e2e_routing-t.cpp index 83f220c730..ecaa815539 100644 --- a/test/tap/tests/test_mysqlx_e2e_routing-t.cpp +++ b/test/tap/tests/test_mysqlx_e2e_routing-t.cpp @@ -280,8 +280,12 @@ int main() { E2EConfig cfg; const char* proxysql_port_env = std::getenv("MYSQLX_E2E_PROXYSQL_PORT"); if (!proxysql_port_env) { - skip_all("MYSQLX_E2E_PROXYSQL_PORT not set; skipping E2E routing test"); - return exit_status(); + // Group setup bug: test/tap/groups/mysqlx-e2e/env.sh defines + // MYSQLX_E2E_PROXYSQL_PORT and CI-mysqlx.yml's e2e-tests job + // sources it. If the var is missing at runtime, the harness + // didn't source the env file — fail loud so the gap is fixed, + // not silently hide the regression as the previous skip_all did. + BAIL_OUT("MYSQLX_E2E_PROXYSQL_PORT not set — group env (test/tap/groups/mysqlx-e2e/env.sh) was not sourced before invoking this test"); } cfg.host = env_or("MYSQLX_E2E_HOST", "127.0.0.1"); cfg.port = static_cast(std::atoi(proxysql_port_env)); diff --git a/test/tap/tests/test_mysqlx_route_drop_inflight-t.cpp b/test/tap/tests/test_mysqlx_route_drop_inflight-t.cpp index 0880f1af1f..dfc8864264 100644 --- a/test/tap/tests/test_mysqlx_route_drop_inflight-t.cpp +++ b/test/tap/tests/test_mysqlx_route_drop_inflight-t.cpp @@ -317,15 +317,18 @@ int main() { const int N_CLIENTS = 5; - // Pre-flight: skip if we cannot reach the X-Protocol listener at all. - // This keeps the test inert in groups (e.g. mysqlx-e2e) where the - // soak setup hasn't provisioned route r1 inside ProxySQL. + // Pre-flight: confirm the X-Protocol listener at cfg.host:cfg.port is + // reachable. This test is registered in groups.json under mysqlx-soak-g1, + // whose harness (test/tap/groups/mysqlx-soak/setup-infras.bash and + // CI-mysqlx.yml's soak-tests job) starts ProxySQL with the mysqlx plugin + // loaded and provisions the route. If the listener is missing at the + // expected port, the group setup is broken — fail loud so it gets + // fixed, not silently hide the regression as the previous skip_all did. { int probe = tcp_connect(cfg.host, cfg.port); if (probe < 0) { - skip_all("X-Protocol listener %s:%u not reachable; skipping (route-drop test requires mysqlx-soak setup)", + BAIL_OUT("X-Protocol listener %s:%u not reachable — mysqlx-soak group setup did not provision the route, or the test was run outside that group", cfg.host.c_str(), cfg.port); - return exit_status(); } close(probe); } diff --git a/test/tap/tests/test_mysqlx_sigterm_inflight-t.cpp b/test/tap/tests/test_mysqlx_sigterm_inflight-t.cpp deleted file mode 100644 index 130139d37e..0000000000 --- a/test/tap/tests/test_mysqlx_sigterm_inflight-t.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/** - * test_mysqlx_sigterm_inflight-t.cpp - * - * Behavioural validation TAP scaffolding for issue #5678 part (a): - * - * When ProxySQL receives SIGTERM with X-Protocol clients connected - * and dispatching queries, each Mysqlx_Thread::run() worker calls - * MysqlxSession::shutdown_notify_client() on every owned session on - * the way out of the loop. shutdown_notify_client() must: - * - enqueue a fatal Mysqlx::Error frame (code 1053, "Server is - * shutting down") via send_error(..., fatal=true); - * - drain the queued frame to the wire via DataStream::write_to_net(); - * - if TLS is active, set quiet shutdown + call SSL_shutdown so the - * peer's TLS stack sees a clean close_notify. - * - * The connected client therefore observes a Mysqlx::Error frame - * (last app-level frame) followed by an orderly EOF -- NOT a TCP RST - * from kernel-level socket teardown. - * - * Contract reference: - * plugins/mysqlx/src/mysqlx_session.cpp:1875 shutdown_notify_client() - * plugins/mysqlx/src/mysqlx_thread.cpp:120-128 run() shutdown path - * - * Why this test does NOT run automatically in CI: - * - * ProxySQL runs in its own Docker container (proxysql.INFRA_ID), - * the test runs in a separate test-runner.INFRA_ID container, and - * the test-runner does NOT have docker socket access. Even if it - * did, sending SIGTERM mid-test would tear down the proxysql - * container that the surrounding run-tests-isolated.bash harness - * expects to remain up for the entire group. There is no harness - * mode for "this test owns proxysql lifecycle"; adding one is a - * substantially larger change than the issue asks for. - * - * The equivalent automated coverage already exists out-of-band: - * - * test/scripts/mysqlx/behavioral_validation.py --scenario sigterm - * - * That harness (1) opens N X-Protocol sessions via mysql-connector- - * python, (2) os.kill(pid, SIGTERM) against the proxysql process - * directly, and (3) verifies each client received a Mysqlx::Error - * frame with errno=1053 rather than a TCP RST. It is invoked - * manually against staging or local-loop infrastructure where - * tearing down the proxysql process is acceptable. - * - * Manual procedure (mirrors what the issue asks operators to - * demonstrate): - * - * 1. Build proxysql + the mysqlx plugin and stand up a backend - * (e.g. via test/tap/groups/mysqlx-soak/setup-infras.bash inside - * a local docker stack, or via the dbdeployer recipe in - * test/tap/groups/mysqlx-e2e/setup-infras.bash with the plugin - * manually configured). - * 2. From any host that can reach proxysql, run: - * - * cd test/scripts/mysqlx - * pip install mysql-connector-python - * python3 behavioral_validation.py \ - * --scenario sigterm \ - * --proxysql-host HOST --proxysql-port ROUTE_PORT \ - * --user alice --password alicepass \ - * --proxysql-pid-file /var/run/proxysql.pid - * - * Expected output line: "PASS: every client received a clean - * shutdown notification" (each session reports errno=1053). - * - * 3. Failure modes that indicate a regression: - * - errno = None / OperationalError without errno -> TCP RST - * - errno != 1053 / different message -> wrong frame - * - hung clients -> SSL_shutdown - * ordering issue - * - * This test compiles + links so the scaffolding is on the same build - * cadence as the rest of the mysqlx TAP suite (groups.json registration - * exercises the binary's existence on every CI run); it issues - * skip_all() at runtime to keep CI green. - * - * NOSONAR(cpp:S2068) annotation is applied where test password literals - * appear inline. - */ - -#include "tap.h" - -#include -#include - -int main() { - // Allow operators who DO want to run this test (e.g. on a single-host - // dev loop where proxysql is the only process and SIGTERM is fine) - // to opt in via MYSQLX_SIGTERM_INFLIGHT_OPT_IN=1. Without the opt-in - // we skip_all so CI never tears down proxysql mid-suite. - const char* opt_in = std::getenv("MYSQLX_SIGTERM_INFLIGHT_OPT_IN"); - if (opt_in == nullptr || std::string(opt_in) != "1") { - skip_all( - "manual procedure -- see header comment. " - "To run automated equivalent: " - "test/scripts/mysqlx/behavioral_validation.py --scenario sigterm. " - "To opt in here: set MYSQLX_SIGTERM_INFLIGHT_OPT_IN=1 (will SIGTERM proxysql)." - ); - return exit_status(); - } - - // If we ever wire this up, the implementation would mirror - // scenario_sigterm() in test/scripts/mysqlx/behavioral_validation.py: - // - // 1. Open 5 X-Protocol clients in worker threads, each running - // a steady SELECT 1 loop (use the same handshake helpers as - // test_mysqlx_e2e_routing-t.cpp + test_mysqlx_route_drop_inflight-t.cpp). - // 2. After ~2s of steady traffic, locate the proxysql pid: - // - read MYSQLX_SIGTERM_INFLIGHT_PIDFILE (env-driven, no - // baked-in path) - // - or fall back to pidof("proxysql") - // Then kill(pid, SIGTERM). - // 3. For each client thread, the next mysqlx_read_frame() call - // should return a frame with message_type == MSG_SRV_ERROR - // whose decoded Mysqlx.Error has code() == 1053. The bytes - // AFTER that frame should be a clean EOF (recv() returns 0) - // not a kernel-level RST (which manifests as ECONNRESET on - // the next syscall). - // - // Even with opt-in, today this binary does not implement the body - // because running it without infrastructure that expects proxysql - // to die would corrupt the surrounding test group. Refuse loudly. - plan(1); - ok(false, - "MYSQLX_SIGTERM_INFLIGHT_OPT_IN is set but the in-process automated path is " - "not implemented. Use test/scripts/mysqlx/behavioral_validation.py --scenario sigterm."); - return exit_status(); -} diff --git a/test/tap/tests/test_set_statement_for-t.cpp b/test/tap/tests/test_set_statement_for-t.cpp index 22143dbcc9..284d4c2eaf 100644 --- a/test/tap/tests/test_set_statement_for-t.cpp +++ b/test/tap/tests/test_set_statement_for-t.cpp @@ -20,53 +20,26 @@ * - No hostgroup lock persists after the query * - Multiple variables in a single SET STATEMENT work * - Case-insensitive matching works + * - Whitespace tolerance after FOR (multi-line, tab, CRLF, multi-space) * - * All tests are skipped on non-MariaDB backends. + * This test requires a MariaDB backend. It should only be registered in + * TAP groups whose infra provides one (e.g. mariadb10-galera-*). The + * earlier runtime self-skip on non-MariaDB backends has been removed — + * if the test is misregistered into a MySQL group, that's a groups.json + * bug and should surface as a loud failure, not a silent skip. * * @date 2026-05-01 */ #include #include +#include #include "mysql.h" #include "proxysql_utils.h" #include "tap.h" #include "command_line.h" #include "utils.h" -/** - * @brief Detect if the backend behind ProxySQL is MariaDB. - * @return 1 if MariaDB, 0 if not MariaDB, -1 on connection/query error. - */ -static int detect_mariadb(CommandLine& cl) { - MYSQL* mysql = mysql_init(NULL); - if (!mysql) return -1; - if (!mysql_real_connect(mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { - fprintf(stderr, "File %s, line %d, Error: %s\n", - __FILE__, __LINE__, mysql_error(mysql)); - mysql_close(mysql); - return -1; - } - - int is_mariadb = 0; - if (mysql_query(mysql, "SELECT @@version") != 0) { - fprintf(stderr, "File %s, line %d, Error: %s\n", - __FILE__, __LINE__, mysql_error(mysql)); - mysql_close(mysql); - return -1; - } - MYSQL_RES* result = mysql_store_result(mysql); - MYSQL_ROW row; - while ((row = mysql_fetch_row(result))) { - if (strstr(row[0], "Maria")) { - is_mariadb = 1; - } - } - mysql_free_result(result); - mysql_close(mysql); - return is_mariadb; -} - int main(int argc, char** argv) { CommandLine cl; @@ -75,18 +48,7 @@ int main(int argc, char** argv) { return -1; } - // SET STATEMENT ... FOR is a MariaDB-specific feature. - // Skip all tests if the backend is not MariaDB, but fail on errors. - int is_mariadb = detect_mariadb(cl); - if (is_mariadb == -1) { - diag("Error: Failed to connect to ProxySQL or query backend version."); - return -1; - } - if (is_mariadb == 0) { - skip_all("SET STATEMENT ... FOR requires a MariaDB backend"); - } - - plan(5); + plan(5 + 8); diag("=== Test: MariaDB SET STATEMENT ... FOR passthrough ==="); diag("Issue #5686: ProxySQL should forward SET STATEMENT ... FOR"); @@ -170,6 +132,74 @@ int main(int argc, char** argv) { } diag(""); + + // Tests 5-8: whitespace tolerance after 'FOR'. The original PR #5708 + // detection used strcasestr(nq, " FOR "), which is a literal-substring + // match requiring a space BOTH before and after 'FOR'. A mysql client + // query typed across multiple lines (the customer-reported failure mode + // of issue #5686 follow-up) has '\n' immediately after 'FOR' and the + // detection misses; the SET parser then falls through to + // unable_to_parse_set_statement() and locks the session to its default + // hostgroup, surfacing as error 9006 on the next query that routes to + // a different hostgroup. The fix uses the digest text (which is + // whitespace-normalised) instead. These tests cover four common + // post-'FOR' whitespace shapes that the original test missed. + // + // Each test: (a) the SET STATEMENT must succeed, (b) a plain SELECT + // immediately afterwards must succeed (proving no hostgroup lock + // persists). On the buggy code path, assertion (b) would manifest as + // error 9006 on any test environment where the follow-up SELECT routes + // to a different hostgroup than the session's default; on a single- + // hostgroup environment it would manifest as the SET STATEMENT itself + // silently going through a different parse path that may or may not + // produce the expected backend response — either way, the assertions + // below will fail loudly. + + struct ws_case_t { + const char* label; + const char* query; + }; + std::vector ws_cases = { + { "newline after FOR", + "SET STATEMENT max_statement_time=60 FOR\n SELECT 1 AS val" }, + { "tab after FOR", + "SET STATEMENT max_statement_time=60 FOR\tSELECT 1 AS val" }, + { "CRLF after FOR", + "SET STATEMENT max_statement_time=60 FOR\r\n SELECT 1 AS val" }, + { "multiple spaces after FOR", + "SET STATEMENT max_statement_time=60 FOR SELECT 1 AS val" }, + }; + + int idx = 5; + for (const auto& c : ws_cases) { + idx++; + diag("Test %d: SET STATEMENT with %s", idx, c.label); + int rc = mysql_query(proxysql, c.query); + ok(rc == 0, "SET STATEMENT (%s) should succeed", c.label); + if (rc != 0) { + diag(" ERROR on SET STATEMENT: %s", mysql_error(proxysql)); + } else { + MYSQL_RES* res = mysql_store_result(proxysql); + if (res) mysql_free_result(res); + } + // Follow-up plain SELECT to verify no hostgroup lock persists. + // On the buggy code path, the SET STATEMENT would have triggered + // unable_to_parse_set_statement() and locked the hostgroup; a + // subsequent query that routes elsewhere would then fail with + // error 9006 ("connection is locked to hostgroup X but trying + // to reach hostgroup Y"). + rc = mysql_query(proxysql, "SELECT 2 AS no_lock_probe"); + ok(rc == 0, "follow-up SELECT after %s should succeed (no hostgroup lock)", c.label); + if (rc != 0) { + diag(" ERROR on follow-up SELECT: %s (this indicates the hostgroup was locked)", + mysql_error(proxysql)); + } else { + MYSQL_RES* res = mysql_store_result(proxysql); + if (res) mysql_free_result(res); + } + diag(""); + } + diag("=== All tests completed ==="); mysql_close(proxysql); diff --git a/test/tap/tests/tokenizer_payloads/pgsql_regular_tokenizer_digests.hjson b/test/tap/tests/tokenizer_payloads/pgsql_regular_tokenizer_digests.hjson index e7e5764841..87250e5df5 100644 --- a/test/tap/tests/tokenizer_payloads/pgsql_regular_tokenizer_digests.hjson +++ b/test/tap/tests/tokenizer_payloads/pgsql_regular_tokenizer_digests.hjson @@ -278,7 +278,7 @@ "select '{\"key\":\"value\"}'::json -> 'key'", "select '[1,2,3]'::json -> 1" ], - "s1": "select ?-> ?", + "s1": "select ? -> ?", "s2": "select ?-> ?", "s3": "select ?-> ?", "s4": "select ?-> ?" @@ -288,10 +288,10 @@ "q": [ "select '{\"key\":\"value\"}'::jsonb @> '{\"key\":\"value\"}'" ], - "s1": "select ?@> ?", - "s2": "select ?@> ?", - "s3": "select ?@> ?", - "s4": "select ?@> ?" + "s1": "select ? @> ?", + "s2": "select ? @> ?", + "s3": "select ? @> ?", + "s4": "select ? @> ?" }, // ======================== // SECTION 8: DOLLAR PARAMETERS @@ -656,7 +656,7 @@ "select '$1,000.00'::money + '$12.34'::money", "select '12.34'::money + '5.66'::money" ], - "s1": "select ?+ ?", + "s1": "select ? + ?", "s2": "select ?+?", "s3": "select ?+?", "s4": "select ?+?" diff --git a/test/tap/tests/unit/genai_discovery_schema_unit-t.cpp b/test/tap/tests/unit/genai_discovery_schema_unit-t.cpp index ba137f6ce1..b95aa0f821 100644 --- a/test/tap/tests/unit/genai_discovery_schema_unit-t.cpp +++ b/test/tap/tests/unit/genai_discovery_schema_unit-t.cpp @@ -31,7 +31,7 @@ * - MCP_Query_Digest_Stats::add_timing() * - MCP_Query_Processor_Output init/destroy * - * @note Requires PROXYSQLGENAI=1 build (auto-detected from libproxysql.a). + * @note Requires PROXYSQL40=1 build (includes genai plugin; auto-detected from libproxysql.a). */ #include "tap.h" diff --git a/test/tap/tests/unit/genai_fts_string_unit-t.cpp b/test/tap/tests/unit/genai_fts_string_unit-t.cpp index 96beda5362..2ae7a5354b 100644 --- a/test/tap/tests/unit/genai_fts_string_unit-t.cpp +++ b/test/tap/tests/unit/genai_fts_string_unit-t.cpp @@ -12,7 +12,7 @@ * These are private instance methods but do not access the database, * so we use the private-to-public trick to test them directly. * - * Requires: PROXYSQLGENAI=1 build + * Requires: PROXYSQL40=1 build (includes genai plugin) */ #ifdef PROXYSQL40 diff --git a/test/tap/tests/unit/genai_llm_clients_unit-t.cpp b/test/tap/tests/unit/genai_llm_clients_unit-t.cpp index f7efaf810c..b94c8ae6a0 100644 --- a/test/tap/tests/unit/genai_llm_clients_unit-t.cpp +++ b/test/tap/tests/unit/genai_llm_clients_unit-t.cpp @@ -6,7 +6,7 @@ * - is_retryable_error() -- HTTP status code / curl error classification * - LLM_WriteCallback() -- libcurl write callback * - * Compiled only when PROXYSQLGENAI=1 (auto-detected from libproxysql.a). + * Built with all v4.0 plugins under PROXYSQL40=1 (auto-detected from libproxysql.a). */ #include "tap.h" diff --git a/test/tap/tests/unit/genai_mcp_endpoint_unit-t.cpp b/test/tap/tests/unit/genai_mcp_endpoint_unit-t.cpp index 234ec9e89c..c0a509c7a1 100644 --- a/test/tap/tests/unit/genai_mcp_endpoint_unit-t.cpp +++ b/test/tap/tests/unit/genai_mcp_endpoint_unit-t.cpp @@ -7,7 +7,7 @@ * - create_jsonrpc_response() -- JSON-RPC 2.0 success response formatting * - create_jsonrpc_error() -- JSON-RPC 2.0 error response formatting * - * Compiled only when PROXYSQLGENAI=1 (auto-detected from libproxysql.a). + * Built with all v4.0 plugins under PROXYSQL40=1 (auto-detected from libproxysql.a). */ #include "tap.h" diff --git a/test/tap/tests/unit/genai_mcp_thread_unit-t.cpp b/test/tap/tests/unit/genai_mcp_thread_unit-t.cpp index e444601e03..22d62b82c6 100644 --- a/test/tap/tests/unit/genai_mcp_thread_unit-t.cpp +++ b/test/tap/tests/unit/genai_mcp_thread_unit-t.cpp @@ -14,7 +14,7 @@ * - get_target_auth_context() with empty map * - get_all_target_auth_contexts() with empty map * - * Compiled only when PROXYSQLGENAI=1 (auto-detected from libproxysql.a). + * Built with all v4.0 plugins under PROXYSQL40=1 (auto-detected from libproxysql.a). */ #include "tap.h" diff --git a/test/tap/tests/unit/genai_mysql_catalog_unit-t.cpp b/test/tap/tests/unit/genai_mysql_catalog_unit-t.cpp index adcd25c254..7937221f33 100644 --- a/test/tap/tests/unit/genai_mysql_catalog_unit-t.cpp +++ b/test/tap/tests/unit/genai_mysql_catalog_unit-t.cpp @@ -17,7 +17,7 @@ * - merge() — merge multiple entries into a domain summary * - Schema isolation — entries scoped by schema name * - * @note Requires PROXYSQLGENAI=1 build (auto-detected from libproxysql.a). + * @note Requires PROXYSQL40=1 build (includes genai plugin; auto-detected from libproxysql.a). */ #include "tap.h" diff --git a/test/tap/tests/unit/genai_query_handler_unit-t.cpp b/test/tap/tests/unit/genai_query_handler_unit-t.cpp index 376aae8b92..dec56cac64 100644 --- a/test/tap/tests/unit/genai_query_handler_unit-t.cpp +++ b/test/tap/tests/unit/genai_query_handler_unit-t.cpp @@ -11,7 +11,7 @@ * - is_runtime_online_status() -- status comparison * - uppercase_or_unknown() -- case conversion with NULL fallback * - * Compiled only when PROXYSQLGENAI=1 (auto-detected from libproxysql.a). + * Built with all v4.0 plugins under PROXYSQL40=1 (auto-detected from libproxysql.a). */ #include "tap.h" diff --git a/test/tap/tests/unit/genai_stats_parsing_unit-t.cpp b/test/tap/tests/unit/genai_stats_parsing_unit-t.cpp index c6b76605e4..f26343daaa 100644 --- a/test/tap/tests/unit/genai_stats_parsing_unit-t.cpp +++ b/test/tap/tests/unit/genai_stats_parsing_unit-t.cpp @@ -10,7 +10,7 @@ * - get_interval_config() -- interval string->seconds mapping * - calculate_percentile() -- histogram percentile estimation * - * Compiled only when PROXYSQLGENAI=1 (auto-detected from libproxysql.a). + * Built with all v4.0 plugins under PROXYSQL40=1 (auto-detected from libproxysql.a). */ #include "tap.h" diff --git a/test/tap/tests/unit/genai_thread_unit-t.cpp b/test/tap/tests/unit/genai_thread_unit-t.cpp index 356548882a..ab50a2ce7c 100644 --- a/test/tap/tests/unit/genai_thread_unit-t.cpp +++ b/test/tap/tests/unit/genai_thread_unit-t.cpp @@ -15,7 +15,7 @@ * The constructor calls curl_global_init() but does NOT start threads * or epoll, so it is safe to instantiate in a unit test. * - * @note Requires PROXYSQLGENAI=1 build (auto-detected from libproxysql.a). + * @note Requires PROXYSQL40=1 build (includes genai plugin; auto-detected from libproxysql.a). */ #include "tap.h"