From 95deb17c78c065dd0b884bc9dd1cb5e50b4df1be Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Fri, 28 Aug 2026 12:21:36 +0100 Subject: [PATCH 1/5] Flyway: single-pass CREATE INDEX CONCURRENTLY support (pin 13.4.0 + session-level lock + auto-repair) Pin flyway/flyway to 13.4.0 everywhere, add -postgresql.transactional.lock=false to every migrate invocation (Flyway's default transactional advisory lock since 9.1.2 deadlocks against CIC), and run repair unconditionally before migrate in prod deploy to auto-clear a failed CIC migration's success=false history row. Documents the versioning trap for CIC data-fix migrations in AGENTS.md. Closes #407 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Ksbj9x1jrixy8qhSE5H46 --- .github/workflows/_bot-deploy.yml | 17 ++++++++++++++++- .github/workflows/alita-real-test.yml | 3 ++- .github/workflows/coupon-real-test.yml | 3 ++- AGENTS.md | 8 ++++++++ src/alita-bot/docker-compose.dev.yml | 4 ++-- src/coupon-hub-bot/docker-compose.dev.yml | 4 ++-- tests/BotTestInfra/ContainerHelpers.fs | 5 +++-- 7 files changed, 35 insertions(+), 9 deletions(-) diff --git a/.github/workflows/_bot-deploy.yml b/.github/workflows/_bot-deploy.yml index 56afabb8..d766fd5e 100644 --- a/.github/workflows/_bot-deploy.yml +++ b/.github/workflows/_bot-deploy.yml @@ -60,7 +60,7 @@ on: description: "For the SRE agent (_sre-agent.yml), invoked on deploy/verify failure" env: - DOCKER_IMAGE: flyway/flyway + DOCKER_IMAGE: flyway/flyway:13.4.0 SCHEMAS: public jobs: @@ -182,6 +182,20 @@ jobs: WIREGUARD_CONFIG: ${{ secrets.WIREGUARD_CONFIG }} SPLIT_TUNNEL: "false" + # Unconditional: no-op if the last run's migrate succeeded (checksum-safe), + # required to clear a failed CIC migration's success=false history row. + - name: Repair production DB migration history + run: >- + docker run --rm + --network host + --volume ${{ github.workspace }}/${{ inputs.migrations-path }}:/flyway/sql:ro + --volume ${{ github.workspace }}/reports:/flyway/reports + "${{ env.DOCKER_IMAGE }}" + -url="jdbc:postgresql://${{ secrets.DB_PROD_HOST }}/${{ inputs.db-name }}" + -user="${{ secrets.DB_PROD_USERNAME }}" + -password="${{ secrets.DB_PROD_PASSWORD }}" + repair -schemas="${{ env.SCHEMAS }}" + - name: Apply migrations to production DB run: >- docker run --rm @@ -192,6 +206,7 @@ jobs: -url="jdbc:postgresql://${{ secrets.DB_PROD_HOST }}/${{ inputs.db-name }}" -user="${{ secrets.DB_PROD_USERNAME }}" -password="${{ secrets.DB_PROD_PASSWORD }}" + -postgresql.transactional.lock=false migrate -schemas="${{ env.SCHEMAS }}" - name: Disconnect VPN diff --git a/.github/workflows/alita-real-test.yml b/.github/workflows/alita-real-test.yml index a495b02d..99e5cd17 100644 --- a/.github/workflows/alita-real-test.yml +++ b/.github/workflows/alita-real-test.yml @@ -137,9 +137,10 @@ jobs: run: >- docker run --rm --network host --volume ${{ github.workspace }}/src/alita-bot/migrations:/flyway/sql:ro - flyway/flyway + flyway/flyway:13.4.0 -url="jdbc:postgresql://127.0.0.1:15433/alita_bot" -user=admin -password=admin + -postgresql.transactional.lock=false migrate -schemas=public - name: Seed bot_setting diff --git a/.github/workflows/coupon-real-test.yml b/.github/workflows/coupon-real-test.yml index ad197499..0d7f2d26 100644 --- a/.github/workflows/coupon-real-test.yml +++ b/.github/workflows/coupon-real-test.yml @@ -248,9 +248,10 @@ jobs: run: >- docker run --rm --network host --volume ${{ github.workspace }}/src/coupon-hub-bot/migrations:/flyway/sql:ro - flyway/flyway + flyway/flyway:13.4.0 -url="jdbc:postgresql://127.0.0.1:15434/coupon_hub_bot" -user=admin -password=admin + -postgresql.transactional.lock=false migrate -schemas=public - name: Seed bot_setting diff --git a/AGENTS.md b/AGENTS.md index 74812df7..6d7fc20f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,6 +93,14 @@ These tests are for **deliberate feature work and manual dev-iteration only** (` - CouponHubBot DB: `coupon_hub_bot`, role: `coupon_hub_bot_service` - AlitaBot DB: `alita_bot`, role: `alita_bot_service` (pgvector-backed — `message_embedding`/`interaction_memory` use the `vector` extension, `pgvector/pgvector:pg17` everywhere AlitaBot provisions its own Postgres) +### `CREATE INDEX CONCURRENTLY` migrations + +Deploy (`flyway/flyway:13.4.0`) runs `migrate` with `-postgresql.transactional.lock=false` (session-level advisory lock, not Flyway's default transactional one since 9.1.2) so a CIC migration doesn't deadlock against Flyway's own coordination lock. `repair` runs unconditionally before every `migrate`. + +- Author CIC migrations like any other, in the normal versioned migrations dir — no `.sql.conf`, Flyway auto-detects them non-transactional. Make them idempotent: `DROP INDEX CONCURRENTLY IF EXISTS ; CREATE [UNIQUE] INDEX CONCURRENTLY ON ...;`. +- **If a CIC migration `Vn` fails on a data problem**, the data-fix migration's version must sit strictly between the last successfully-applied version and `Vn` (e.g. `V(n-1).1`) — never above `Vn`. Pending migrations run in ascending version order, so a higher-numbered fix never runs before `Vn`'s retry, and `Vn` fails identically on every deploy forever. +- Unconditional `repair` means editing an already-applied migration's file gets its checksum silently realigned instead of failing validation — don't edit applied migrations. + ## Settings configuration - All **non-secret** bot configuration lives in the `bot_setting` table. Env vars are only for secrets (`BOT_TELEGRAM_TOKEN`, `BOT_AUTH_TOKEN`, `AZURE_OCR_KEY`, `GITHUB_TOKEN`, `DATABASE_URL`, etc.). diff --git a/src/alita-bot/docker-compose.dev.yml b/src/alita-bot/docker-compose.dev.yml index f2d696c9..3e778d47 100644 --- a/src/alita-bot/docker-compose.dev.yml +++ b/src/alita-bot/docker-compose.dev.yml @@ -28,7 +28,7 @@ services: retries: 5 flyway: - image: flyway/flyway + image: flyway/flyway:13.4.0 container_name: alita-bot-flyway-dev depends_on: postgres: @@ -39,7 +39,7 @@ services: FLYWAY_URL: jdbc:postgresql://postgres:5432/alita_bot FLYWAY_USER: admin FLYWAY_PASSWORD: admin - command: migrate -schemas=public + command: -postgresql.transactional.lock=false migrate -schemas=public restart: "no" seed-bot-settings: diff --git a/src/coupon-hub-bot/docker-compose.dev.yml b/src/coupon-hub-bot/docker-compose.dev.yml index 7d8ef0e0..0dae5ceb 100644 --- a/src/coupon-hub-bot/docker-compose.dev.yml +++ b/src/coupon-hub-bot/docker-compose.dev.yml @@ -22,7 +22,7 @@ services: retries: 5 flyway: - image: flyway/flyway + image: flyway/flyway:13.4.0 container_name: coupon-hub-flyway-dev depends_on: postgres: @@ -33,7 +33,7 @@ services: FLYWAY_URL: jdbc:postgresql://postgres:5432/coupon_hub_bot FLYWAY_USER: admin FLYWAY_PASSWORD: admin - command: migrate -schemas=public + command: -postgresql.transactional.lock=false migrate -schemas=public restart: "no" seed-bot-settings: diff --git a/tests/BotTestInfra/ContainerHelpers.fs b/tests/BotTestInfra/ContainerHelpers.fs index 9aad7ba0..6b83d39f 100644 --- a/tests/BotTestInfra/ContainerHelpers.fs +++ b/tests/BotTestInfra/ContainerHelpers.fs @@ -92,7 +92,7 @@ let createPostgresContainer (network: INetwork) (alias: string) (pgImage: string .Build() let createFlywayContainer (network: INetwork) (migrationsPath: string) (dbAlias: string) (dbName: string) (dbContainer: PostgreSqlContainer) = - ContainerBuilder("flyway/flyway") + ContainerBuilder("flyway/flyway:13.4.0") .WithNetwork(network) .WithBindMount(migrationsPath, "/flyway/sql", AccessMode.ReadOnly) // Without label=disable, hosts with SELinux enforcing block the container from @@ -102,7 +102,8 @@ let createFlywayContainer (network: INetwork) (migrationsPath: string) (dbAlias: .WithEnvironment("FLYWAY_URL", $"jdbc:postgresql://{dbAlias}:5432/{dbName}") .WithEnvironment("FLYWAY_USER", "admin") .WithEnvironment("FLYWAY_PASSWORD", "admin") - .WithCommand("migrate", "-schemas=public") + // Must match prod's migrate flags, or CIC migrations deadlock only here, not in prod. + .WithCommand("-postgresql.transactional.lock=false", "migrate", "-schemas=public") .WithWaitStrategy( Wait.ForUnixContainer().AddCustomWaitStrategy( { new IWaitUntil with From 7128cec9fe22054511cdddbfd5daaf0ae189be26 Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Fri, 28 Aug 2026 12:42:56 +0100 Subject: [PATCH 2/5] Add PR-time check for migration file immutability Unconditional flyway repair (#438) silently realigns checksums of edited applied migrations instead of failing validation, so the no-edits invariant must now be enforced at PR time: fail on any edit/delete/rename of an existing migration file, diffed against the merge-base with the target branch, across all three bots' migrations dirs. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Ksbj9x1jrixy8qhSE5H46 --- .github/workflows/migration-immutability.yml | 26 ++++++++++++++ scripts/check-migration-immutability.sh | 38 ++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 .github/workflows/migration-immutability.yml create mode 100755 scripts/check-migration-immutability.sh diff --git a/.github/workflows/migration-immutability.yml b/.github/workflows/migration-immutability.yml new file mode 100644 index 00000000..0d55a6bd --- /dev/null +++ b/.github/workflows/migration-immutability.yml @@ -0,0 +1,26 @@ +name: "Lint: Migration immutability" + +on: + pull_request: + paths: + - 'src/vahter-bot/migrations/**' + - 'src/coupon-hub-bot/migrations/**' + - 'src/alita-bot/migrations/**' + +permissions: + contents: read + +jobs: + migration-immutability: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch base ref + run: git fetch origin "${{ github.base_ref }}" + + - name: Check migration immutability + run: scripts/check-migration-immutability.sh --range "origin/${{ github.base_ref }}...HEAD" diff --git a/scripts/check-migration-immutability.sh b/scripts/check-migration-immutability.sh new file mode 100755 index 00000000..34afefdb --- /dev/null +++ b/scripts/check-migration-immutability.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Fails if a PR edits/deletes/renames a migration file already on the base branch. +# --range A...B diffs against the merge-base, so files added-then-edited within the same PR still pass. +set -euo pipefail + +MIGRATION_DIRS=( + "src/vahter-bot/migrations" + "src/coupon-hub-bot/migrations" + "src/alita-bot/migrations" +) + +mode="${1:-}" +case "$mode" in + --range) + range="${2:?--range requires }" + a_ref="${range%%...*}" + b_ref="${range##*...}" + before_ref=$(git merge-base "$a_ref" "$b_ref") + after_ref="$b_ref" + ;; + *) + echo "usage: $0 --range " >&2 + exit 2 + ;; +esac + +offenders=$(git diff --no-ext-diff --no-color -M --diff-filter=MDR --name-status \ + "$before_ref" "$after_ref" -- "${MIGRATION_DIRS[@]}") + +if [ -z "$offenders" ]; then + echo "migration-immutability: OK — no edited/deleted/renamed migration files" + exit 0 +fi + +echo "::error::Applied migrations are immutable — add a new migration instead of editing/deleting/renaming an existing one. Offending files:" +echo "$offenders" +exit 1 From ac3f052a8321c2ce006b2ebe205175241cd91c1d Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Fri, 28 Aug 2026 12:55:55 +0100 Subject: [PATCH 3/5] test: intentionally violate migration immutability (to be reverted) Edits, deletes, and renames an existing migration in each bot's dir to verify migration-immutability.yml catches all three cases in real CI. This commit is reverted by the next one before merge. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Ksbj9x1jrixy8qhSE5H46 --- src/coupon-hub-bot/migrations/V3__missing_grants.sql | 6 ------ ...issions_2.sql => V15__missing_permissions_2_renamed.sql} | 0 .../migrations/V44__msg_text_repetition_index.sql | 1 + 3 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 src/coupon-hub-bot/migrations/V3__missing_grants.sql rename src/vahter-bot/migrations/{V15__missing_permissions_2.sql => V15__missing_permissions_2_renamed.sql} (100%) diff --git a/src/coupon-hub-bot/migrations/V3__missing_grants.sql b/src/coupon-hub-bot/migrations/V3__missing_grants.sql deleted file mode 100644 index ba025361..00000000 --- a/src/coupon-hub-bot/migrations/V3__missing_grants.sql +++ /dev/null @@ -1,6 +0,0 @@ -GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public.coupon TO coupon_hub_bot_service; -GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public.coupon_event TO coupon_hub_bot_service; -GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public.pending_add TO coupon_hub_bot_service; -GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public."user" TO coupon_hub_bot_service; - -GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO coupon_hub_bot_service; diff --git a/src/vahter-bot/migrations/V15__missing_permissions_2.sql b/src/vahter-bot/migrations/V15__missing_permissions_2_renamed.sql similarity index 100% rename from src/vahter-bot/migrations/V15__missing_permissions_2.sql rename to src/vahter-bot/migrations/V15__missing_permissions_2_renamed.sql diff --git a/src/vahter-bot/migrations/V44__msg_text_repetition_index.sql b/src/vahter-bot/migrations/V44__msg_text_repetition_index.sql index e51f7c43..13e72016 100644 --- a/src/vahter-bot/migrations/V44__msg_text_repetition_index.sql +++ b/src/vahter-bot/migrations/V44__msg_text_repetition_index.sql @@ -5,3 +5,4 @@ CREATE INDEX IF NOT EXISTS idx_event_msg_text_md5_created_at ON event (msg_text_md5, created_at) WHERE event_type = 'MessageReceived' AND msg_text_md5 IS NOT NULL; +-- CI test: this line proves migration-immutability detects an edit -- From 920ffe01b051b16ed2279cecc31814a838d3029f Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Fri, 28 Aug 2026 12:56:37 +0100 Subject: [PATCH 4/5] Revert "test: intentionally violate migration immutability (to be reverted)" This reverts commit ac3f052a8321c2ce006b2ebe205175241cd91c1d. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Ksbj9x1jrixy8qhSE5H46 --- src/coupon-hub-bot/migrations/V3__missing_grants.sql | 6 ++++++ ...issions_2_renamed.sql => V15__missing_permissions_2.sql} | 0 .../migrations/V44__msg_text_repetition_index.sql | 1 - 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 src/coupon-hub-bot/migrations/V3__missing_grants.sql rename src/vahter-bot/migrations/{V15__missing_permissions_2_renamed.sql => V15__missing_permissions_2.sql} (100%) diff --git a/src/coupon-hub-bot/migrations/V3__missing_grants.sql b/src/coupon-hub-bot/migrations/V3__missing_grants.sql new file mode 100644 index 00000000..ba025361 --- /dev/null +++ b/src/coupon-hub-bot/migrations/V3__missing_grants.sql @@ -0,0 +1,6 @@ +GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public.coupon TO coupon_hub_bot_service; +GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public.coupon_event TO coupon_hub_bot_service; +GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public.pending_add TO coupon_hub_bot_service; +GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public."user" TO coupon_hub_bot_service; + +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO coupon_hub_bot_service; diff --git a/src/vahter-bot/migrations/V15__missing_permissions_2_renamed.sql b/src/vahter-bot/migrations/V15__missing_permissions_2.sql similarity index 100% rename from src/vahter-bot/migrations/V15__missing_permissions_2_renamed.sql rename to src/vahter-bot/migrations/V15__missing_permissions_2.sql diff --git a/src/vahter-bot/migrations/V44__msg_text_repetition_index.sql b/src/vahter-bot/migrations/V44__msg_text_repetition_index.sql index 13e72016..e51f7c43 100644 --- a/src/vahter-bot/migrations/V44__msg_text_repetition_index.sql +++ b/src/vahter-bot/migrations/V44__msg_text_repetition_index.sql @@ -5,4 +5,3 @@ CREATE INDEX IF NOT EXISTS idx_event_msg_text_md5_created_at ON event (msg_text_md5, created_at) WHERE event_type = 'MessageReceived' AND msg_text_md5 IS NOT NULL; --- CI test: this line proves migration-immutability detects an edit -- From c093a8cc8b6a20e9c401ba86259e974d0a6657d0 Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Fri, 28 Aug 2026 13:05:17 +0100 Subject: [PATCH 5/5] Remove paths filter from migration-immutability workflow A path-filtered required check never runs on PRs that don't touch migrations, blocking merge forever on a check that never reports. Run on every pull_request instead; the script exits 0 immediately when no migration files changed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Ksbj9x1jrixy8qhSE5H46 --- .github/workflows/migration-immutability.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/migration-immutability.yml b/.github/workflows/migration-immutability.yml index 0d55a6bd..ca159617 100644 --- a/.github/workflows/migration-immutability.yml +++ b/.github/workflows/migration-immutability.yml @@ -2,10 +2,6 @@ name: "Lint: Migration immutability" on: pull_request: - paths: - - 'src/vahter-bot/migrations/**' - - 'src/coupon-hub-bot/migrations/**' - - 'src/alita-bot/migrations/**' permissions: contents: read