Flyway: single-pass CREATE INDEX CONCURRENTLY support (pin 13.4.0 + session-level lock + auto-repair)

.github/workflows/_bot-deploy.yml

@@ -60,7 +60,7 @@ on:
         description: "For the SRE agent (_sre-agent.yml), invoked on deploy/verify failure"
 
 env:
-  DOCKER_IMAGE: flyway/flyway
+  DOCKER_IMAGE: flyway/flyway:13.4.0
   SCHEMAS: public
 
 jobs:
@@ -182,6 +182,20 @@ jobs:
         WIREGUARD_CONFIG: ${{ secrets.WIREGUARD_CONFIG }}
         SPLIT_TUNNEL: "false"
 
+    # Unconditional: no-op if the last run's migrate succeeded (checksum-safe),
+    # required to clear a failed CIC migration's success=false history row.
+    - name: Repair production DB migration history
+      run: >-
+        docker run --rm
+        --network host
+        --volume ${{ github.workspace }}/${{ inputs.migrations-path }}:/flyway/sql:ro
+        --volume ${{ github.workspace }}/reports:/flyway/reports
+        "${{ env.DOCKER_IMAGE }}"
+        -url="jdbc:postgresql://${{ secrets.DB_PROD_HOST }}/${{ inputs.db-name }}"
+        -user="${{ secrets.DB_PROD_USERNAME }}"
+        -password="${{ secrets.DB_PROD_PASSWORD }}"
+        repair -schemas="${{ env.SCHEMAS }}"
+
     - name: Apply migrations to production DB
       run: >-
         docker run --rm
@@ -192,6 +206,7 @@ jobs:
         -url="jdbc:postgresql://${{ secrets.DB_PROD_HOST }}/${{ inputs.db-name }}"
         -user="${{ secrets.DB_PROD_USERNAME }}"
         -password="${{ secrets.DB_PROD_PASSWORD }}"
+        -postgresql.transactional.lock=false
         migrate -schemas="${{ env.SCHEMAS }}"
 
     - name: Disconnect VPN

.github/workflows/alita-real-test.yml

@@ -137,9 +137,10 @@ jobs:
       run: >-
         docker run --rm --network host
         --volume ${{ github.workspace }}/src/alita-bot/migrations:/flyway/sql:ro
-        flyway/flyway
+        flyway/flyway:13.4.0
         -url="jdbc:postgresql://127.0.0.1:15433/alita_bot"
         -user=admin -password=admin
+        -postgresql.transactional.lock=false
         migrate -schemas=public
 
     - name: Seed bot_setting

.github/workflows/coupon-real-test.yml

@@ -248,9 +248,10 @@ jobs:
       run: >-
         docker run --rm --network host
         --volume ${{ github.workspace }}/src/coupon-hub-bot/migrations:/flyway/sql:ro
-        flyway/flyway
+        flyway/flyway:13.4.0
         -url="jdbc:postgresql://127.0.0.1:15434/coupon_hub_bot"
         -user=admin -password=admin
+        -postgresql.transactional.lock=false
         migrate -schemas=public
 
     - name: Seed bot_setting

.github/workflows/migration-immutability.yml

@@ -0,0 +1,22 @@
+name: "Lint: Migration immutability"
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  migration-immutability:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Fetch base ref
+      run: git fetch origin "${{ github.base_ref }}"
+
+    - name: Check migration immutability
+      run: scripts/check-migration-immutability.sh --range "origin/${{ github.base_ref }}...HEAD"

AGENTS.md

@@ -93,6 +93,14 @@ These tests are for **deliberate feature work and manual dev-iteration only** (`
 - CouponHubBot DB: `coupon_hub_bot`, role: `coupon_hub_bot_service`
 - AlitaBot DB: `alita_bot`, role: `alita_bot_service` (pgvector-backed — `message_embedding`/`interaction_memory` use the `vector` extension, `pgvector/pgvector:pg17` everywhere AlitaBot provisions its own Postgres)
 
+### `CREATE INDEX CONCURRENTLY` migrations
+
+Deploy (`flyway/flyway:13.4.0`) runs `migrate` with `-postgresql.transactional.lock=false` (session-level advisory lock, not Flyway's default transactional one since 9.1.2) so a CIC migration doesn't deadlock against Flyway's own coordination lock. `repair` runs unconditionally before every `migrate`.
+
+- Author CIC migrations like any other, in the normal versioned migrations dir — no `.sql.conf`, Flyway auto-detects them non-transactional. Make them idempotent: `DROP INDEX CONCURRENTLY IF EXISTS <name>; CREATE [UNIQUE] INDEX CONCURRENTLY <name> ON ...;`.
+- **If a CIC migration `Vn` fails on a data problem**, the data-fix migration's version must sit strictly between the last successfully-applied version and `Vn` (e.g. `V(n-1).1`) — never above `Vn`. Pending migrations run in ascending version order, so a higher-numbered fix never runs before `Vn`'s retry, and `Vn` fails identically on every deploy forever.
+- Unconditional `repair` means editing an already-applied migration's file gets its checksum silently realigned instead of failing validation — don't edit applied migrations.
+
 ## Settings configuration
 
 - All **non-secret** bot configuration lives in the `bot_setting` table. Env vars are only for secrets (`BOT_TELEGRAM_TOKEN`, `BOT_AUTH_TOKEN`, `AZURE_OCR_KEY`, `GITHUB_TOKEN`, `DATABASE_URL`, etc.).

scripts/check-migration-immutability.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Fails if a PR edits/deletes/renames a migration file already on the base branch.
+# --range A...B diffs against the merge-base, so files added-then-edited within the same PR still pass.
+set -euo pipefail
+
+MIGRATION_DIRS=(
+  "src/vahter-bot/migrations"
+  "src/coupon-hub-bot/migrations"
+  "src/alita-bot/migrations"
+)
+
+mode="${1:-}"
+case "$mode" in
+  --range)
+    range="${2:?--range requires <A...B>}"
+    a_ref="${range%%...*}"
+    b_ref="${range##*...}"
+    before_ref=$(git merge-base "$a_ref" "$b_ref")
+    after_ref="$b_ref"
+    ;;
+  *)
+    echo "usage: $0 --range <A...B>" >&2
+    exit 2
+    ;;
+esac
+
+offenders=$(git diff --no-ext-diff --no-color -M --diff-filter=MDR --name-status \
+  "$before_ref" "$after_ref" -- "${MIGRATION_DIRS[@]}")
+
+if [ -z "$offenders" ]; then
+  echo "migration-immutability: OK — no edited/deleted/renamed migration files"
+  exit 0
+fi
+
+echo "::error::Applied migrations are immutable — add a new migration instead of editing/deleting/renaming an existing one. Offending files:"
+echo "$offenders"
+exit 1

src/alita-bot/docker-compose.dev.yml

@@ -28,7 +28,7 @@ services:
       retries: 5
 
   flyway:
-    image: flyway/flyway
+    image: flyway/flyway:13.4.0
     container_name: alita-bot-flyway-dev
     depends_on:
       postgres:
@@ -39,7 +39,7 @@ services:
       FLYWAY_URL: jdbc:postgresql://postgres:5432/alita_bot
       FLYWAY_USER: admin
       FLYWAY_PASSWORD: admin
-    command: migrate -schemas=public
+    command: -postgresql.transactional.lock=false migrate -schemas=public
     restart: "no"
 
   seed-bot-settings:

src/coupon-hub-bot/docker-compose.dev.yml

@@ -22,7 +22,7 @@ services:
       retries: 5
 
   flyway:
-    image: flyway/flyway
+    image: flyway/flyway:13.4.0
     container_name: coupon-hub-flyway-dev
     depends_on:
       postgres:
@@ -33,7 +33,7 @@ services:
       FLYWAY_URL: jdbc:postgresql://postgres:5432/coupon_hub_bot
       FLYWAY_USER: admin
       FLYWAY_PASSWORD: admin
-    command: migrate -schemas=public
+    command: -postgresql.transactional.lock=false migrate -schemas=public
     restart: "no"
 
   seed-bot-settings:

tests/BotTestInfra/ContainerHelpers.fs

@@ -92,7 +92,7 @@ let createPostgresContainer (network: INetwork) (alias: string) (pgImage: string
         .Build()
 
 let createFlywayContainer (network: INetwork) (migrationsPath: string) (dbAlias: string) (dbName: string) (dbContainer: PostgreSqlContainer) =
-    ContainerBuilder("flyway/flyway")
+    ContainerBuilder("flyway/flyway:13.4.0")
         .WithNetwork(network)
         .WithBindMount(migrationsPath, "/flyway/sql", AccessMode.ReadOnly)
         // Without label=disable, hosts with SELinux enforcing block the container from
@@ -102,7 +102,8 @@ let createFlywayContainer (network: INetwork) (migrationsPath: string) (dbAlias:
         .WithEnvironment("FLYWAY_URL", $"jdbc:postgresql://{dbAlias}:5432/{dbName}")
         .WithEnvironment("FLYWAY_USER", "admin")
         .WithEnvironment("FLYWAY_PASSWORD", "admin")
-        .WithCommand("migrate", "-schemas=public")
+        // Must match prod's migrate flags, or CIC migrations deadlock only here, not in prod.
+        .WithCommand("-postgresql.transactional.lock=false", "migrate", "-schemas=public")
         .WithWaitStrategy(
             Wait.ForUnixContainer().AddCustomWaitStrategy(
                 { new IWaitUntil with