diff --git a/.github/workflows/_bot-deploy.yml b/.github/workflows/_bot-deploy.yml index 56afabb8..d766fd5e 100644 --- a/.github/workflows/_bot-deploy.yml +++ b/.github/workflows/_bot-deploy.yml @@ -60,7 +60,7 @@ on: description: "For the SRE agent (_sre-agent.yml), invoked on deploy/verify failure" env: - DOCKER_IMAGE: flyway/flyway + DOCKER_IMAGE: flyway/flyway:13.4.0 SCHEMAS: public jobs: @@ -182,6 +182,20 @@ jobs: WIREGUARD_CONFIG: ${{ secrets.WIREGUARD_CONFIG }} SPLIT_TUNNEL: "false" + # Unconditional: no-op if the last run's migrate succeeded (checksum-safe), + # required to clear a failed CIC migration's success=false history row. + - name: Repair production DB migration history + run: >- + docker run --rm + --network host + --volume ${{ github.workspace }}/${{ inputs.migrations-path }}:/flyway/sql:ro + --volume ${{ github.workspace }}/reports:/flyway/reports + "${{ env.DOCKER_IMAGE }}" + -url="jdbc:postgresql://${{ secrets.DB_PROD_HOST }}/${{ inputs.db-name }}" + -user="${{ secrets.DB_PROD_USERNAME }}" + -password="${{ secrets.DB_PROD_PASSWORD }}" + repair -schemas="${{ env.SCHEMAS }}" + - name: Apply migrations to production DB run: >- docker run --rm @@ -192,6 +206,7 @@ jobs: -url="jdbc:postgresql://${{ secrets.DB_PROD_HOST }}/${{ inputs.db-name }}" -user="${{ secrets.DB_PROD_USERNAME }}" -password="${{ secrets.DB_PROD_PASSWORD }}" + -postgresql.transactional.lock=false migrate -schemas="${{ env.SCHEMAS }}" - name: Disconnect VPN diff --git a/.github/workflows/alita-real-test.yml b/.github/workflows/alita-real-test.yml index a495b02d..99e5cd17 100644 --- a/.github/workflows/alita-real-test.yml +++ b/.github/workflows/alita-real-test.yml @@ -137,9 +137,10 @@ jobs: run: >- docker run --rm --network host --volume ${{ github.workspace }}/src/alita-bot/migrations:/flyway/sql:ro - flyway/flyway + flyway/flyway:13.4.0 -url="jdbc:postgresql://127.0.0.1:15433/alita_bot" -user=admin -password=admin + -postgresql.transactional.lock=false migrate -schemas=public - name: Seed bot_setting diff --git a/.github/workflows/coupon-real-test.yml b/.github/workflows/coupon-real-test.yml index ad197499..0d7f2d26 100644 --- a/.github/workflows/coupon-real-test.yml +++ b/.github/workflows/coupon-real-test.yml @@ -248,9 +248,10 @@ jobs: run: >- docker run --rm --network host --volume ${{ github.workspace }}/src/coupon-hub-bot/migrations:/flyway/sql:ro - flyway/flyway + flyway/flyway:13.4.0 -url="jdbc:postgresql://127.0.0.1:15434/coupon_hub_bot" -user=admin -password=admin + -postgresql.transactional.lock=false migrate -schemas=public - name: Seed bot_setting diff --git a/.github/workflows/migration-immutability.yml b/.github/workflows/migration-immutability.yml new file mode 100644 index 00000000..ca159617 --- /dev/null +++ b/.github/workflows/migration-immutability.yml @@ -0,0 +1,22 @@ +name: "Lint: Migration immutability" + +on: + pull_request: + +permissions: + contents: read + +jobs: + migration-immutability: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch base ref + run: git fetch origin "${{ github.base_ref }}" + + - name: Check migration immutability + run: scripts/check-migration-immutability.sh --range "origin/${{ github.base_ref }}...HEAD" diff --git a/AGENTS.md b/AGENTS.md index 74812df7..6d7fc20f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,6 +93,14 @@ These tests are for **deliberate feature work and manual dev-iteration only** (` - CouponHubBot DB: `coupon_hub_bot`, role: `coupon_hub_bot_service` - AlitaBot DB: `alita_bot`, role: `alita_bot_service` (pgvector-backed — `message_embedding`/`interaction_memory` use the `vector` extension, `pgvector/pgvector:pg17` everywhere AlitaBot provisions its own Postgres) +### `CREATE INDEX CONCURRENTLY` migrations + +Deploy (`flyway/flyway:13.4.0`) runs `migrate` with `-postgresql.transactional.lock=false` (session-level advisory lock, not Flyway's default transactional one since 9.1.2) so a CIC migration doesn't deadlock against Flyway's own coordination lock. `repair` runs unconditionally before every `migrate`. + +- Author CIC migrations like any other, in the normal versioned migrations dir — no `.sql.conf`, Flyway auto-detects them non-transactional. Make them idempotent: `DROP INDEX CONCURRENTLY IF EXISTS ; CREATE [UNIQUE] INDEX CONCURRENTLY ON ...;`. +- **If a CIC migration `Vn` fails on a data problem**, the data-fix migration's version must sit strictly between the last successfully-applied version and `Vn` (e.g. `V(n-1).1`) — never above `Vn`. Pending migrations run in ascending version order, so a higher-numbered fix never runs before `Vn`'s retry, and `Vn` fails identically on every deploy forever. +- Unconditional `repair` means editing an already-applied migration's file gets its checksum silently realigned instead of failing validation — don't edit applied migrations. + ## Settings configuration - All **non-secret** bot configuration lives in the `bot_setting` table. Env vars are only for secrets (`BOT_TELEGRAM_TOKEN`, `BOT_AUTH_TOKEN`, `AZURE_OCR_KEY`, `GITHUB_TOKEN`, `DATABASE_URL`, etc.). diff --git a/scripts/check-migration-immutability.sh b/scripts/check-migration-immutability.sh new file mode 100755 index 00000000..34afefdb --- /dev/null +++ b/scripts/check-migration-immutability.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Fails if a PR edits/deletes/renames a migration file already on the base branch. +# --range A...B diffs against the merge-base, so files added-then-edited within the same PR still pass. +set -euo pipefail + +MIGRATION_DIRS=( + "src/vahter-bot/migrations" + "src/coupon-hub-bot/migrations" + "src/alita-bot/migrations" +) + +mode="${1:-}" +case "$mode" in + --range) + range="${2:?--range requires }" + a_ref="${range%%...*}" + b_ref="${range##*...}" + before_ref=$(git merge-base "$a_ref" "$b_ref") + after_ref="$b_ref" + ;; + *) + echo "usage: $0 --range " >&2 + exit 2 + ;; +esac + +offenders=$(git diff --no-ext-diff --no-color -M --diff-filter=MDR --name-status \ + "$before_ref" "$after_ref" -- "${MIGRATION_DIRS[@]}") + +if [ -z "$offenders" ]; then + echo "migration-immutability: OK — no edited/deleted/renamed migration files" + exit 0 +fi + +echo "::error::Applied migrations are immutable — add a new migration instead of editing/deleting/renaming an existing one. Offending files:" +echo "$offenders" +exit 1 diff --git a/src/alita-bot/docker-compose.dev.yml b/src/alita-bot/docker-compose.dev.yml index f2d696c9..3e778d47 100644 --- a/src/alita-bot/docker-compose.dev.yml +++ b/src/alita-bot/docker-compose.dev.yml @@ -28,7 +28,7 @@ services: retries: 5 flyway: - image: flyway/flyway + image: flyway/flyway:13.4.0 container_name: alita-bot-flyway-dev depends_on: postgres: @@ -39,7 +39,7 @@ services: FLYWAY_URL: jdbc:postgresql://postgres:5432/alita_bot FLYWAY_USER: admin FLYWAY_PASSWORD: admin - command: migrate -schemas=public + command: -postgresql.transactional.lock=false migrate -schemas=public restart: "no" seed-bot-settings: diff --git a/src/coupon-hub-bot/docker-compose.dev.yml b/src/coupon-hub-bot/docker-compose.dev.yml index 7d8ef0e0..0dae5ceb 100644 --- a/src/coupon-hub-bot/docker-compose.dev.yml +++ b/src/coupon-hub-bot/docker-compose.dev.yml @@ -22,7 +22,7 @@ services: retries: 5 flyway: - image: flyway/flyway + image: flyway/flyway:13.4.0 container_name: coupon-hub-flyway-dev depends_on: postgres: @@ -33,7 +33,7 @@ services: FLYWAY_URL: jdbc:postgresql://postgres:5432/coupon_hub_bot FLYWAY_USER: admin FLYWAY_PASSWORD: admin - command: migrate -schemas=public + command: -postgresql.transactional.lock=false migrate -schemas=public restart: "no" seed-bot-settings: diff --git a/tests/BotTestInfra/ContainerHelpers.fs b/tests/BotTestInfra/ContainerHelpers.fs index 9aad7ba0..6b83d39f 100644 --- a/tests/BotTestInfra/ContainerHelpers.fs +++ b/tests/BotTestInfra/ContainerHelpers.fs @@ -92,7 +92,7 @@ let createPostgresContainer (network: INetwork) (alias: string) (pgImage: string .Build() let createFlywayContainer (network: INetwork) (migrationsPath: string) (dbAlias: string) (dbName: string) (dbContainer: PostgreSqlContainer) = - ContainerBuilder("flyway/flyway") + ContainerBuilder("flyway/flyway:13.4.0") .WithNetwork(network) .WithBindMount(migrationsPath, "/flyway/sql", AccessMode.ReadOnly) // Without label=disable, hosts with SELinux enforcing block the container from @@ -102,7 +102,8 @@ let createFlywayContainer (network: INetwork) (migrationsPath: string) (dbAlias: .WithEnvironment("FLYWAY_URL", $"jdbc:postgresql://{dbAlias}:5432/{dbName}") .WithEnvironment("FLYWAY_USER", "admin") .WithEnvironment("FLYWAY_PASSWORD", "admin") - .WithCommand("migrate", "-schemas=public") + // Must match prod's migrate flags, or CIC migrations deadlock only here, not in prod. + .WithCommand("-postgresql.transactional.lock=false", "migrate", "-schemas=public") .WithWaitStrategy( Wait.ForUnixContainer().AddCustomWaitStrategy( { new IWaitUntil with