Switch to using goose for migrations

.claude/skills/kagent-dev/SKILL.md

@@ -79,10 +79,31 @@ After SQL changes, run `sqlc generate` in `go/core/internal/database` and commit
 
 ## Database changes
 
-- Add paired migrations and sqlc queries.
-- Preserve transaction boundaries for idempotency, ownership, lifecycle fencing, and task ordering.
-- Use PostgreSQL constraints for invariants that can be enforced atomically.
-- Keep migrations schema-agnostic and safe for multiple controller replicas.
+- PostgreSQL migrations use Goose with embedded SQL files.
+- Add each migration as `NNNNNN_description.sql`.
+- Do not add legacy split files ending in `.up.sql` or `.down.sql`.
+- Include one `-- +goose Up` section and one `-- +goose Down` section.
+- Goose makes the Down section optional, but Kagent requires it for the database CLI.
+- Do not use `-- +goose NO TRANSACTION`.
+- Goose must commit each schema change and its migration record in one transaction.
+- Never change, rename, or delete a migration after it merges.
+- Fix an accepted migration with a new migration.
+- Keep migration SQL schema-agnostic.
+- Change only objects that the migration source owns.
+- Do not use existence guards for source-owned objects; a migration ledger mismatch must fail.
+- Use existence guards only for shared bootstrap resources such as PostgreSQL extensions.
+- Each migration source must use its own migration table and advisory lock.
+- Register dependent sources after the sources that they need.
+- Do not add automatic down migrations when a later source fails.
+- A restart must continue from each source's last committed version.
+- Allow non-destructive startup when the database is ahead of the binary for rolling compatibility.
+- The Goose cutover requires a fresh PostgreSQL database.
+- Do not add a golang-migrate bridge for the cutover.
+- Keep `schema_migrations` for the core source.
+- Keep `vector_schema_migrations` for the vector source.
+- Run `make -C go sqlc-generate` after a migration change.
+- Test the Up and Down sections against PostgreSQL.
+- Use PostgreSQL constraints for invariants that the database can enforce atomically.
 
 ## Testing and CI
 

.github/actions/upgrade-test-setup/action.yaml

@@ -2,8 +2,8 @@ name: Upgrade Test Setup
 
 description: >-
   Shared prelude for the upgrade-tests and rolling-upgrade-tests jobs: resolve
-  the upgrade-from version (with the prev-stable == adjacent skip) and bring up
-  the build/cluster toolchain. Exposes the resolved version and skip flag so the
+  the upgrade-from version, skip redundant and pre-Goose legs, and bring up the
+  build/cluster toolchain. Exposes the resolved version and skip flag so the
   caller can gate its test step. The caller MUST run actions/checkout (with
   fetch-depth: 0 + fetch-tags) before this action — a local action is loaded
   from the checked-out workspace and the version resolvers need full history.
@@ -15,7 +15,7 @@ inputs:
 
 outputs:
   skip:
-    description: '"true" when this leg is redundant (prev-stable == adjacent) and the caller should skip its test step.'
+    description: '"true" when this leg is redundant or predates Goose and the caller should skip its test step.'
     value: ${{ steps.resolve.outputs.skip }}
   version:
     description: The resolved upgrade-from version (empty when skip is true).
@@ -66,6 +66,11 @@ runs:
           fi
           V="$(./scripts/upgrade-from-version.sh)"
         fi
+        if [[ "$V" == 0.10.* ]]; then
+          echo "previous release v$V predates Goose; skipping upgrade tests."
+          echo "skip=true" >> "$GITHUB_OUTPUT"
+          exit 0
+        fi
         echo "version=$V" >> "$GITHUB_OUTPUT"
         echo "=== Upgrade test (${{ inputs.upgrade-from }} leg): will upgrade FROM v$V TO the current build — building images next ==="
         echo "::notice title=Upgrade from::v$V (${{ inputs.upgrade-from }} leg) -> current build"

.github/workflows/ci.yaml

@@ -207,9 +207,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # prev-stable: the previous minor line's latest published version,
+        #   including a prerelease before GA — runs on every base.
         # adjacent: the same-line latest published version, including prereleases
         #   — only meaningful on a release branch and skipped on main.
-        upgrade-from: [adjacent]
+        upgrade-from: [adjacent, prev-stable]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
@@ -253,9 +255,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # prev-stable: the previous minor line's latest published version,
+        #   including a prerelease before GA — runs on every base.
         # adjacent: the same-line latest published version, including prereleases
         #   — only meaningful on a release branch and skipped on main.
-        upgrade-from: [adjacent]
+        upgrade-from: [adjacent, prev-stable]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6

.github/workflows/migration-immutability.yaml

@@ -16,20 +16,40 @@ jobs:
 
       - name: Fail if any existing migration file was modified
         run: |
-          # List files under go/core/pkg/migrations/ that were changed relative
-          # to the merge base of this PR.  We only care about modifications (M)
-          # and renames (R); additions (A) are fine.
           BASE=$(git merge-base HEAD origin/${{ github.base_ref }})
-          MODIFIED=$(git diff --name-only --diff-filter=MR "$BASE" HEAD \
-            -- 'go/core/pkg/migrations/**/*.sql')
+          CHANGED=$(git diff --no-renames --name-status --diff-filter=MDR "$BASE" HEAD \
+            -- 'go/core/pkg/migrations/**/*.sql' \
+            | awk '$1 != "D" || $2 !~ /\.(up|down)\.sql$/ { print $2 }')
 
-          if [ -n "$MODIFIED" ]; then
-            echo "ERROR: The following migration files were modified."
+          if [ -n "$CHANGED" ]; then
+            echo "ERROR: These migration files changed or were removed."
             echo "Migration files are immutable once merged."
             echo "Fix bugs with a new migration instead."
             echo ""
-            echo "$MODIFIED"
+            echo "$CHANGED"
             exit 1
           fi
 
           echo "OK: no existing migration files were modified."
+
+      - name: Reject non-transactional migrations
+        run: |
+          SPLIT=$(find go/core/pkg/migrations -type f \
+            \( -name '*.up.sql' -o -name '*.down.sql' \))
+
+          if [ -n "$SPLIT" ]; then
+            echo "ERROR: Legacy split migration files are not supported."
+            echo "$SPLIT"
+            exit 1
+          fi
+
+          FORBIDDEN=$(find go/core/pkg/migrations -type f -name '*.sql' \
+            -exec grep -Eil -- '^--[[:space:]]*\+goose[[:space:]]*NO[[:space:]]+TRANSACTION[[:space:]]*$' {} + || true)
+
+          if [ -n "$FORBIDDEN" ]; then
+            echo "ERROR: These migrations disable transactions."
+            echo "$FORBIDDEN"
+            exit 1
+          fi
+
+          echo "OK: all active migrations use transactions."

Makefile

@@ -439,17 +439,10 @@ helm-uninstall: ## Uninstall kagent and kagent-crds Helm releases from the kind
 	helm uninstall kagent --namespace kagent --kube-context kind-$(KIND_CLUSTER_NAME) --wait
 	helm uninstall kagent-crds --namespace kagent --kube-context kind-$(KIND_CLUSTER_NAME) --wait
 
-# Upgrade test targets install the previous released kagent chart from the public
-# OCI registry, build the current images, then run the assertions in
-# go/core/test/upgrade. These tests are deliberately kept out of test/e2e: they
-# mutate the cluster (upgrade then reverse-migrate it) and so cannot share the
-# e2e suite's cluster. The Go test performs the actual upgrade to the current
-# build by invoking `make helm-install-provider`. UPGRADE_FROM_VERSION defaults to
-# the latest version reachable from HEAD (scripts/upgrade-from-version.sh); CI runs
-# this against two targets via a matrix — that adjacent version and the previous
-# release line's latest published version (scripts/prev-stable-version.sh) — and
-# you can pin either locally, e.g.
-# `UPGRADE_FROM_VERSION=$$(./scripts/prev-stable-version.sh)`.
+# Upgrade tests install a previous Kagent chart and upgrade it to the current build.
+# The tests use a separate cluster because they change the database and deployment.
+# The tests skip releases that do not use Goose.
+# UPGRADE_FROM_VERSION selects the previous release.
 # The previous install pins the bundled Postgres image to whatever the
 # upgrade-from release's own install target shipped (resolved inside
 # install-previous-release), so the baseline matches how that release actually
@@ -504,37 +497,29 @@ install-previous-release: ## Install the previous released kagent + kagent-crds
 		--set providers.openAI.apiKey="$${OPENAI_API_KEY:-test}" \
 		$$db_flags $(UPGRADE_PREV_EXTRA_ARGS)
 
-# run-upgrade-tests installs the previous release, builds the current images, and
-# runs the DB-layer upgrade scenario in TestUpgrade: seed -> upgrade -> controller
-# rollout (no crash) -> data survival -> schema-equivalence (upgraded == clean
-# install) -> reverse schema to target (down files) + data survival. At each
-# state it also runs a version-matched invoke e2e slice (TestE2EInvokeInlineAgent)
-# so the serving controller's real query paths are exercised, not just psql: the
-# HEAD tree post-upgrade, and the previous release's own tree (a git worktree at
-# its tag, in .upgrade-prev) for the old-code-against-new-schema and post-rollback
-# states. KAGENT_LOCAL_HOST (kind gateway IP) lets the agent reach the in-process
-# mock LLM; without it the invoke slices self-skip and only the DB round-trip runs.
+# run-upgrade-tests installs the previous release and upgrades it to the current build.
+# The test skips releases that do not use Goose.
+# Later Goose releases test one Substrate agent across the target migrations,
+# data survival, schema equality, previous/current controller startup, and a
+# complete application and schema rollback to the previous release.
+# KAGENT_LOCAL_HOST lets the agent reach the local mock LLM.
 # Prerequisite (provided by CI as a separate step; run it locally first): a kind
 # cluster (make create-kind-cluster). The controller tolerates the missing
 # agent-sandbox CRD (the owned-resource watch is skipped), and these tests create
 # no SandboxAgents, so agent-sandbox is not required.
 .PHONY: announce-upgrade-from
 announce-upgrade-from: ## Print the upgrade-from -> to versions (runs before the build so it is clear up front)
-	@echo "=== Upgrade test: FROM $(UPGRADE_FROM_VERSION) TO $(VERSION) — building current images next ==="
+	@echo "=== Upgrade test: FROM $(UPGRADE_FROM_VERSION) TO $(VERSION). Building current images next. ==="
 
 .PHONY: run-upgrade-tests
-run-upgrade-tests: announce-upgrade-from build install-previous-release ## Install the previous release, build current images, and run the upgrade + version-matched invoke tests
+run-upgrade-tests: announce-upgrade-from build install-previous-release ## Test an upgrade between Goose releases
 	@echo "=== Upgrade test: $(UPGRADE_FROM_VERSION) -> $(VERSION) (registry=$(DOCKER_REGISTRY)) ==="
 	@set -e; \
-	git worktree remove --force "$(CURDIR)/.upgrade-prev" 2>/dev/null || true; \
-	git worktree add --detach "$(CURDIR)/.upgrade-prev" "v$(UPGRADE_FROM_VERSION)"; \
-	trap 'git worktree remove --force "$(CURDIR)/.upgrade-prev" 2>/dev/null || true' EXIT; \
 	kind_gw="$$($(CONTAINER_RUNTIME) network inspect kind -f '{{range .IPAM.Config}}{{if .Gateway}}{{.Gateway}}{{"\n"}}{{end}}{{end}}' | grep -E '^[0-9]+\.' | head -1)"; \
 	echo "kind gateway (KAGENT_LOCAL_HOST): $$kind_gw"; \
 	cd go && \
 	RUN_UPGRADE_TESTS=true \
 	REPO_ROOT=$(CURDIR) \
-	PREV_E2E_DIR=$(CURDIR)/.upgrade-prev \
 	KAGENT_LOCAL_HOST="$$kind_gw" \
 	UPGRADE_FROM_VERSION=$(UPGRADE_FROM_VERSION) \
 	VERSION=$(VERSION) \

go/core/pkg/app/app.go

@@ -167,13 +167,18 @@ func Run(ctx context.Context, opts Options) error {
 	if err != nil {
 		return err
 	}
+	vectorEnabled := envBool("DATABASE_VECTOR_ENABLED")
 	// Appended, not merged: the built-in tracks must reach their final version
 	// before a library consumer's tables, which may reference them.
-	sources := append(migrations.BuiltinSources(false), opts.ExtraMigrations...)
-	if err := migrations.RunUp(ctx, dbURL, sources); err != nil {
+	sources := append(migrations.BuiltinSources(vectorEnabled), opts.ExtraMigrations...)
+	if envBool("SKIP_MIGRATIONS") {
+		if err := migrations.VerifyMigrated(ctx, dbURL, sources); err != nil {
+			return fmt.Errorf("verify database migrations: %w", err)
+		}
+	} else if err := migrations.RunUp(ctx, dbURL, sources); err != nil {
 		return fmt.Errorf("run database migrations: %w", err)
 	}
-	db, err := database.Connect(ctx, &database.PostgresConfig{URL: dbURL})
+	db, err := database.Connect(ctx, &database.PostgresConfig{URL: dbURL, VectorEnabled: vectorEnabled})
 	if err != nil {
 		return err
 	}