diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5997d71..8282795 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,7 +182,7 @@ jobs: --ignorefile .trivyignore.yaml \ --exit-code "$code" \ --format table \ - "ghcr.io/simtabi/dbpantry-${ENGINE}:${VERSION}" + "localhost/dbpantry-${ENGINE}:${VERSION}" - name: Report unfixed findings (never fails) if: always() @@ -191,7 +191,7 @@ jobs: VERSION: ${{ matrix.entry.version }} run: | trivy image --severity HIGH,CRITICAL --exit-code 0 --format table \ - "ghcr.io/simtabi/dbpantry-${ENGINE}:${VERSION}" || true + "localhost/dbpantry-${ENGINE}:${VERSION}" || true # --------------------------------------------------------------------------- # When the scheduled build fails, say so where people look (design.md D-71). diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bbfb01a..787fd37 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -209,7 +209,7 @@ jobs: run: | trivy image --severity HIGH,CRITICAL --ignore-unfixed \ --ignorefile .trivyignore.yaml --exit-code 1 --format table \ - "ghcr.io/simtabi/dbpantry-${ENGINE}:${VERSION}" + "localhost/dbpantry-${ENGINE}:${VERSION}" # --------------------------------------------------------------------------- # Gate 2: the acceptance harness, on both architectures. diff --git a/CHANGELOG.md b/CHANGELOG.md index f21caf0..ab6f9fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ waiving them. ### Upgrade +- **Images are now named `localhost/dbpantry-*`.** Images you built before keep their old + `ghcr.io/simtabi/dbpantry-*` names, which nothing runs any more. Run `make build` once; the next + `make up` recreates the containers on the new names. Volumes and data are untouched. To build, + run and deploy under your own registry instead, export `MDB_IMAGE_PREFIX` (for example + `registry.example.com/team`) before `make build` and `make up` (design.md D-78). - **Prod: PostgreSQL's own port stops being published.** With the prod overlay applied (see Changed), a prod stack under `MDB_PUBLISH=direct` publishes the pooler, `pg-pooler` on `MDB_PORT_BASE + 100` (54100 by default), and no engine port: 54000 and the other engines' ports go away. Point applications at 54100 with @@ -29,9 +34,20 @@ waiving them. - **Container images are no longer published.** A release publishes the GitHub release only, with the notes and the source tarball (owner, 2026-11-13). The `ghcr.io/simtabi/dbpantry-*` images are frozen: no new tag is pushed and nothing is deleted from ghcr.io. Build the images with - `make build`. The Kubernetes backend and the Helm chart still name `ghcr.io/simtabi` as their - registry, so push your own build there or point `global.imageRegistry` at your own registry - (design.md D-77). + `make build` (design.md D-77). +- **Images are built, run and deployed as `localhost/dbpantry-:`,** and nothing + falls back to a remote registry. One default, `MDB_IMAGE_PREFIX=localhost` in + `scripts/env-lib.sh`, replaces every hard-coded registry: Compose, the generated engine and pooler + services, the patroni build, the Makefile, `scripts/{build,upgrade,verify-backups,transfer,scan}`, + the verify checks and the CI scan steps. The Helm chart's `global.imageRegistry` defaults to + `localhost`, `make up` on Kubernetes passes it the prefix the images were built under, and kind + loading follows it; `imagePullPolicy` stays `IfNotPresent`. Setting `MDB_IMAGE_PREFIX` (or + `global.imageRegistry` when installing the chart by hand) prefixes every image. The Kubernetes + guide shows `kind load docker-image`, `minikube image load` and pushing to your own registry. + Check 73 fails if any default names a remote registry or a set prefix stops reaching every image, + and check 49 now resolves every `${VAR:-default}` in an image reference. Checks 42 and 72 recognise + our images as `localhost/dbpantry-*`, and `make init-prod` resolves the caddy image the way Compose + does (`mdb_compose_ref`) before running it (design.md D-78). - **A release runs on a pushed `vX.Y.Z` tag, or on request for a tag already on GitHub:** `gh workflow run release.yml --repo simtabi/dbpantry -f tag=vX.Y.Z`. This replaces the dispatch gated on `-f confirm=release-go` (D-76). Every job checks out the tag, and the first step refuses diff --git a/Makefile b/Makefile index 20c8cb3..f1a67bb 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,10 @@ env_project := $(call env_get,COMPOSE_PROJECT_NAME) COMPOSE_PROJECT_NAME ?= $(or $(env_project),mdb) export COMPOSE_PROJECT_NAME ENV_FILE_PROD ?= .env.prod +# The prefix our images are built and run under: MDB_IMAGE_PREFIX from the +# environment, else the one default in scripts/env-lib.sh (localhost). Set but +# empty counts as unset, as it does for the scripts. +MDB_IMAGE_PREFIX := $(or $(MDB_IMAGE_PREFIX),$(shell bash -c '. scripts/env-lib.sh && mdb_image_prefix' 2>/dev/null)) # Every container call goes through scripts/runtime, which picks Docker, Podman # or nerdctl (CONTAINER_RUNTIME=auto|docker|podman|nerdctl; design.md D-60). RUNTIME := scripts/runtime @@ -187,7 +191,7 @@ shell: ## Open the cli image (tools with no server required) --network $(COMPOSE_PROJECT_NAME)_net \ -v "$(CURDIR)/certs:/certs:ro" \ -v "$(CURDIR)/backups:/work/backups" \ - ghcr.io/simtabi/dbpantry-cli:dev + $(MDB_IMAGE_PREFIX)/dbpantry-cli:dev .PHONY: new-project new-project: ## Provision DB + roles + extensions, print the Laravel .env block diff --git a/compose.ha.yml b/compose.ha.yml index 1673a12..c6e2a20 100644 --- a/compose.ha.yml +++ b/compose.ha.yml @@ -42,7 +42,8 @@ x-patroni: &patroni dockerfile: patroni/Dockerfile args: ENGINE_VERSION: ${MDB_PG_VERSION:-17} - image: ghcr.io/simtabi/dbpantry-patroni:${MDB_PG_VERSION:-17} + IMAGE_PREFIX: ${MDB_IMAGE_PREFIX:-localhost} + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-patroni:${MDB_PG_VERSION:-17} profiles: [ha] restart: unless-stopped networks: [mdb] @@ -210,7 +211,7 @@ services: # --------------------------------------------------------------------------- pgbouncer: profiles: [ha-pooler] - image: ghcr.io/simtabi/dbpantry-pgbouncer:dev + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-pgbouncer:dev restart: unless-stopped depends_on: haproxy: {condition: service_healthy} diff --git a/deploy/helm/dbpantry/values.yaml b/deploy/helm/dbpantry/values.yaml index 826b40f..d9f08cb 100644 --- a/deploy/helm/dbpantry/values.yaml +++ b/deploy/helm/dbpantry/values.yaml @@ -6,9 +6,13 @@ # engines you want and create the Secret named in global.secretName first. global: - # Registry and repository prefix for the dbpantry images. The tag is each - # engine's version (engines..tag). - imageRegistry: ghcr.io/simtabi + # Registry and repository prefix for the dbpantry images, which are named + # /dbpantry-: (the tag is engines..tag). + # localhost is the name `make build` gives them, so a kind or minikube + # cluster runs images loaded into it (docs/guides/kubernetes.md). Set it to + # your own registry (registry.example.com/team) after pushing them there. + # No dbpantry image is published to any public registry. + imageRegistry: localhost imagePullPolicy: IfNotPresent imagePullSecrets: [] # "" uses the cluster's default StorageClass. diff --git a/design.md b/design.md index 27fefe1..f109ed5 100644 --- a/design.md +++ b/design.md @@ -9,7 +9,7 @@ Everything below was verified during this session against live registries, the G | Field | Value | |---|---| | Repository | `github.com/simtabi/dbpantry` (was `dbpantry-docker`, and before that `db-toolkit-docker`) | -| Images | `ghcr.io/simtabi/dbpantry-{pg,mysql,mariadb,cli}` | +| Images | `localhost/dbpantry-{pg,mysql,mariadb,cli}`, built locally (`MDB_IMAGE_PREFIX`, D-78); not published | | Docs | [`docs/`](docs/) in this repository; no hosted site yet | | Product page | none yet | | Env prefix | `MDB_` | @@ -22,9 +22,9 @@ Resolves the three-way conflict recorded in decision **D-02**. | Service | Image | Profile | Internal port | Host port env | Published by default | |---|---|---|---|---|---| -| `pg` | `ghcr.io/simtabi/dbpantry-pg:` | `pg` | 5432 | `MDB_PG_HOST_PORT` | no | -| `mysql` | `ghcr.io/simtabi/dbpantry-mysql:` | `mysql` | 3306 | `MDB_MYSQL_HOST_PORT` | no | -| `mariadb` | `ghcr.io/simtabi/dbpantry-mariadb:` | `mariadb` | 3306 | `MDB_MARIADB_HOST_PORT` (3307) | no | +| `pg` | `localhost/dbpantry-pg:` | `pg` | 5432 | `MDB_PG_HOST_PORT` | no | +| `mysql` | `localhost/dbpantry-mysql:` | `mysql` | 3306 | `MDB_MYSQL_HOST_PORT` | no | +| `mariadb` | `localhost/dbpantry-mariadb:` | `mariadb` | 3306 | `MDB_MARIADB_HOST_PORT` (3307) | no | | `adminer` | `adminer:5.4.1` | `ui` | 8080 | `MDB_ADMINER_HOST_PORT` | no (via Caddy) | | `phpmyadmin` | `phpmyadmin:5.2.3` | `ui` | 80 | `MDB_PMA_HOST_PORT` | no (via Caddy) | | `pgadmin` | `dpage/pgadmin4:9.9` | `ui` | 80 | `MDB_PGADMIN_HOST_PORT` | no (via Caddy) | @@ -36,7 +36,7 @@ Resolves the three-way conflict recorded in decision **D-02**. | `pgbouncer` | `dbpantry-pgbouncer` (built) | `prod` (required), `ha` | 6432 | `MDB_PGBOUNCER_HOST_PORT` | no | | `etcd` | `gcr.io/etcd-development/etcd:v3.6.6` | `ha` | 2379/2380 | — | no | | `haproxy` | `haproxy:3.2-alpine` | `ha` | 5432 w / 5433 r | `MDB_HAPROXY_*_PORT` | no | -| `cli` | `ghcr.io/simtabi/dbpantry-cli:` | on demand | — | — | no | +| `cli` | `localhost/dbpantry-cli:` | on demand | — | — | no | The exporter rows match `docker-compose.yml` as of 2026-11-03. The compose files are the source of truth for every image here; the `etcd`, `haproxy` and `caddy` rows above still show the step 1 plan and differ from `compose.ha.yml` (`quay.io/coreos/etcd:v3.5.21`, `haproxy:3.0-alpine`) and `docker-compose.yml` (the built `dbpantry-caddy` image). @@ -492,6 +492,8 @@ Relied on: pgBouncer configuration reference (`, because a dispatch's `GITHUB_SHA` is its branch's head, not the tag's commit: the tag-on-`main` check now tests `HEAD` after that checkout. No pre-release tag filter is declared, since dbpantry refuses pre-releases. dbpantry builds no release binaries, so it keeps its own jobs rather than calling `simtabi/.github`'s shared `go-release.yml`, which runs GoReleaser. The Kubernetes backend (`scripts/backends/kubernetes.sh`) and the Helm chart (`global.imageRegistry: ghcr.io/simtabi`) still name the frozen registry; changing them is a separate decision. The guard is `src/internal/release`'s `TestReleaseRunsOnAVersionTagOrOnRequest`, which fails on any other trigger, a branch or non-version tag filter, a missing or optional `tag` input, a first step that is not the refusal, a checkout of anything but the tag, a job that does not need the gating job, or any image publish (registry login or push, `buildx --push`, `imagetools create`, `push-to-registry: true`, a `docker/` publish action, `packages: write`); `TestReleasePolicyRejectsAWrongRelease` proves each can fail, and that a local build and scan passes. +**D-78: our images are named localhost/dbpantry-*, from one default, and nothing names a remote registry.** Follows D-77. With images no longer published, `ghcr.io/simtabi/dbpantry-*` stayed the name every image was built, run and deployed under: `scripts/build` defaulted `MDB_IMAGE_PREFIX` to it, Compose, the generated engine services, the pooler descriptor, the patroni Dockerfile, the Makefile, five scripts, six checks and both workflows wrote it out, the Helm chart's `global.imageRegistry` defaulted to it, and the Kubernetes backend loaded it into kind. A cluster without an image therefore pulled from a registry that holds nothing. The owner chose `localhost/` (2026-11-13) over an unqualified `dbpantry/`, which check 49 refuses because Podman will not resolve a short name (D-60) and which Docker would expand to Docker Hub on a pull; `localhost` is fully qualified, is the prefix Podman gives local builds, never reaches a remote registry, and loads into kind and minikube as it is. The default is written once, `MDB_IMAGE_PREFIX_DEFAULT=localhost` in `scripts/env-lib.sh`, with `mdb_image_prefix`, `mdb_image` and `mdb_image_ref` (which moves a descriptor's `localhost/dbpantry-pgbouncer:dev` to the prefix in force); the Makefile reads it from there. Files that cannot read it repeat it: Compose as `${MDB_IMAGE_PREFIX:-localhost}`, the patroni Dockerfile's `ARG IMAGE_PREFIX`, the chart's `global.imageRegistry`, the pooler descriptor, and the CI scan steps, which build with the default. `MDB_IMAGE_PREFIX` stays an environment variable, as before, not a `.env` setting. On Kubernetes, `scripts/helm-values` renders `global.imageRegistry` from it and `_mdb_k8s_local_images` reads that back, so kind loads exactly the names the chart runs; `imagePullPolicy` stays `IfNotPresent`. Check 49's Compose parser resolved only a leading `${VAR:-default}`, which misread a prefixed reference as `localhost}/...`; it now resolves every one, innermost first, and still counts 55 references. Check 73 holds every default to `localhost`, fails on a dotted registry before `dbpantry-` in any tracked file but prose, limits `ghcr.io/simtabi` to nine files that describe the frozen images or their history (each at its measured count), and proves that a set prefix reaches `mdb_image`, the pooler, `make shell`, helm-values, kind loading and the rendered chart. Images built before keep their old names: `make build` once, and the next `make up` recreates the containers; volumes are untouched. The first CI run of this change failed four checks, each a reader of the old name the literal search missed: check 42 allowlisted our images as `ghcr\.io/simtabi/[a-z-]+` and check 72 recognised them by the same regex, so both now name `localhost/dbpantry-*` (72 under the prefix in force); `scripts/init` passed the caddy service's raw `image:` line to `run`, which is now Compose syntax, so it resolves it with `mdb_compose_ref` and `make init-prod` (check 26) works again; and check 73 itself listed one image in CI because `make verify` exports `COMPOSE_PROFILES`, which `helm-values` ranks above the `MDB_PROFILES` the check set. Check 73 now sets both, counts the escaped spelling too, and fails on any script that runs a Compose `image:` line without `mdb_compose_ref`. A check run directly is not the check `make verify` runs: reproduce through `make verify CHECKS=`. + ## 8. arm64 deliverability **Nothing in the spec is undeliverable on arm64.** The named unknown anticipated gaps that do not exist: diff --git a/docker-compose.yml b/docker-compose.yml index 8c85393..ec2ba89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,7 +119,7 @@ services: build: context: ./images dockerfile: caddy/Dockerfile - image: ghcr.io/simtabi/dbpantry-caddy:dev + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-caddy:dev restart: unless-stopped <<: *hardening # Binding 80/443 needs this one capability back; everything else stays @@ -159,7 +159,7 @@ services: # --------------------------------------------------------------------------- backup-pg: profiles: [backup] - image: ghcr.io/simtabi/dbpantry-pg:${MDB_PG_VERSION:-17} + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-pg:${MDB_PG_VERSION:-17} restart: unless-stopped <<: *hardening entrypoint: ["/mdb/scripts/backup", "--engine", "pg", "--schedule"] @@ -213,7 +213,7 @@ services: backup-mysql: profiles: [backup] - image: ghcr.io/simtabi/dbpantry-mysql:${MDB_MYSQL_VERSION:-8.4} + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-mysql:${MDB_MYSQL_VERSION:-8.4} restart: unless-stopped <<: *hardening entrypoint: ["/mdb/scripts/backup", "--engine", "mysql", "--schedule"] @@ -266,7 +266,7 @@ services: backup-mariadb: profiles: [backup] - image: ghcr.io/simtabi/dbpantry-mariadb:${MDB_MARIADB_VERSION:-11.8} + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-mariadb:${MDB_MARIADB_VERSION:-11.8} restart: unless-stopped <<: *hardening entrypoint: ["/mdb/scripts/backup", "--engine", "mariadb", "--schedule"] diff --git a/docs/guides/architecture.md b/docs/guides/architecture.md index 9e5f169..621d691 100644 --- a/docs/guides/architecture.md +++ b/docs/guides/architecture.md @@ -47,7 +47,7 @@ MDB_ENGINE_DUMP_GLOBALS='pg_dumpall --globals-only' MDB_ENGINE_RESTORE='pg_restore --no-owner' MDB_ENGINE_POOLING=external -MDB_ENGINE_POOLER_IMAGE=ghcr.io/simtabi/dbpantry-pgbouncer:dev +MDB_ENGINE_POOLER_IMAGE=localhost/dbpantry-pgbouncer:dev MDB_ENGINE_POOLING_REQUIRED_IN_PROD=true MDB_ENGINE_LICENSE=PostgreSQL diff --git a/docs/guides/kubernetes.md b/docs/guides/kubernetes.md index ff21f0d..4b1d2e8 100644 --- a/docs/guides/kubernetes.md +++ b/docs/guides/kubernetes.md @@ -18,6 +18,7 @@ kind 0.33 (all five engines on kind; PG and MySQL on both). | kubectl | within one minor version of the cluster | on PATH | | Helm | 3.x or 4.x | on PATH; tested with 4.3.1 | | kind | 0.20 or later | only for kind clusters, to load the local images | +| minikube | any | only for minikube clusters; load the images with `minikube image load` (see [Images](#images)) | | A StorageClass | any `ReadWriteOnce` class | the cluster's default is used unless `MDB_K8S_STORAGE_CLASS` names one | ## Quick start @@ -28,7 +29,7 @@ MDB_BACKEND=kubernetes MDB_K8S_CONTEXT=orbstack # required: `kubectl config get-contexts` MDB_PROFILES="pg,mysql" -make build # local images (skip on a real cluster that pulls from GHCR) +make build # the images, as localhost/dbpantry-*; a real cluster needs them pushed (see Images) make up # secrets, certs and overrides, then helm upgrade --install --wait make psql make backup # dumps stream through kubectl exec into backups/, as on Compose @@ -46,6 +47,7 @@ development database to production. | `MDB_K8S_STORAGE_CLASS` | the cluster's default | StorageClass of every volume | | `MDB_K8S_VALUES` | none | extra values files, colon-separated, applied last | | `MDB_K8S_LOAD_IMAGES` | `auto` | copy local images into the cluster: `auto` on `kind-*` contexts only | +| `MDB_IMAGE_PREFIX` | `localhost` | the name prefix the images are built and run under; set it to your own registry (environment variable, see [Images](#images)) | | `MDB_K8S_CNPG_READY_TIMEOUT` | `MDB_WAIT_TIMEOUT`, else `600` | seconds `make up` waits for a CloudNativePG Cluster's instances to be ready | ## What `make up` does @@ -63,9 +65,10 @@ development database to production. `dbpantry-overrides-`. It uses `kubectl apply`, so running it again after `make rotate-secrets` updates them. 3. On a kind context, `kind load docker-image` for each locally built engine - image. OrbStack and Docker Desktop share Docker's image store already; a - real cluster pulls from the registry, so push your images there or use the - published ones on GHCR. + image, under the same prefix the chart runs (`global.imageRegistry`, from + `MDB_IMAGE_PREFIX`). OrbStack and Docker Desktop share Docker's image store + already; minikube and a real cluster need the images loaded or pushed first + (see [Images](#images)). 4. `helm upgrade --install --wait` with that values file and any `MDB_K8S_VALUES`. 5. With a CloudNativePG Cluster, waits until `.status.readyInstances` reaches @@ -79,6 +82,55 @@ so `make up` brings the same data back. Only `make destroy` deletes them, with the same typed confirmation as on Compose, and it keeps the `pitr-*` recovery volumes unless you add `--include-recovery`. +## Images + +No dbpantry image is published to a public registry: the cluster runs the +images `make build` built, named `/dbpantry-:`. The +prefix is `MDB_IMAGE_PREFIX`, `localhost` by default, and `make up` hands it to +the chart as `global.imageRegistry`, so the chart runs exactly what was built. +`imagePullPolicy` is `IfNotPresent`, so an image already in the cluster's store +is used as it is. The old `ghcr.io/simtabi/dbpantry-*` images are frozen +([Release](release.md)), and nothing falls back to them. + +Build the images first, on the same architecture as the cluster's nodes: + +```bash +make build # every enabled engine, plus pgbouncer and patroni +``` + +**kind** loads them for you: on a `kind-*` context `make up` runs +`kind load docker-image` for each image it is about to deploy +(`MDB_K8S_LOAD_IMAGES=auto`). To load one by hand: + +```bash +kind load docker-image localhost/dbpantry-pg:17 --name kind # the cluster of context kind-kind +``` + +**minikube** shares no image store with your Docker, so load them before +`make up`: + +```bash +minikube image load localhost/dbpantry-pg:17 +minikube image load localhost/dbpantry-patroni:17 # with the ha profile +``` + +**Any other cluster** pulls from a registry, so build under your registry's +name, push, and deploy with the same prefix. `MDB_IMAGE_PREFIX` is read from +the environment, not from `.env`: + +```bash +export MDB_IMAGE_PREFIX=registry.example.com/team +make build +docker push registry.example.com/team/dbpantry-pg:17 +make up +``` + +`make up` then renders `global.imageRegistry: registry.example.com/team`, and +every dbpantry image in the release carries that prefix. Add the pull secret +with `global.imagePullSecrets` in a values file named by `MDB_K8S_VALUES`. +Installing the chart by hand works the same way: +`helm install dbpantry deploy/helm/dbpantry --set global.imageRegistry=registry.example.com/team ...`. + ## What gets deployed One StatefulSet per enabled engine, with one replica: @@ -215,7 +267,8 @@ The Services are `ClusterIP`. Set `service.type: LoadBalancer` in an |---|---| | `needs MDB_K8S_CONTEXT` | set it in `.env`; see `kubectl config get-contexts` | | Helm says `TLS handshake timeout` while kubectl works | the API server name resolves to an IPv6 address that does not answer, and Helm does not fall back to IPv4. Seen with OrbStack's `k8s.orb.local`. Point a kubeconfig context at `https://127.0.0.1:26443` with `tls-server-name: k8s.orb.local` | -| `ErrImagePull` on kind | build the images first, or set `MDB_K8S_LOAD_IMAGES=true` | +| `ErrImagePull` / `ErrImageNeverPull` on kind or minikube | the image is not in the cluster's store: `make build`, then load it (kind: `make up` does, or `MDB_K8S_LOAD_IMAGES=true`; minikube: `minikube image load`). See [Images](#images) | +| `ErrImagePull` on another cluster | `global.imageRegistry` is `localhost`, which no cluster can pull from: push the images and set `MDB_IMAGE_PREFIX` (see [Images](#images)) | | `another operation (install/upgrade/rollback) is in progress` | an interrupted `make up` left the release `pending-upgrade`. Run `helm --kube-context -n history `, then `helm ... rollback `, and run `make up` again | | CloudNativePG: `conflict with "kubectl-annotate" ... cnpg.io/hibernation` | a Cluster hibernated by an older dbpantry. Run `kubectl annotate clusters.postgresql.cnpg.io -pg cnpg.io/hibernation-`, then `make up` (current versions do this themselves) | | HA: a replica logs `no pg_hba.conf entry for replication` | a dbpantry-patroni image from before D-65; rebuild or pull it and delete the pod | diff --git a/docs/guides/licensing.md b/docs/guides/licensing.md index 44b2f69..d22d6be 100644 --- a/docs/guides/licensing.md +++ b/docs/guides/licensing.md @@ -83,17 +83,18 @@ withdraw. ## Verifying what you run -Every published image carries an accurate -`org.opencontainers.image.licenses` label for its own contents, plus a build -provenance attestation: +Every image `make build` produces carries an accurate +`org.opencontainers.image.licenses` label for its own contents: ```bash -docker inspect ghcr.io/simtabi/dbpantry-pg:17 \ +docker inspect localhost/dbpantry-pg:17 \ --format '{{ index .Config.Labels "org.opencontainers.image.licenses" }}' - -gh attestation verify oci://ghcr.io/simtabi/dbpantry-pg:17 --owner simtabi ``` +The images are no longer published, so there is no registry attestation to +verify; the frozen `ghcr.io/simtabi/dbpantry-*` images are described in +[Release](release.md). + --- [← Docs index](../../README.md#documentation) diff --git a/docs/guides/runtime-apple-container.md b/docs/guides/runtime-apple-container.md index af59165..0405c4c 100644 --- a/docs/guides/runtime-apple-container.md +++ b/docs/guides/runtime-apple-container.md @@ -72,7 +72,7 @@ because its BuildKit builder otherwise keeps the result in the build cache), or copy an image across: ```bash -docker save -o /tmp/pg17.tar ghcr.io/simtabi/dbpantry-pg:17 # on the other engine +docker save -o /tmp/pg17.tar localhost/dbpantry-pg:17 # on the other engine DOCKER_HOST=unix:///opt/homebrew/var/run/socktainer/.socktainer/container.sock \ docker load -i /tmp/pg17.tar ``` diff --git a/docs/guides/runtime-podman.md b/docs/guides/runtime-podman.md index e9e1ba6..ddacb76 100644 --- a/docs/guides/runtime-podman.md +++ b/docs/guides/runtime-podman.md @@ -140,7 +140,8 @@ make verify CHECKS=08,15,22,28 ``` Podman keeps its own image store, separate from Docker's, so the engine images -have to be built (or pulled from GHCR) again under Podman. +have to be built again under Podman (`make build`). Their `localhost/` prefix is +the name Podman gives local builds anyway. If `docker-compose` fails pulls with `unauthorized: incorrect username or password` for docker.io images, it is sending stale Docker Hub credentials from diff --git a/docs/reference/acceptance.md b/docs/reference/acceptance.md index 2fd5b0f..65484d3 100644 --- a/docs/reference/acceptance.md +++ b/docs/reference/acceptance.md @@ -49,6 +49,7 @@ so a green `make verify` is the criteria being met rather than a proxy for it. | Images take distribution updates whole, download only through a verified, retried helper; CI gates findings where caused | `70-supply-chain` | | Every profile combination renders with one authenticating pooler per upstream; prod publishes only poolers and the front door | `71-compose-profiles-render` | | Every service drops privileges, has a health signal, is limited under prod and takes secrets as files; exceptions must stay needed | `72-service-hardening` | +| Our images are named `localhost/` by default, nothing names a remote registry for them, and a set `MDB_IMAGE_PREFIX` or `global.imageRegistry` reaches every one | `73-images-local-by-default` | | No leftover old product name outside its history | `44-no-old-name` | | Cassandra's replaced JARs are pinned by checksum and installed | `45-cassandra-jars` | | Every relative link in the Markdown docs resolves | `46-doc-links-resolve` | diff --git a/docs/reorganization-plan.md b/docs/reorganization-plan.md index f00aa0f..98f7701 100644 --- a/docs/reorganization-plan.md +++ b/docs/reorganization-plan.md @@ -19,7 +19,7 @@ on stays where it is. | `engines//engine.conf` + `compose-*.yml`, `engines/_family/*.sh` | `scripts/gen-compose`, `scripts/engine-lib.sh` | `compose.engines.yml`, `compose.proxy.yml`, `config/caddy/layer4.caddy` (generated) | | `config/caddy/*.tmpl` | `scripts/render-config` → `scripts/env-render` | `config/caddy/Caddyfile`, `config/caddy/servers.json` (generated) | | `config/haproxy/haproxy.cfg`, `config/proxysql/bootstrap.sh` | compose (bind mounts) | HA front door, ProxySQL pooler | -| `images//`, `images/_shared/`, `images/bases.tsv` | `scripts/build`, compose `build:` | `ghcr.io/simtabi/dbpantry-*` images | +| `images//`, `images/_shared/`, `images/bases.tsv` | `scripts/build`, compose `build:` | `localhost/dbpantry-*` images (`MDB_IMAGE_PREFIX`) | | `docker-compose.yml` + `compose.*.yml` | `docker compose` via `COMPOSE_FILE` | running stack, named volumes | | running stack | `scripts/backup`, `scripts/pitr`, `scripts/transfer` | `backups/` (gitignored) | | `scripts/certs` | — | `certs/` (gitignored) | diff --git a/engines/postgres/engine.conf b/engines/postgres/engine.conf index 8f96ef5..a61d7cc 100644 --- a/engines/postgres/engine.conf +++ b/engines/postgres/engine.conf @@ -49,7 +49,9 @@ MDB_ENGINE_POOLING=external # image, and edoburu/pgbouncer was this toolkit's only community rebuild. # images/pgbouncer/Dockerfile builds it from a Docker Official base plus the # PGDG package. See docs/reference/image-provenance.md. -MDB_ENGINE_POOLER_IMAGE='ghcr.io/simtabi/dbpantry-pgbouncer:dev' +# Written with the default prefix; scripts/gen-compose and the checks apply +# MDB_IMAGE_PREFIX to it (mdb_image_ref in scripts/env-lib.sh). +MDB_ENGINE_POOLER_IMAGE='localhost/dbpantry-pgbouncer:dev' MDB_ENGINE_POOLER_BUILD=pgbouncer/Dockerfile # Every pooler starts through a script in the repository. Note it must exec the # image's own entrypoint AND its CMD: overriding a service's entrypoint discards diff --git a/images/patroni/Dockerfile b/images/patroni/Dockerfile index 51c1692..f610459 100644 --- a/images/patroni/Dockerfile +++ b/images/patroni/Dockerfile @@ -12,7 +12,7 @@ # bundle rather than left to race Patroni for the same data directory. ARG ENGINE_VERSION=17 -ARG IMAGE_PREFIX=ghcr.io/simtabi +ARG IMAGE_PREFIX=localhost FROM ${IMAGE_PREFIX}/dbpantry-pg:${ENGINE_VERSION} # Pinned. Patroni's behaviour on failover is the whole point of this image, so diff --git a/scripts/backends/kubernetes.sh b/scripts/backends/kubernetes.sh index 242ba6e..a6431eb 100644 --- a/scripts/backends/kubernetes.sh +++ b/scripts/backends/kubernetes.sh @@ -289,16 +289,19 @@ _mdb_be_kubernetes_image_load() { esac } -# The images the release will run that are built here (MongoDB's is upstream). +# The images the release will run that are built here (MongoDB's is upstream), +# under the prefix they were built with: the global.imageRegistry helm-values +# rendered from MDB_IMAGE_PREFIX, the same name the chart then runs. _mdb_k8s_local_images() { local values="$1" - awk ' + awk -v prefix="$(mdb_image_prefix)" ' + /^ imageRegistry: / { prefix = $2; gsub(/"/, "", prefix) } /^ [a-z]+:$/ { eng = $1; sub(/:/, "", eng); en = 0; ha = 0; tag = "" } /^ enabled: true$/ { en = 1 } /^ enabled: true$/ { ha = 1 } /^ tag: / { tag = $2; gsub(/"/, "", tag) } - /^ env:$/ && en && tag != "" { print "ghcr.io/simtabi/dbpantry-" eng ":" tag } - /^ provider: "?patroni"?$/ && en && ha && tag != "" { print "ghcr.io/simtabi/dbpantry-patroni:" tag } + /^ env:$/ && en && tag != "" { print prefix "/dbpantry-" eng ":" tag } + /^ provider: "?patroni"?$/ && en && ha && tag != "" { print prefix "/dbpantry-patroni:" tag } ' "$values" } diff --git a/scripts/build b/scripts/build index 3625e5b..d12c30d 100755 --- a/scripts/build +++ b/scripts/build @@ -22,7 +22,7 @@ cd "$ROOT" || exit 1 ENV_FILE="${MDB_ENV_FILE:-$ROOT/.env}" [[ -f "$ENV_FILE" ]] || ENV_FILE="$ROOT/.env.example" -PREFIX="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" +PREFIX="$(mdb_image_prefix)" get() { mdb_env_get "$1" "$ENV_FILE"; } diff --git a/scripts/env-lib.sh b/scripts/env-lib.sh index 4f76f87..3497c33 100644 --- a/scripts/env-lib.sh +++ b/scripts/env-lib.sh @@ -90,3 +90,50 @@ mdb_env_needs_quotes() { } export -f _mdb_env_unsafe mdb_env_unquote mdb_env_get mdb_env_quote mdb_env_needs_quotes 2>/dev/null || true + +# The images dbpantry builds are named /dbpantry-:. +# The prefix is MDB_IMAGE_PREFIX, or localhost when it is unset (design.md +# D-78): a fully qualified local name, so Docker, Podman and nerdctl resolve it +# the same way (check 49), it never falls back to a remote registry, and kind or +# minikube can load it. Set MDB_IMAGE_PREFIX to your own registry (for example +# registry.example.com/team) to build, run and deploy under that name instead. +# Compose files, the Helm chart, the Makefile and the patroni Dockerfile carry +# the same default; check 73 fails if any of them names another. +MDB_IMAGE_PREFIX_DEFAULT=localhost + +# The prefix in force, without a trailing slash. +mdb_image_prefix() { + local p="${MDB_IMAGE_PREFIX:-$MDB_IMAGE_PREFIX_DEFAULT}" + printf '%s\n' "${p%/}" +} + +# The full name of one of our images: mdb_image pg 17 -> localhost/dbpantry-pg:17 +mdb_image() { + printf '%s/dbpantry-%s:%s\n' "$(mdb_image_prefix)" "$1" "$2" +} + +# A reference written with the default prefix (an engine descriptor's +# localhost/dbpantry-pgbouncer:dev) under the prefix in force. Any other +# reference, such as an upstream image, is returned unchanged. +mdb_image_ref() { + case "$1" in + "$MDB_IMAGE_PREFIX_DEFAULT"/dbpantry-*) printf '%s/%s\n' "$(mdb_image_prefix)" "${1#"$MDB_IMAGE_PREFIX_DEFAULT"/}" ;; + *) printf '%s\n' "$1" ;; + esac +} + +# Resolve a Compose-style reference the way Compose does: every ${VAR:-default} +# becomes VAR from the environment, or the default when it is unset or empty, +# innermost first. For a script that reads an image: line out of a Compose file +# and runs it directly, so ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-caddy:dev +# becomes localhost/dbpantry-caddy:dev (or the prefix in force). +mdb_compose_ref() { + # shellcheck disable=SC2016 # a pattern for a literal ${...}, not an expansion + local ref="$1" re='\$\{([A-Za-z_][A-Za-z0-9_]*):-([^${}]*)\}' name fallback val + while [[ "$ref" =~ $re ]]; do + name="${BASH_REMATCH[1]}"; fallback="${BASH_REMATCH[2]}" + val="${!name:-$fallback}" + ref="${ref/"${BASH_REMATCH[0]}"/$val}" + done + printf '%s\n' "$ref" +} diff --git a/scripts/gen-compose b/scripts/gen-compose index d340d90..75b8976 100755 --- a/scripts/gen-compose +++ b/scripts/gen-compose @@ -146,7 +146,7 @@ for engine in $(engine_list); do image_line="\${MDB_${upper}_IMAGE:-\${MDB_${upper}_BASE_IMAGE:-${pinned}}}" build_block="" else - image_line="ghcr.io/simtabi/dbpantry-${MDB_ENGINE_NAME}:\${MDB_${upper}_VERSION:-${MDB_ENGINE_DEFAULT_VERSION}}" + image_line="\${MDB_IMAGE_PREFIX:-${MDB_IMAGE_PREFIX_DEFAULT}}/dbpantry-${MDB_ENGINE_NAME}:\${MDB_${upper}_VERSION:-${MDB_ENGINE_DEFAULT_VERSION}}" # BASE_IMAGE is passed so a compose build of a non-default version # starts from that version's base rather than the Dockerfile's default. build_block=" build: @@ -254,7 +254,11 @@ for engine in $(engine_list); do # project over the dangling depends_on. scripts/profiles expands # `pooler` into the ones that match the selected engines. printf ' profiles: [%s-pooler]\n' "$MDB_ENGINE_NAME" - printf ' image: %s\n' "$MDB_ENGINE_POOLER_IMAGE" + # Our own pooler image follows MDB_IMAGE_PREFIX as the engines do. + case "$MDB_ENGINE_POOLER_IMAGE" in + "$MDB_IMAGE_PREFIX_DEFAULT"/dbpantry-*) printf ' image: ${MDB_IMAGE_PREFIX:-%s}/%s\n' "$MDB_IMAGE_PREFIX_DEFAULT" "${MDB_ENGINE_POOLER_IMAGE#"$MDB_IMAGE_PREFIX_DEFAULT"/}" ;; + *) printf ' image: %s\n' "$MDB_ENGINE_POOLER_IMAGE" ;; + esac printf ' restart: unless-stopped\n' printf ' depends_on: [%s]\n' "$MDB_ENGINE_NAME" printf ' security_opt:\n - no-new-privileges:true\n' diff --git a/scripts/helm-values b/scripts/helm-values index c53143b..46d05cb 100755 --- a/scripts/helm-values +++ b/scripts/helm-values @@ -16,6 +16,7 @@ # MDB__MEM_LIMIT / _CPU_LIMIT memory and cpu (requests and limits) # MDB_PG_SHM the size of PostgreSQL's /dev/shm # MDB_K8S_STORAGE_CLASS global.storageClass +# MDB_IMAGE_PREFIX global.imageRegistry (default localhost) # the ha profile PostgreSQL HA (engines.pg.ha): Patroni, or # CloudNativePG with MDB_K8S_HA_PROVIDER=cnpg; # MDB_HA_CLUSTER_NAME, MDB_PG_SYNC_MODE, @@ -165,6 +166,9 @@ render() { printf '# Rendered by scripts/helm-values from %s. Do not edit: change .env.\n' "$(basename "$ENV_FILE")" sc="$(setting MDB_K8S_STORAGE_CLASS)" printf 'global:\n' + # The prefix the images were built under (MDB_IMAGE_PREFIX, default + # localhost), so the chart runs exactly what `make build` produced. + printf ' imageRegistry: %s\n' "$(q "$(mdb_image_prefix)")" printf ' storageClass: %s\n' "$(q "$sc")" printf 'engines:\n' for eng in $ENGINES; do diff --git a/scripts/init b/scripts/init index f70cc46..057dc36 100755 --- a/scripts/init +++ b/scripts/init @@ -133,8 +133,10 @@ if (( PROD )); then # to be /image: caddy:/, which stopped matching the moment the image became # one we build -- and the only symptom was "could not compute the UI # password hash (is the container runtime running? scripts/runtime info)", which blames Docker. - caddy_image="$(awk '/^ caddy:/{f=1} f && /^[[:space:]]+image:/{print $2; exit}' \ - "$ROOT/docker-compose.yml")" + # The line is Compose syntax, ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-caddy:dev, + # so it is resolved as Compose resolves it before it is run directly. + caddy_image="$(mdb_compose_ref "$(awk '/^ caddy:/{f=1} f && /^[[:space:]]+image:/{print $2; exit}' \ + "$ROOT/docker-compose.yml")")" ui_hash="$(mdb_ctr run --rm "$caddy_image" caddy hash-password --plaintext "$ui_password" 2>/dev/null || true)" if [[ -z "$ui_hash" ]]; then printf ' could not compute the UI password hash (is docker running?)\n' >&2 diff --git a/scripts/scan b/scripts/scan index 5204a25..9c62ffd 100755 --- a/scripts/scan +++ b/scripts/scan @@ -69,13 +69,13 @@ failed=0 scanned=0 for engine in "${targets[@]}"; do if [[ "$engine" == "cli" ]]; then - img="ghcr.io/simtabi/dbpantry-cli:dev" + img="$(mdb_image cli dev)" else engine_load "$engine" >/dev/null 2>&1 || continue upper="$(printf '%s' "$engine" | tr '[:lower:]-' '[:upper:]_')" ver="$(mdb_env_get "MDB_${upper}_VERSION" .env)" ver="${ver:-$MDB_ENGINE_DEFAULT_VERSION}" - img="ghcr.io/simtabi/dbpantry-${engine}:${ver}" + img="$(mdb_image "$engine" "$ver")" fi if ! mdb_ctr image inspect "$img" >/dev/null 2>&1; then diff --git a/scripts/transfer b/scripts/transfer index 947ec38..ed4fbc4 100755 --- a/scripts/transfer +++ b/scripts/transfer @@ -80,14 +80,14 @@ if [[ -n "$FROM_HOST" ]]; then # host doing the migration. mdb_ctr run --rm -i --network host \ -v "$ROOT/backups:/backups" \ - ghcr.io/simtabi/dbpantry-cli:dev \ + "$(mdb_image cli dev)" \ pg_dump -h "$FROM_HOST" -U "$FROM_USER" -Fc "$DB" > "$FILE" \ || die "could not dump from $FROM_HOST" ;; mysql|mariadb) mdb_ctr run --rm -i --network host \ -v "$ROOT/backups:/backups" \ - ghcr.io/simtabi/dbpantry-cli:dev \ + "$(mdb_image cli dev)" \ mariadb-dump -h "$FROM_HOST" -u "$FROM_USER" -p \ --single-transaction --routines --triggers --events --quick "$DB" > "$FILE" \ || die "could not dump from $FROM_HOST" diff --git a/scripts/upgrade b/scripts/upgrade index 1e85a8b..9373f89 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -35,7 +35,7 @@ cd "$ROOT" || exit 1 mdb_backend_require compose || exit 64 PROJECT="$(mdb_project_name)" -PREFIX="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" +PREFIX="$(mdb_image_prefix)" # The working dumps. backups/upgrade in the checkout unless MDB_UPGRADE_DIR names # another directory (the verify harness points it at a private temporary one, so # a check never writes or cleans the operator's backups/). diff --git a/scripts/verify-backups b/scripts/verify-backups index 1d66927..5e5adf5 100755 --- a/scripts/verify-backups +++ b/scripts/verify-backups @@ -26,7 +26,7 @@ ENV_FILE="${MDB_ENV_FILE:-$ROOT/.env}" [[ -f "$ENV_FILE" ]] || ENV_FILE="$ROOT/.env.example" BACKUP_DIR="$(mdb_backup_dir "$ROOT" "$ENV_FILE")" -PREFIX="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" +PREFIX="$(mdb_image_prefix)" get() { mdb_env_get "$1" "$ENV_FILE"; } log() { printf '[verify-backups] %s\n' "$*"; } diff --git a/scripts/verify/02-pinned-versions.sh b/scripts/verify/02-pinned-versions.sh index e1dd192..240d045 100755 --- a/scripts/verify/02-pinned-versions.sh +++ b/scripts/verify/02-pinned-versions.sh @@ -48,7 +48,7 @@ for f in "${candidates[@]}"; do # published. The rule exists to pin what we consume, not what we # produce; their bases are pinned in images/bases.tsv, validated # below. - *ghcr.io/simtabi/dbpantry-*) continue ;; + *"/dbpantry-"*) continue ;; esac printf ' %s: not digest-pinned: %s\n' "${f#./}" "$line" >&2 (( violations++ )) || true diff --git a/scripts/verify/08-pg-standalone-run.sh b/scripts/verify/08-pg-standalone-run.sh index b54112c..8a381a0 100755 --- a/scripts/verify/08-pg-standalone-run.sh +++ b/scripts/verify/08-pg-standalone-run.sh @@ -7,8 +7,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" need_runtime -# SPEC section 18: "docker run ghcr.io/simtabi/dbpantry-pg with only a -# password env yields a fully configured server, no compose required." +# SPEC section 18: "docker run /dbpantry-pg with only a password env +# yields a fully configured server, no compose required." The image is the +# local build, localhost/dbpantry-pg (image_name, MDB_IMAGE_PREFIX). # This is the check that keeps layer 2 of the architecture honest. img="$(image_name pg)" diff --git a/scripts/verify/14-version-switch.sh b/scripts/verify/14-version-switch.sh index 0c442d1..d9834b3 100755 --- a/scripts/verify/14-version-switch.sh +++ b/scripts/verify/14-version-switch.sh @@ -15,7 +15,7 @@ need_runtime # 17 -> 18 are the two maintained majors (design.md D-57). 18 also moved the # data mount one level up, to /var/lib/postgresql, so each side mounts its own. -prefix="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" +prefix="$(mdb_image_prefix)" old_img="$prefix/dbpantry-pg:17" new_img="$prefix/dbpantry-pg:18" diff --git a/scripts/verify/29-upgrade-migrates-data.sh b/scripts/verify/29-upgrade-migrates-data.sh index 112290a..f44c233 100755 --- a/scripts/verify/29-upgrade-migrates-data.sh +++ b/scripts/verify/29-upgrade-migrates-data.sh @@ -17,7 +17,7 @@ cd "$MDB_ROOT" || exit 1 # PG 18's move of the data mount to /var/lib/postgresql, which `make upgrade` # has to get right on each side. -prefix="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" +prefix="$(mdb_image_prefix)" old_img="$prefix/dbpantry-pg:17" new_img="$prefix/dbpantry-pg:18" need_image "$old_img" diff --git a/scripts/verify/32-every-engine-auth.sh b/scripts/verify/32-every-engine-auth.sh index 91b6a3d..97faa5a 100755 --- a/scripts/verify/32-every-engine-auth.sh +++ b/scripts/verify/32-every-engine-auth.sh @@ -44,7 +44,7 @@ for engine in $(engine_list); do fi ver="$(engine_version "$engine")" - img="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}/dbpantry-${engine}:${ver}" + img="$(mdb_image "$engine" "$ver")" # A referenced engine is not published by us, so it runs the pinned # upstream image (SPEC section 22.5). diff --git a/scripts/verify/42-image-provenance.sh b/scripts/verify/42-image-provenance.sh index ceb12aa..8a2806b 100755 --- a/scripts/verify/42-image-provenance.sh +++ b/scripts/verify/42-image-provenance.sh @@ -19,8 +19,10 @@ cd "$MDB_ROOT" || exit 1 need_file "$MDB_ROOT/docs/reference/image-provenance.md" -# Namespaces belonging to the projects whose software they package. -UPSTREAM='^(pgvector/pgvector|dpage/pgadmin4|proxysql/proxysql|prom/[a-z-]+|aquasec/trivy|ghcr\.io/ferretdb/[a-z-]+|quay\.io/coreos/etcd|quay\.io/prometheuscommunity/[a-z-]+|ghcr\.io/simtabi/[a-z-]+)$' +# Namespaces belonging to the projects whose software they package, and our own +# images, which every descriptor names under the default prefix (localhost/, +# design.md D-78). +UPSTREAM='^(pgvector/pgvector|dpage/pgadmin4|proxysql/proxysql|prom/[a-z-]+|aquasec/trivy|ghcr\.io/ferretdb/[a-z-]+|quay\.io/coreos/etcd|quay\.io/prometheuscommunity/[a-z-]+|localhost/dbpantry-[a-z-]+)$' violations=0 checked=0 diff --git a/scripts/verify/49-images-fully-qualified.sh b/scripts/verify/49-images-fully-qualified.sh index 11e8e3a..099726a 100755 --- a/scripts/verify/49-images-fully-qualified.sh +++ b/scripts/verify/49-images-fully-qualified.sh @@ -36,8 +36,15 @@ judge() { # image: lines, including the defaults inside ${VAR:-default}. while IFS=: read -r file _ rest; do ref="$(printf '%s' "$rest" | sed -E 's/^[[:space:]]*image:[[:space:]]*//; s/^"//; s/"$//')" - # shellcheck disable=SC2016 # a literal ${, not an expansion - while [[ "$ref" == '${'*':-'* ]]; do ref="${ref#*:-}"; ref="${ref%\}}"; done + # Every ${VAR:-default} is replaced by its default, innermost first, so a + # nested ${MDB_X_IMAGE:-${MDB_X_BASE_IMAGE:-docker.io/...}} and a prefixed + # ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-pg:${MDB_PG_VERSION:-17} are both + # judged by the name they resolve to when nothing is set. + prev="" + while [[ "$ref" != "$prev" ]]; do + prev="$ref" + ref="$(printf '%s' "$ref" | sed -E 's/\$\{[A-Za-z_][A-Za-z0-9_]*:-([^${}]*)\}/\1/g')" + done judge "$file" "$ref" done < <(grep -nE '^[[:space:]]*image:' docker-compose.yml compose.*.yml 2>/dev/null) diff --git a/scripts/verify/55-kubernetes-ha.sh b/scripts/verify/55-kubernetes-ha.sh index 3b3b35d..aea540b 100755 --- a/scripts/verify/55-kubernetes-ha.sh +++ b/scripts/verify/55-kubernetes-ha.sh @@ -95,7 +95,7 @@ ha=(--set engines.pg.enabled=true --set engines.pg.ha.enabled=true) helm template r "$chart" "${ha[@]}" >"$tmp/patroni.yaml" 2>"$tmp/err" || vfail "helm template (patroni): $(cat "$tmp/err")" n="$(grep -c '^kind: StatefulSet' "$tmp/patroni.yaml")" [[ "$n" == 1 ]] || vfail "HA should replace the standalone StatefulSet; rendered $n" -grep -q 'image: ghcr.io/simtabi/dbpantry-patroni:17' "$tmp/patroni.yaml" || vfail "the HA StatefulSet does not run dbpantry-patroni" +grep -q 'image: localhost/dbpantry-patroni:17' "$tmp/patroni.yaml" || vfail "the HA StatefulSet does not run dbpantry-patroni" grep -q '^kind: ClusterRole' "$tmp/patroni.yaml" && vfail "Patroni was given cluster-wide rights" grep -q '^kind: Role$' "$tmp/patroni.yaml" || vfail "Patroni has no Role for its DCS" awk '/^kind: Service$/{s=1} s&&/^ name: pg$/{n=1} n&&/role: primary/{print "ok"; exit}' "$tmp/patroni.yaml" | grep -q ok \ diff --git a/scripts/verify/68-pooler-tls.sh b/scripts/verify/68-pooler-tls.sh index c217be8..a5917ed 100755 --- a/scripts/verify/68-pooler-tls.sh +++ b/scripts/verify/68-pooler-tls.sh @@ -43,6 +43,7 @@ need_image "$img" pooler_img="$(grep -oE "^MDB_ENGINE_POOLER_IMAGE='[^']+'" engines/postgres/engine.conf \ | sed "s/^MDB_ENGINE_POOLER_IMAGE='//; s/'$//")" +pooler_img="$(mdb_image_ref "$pooler_img")" [[ -n "$pooler_img" ]] || vfail "engines/postgres/engine.conf declares no pooler image" need_image "$pooler_img" diff --git a/scripts/verify/72-service-hardening.sh b/scripts/verify/72-service-hardening.sh index 9ce7e4d..731f4e1 100755 --- a/scripts/verify/72-service-hardening.sh +++ b/scripts/verify/72-service-hardening.sh @@ -49,10 +49,13 @@ for df in images/*/Dockerfile; do grep -q '^HEALTHCHECK' "$df" && built_health+="$(basename "$(dirname "$df")") " done -out="$(python3 - "$s/all.json" "$built_health" <<'PY' +out="$(python3 - "$s/all.json" "$built_health" "$(mdb_image_prefix)" <<'PY' import json, re, sys svc = json.load(open(sys.argv[1]))["services"] built = set(sys.argv[2].split()) +# Our images are /dbpantry-, the prefix Compose resolved them under +# (MDB_IMAGE_PREFIX, default localhost; design.md D-78). +ours = re.compile(re.escape(sys.argv[3]) + r"/dbpantry-([a-z0-9-]+):") # rule -> {service: reason}. Each is a measured gap with its reason; the check # fails if one stops being a gap. @@ -93,7 +96,7 @@ for name, d in sorted(svc.items()): gap("privileges", name, f"{name}: missing no-new-privileges or cap_drop ALL") hc = d.get("healthcheck") or {} image = str(d.get("image", "")) - m = re.match(r"ghcr\.io/simtabi/dbpantry-([a-z0-9-]+):", image) + m = ours.match(image) health_ok = (hc and not hc.get("disable")) or (m and m.group(1) in built) if not health_ok: gap("health", name, f"{name}: no healthcheck in Compose or in its image") diff --git a/scripts/verify/73-images-local-by-default.sh b/scripts/verify/73-images-local-by-default.sh new file mode 100755 index 0000000..12ffea5 --- /dev/null +++ b/scripts/verify/73-images-local-by-default.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# verify: our images are named localhost/ by default, never after a remote registry, and a set MDB_IMAGE_PREFIX or global.imageRegistry names every one +# tags: fast structure +# phase: 1 + +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# design.md D-78. dbpantry publishes no image (D-77), yet until 2026-11-13 every +# image was built, run and deployed as ghcr.io/simtabi/dbpantry-*: the Helm +# chart's default registry, the Compose files, the Kubernetes backend's kind +# list and a dozen scripts each carried that name. A cluster missing an image +# therefore pulled from a registry that holds nothing. The default is now +# localhost, written once in scripts/env-lib.sh (MDB_IMAGE_PREFIX_DEFAULT) and +# repeated only where a file cannot read it: Compose, the chart, the patroni +# Dockerfile and an engine descriptor. This check holds all of them to it. + +cd "$MDB_ROOT" || exit 1 +scripts/gen-compose >/dev/null 2>&1 || vfail "scripts/gen-compose failed" + +def=localhost +remote='registry.example.com/team' + +# --- 1. every default is localhost ------------------------------------------------ +grep -qx "MDB_IMAGE_PREFIX_DEFAULT=$def" scripts/env-lib.sh \ + || vfail "scripts/env-lib.sh does not set MDB_IMAGE_PREFIX_DEFAULT=$def" +grep -qx " imageRegistry: $def" deploy/helm/dbpantry/values.yaml \ + || vfail "the chart's global.imageRegistry default is not $def: $(grep -m1 'imageRegistry:' deploy/helm/dbpantry/values.yaml)" +while IFS= read -r line; do + [[ "$line" == *"ARG IMAGE_PREFIX=$def" ]] || vfail "a Dockerfile defaults IMAGE_PREFIX to something else: $line" +done < <(grep -H '^ARG IMAGE_PREFIX=' images/*/Dockerfile) + +# Compose: every image of ours is ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-*, in +# the tracked files and the ones scripts/gen-compose just wrote. +ours=0 +while IFS=: read -r file _ ref; do + ours=$(( ours + 1 )) + ref="${ref#"${ref%%[![:space:]]*}"}" + # shellcheck disable=SC2016 # a literal Compose ${...}, not an expansion + [[ "$ref" == 'image: ${MDB_IMAGE_PREFIX:-'"$def"'}/dbpantry-'* ]] \ + || vfail "$file: $ref does not take its prefix from \${MDB_IMAGE_PREFIX:-$def}" +done < <(grep -nE '^[[:space:]]*image:.*/dbpantry-' docker-compose.yml compose.*.yml) +# docker-compose.yml (4), compose.ha.yml (2) and the generated engines (5) on +# 2026-11-13; fewer means this is not reading the Compose files. +(( ours >= 8 )) || vfail "found $ours dbpantry image line(s) in the Compose files, want at least 8" +others="$(grep -hoE 'MDB_IMAGE_PREFIX:-[^}]*' docker-compose.yml compose.*.yml | sort -u)" +[[ "$others" == "MDB_IMAGE_PREFIX:-$def" ]] || vfail "a Compose default prefix is not $def: $others" + +# Engine descriptors write our images with the default prefix. +while IFS= read -r line; do + ref="$(printf '%s' "$line" | sed -E "s/.*_IMAGE='//; s/'$//")" + [[ "$ref" == "$def/"* ]] || vfail "${line%%:*}: $ref is not named $def/" +done < <(grep -HoE "_IMAGE='[^']*/dbpantry-[^']*'" engines/*/engine.conf) +vinfo "every default prefix is $def: env-lib, the chart, Compose ($ours image lines), the Dockerfiles and the descriptors" + +# --- 2. nothing names a remote registry for our images ------------------------------ +# A dotted host (or host:port) before dbpantry-, in anything but prose. The +# Markdown docs may show registry.example.com as the user's own registry. +scanned=0 +bad=0 +while IFS= read -r -d '' f; do + case "$f" in *.md|scripts/verify/73-images-local-by-default.sh) continue ;; esac + scanned=$(( scanned + 1 )) + if grep -nE '([A-Za-z0-9-]+\.)+[A-Za-z]{2,}(:[0-9]+)?/([A-Za-z0-9._-]+/)*dbpantry-[a-z]' "$f" \ + | grep -v 'ghcr.io/simtabi' >/dev/null; then + printf ' %s names a remote registry for a dbpantry image\n' "$f" >&2 + bad=$(( bad + 1 )) + fi +done < <(git ls-files -z) +(( scanned >= 300 )) || vfail "scanned $scanned tracked file(s), want at least 300: git ls-files is not listing the repository" +(( bad == 0 )) || vfail "$bad file(s) name a remote registry for our images (listed above)" + +# ghcr.io/simtabi may only describe the frozen images, or the history that +# published them. Each file is held to the number of lines measured on +# 2026-11-13, so a new mention fails here and an old one that goes stale +# fails too. +frozen_lines() { + case "$1" in + .github/workflows/release.yml) echo 1 ;; # the header: images on ghcr.io/simtabi are frozen + CHANGELOG.md) echo 3 ;; # history, and the D-77/D-78 entries + README.md) echo 1 ;; # the images are frozen + design.md) echo 4 ;; # D-02, D-23 and D-77 history, D-78 + docs/guides/kubernetes.md) echo 1 ;; # nothing falls back to the frozen images + docs/guides/licensing.md) echo 1 ;; # no registry attestation; the images are frozen + docs/guides/release.md) echo 2 ;; # the frozen tag scheme + docs/reference/spec.md) echo 4 ;; # the original SPEC, kept as written + src/internal/release/workflow_test.go) echo 4 ;; # samples of the image pushes the release guard refuses + *) echo 0 ;; + esac +} +listed=0 +while IFS= read -r -d '' f; do + [[ "$f" == scripts/verify/73-images-local-by-default.sh ]] && continue + # Escaped too: a regex allowlist spells it ghcr\.io/simtabi, which is how + # checks 42 and 72 kept the old name past the first version of this check. + n="$(grep -cE 'ghcr\\?\.io/simtabi' "$f" || true)" + want="$(frozen_lines "$f")" + (( want > 0 )) && listed=$(( listed + 1 )) + (( n == want )) && continue + (( want == 0 )) && vfail "$f mentions ghcr.io/simtabi $n time(s); our images are localhost/ (D-78)" + vfail "$f mentions ghcr.io/simtabi $n time(s), measured $want: update the entry or the file" +done < <(git ls-files -z) +(( listed == 9 )) || vfail "found $listed of the 9 files allowed to describe the frozen images: an entry names a missing file" +vinfo "$scanned tracked file(s): no remote registry for our images; ghcr.io/simtabi only where the frozen images are described" + +# A script that runs an image named on a Compose image: line must resolve it the +# way Compose does first (mdb_compose_ref): the line reads +# ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-..., which is not an image name. +# scripts/init's caddy lookup handed that string to `run` and failed +# `make init-prod` (check 26) in the first version of this change. +readers=0 +while IFS= read -r line; do + readers=$(( readers + 1 )) + [[ "$line" == *mdb_compose_ref* ]] \ + || vfail "a script reads a Compose image: line without mdb_compose_ref: $line" +done < <(grep -rnE "(awk|grep|sed).*image:" scripts --exclude-dir=verify | grep -vE '^[^:]+:[0-9]+:[[:space:]]*#') +vinfo "$readers script(s) read a Compose image: line, each resolving it with mdb_compose_ref" + +# --- 3. a set prefix names every image ----------------------------------------------- +lib_out() { (. scripts/env-lib.sh && "$@"); } +[[ "$(MDB_IMAGE_PREFIX='' lib_out mdb_image pg 17)" == "$def/dbpantry-pg:17" ]] \ + || vfail "mdb_image with no prefix set is not $def/dbpantry-pg:17" +[[ "$(MDB_IMAGE_PREFIX="$remote/" lib_out mdb_image pg 17)" == "$remote/dbpantry-pg:17" ]] \ + || vfail "mdb_image does not apply MDB_IMAGE_PREFIX" +[[ "$(MDB_IMAGE_PREFIX="$remote" lib_out mdb_image_ref "$def/dbpantry-pgbouncer:dev")" == "$remote/dbpantry-pgbouncer:dev" ]] \ + || vfail "mdb_image_ref does not move the descriptor's pooler image to MDB_IMAGE_PREFIX" +[[ "$(MDB_IMAGE_PREFIX="$remote" lib_out mdb_image_ref docker.io/library/mongo:7.0)" == docker.io/library/mongo:7.0 ]] \ + || vfail "mdb_image_ref rewrote an upstream image" +# shellcheck disable=SC2016 # literal Compose references +{ + [[ "$(MDB_IMAGE_PREFIX='' lib_out mdb_compose_ref '${MDB_IMAGE_PREFIX:-localhost}/dbpantry-caddy:dev')" == "$def/dbpantry-caddy:dev" ]] \ + || vfail "mdb_compose_ref does not resolve the Compose default prefix" + [[ "$(MDB_IMAGE_PREFIX="$remote" lib_out mdb_compose_ref '${MDB_IMAGE_PREFIX:-localhost}/dbpantry-caddy:dev')" == "$remote/dbpantry-caddy:dev" ]] \ + || vfail "mdb_compose_ref ignores MDB_IMAGE_PREFIX" +} + +# The Makefile's own use (make shell). +grep -qE "[[:space:]]$def/dbpantry-cli:dev" < <(MDB_IMAGE_PREFIX='' make -s -n shell 2>/dev/null) \ + || vfail "make shell does not run $def/dbpantry-cli:dev by default" +grep -qE "[[:space:]]$remote/dbpantry-cli:dev" < <(MDB_IMAGE_PREFIX="$remote" make -s -n shell 2>/dev/null) \ + || vfail "make shell ignores MDB_IMAGE_PREFIX" + +# Kubernetes: helm-values hands the prefix to the chart, and the backend loads +# exactly the names the chart will run. +k8s_images() { + local values="$1" + ( + mdb_backend_register() { :; } + . scripts/env-lib.sh + . scripts/backends/kubernetes.sh + _mdb_k8s_local_images "$values" + ) +} +tmp="$(mktemp -d "${TMPDIR:-/tmp}/mdb-images.XXXXXX")" +add_cleanup "rm -rf '$tmp'" +for prefix in "$def" "$remote"; do + # COMPOSE_PROFILES as well as MDB_PROFILES: helm-values ranks it first, and + # `make verify` exports the checkout's own (pg,ui), which enabled only pg in + # CI while a direct run of this check enabled all three. + MDB_IMAGE_PREFIX="$( [[ "$prefix" == "$def" ]] || printf '%s' "$prefix")" \ + COMPOSE_PROFILES=pg,mysql,ha MDB_PROFILES=pg,mysql,ha \ + scripts/helm-values -o "$tmp/values.yaml" 2>/dev/null || vfail "scripts/helm-values failed" + grep -qx " imageRegistry: \"$prefix\"" "$tmp/values.yaml" \ + || vfail "helm-values does not render global.imageRegistry \"$prefix\"" + n=0 + while IFS= read -r img; do + n=$(( n + 1 )) + [[ "$img" == "$prefix/dbpantry-"* ]] || vfail "the Kubernetes backend would load $img, not $prefix/dbpantry-*" + done < <(k8s_images "$tmp/values.yaml") + (( n >= 3 )) || vfail "the Kubernetes backend listed $n image(s) for pg, mysql and ha, want at least 3" + + if command -v helm >/dev/null 2>&1; then + set_args=() + [[ "$prefix" == "$def" ]] || set_args=(--set "global.imageRegistry=$prefix") + helm template r deploy/helm/dbpantry --set engines.pg.enabled=true --set engines.mysql.enabled=true \ + --set engines.pg.ha.enabled=true ${set_args[@]+"${set_args[@]}"} >"$tmp/out.yaml" 2>"$tmp/err" \ + || vfail "helm template: $(cat "$tmp/err")" + imgs="$(grep -E '^[[:space:]]+image: ' "$tmp/out.yaml" | grep '/dbpantry-' || true)" + [[ "$(printf '%s\n' "$imgs" | grep -c .)" -ge 2 ]] || vfail "helm template rendered no dbpantry image" + printf '%s\n' "$imgs" | grep -vE "image: \"?$prefix/dbpantry-" && vfail "helm template ran an image outside $prefix/ (above)" + grep -q 'ghcr.io' "$tmp/out.yaml" && vfail "helm template rendered a ghcr.io image" + grep -qE 'imagePullPolicy: IfNotPresent' "$tmp/out.yaml" || vfail "the chart no longer uses imagePullPolicy: IfNotPresent" + fi +done +if command -v helm >/dev/null 2>&1; then + vinfo "unset, every image is $def/dbpantry-*; with $remote set, every image (Make, helm-values, kind loading, the rendered chart) carries it" +else + vinfo "unset, every image is $def/dbpantry-*; with $remote set, Make, helm-values and kind loading carry it (helm is not installed, so the chart was not rendered)" +fi diff --git a/scripts/verify/lib.sh b/scripts/verify/lib.sh index 2ca4a21..7959b12 100755 --- a/scripts/verify/lib.sh +++ b/scripts/verify/lib.sh @@ -125,14 +125,14 @@ engine_selected() { image_exists() { mdb_ctr image inspect "$1" >/dev/null 2>&1; } # Fully-qualified name for one of our engine images. -# image_name pg -> ghcr.io/simtabi/dbpantry-pg:17 -# image_name mariadb -> ghcr.io/simtabi/dbpantry-mariadb:11.8 +# image_name pg -> localhost/dbpantry-pg:17 +# image_name mariadb -> localhost/dbpantry-mariadb:11.8 +# (under MDB_IMAGE_PREFIX when it is set; see mdb_image in scripts/env-lib.sh) image_name() { - local engine="$1" prefix ver - prefix="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" + local engine="$1" ver ver="$(engine_version "$engine")" [[ -n "$ver" ]] || vfail "unknown engine: $engine" - printf '%s/dbpantry-%s:%s\n' "$prefix" "$engine" "$ver" + mdb_image "$engine" "$ver" } # The configured version for an engine: MDB__VERSION if set, otherwise