From 1b6cc5a24417baaf1fdde45ab4333467e4338eac Mon Sep 17 00:00:00 2001 From: Imani Manyara Date: Fri, 13 Nov 2026 13:11:45 -0500 Subject: [PATCH 1/2] Name our images localhost/dbpantry-* and never fall back to ghcr.io dbpantry publishes no image (D-77), but every image was still built, run and deployed as ghcr.io/simtabi/dbpantry-*, so a cluster missing one pulled from a registry that holds nothing. One default now names them: MDB_IMAGE_PREFIX_DEFAULT=localhost in scripts/env-lib.sh, with mdb_image_prefix, mdb_image and mdb_image_ref. It replaces every hard-coded registry in Compose, the generated engine and pooler services, the patroni build, the Makefile, five scripts, six checks and both scan steps. The Helm chart defaults global.imageRegistry to localhost, helm-values passes it the prefix in force, and kind loading follows it. A set MDB_IMAGE_PREFIX or global.imageRegistry still prefixes every image. localhost rather than an unqualified name: check 49 and Podman need a fully qualified reference, and Docker would expand a short one to Docker Hub. Check 49 now resolves every ${VAR:-default} in a reference. New check 73 holds every default to localhost, fails on a remote registry for our images, and proves a set prefix reaches every image. Recorded as D-78; existing local images need one make build. --- .github/workflows/ci.yml | 4 +- .github/workflows/release.yml | 2 +- CHANGELOG.md | 20 ++- Makefile | 6 +- compose.ha.yml | 5 +- deploy/helm/dbpantry/values.yaml | 10 +- design.md | 12 +- docker-compose.yml | 8 +- docs/guides/architecture.md | 2 +- docs/guides/kubernetes.md | 63 +++++++- docs/guides/licensing.md | 13 +- docs/guides/runtime-apple-container.md | 2 +- docs/guides/runtime-podman.md | 3 +- docs/reference/acceptance.md | 1 + docs/reorganization-plan.md | 2 +- engines/postgres/engine.conf | 4 +- images/patroni/Dockerfile | 2 +- scripts/backends/kubernetes.sh | 11 +- scripts/build | 2 +- scripts/env-lib.sh | 31 ++++ scripts/gen-compose | 8 +- scripts/helm-values | 4 + scripts/scan | 4 +- scripts/transfer | 4 +- scripts/upgrade | 2 +- scripts/verify-backups | 2 +- scripts/verify/02-pinned-versions.sh | 2 +- scripts/verify/08-pg-standalone-run.sh | 5 +- scripts/verify/14-version-switch.sh | 2 +- scripts/verify/29-upgrade-migrates-data.sh | 2 +- scripts/verify/32-every-engine-auth.sh | 2 +- scripts/verify/49-images-fully-qualified.sh | 11 +- scripts/verify/55-kubernetes-ha.sh | 2 +- scripts/verify/68-pooler-tls.sh | 1 + scripts/verify/73-images-local-by-default.sh | 162 +++++++++++++++++++ scripts/verify/lib.sh | 10 +- 36 files changed, 361 insertions(+), 65 deletions(-) create mode 100755 scripts/verify/73-images-local-by-default.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5997d71..8282795 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,7 +182,7 @@ jobs: --ignorefile .trivyignore.yaml \ --exit-code "$code" \ --format table \ - "ghcr.io/simtabi/dbpantry-${ENGINE}:${VERSION}" + "localhost/dbpantry-${ENGINE}:${VERSION}" - name: Report unfixed findings (never fails) if: always() @@ -191,7 +191,7 @@ jobs: VERSION: ${{ matrix.entry.version }} run: | trivy image --severity HIGH,CRITICAL --exit-code 0 --format table \ - "ghcr.io/simtabi/dbpantry-${ENGINE}:${VERSION}" || true + "localhost/dbpantry-${ENGINE}:${VERSION}" || true # --------------------------------------------------------------------------- # When the scheduled build fails, say so where people look (design.md D-71). diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bbfb01a..787fd37 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -209,7 +209,7 @@ jobs: run: | trivy image --severity HIGH,CRITICAL --ignore-unfixed \ --ignorefile .trivyignore.yaml --exit-code 1 --format table \ - "ghcr.io/simtabi/dbpantry-${ENGINE}:${VERSION}" + "localhost/dbpantry-${ENGINE}:${VERSION}" # --------------------------------------------------------------------------- # Gate 2: the acceptance harness, on both architectures. diff --git a/CHANGELOG.md b/CHANGELOG.md index f21caf0..588adda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ waiving them. ### Upgrade +- **Images are now named `localhost/dbpantry-*`.** Images you built before keep their old + `ghcr.io/simtabi/dbpantry-*` names, which nothing runs any more. Run `make build` once; the next + `make up` recreates the containers on the new names. Volumes and data are untouched. To build, + run and deploy under your own registry instead, export `MDB_IMAGE_PREFIX` (for example + `registry.example.com/team`) before `make build` and `make up` (design.md D-78). - **Prod: PostgreSQL's own port stops being published.** With the prod overlay applied (see Changed), a prod stack under `MDB_PUBLISH=direct` publishes the pooler, `pg-pooler` on `MDB_PORT_BASE + 100` (54100 by default), and no engine port: 54000 and the other engines' ports go away. Point applications at 54100 with @@ -29,9 +34,18 @@ waiving them. - **Container images are no longer published.** A release publishes the GitHub release only, with the notes and the source tarball (owner, 2026-11-13). The `ghcr.io/simtabi/dbpantry-*` images are frozen: no new tag is pushed and nothing is deleted from ghcr.io. Build the images with - `make build`. The Kubernetes backend and the Helm chart still name `ghcr.io/simtabi` as their - registry, so push your own build there or point `global.imageRegistry` at your own registry - (design.md D-77). + `make build` (design.md D-77). +- **Images are built, run and deployed as `localhost/dbpantry-:`,** and nothing + falls back to a remote registry. One default, `MDB_IMAGE_PREFIX=localhost` in + `scripts/env-lib.sh`, replaces every hard-coded registry: Compose, the generated engine and pooler + services, the patroni build, the Makefile, `scripts/{build,upgrade,verify-backups,transfer,scan}`, + the verify checks and the CI scan steps. The Helm chart's `global.imageRegistry` defaults to + `localhost`, `make up` on Kubernetes passes it the prefix the images were built under, and kind + loading follows it; `imagePullPolicy` stays `IfNotPresent`. Setting `MDB_IMAGE_PREFIX` (or + `global.imageRegistry` when installing the chart by hand) prefixes every image. The Kubernetes + guide shows `kind load docker-image`, `minikube image load` and pushing to your own registry. + Check 73 fails if any default names a remote registry or a set prefix stops reaching every image, + and check 49 now resolves every `${VAR:-default}` in an image reference (design.md D-78). - **A release runs on a pushed `vX.Y.Z` tag, or on request for a tag already on GitHub:** `gh workflow run release.yml --repo simtabi/dbpantry -f tag=vX.Y.Z`. This replaces the dispatch gated on `-f confirm=release-go` (D-76). Every job checks out the tag, and the first step refuses diff --git a/Makefile b/Makefile index 20c8cb3..f1a67bb 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,10 @@ env_project := $(call env_get,COMPOSE_PROJECT_NAME) COMPOSE_PROJECT_NAME ?= $(or $(env_project),mdb) export COMPOSE_PROJECT_NAME ENV_FILE_PROD ?= .env.prod +# The prefix our images are built and run under: MDB_IMAGE_PREFIX from the +# environment, else the one default in scripts/env-lib.sh (localhost). Set but +# empty counts as unset, as it does for the scripts. +MDB_IMAGE_PREFIX := $(or $(MDB_IMAGE_PREFIX),$(shell bash -c '. scripts/env-lib.sh && mdb_image_prefix' 2>/dev/null)) # Every container call goes through scripts/runtime, which picks Docker, Podman # or nerdctl (CONTAINER_RUNTIME=auto|docker|podman|nerdctl; design.md D-60). RUNTIME := scripts/runtime @@ -187,7 +191,7 @@ shell: ## Open the cli image (tools with no server required) --network $(COMPOSE_PROJECT_NAME)_net \ -v "$(CURDIR)/certs:/certs:ro" \ -v "$(CURDIR)/backups:/work/backups" \ - ghcr.io/simtabi/dbpantry-cli:dev + $(MDB_IMAGE_PREFIX)/dbpantry-cli:dev .PHONY: new-project new-project: ## Provision DB + roles + extensions, print the Laravel .env block diff --git a/compose.ha.yml b/compose.ha.yml index 1673a12..c6e2a20 100644 --- a/compose.ha.yml +++ b/compose.ha.yml @@ -42,7 +42,8 @@ x-patroni: &patroni dockerfile: patroni/Dockerfile args: ENGINE_VERSION: ${MDB_PG_VERSION:-17} - image: ghcr.io/simtabi/dbpantry-patroni:${MDB_PG_VERSION:-17} + IMAGE_PREFIX: ${MDB_IMAGE_PREFIX:-localhost} + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-patroni:${MDB_PG_VERSION:-17} profiles: [ha] restart: unless-stopped networks: [mdb] @@ -210,7 +211,7 @@ services: # --------------------------------------------------------------------------- pgbouncer: profiles: [ha-pooler] - image: ghcr.io/simtabi/dbpantry-pgbouncer:dev + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-pgbouncer:dev restart: unless-stopped depends_on: haproxy: {condition: service_healthy} diff --git a/deploy/helm/dbpantry/values.yaml b/deploy/helm/dbpantry/values.yaml index 826b40f..d9f08cb 100644 --- a/deploy/helm/dbpantry/values.yaml +++ b/deploy/helm/dbpantry/values.yaml @@ -6,9 +6,13 @@ # engines you want and create the Secret named in global.secretName first. global: - # Registry and repository prefix for the dbpantry images. The tag is each - # engine's version (engines..tag). - imageRegistry: ghcr.io/simtabi + # Registry and repository prefix for the dbpantry images, which are named + # /dbpantry-: (the tag is engines..tag). + # localhost is the name `make build` gives them, so a kind or minikube + # cluster runs images loaded into it (docs/guides/kubernetes.md). Set it to + # your own registry (registry.example.com/team) after pushing them there. + # No dbpantry image is published to any public registry. + imageRegistry: localhost imagePullPolicy: IfNotPresent imagePullSecrets: [] # "" uses the cluster's default StorageClass. diff --git a/design.md b/design.md index 27fefe1..0d0edab 100644 --- a/design.md +++ b/design.md @@ -9,7 +9,7 @@ Everything below was verified during this session against live registries, the G | Field | Value | |---|---| | Repository | `github.com/simtabi/dbpantry` (was `dbpantry-docker`, and before that `db-toolkit-docker`) | -| Images | `ghcr.io/simtabi/dbpantry-{pg,mysql,mariadb,cli}` | +| Images | `localhost/dbpantry-{pg,mysql,mariadb,cli}`, built locally (`MDB_IMAGE_PREFIX`, D-78); not published | | Docs | [`docs/`](docs/) in this repository; no hosted site yet | | Product page | none yet | | Env prefix | `MDB_` | @@ -22,9 +22,9 @@ Resolves the three-way conflict recorded in decision **D-02**. | Service | Image | Profile | Internal port | Host port env | Published by default | |---|---|---|---|---|---| -| `pg` | `ghcr.io/simtabi/dbpantry-pg:` | `pg` | 5432 | `MDB_PG_HOST_PORT` | no | -| `mysql` | `ghcr.io/simtabi/dbpantry-mysql:` | `mysql` | 3306 | `MDB_MYSQL_HOST_PORT` | no | -| `mariadb` | `ghcr.io/simtabi/dbpantry-mariadb:` | `mariadb` | 3306 | `MDB_MARIADB_HOST_PORT` (3307) | no | +| `pg` | `localhost/dbpantry-pg:` | `pg` | 5432 | `MDB_PG_HOST_PORT` | no | +| `mysql` | `localhost/dbpantry-mysql:` | `mysql` | 3306 | `MDB_MYSQL_HOST_PORT` | no | +| `mariadb` | `localhost/dbpantry-mariadb:` | `mariadb` | 3306 | `MDB_MARIADB_HOST_PORT` (3307) | no | | `adminer` | `adminer:5.4.1` | `ui` | 8080 | `MDB_ADMINER_HOST_PORT` | no (via Caddy) | | `phpmyadmin` | `phpmyadmin:5.2.3` | `ui` | 80 | `MDB_PMA_HOST_PORT` | no (via Caddy) | | `pgadmin` | `dpage/pgadmin4:9.9` | `ui` | 80 | `MDB_PGADMIN_HOST_PORT` | no (via Caddy) | @@ -36,7 +36,7 @@ Resolves the three-way conflict recorded in decision **D-02**. | `pgbouncer` | `dbpantry-pgbouncer` (built) | `prod` (required), `ha` | 6432 | `MDB_PGBOUNCER_HOST_PORT` | no | | `etcd` | `gcr.io/etcd-development/etcd:v3.6.6` | `ha` | 2379/2380 | — | no | | `haproxy` | `haproxy:3.2-alpine` | `ha` | 5432 w / 5433 r | `MDB_HAPROXY_*_PORT` | no | -| `cli` | `ghcr.io/simtabi/dbpantry-cli:` | on demand | — | — | no | +| `cli` | `localhost/dbpantry-cli:` | on demand | — | — | no | The exporter rows match `docker-compose.yml` as of 2026-11-03. The compose files are the source of truth for every image here; the `etcd`, `haproxy` and `caddy` rows above still show the step 1 plan and differ from `compose.ha.yml` (`quay.io/coreos/etcd:v3.5.21`, `haproxy:3.0-alpine`) and `docker-compose.yml` (the built `dbpantry-caddy` image). @@ -492,6 +492,8 @@ Relied on: pgBouncer configuration reference (`, because a dispatch's `GITHUB_SHA` is its branch's head, not the tag's commit: the tag-on-`main` check now tests `HEAD` after that checkout. No pre-release tag filter is declared, since dbpantry refuses pre-releases. dbpantry builds no release binaries, so it keeps its own jobs rather than calling `simtabi/.github`'s shared `go-release.yml`, which runs GoReleaser. The Kubernetes backend (`scripts/backends/kubernetes.sh`) and the Helm chart (`global.imageRegistry: ghcr.io/simtabi`) still name the frozen registry; changing them is a separate decision. The guard is `src/internal/release`'s `TestReleaseRunsOnAVersionTagOrOnRequest`, which fails on any other trigger, a branch or non-version tag filter, a missing or optional `tag` input, a first step that is not the refusal, a checkout of anything but the tag, a job that does not need the gating job, or any image publish (registry login or push, `buildx --push`, `imagetools create`, `push-to-registry: true`, a `docker/` publish action, `packages: write`); `TestReleasePolicyRejectsAWrongRelease` proves each can fail, and that a local build and scan passes. +**D-78: our images are named localhost/dbpantry-*, from one default, and nothing names a remote registry.** Follows D-77. With images no longer published, `ghcr.io/simtabi/dbpantry-*` stayed the name every image was built, run and deployed under: `scripts/build` defaulted `MDB_IMAGE_PREFIX` to it, Compose, the generated engine services, the pooler descriptor, the patroni Dockerfile, the Makefile, five scripts, six checks and both workflows wrote it out, the Helm chart's `global.imageRegistry` defaulted to it, and the Kubernetes backend loaded it into kind. A cluster without an image therefore pulled from a registry that holds nothing. The owner chose `localhost/` (2026-11-13) over an unqualified `dbpantry/`, which check 49 refuses because Podman will not resolve a short name (D-60) and which Docker would expand to Docker Hub on a pull; `localhost` is fully qualified, is the prefix Podman gives local builds, never reaches a remote registry, and loads into kind and minikube as it is. The default is written once, `MDB_IMAGE_PREFIX_DEFAULT=localhost` in `scripts/env-lib.sh`, with `mdb_image_prefix`, `mdb_image` and `mdb_image_ref` (which moves a descriptor's `localhost/dbpantry-pgbouncer:dev` to the prefix in force); the Makefile reads it from there. Files that cannot read it repeat it: Compose as `${MDB_IMAGE_PREFIX:-localhost}`, the patroni Dockerfile's `ARG IMAGE_PREFIX`, the chart's `global.imageRegistry`, the pooler descriptor, and the CI scan steps, which build with the default. `MDB_IMAGE_PREFIX` stays an environment variable, as before, not a `.env` setting. On Kubernetes, `scripts/helm-values` renders `global.imageRegistry` from it and `_mdb_k8s_local_images` reads that back, so kind loads exactly the names the chart runs; `imagePullPolicy` stays `IfNotPresent`. Check 49's Compose parser resolved only a leading `${VAR:-default}`, which misread a prefixed reference as `localhost}/...`; it now resolves every one, innermost first, and still counts 55 references. Check 73 holds every default to `localhost`, fails on a dotted registry before `dbpantry-` in any tracked file but prose, limits `ghcr.io/simtabi` to nine files that describe the frozen images or their history (each at its measured count), and proves that a set prefix reaches `mdb_image`, the pooler, `make shell`, helm-values, kind loading and the rendered chart. Images built before keep their old names: `make build` once, and the next `make up` recreates the containers; volumes are untouched. + ## 8. arm64 deliverability **Nothing in the spec is undeliverable on arm64.** The named unknown anticipated gaps that do not exist: diff --git a/docker-compose.yml b/docker-compose.yml index 8c85393..ec2ba89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,7 +119,7 @@ services: build: context: ./images dockerfile: caddy/Dockerfile - image: ghcr.io/simtabi/dbpantry-caddy:dev + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-caddy:dev restart: unless-stopped <<: *hardening # Binding 80/443 needs this one capability back; everything else stays @@ -159,7 +159,7 @@ services: # --------------------------------------------------------------------------- backup-pg: profiles: [backup] - image: ghcr.io/simtabi/dbpantry-pg:${MDB_PG_VERSION:-17} + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-pg:${MDB_PG_VERSION:-17} restart: unless-stopped <<: *hardening entrypoint: ["/mdb/scripts/backup", "--engine", "pg", "--schedule"] @@ -213,7 +213,7 @@ services: backup-mysql: profiles: [backup] - image: ghcr.io/simtabi/dbpantry-mysql:${MDB_MYSQL_VERSION:-8.4} + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-mysql:${MDB_MYSQL_VERSION:-8.4} restart: unless-stopped <<: *hardening entrypoint: ["/mdb/scripts/backup", "--engine", "mysql", "--schedule"] @@ -266,7 +266,7 @@ services: backup-mariadb: profiles: [backup] - image: ghcr.io/simtabi/dbpantry-mariadb:${MDB_MARIADB_VERSION:-11.8} + image: ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-mariadb:${MDB_MARIADB_VERSION:-11.8} restart: unless-stopped <<: *hardening entrypoint: ["/mdb/scripts/backup", "--engine", "mariadb", "--schedule"] diff --git a/docs/guides/architecture.md b/docs/guides/architecture.md index 9e5f169..621d691 100644 --- a/docs/guides/architecture.md +++ b/docs/guides/architecture.md @@ -47,7 +47,7 @@ MDB_ENGINE_DUMP_GLOBALS='pg_dumpall --globals-only' MDB_ENGINE_RESTORE='pg_restore --no-owner' MDB_ENGINE_POOLING=external -MDB_ENGINE_POOLER_IMAGE=ghcr.io/simtabi/dbpantry-pgbouncer:dev +MDB_ENGINE_POOLER_IMAGE=localhost/dbpantry-pgbouncer:dev MDB_ENGINE_POOLING_REQUIRED_IN_PROD=true MDB_ENGINE_LICENSE=PostgreSQL diff --git a/docs/guides/kubernetes.md b/docs/guides/kubernetes.md index ff21f0d..4b1d2e8 100644 --- a/docs/guides/kubernetes.md +++ b/docs/guides/kubernetes.md @@ -18,6 +18,7 @@ kind 0.33 (all five engines on kind; PG and MySQL on both). | kubectl | within one minor version of the cluster | on PATH | | Helm | 3.x or 4.x | on PATH; tested with 4.3.1 | | kind | 0.20 or later | only for kind clusters, to load the local images | +| minikube | any | only for minikube clusters; load the images with `minikube image load` (see [Images](#images)) | | A StorageClass | any `ReadWriteOnce` class | the cluster's default is used unless `MDB_K8S_STORAGE_CLASS` names one | ## Quick start @@ -28,7 +29,7 @@ MDB_BACKEND=kubernetes MDB_K8S_CONTEXT=orbstack # required: `kubectl config get-contexts` MDB_PROFILES="pg,mysql" -make build # local images (skip on a real cluster that pulls from GHCR) +make build # the images, as localhost/dbpantry-*; a real cluster needs them pushed (see Images) make up # secrets, certs and overrides, then helm upgrade --install --wait make psql make backup # dumps stream through kubectl exec into backups/, as on Compose @@ -46,6 +47,7 @@ development database to production. | `MDB_K8S_STORAGE_CLASS` | the cluster's default | StorageClass of every volume | | `MDB_K8S_VALUES` | none | extra values files, colon-separated, applied last | | `MDB_K8S_LOAD_IMAGES` | `auto` | copy local images into the cluster: `auto` on `kind-*` contexts only | +| `MDB_IMAGE_PREFIX` | `localhost` | the name prefix the images are built and run under; set it to your own registry (environment variable, see [Images](#images)) | | `MDB_K8S_CNPG_READY_TIMEOUT` | `MDB_WAIT_TIMEOUT`, else `600` | seconds `make up` waits for a CloudNativePG Cluster's instances to be ready | ## What `make up` does @@ -63,9 +65,10 @@ development database to production. `dbpantry-overrides-`. It uses `kubectl apply`, so running it again after `make rotate-secrets` updates them. 3. On a kind context, `kind load docker-image` for each locally built engine - image. OrbStack and Docker Desktop share Docker's image store already; a - real cluster pulls from the registry, so push your images there or use the - published ones on GHCR. + image, under the same prefix the chart runs (`global.imageRegistry`, from + `MDB_IMAGE_PREFIX`). OrbStack and Docker Desktop share Docker's image store + already; minikube and a real cluster need the images loaded or pushed first + (see [Images](#images)). 4. `helm upgrade --install --wait` with that values file and any `MDB_K8S_VALUES`. 5. With a CloudNativePG Cluster, waits until `.status.readyInstances` reaches @@ -79,6 +82,55 @@ so `make up` brings the same data back. Only `make destroy` deletes them, with the same typed confirmation as on Compose, and it keeps the `pitr-*` recovery volumes unless you add `--include-recovery`. +## Images + +No dbpantry image is published to a public registry: the cluster runs the +images `make build` built, named `/dbpantry-:`. The +prefix is `MDB_IMAGE_PREFIX`, `localhost` by default, and `make up` hands it to +the chart as `global.imageRegistry`, so the chart runs exactly what was built. +`imagePullPolicy` is `IfNotPresent`, so an image already in the cluster's store +is used as it is. The old `ghcr.io/simtabi/dbpantry-*` images are frozen +([Release](release.md)), and nothing falls back to them. + +Build the images first, on the same architecture as the cluster's nodes: + +```bash +make build # every enabled engine, plus pgbouncer and patroni +``` + +**kind** loads them for you: on a `kind-*` context `make up` runs +`kind load docker-image` for each image it is about to deploy +(`MDB_K8S_LOAD_IMAGES=auto`). To load one by hand: + +```bash +kind load docker-image localhost/dbpantry-pg:17 --name kind # the cluster of context kind-kind +``` + +**minikube** shares no image store with your Docker, so load them before +`make up`: + +```bash +minikube image load localhost/dbpantry-pg:17 +minikube image load localhost/dbpantry-patroni:17 # with the ha profile +``` + +**Any other cluster** pulls from a registry, so build under your registry's +name, push, and deploy with the same prefix. `MDB_IMAGE_PREFIX` is read from +the environment, not from `.env`: + +```bash +export MDB_IMAGE_PREFIX=registry.example.com/team +make build +docker push registry.example.com/team/dbpantry-pg:17 +make up +``` + +`make up` then renders `global.imageRegistry: registry.example.com/team`, and +every dbpantry image in the release carries that prefix. Add the pull secret +with `global.imagePullSecrets` in a values file named by `MDB_K8S_VALUES`. +Installing the chart by hand works the same way: +`helm install dbpantry deploy/helm/dbpantry --set global.imageRegistry=registry.example.com/team ...`. + ## What gets deployed One StatefulSet per enabled engine, with one replica: @@ -215,7 +267,8 @@ The Services are `ClusterIP`. Set `service.type: LoadBalancer` in an |---|---| | `needs MDB_K8S_CONTEXT` | set it in `.env`; see `kubectl config get-contexts` | | Helm says `TLS handshake timeout` while kubectl works | the API server name resolves to an IPv6 address that does not answer, and Helm does not fall back to IPv4. Seen with OrbStack's `k8s.orb.local`. Point a kubeconfig context at `https://127.0.0.1:26443` with `tls-server-name: k8s.orb.local` | -| `ErrImagePull` on kind | build the images first, or set `MDB_K8S_LOAD_IMAGES=true` | +| `ErrImagePull` / `ErrImageNeverPull` on kind or minikube | the image is not in the cluster's store: `make build`, then load it (kind: `make up` does, or `MDB_K8S_LOAD_IMAGES=true`; minikube: `minikube image load`). See [Images](#images) | +| `ErrImagePull` on another cluster | `global.imageRegistry` is `localhost`, which no cluster can pull from: push the images and set `MDB_IMAGE_PREFIX` (see [Images](#images)) | | `another operation (install/upgrade/rollback) is in progress` | an interrupted `make up` left the release `pending-upgrade`. Run `helm --kube-context -n history `, then `helm ... rollback `, and run `make up` again | | CloudNativePG: `conflict with "kubectl-annotate" ... cnpg.io/hibernation` | a Cluster hibernated by an older dbpantry. Run `kubectl annotate clusters.postgresql.cnpg.io -pg cnpg.io/hibernation-`, then `make up` (current versions do this themselves) | | HA: a replica logs `no pg_hba.conf entry for replication` | a dbpantry-patroni image from before D-65; rebuild or pull it and delete the pod | diff --git a/docs/guides/licensing.md b/docs/guides/licensing.md index 44b2f69..d22d6be 100644 --- a/docs/guides/licensing.md +++ b/docs/guides/licensing.md @@ -83,17 +83,18 @@ withdraw. ## Verifying what you run -Every published image carries an accurate -`org.opencontainers.image.licenses` label for its own contents, plus a build -provenance attestation: +Every image `make build` produces carries an accurate +`org.opencontainers.image.licenses` label for its own contents: ```bash -docker inspect ghcr.io/simtabi/dbpantry-pg:17 \ +docker inspect localhost/dbpantry-pg:17 \ --format '{{ index .Config.Labels "org.opencontainers.image.licenses" }}' - -gh attestation verify oci://ghcr.io/simtabi/dbpantry-pg:17 --owner simtabi ``` +The images are no longer published, so there is no registry attestation to +verify; the frozen `ghcr.io/simtabi/dbpantry-*` images are described in +[Release](release.md). + --- [← Docs index](../../README.md#documentation) diff --git a/docs/guides/runtime-apple-container.md b/docs/guides/runtime-apple-container.md index af59165..0405c4c 100644 --- a/docs/guides/runtime-apple-container.md +++ b/docs/guides/runtime-apple-container.md @@ -72,7 +72,7 @@ because its BuildKit builder otherwise keeps the result in the build cache), or copy an image across: ```bash -docker save -o /tmp/pg17.tar ghcr.io/simtabi/dbpantry-pg:17 # on the other engine +docker save -o /tmp/pg17.tar localhost/dbpantry-pg:17 # on the other engine DOCKER_HOST=unix:///opt/homebrew/var/run/socktainer/.socktainer/container.sock \ docker load -i /tmp/pg17.tar ``` diff --git a/docs/guides/runtime-podman.md b/docs/guides/runtime-podman.md index e9e1ba6..ddacb76 100644 --- a/docs/guides/runtime-podman.md +++ b/docs/guides/runtime-podman.md @@ -140,7 +140,8 @@ make verify CHECKS=08,15,22,28 ``` Podman keeps its own image store, separate from Docker's, so the engine images -have to be built (or pulled from GHCR) again under Podman. +have to be built again under Podman (`make build`). Their `localhost/` prefix is +the name Podman gives local builds anyway. If `docker-compose` fails pulls with `unauthorized: incorrect username or password` for docker.io images, it is sending stale Docker Hub credentials from diff --git a/docs/reference/acceptance.md b/docs/reference/acceptance.md index 2fd5b0f..65484d3 100644 --- a/docs/reference/acceptance.md +++ b/docs/reference/acceptance.md @@ -49,6 +49,7 @@ so a green `make verify` is the criteria being met rather than a proxy for it. | Images take distribution updates whole, download only through a verified, retried helper; CI gates findings where caused | `70-supply-chain` | | Every profile combination renders with one authenticating pooler per upstream; prod publishes only poolers and the front door | `71-compose-profiles-render` | | Every service drops privileges, has a health signal, is limited under prod and takes secrets as files; exceptions must stay needed | `72-service-hardening` | +| Our images are named `localhost/` by default, nothing names a remote registry for them, and a set `MDB_IMAGE_PREFIX` or `global.imageRegistry` reaches every one | `73-images-local-by-default` | | No leftover old product name outside its history | `44-no-old-name` | | Cassandra's replaced JARs are pinned by checksum and installed | `45-cassandra-jars` | | Every relative link in the Markdown docs resolves | `46-doc-links-resolve` | diff --git a/docs/reorganization-plan.md b/docs/reorganization-plan.md index f00aa0f..98f7701 100644 --- a/docs/reorganization-plan.md +++ b/docs/reorganization-plan.md @@ -19,7 +19,7 @@ on stays where it is. | `engines//engine.conf` + `compose-*.yml`, `engines/_family/*.sh` | `scripts/gen-compose`, `scripts/engine-lib.sh` | `compose.engines.yml`, `compose.proxy.yml`, `config/caddy/layer4.caddy` (generated) | | `config/caddy/*.tmpl` | `scripts/render-config` → `scripts/env-render` | `config/caddy/Caddyfile`, `config/caddy/servers.json` (generated) | | `config/haproxy/haproxy.cfg`, `config/proxysql/bootstrap.sh` | compose (bind mounts) | HA front door, ProxySQL pooler | -| `images//`, `images/_shared/`, `images/bases.tsv` | `scripts/build`, compose `build:` | `ghcr.io/simtabi/dbpantry-*` images | +| `images//`, `images/_shared/`, `images/bases.tsv` | `scripts/build`, compose `build:` | `localhost/dbpantry-*` images (`MDB_IMAGE_PREFIX`) | | `docker-compose.yml` + `compose.*.yml` | `docker compose` via `COMPOSE_FILE` | running stack, named volumes | | running stack | `scripts/backup`, `scripts/pitr`, `scripts/transfer` | `backups/` (gitignored) | | `scripts/certs` | — | `certs/` (gitignored) | diff --git a/engines/postgres/engine.conf b/engines/postgres/engine.conf index 8f96ef5..a61d7cc 100644 --- a/engines/postgres/engine.conf +++ b/engines/postgres/engine.conf @@ -49,7 +49,9 @@ MDB_ENGINE_POOLING=external # image, and edoburu/pgbouncer was this toolkit's only community rebuild. # images/pgbouncer/Dockerfile builds it from a Docker Official base plus the # PGDG package. See docs/reference/image-provenance.md. -MDB_ENGINE_POOLER_IMAGE='ghcr.io/simtabi/dbpantry-pgbouncer:dev' +# Written with the default prefix; scripts/gen-compose and the checks apply +# MDB_IMAGE_PREFIX to it (mdb_image_ref in scripts/env-lib.sh). +MDB_ENGINE_POOLER_IMAGE='localhost/dbpantry-pgbouncer:dev' MDB_ENGINE_POOLER_BUILD=pgbouncer/Dockerfile # Every pooler starts through a script in the repository. Note it must exec the # image's own entrypoint AND its CMD: overriding a service's entrypoint discards diff --git a/images/patroni/Dockerfile b/images/patroni/Dockerfile index 51c1692..f610459 100644 --- a/images/patroni/Dockerfile +++ b/images/patroni/Dockerfile @@ -12,7 +12,7 @@ # bundle rather than left to race Patroni for the same data directory. ARG ENGINE_VERSION=17 -ARG IMAGE_PREFIX=ghcr.io/simtabi +ARG IMAGE_PREFIX=localhost FROM ${IMAGE_PREFIX}/dbpantry-pg:${ENGINE_VERSION} # Pinned. Patroni's behaviour on failover is the whole point of this image, so diff --git a/scripts/backends/kubernetes.sh b/scripts/backends/kubernetes.sh index 242ba6e..a6431eb 100644 --- a/scripts/backends/kubernetes.sh +++ b/scripts/backends/kubernetes.sh @@ -289,16 +289,19 @@ _mdb_be_kubernetes_image_load() { esac } -# The images the release will run that are built here (MongoDB's is upstream). +# The images the release will run that are built here (MongoDB's is upstream), +# under the prefix they were built with: the global.imageRegistry helm-values +# rendered from MDB_IMAGE_PREFIX, the same name the chart then runs. _mdb_k8s_local_images() { local values="$1" - awk ' + awk -v prefix="$(mdb_image_prefix)" ' + /^ imageRegistry: / { prefix = $2; gsub(/"/, "", prefix) } /^ [a-z]+:$/ { eng = $1; sub(/:/, "", eng); en = 0; ha = 0; tag = "" } /^ enabled: true$/ { en = 1 } /^ enabled: true$/ { ha = 1 } /^ tag: / { tag = $2; gsub(/"/, "", tag) } - /^ env:$/ && en && tag != "" { print "ghcr.io/simtabi/dbpantry-" eng ":" tag } - /^ provider: "?patroni"?$/ && en && ha && tag != "" { print "ghcr.io/simtabi/dbpantry-patroni:" tag } + /^ env:$/ && en && tag != "" { print prefix "/dbpantry-" eng ":" tag } + /^ provider: "?patroni"?$/ && en && ha && tag != "" { print prefix "/dbpantry-patroni:" tag } ' "$values" } diff --git a/scripts/build b/scripts/build index 3625e5b..d12c30d 100755 --- a/scripts/build +++ b/scripts/build @@ -22,7 +22,7 @@ cd "$ROOT" || exit 1 ENV_FILE="${MDB_ENV_FILE:-$ROOT/.env}" [[ -f "$ENV_FILE" ]] || ENV_FILE="$ROOT/.env.example" -PREFIX="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" +PREFIX="$(mdb_image_prefix)" get() { mdb_env_get "$1" "$ENV_FILE"; } diff --git a/scripts/env-lib.sh b/scripts/env-lib.sh index 4f76f87..8e971d1 100644 --- a/scripts/env-lib.sh +++ b/scripts/env-lib.sh @@ -90,3 +90,34 @@ mdb_env_needs_quotes() { } export -f _mdb_env_unsafe mdb_env_unquote mdb_env_get mdb_env_quote mdb_env_needs_quotes 2>/dev/null || true + +# The images dbpantry builds are named /dbpantry-:. +# The prefix is MDB_IMAGE_PREFIX, or localhost when it is unset (design.md +# D-78): a fully qualified local name, so Docker, Podman and nerdctl resolve it +# the same way (check 49), it never falls back to a remote registry, and kind or +# minikube can load it. Set MDB_IMAGE_PREFIX to your own registry (for example +# registry.example.com/team) to build, run and deploy under that name instead. +# Compose files, the Helm chart, the Makefile and the patroni Dockerfile carry +# the same default; check 73 fails if any of them names another. +MDB_IMAGE_PREFIX_DEFAULT=localhost + +# The prefix in force, without a trailing slash. +mdb_image_prefix() { + local p="${MDB_IMAGE_PREFIX:-$MDB_IMAGE_PREFIX_DEFAULT}" + printf '%s\n' "${p%/}" +} + +# The full name of one of our images: mdb_image pg 17 -> localhost/dbpantry-pg:17 +mdb_image() { + printf '%s/dbpantry-%s:%s\n' "$(mdb_image_prefix)" "$1" "$2" +} + +# A reference written with the default prefix (an engine descriptor's +# localhost/dbpantry-pgbouncer:dev) under the prefix in force. Any other +# reference, such as an upstream image, is returned unchanged. +mdb_image_ref() { + case "$1" in + "$MDB_IMAGE_PREFIX_DEFAULT"/dbpantry-*) printf '%s/%s\n' "$(mdb_image_prefix)" "${1#"$MDB_IMAGE_PREFIX_DEFAULT"/}" ;; + *) printf '%s\n' "$1" ;; + esac +} diff --git a/scripts/gen-compose b/scripts/gen-compose index d340d90..75b8976 100755 --- a/scripts/gen-compose +++ b/scripts/gen-compose @@ -146,7 +146,7 @@ for engine in $(engine_list); do image_line="\${MDB_${upper}_IMAGE:-\${MDB_${upper}_BASE_IMAGE:-${pinned}}}" build_block="" else - image_line="ghcr.io/simtabi/dbpantry-${MDB_ENGINE_NAME}:\${MDB_${upper}_VERSION:-${MDB_ENGINE_DEFAULT_VERSION}}" + image_line="\${MDB_IMAGE_PREFIX:-${MDB_IMAGE_PREFIX_DEFAULT}}/dbpantry-${MDB_ENGINE_NAME}:\${MDB_${upper}_VERSION:-${MDB_ENGINE_DEFAULT_VERSION}}" # BASE_IMAGE is passed so a compose build of a non-default version # starts from that version's base rather than the Dockerfile's default. build_block=" build: @@ -254,7 +254,11 @@ for engine in $(engine_list); do # project over the dangling depends_on. scripts/profiles expands # `pooler` into the ones that match the selected engines. printf ' profiles: [%s-pooler]\n' "$MDB_ENGINE_NAME" - printf ' image: %s\n' "$MDB_ENGINE_POOLER_IMAGE" + # Our own pooler image follows MDB_IMAGE_PREFIX as the engines do. + case "$MDB_ENGINE_POOLER_IMAGE" in + "$MDB_IMAGE_PREFIX_DEFAULT"/dbpantry-*) printf ' image: ${MDB_IMAGE_PREFIX:-%s}/%s\n' "$MDB_IMAGE_PREFIX_DEFAULT" "${MDB_ENGINE_POOLER_IMAGE#"$MDB_IMAGE_PREFIX_DEFAULT"/}" ;; + *) printf ' image: %s\n' "$MDB_ENGINE_POOLER_IMAGE" ;; + esac printf ' restart: unless-stopped\n' printf ' depends_on: [%s]\n' "$MDB_ENGINE_NAME" printf ' security_opt:\n - no-new-privileges:true\n' diff --git a/scripts/helm-values b/scripts/helm-values index c53143b..46d05cb 100755 --- a/scripts/helm-values +++ b/scripts/helm-values @@ -16,6 +16,7 @@ # MDB__MEM_LIMIT / _CPU_LIMIT memory and cpu (requests and limits) # MDB_PG_SHM the size of PostgreSQL's /dev/shm # MDB_K8S_STORAGE_CLASS global.storageClass +# MDB_IMAGE_PREFIX global.imageRegistry (default localhost) # the ha profile PostgreSQL HA (engines.pg.ha): Patroni, or # CloudNativePG with MDB_K8S_HA_PROVIDER=cnpg; # MDB_HA_CLUSTER_NAME, MDB_PG_SYNC_MODE, @@ -165,6 +166,9 @@ render() { printf '# Rendered by scripts/helm-values from %s. Do not edit: change .env.\n' "$(basename "$ENV_FILE")" sc="$(setting MDB_K8S_STORAGE_CLASS)" printf 'global:\n' + # The prefix the images were built under (MDB_IMAGE_PREFIX, default + # localhost), so the chart runs exactly what `make build` produced. + printf ' imageRegistry: %s\n' "$(q "$(mdb_image_prefix)")" printf ' storageClass: %s\n' "$(q "$sc")" printf 'engines:\n' for eng in $ENGINES; do diff --git a/scripts/scan b/scripts/scan index 5204a25..9c62ffd 100755 --- a/scripts/scan +++ b/scripts/scan @@ -69,13 +69,13 @@ failed=0 scanned=0 for engine in "${targets[@]}"; do if [[ "$engine" == "cli" ]]; then - img="ghcr.io/simtabi/dbpantry-cli:dev" + img="$(mdb_image cli dev)" else engine_load "$engine" >/dev/null 2>&1 || continue upper="$(printf '%s' "$engine" | tr '[:lower:]-' '[:upper:]_')" ver="$(mdb_env_get "MDB_${upper}_VERSION" .env)" ver="${ver:-$MDB_ENGINE_DEFAULT_VERSION}" - img="ghcr.io/simtabi/dbpantry-${engine}:${ver}" + img="$(mdb_image "$engine" "$ver")" fi if ! mdb_ctr image inspect "$img" >/dev/null 2>&1; then diff --git a/scripts/transfer b/scripts/transfer index 947ec38..ed4fbc4 100755 --- a/scripts/transfer +++ b/scripts/transfer @@ -80,14 +80,14 @@ if [[ -n "$FROM_HOST" ]]; then # host doing the migration. mdb_ctr run --rm -i --network host \ -v "$ROOT/backups:/backups" \ - ghcr.io/simtabi/dbpantry-cli:dev \ + "$(mdb_image cli dev)" \ pg_dump -h "$FROM_HOST" -U "$FROM_USER" -Fc "$DB" > "$FILE" \ || die "could not dump from $FROM_HOST" ;; mysql|mariadb) mdb_ctr run --rm -i --network host \ -v "$ROOT/backups:/backups" \ - ghcr.io/simtabi/dbpantry-cli:dev \ + "$(mdb_image cli dev)" \ mariadb-dump -h "$FROM_HOST" -u "$FROM_USER" -p \ --single-transaction --routines --triggers --events --quick "$DB" > "$FILE" \ || die "could not dump from $FROM_HOST" diff --git a/scripts/upgrade b/scripts/upgrade index 1e85a8b..9373f89 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -35,7 +35,7 @@ cd "$ROOT" || exit 1 mdb_backend_require compose || exit 64 PROJECT="$(mdb_project_name)" -PREFIX="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" +PREFIX="$(mdb_image_prefix)" # The working dumps. backups/upgrade in the checkout unless MDB_UPGRADE_DIR names # another directory (the verify harness points it at a private temporary one, so # a check never writes or cleans the operator's backups/). diff --git a/scripts/verify-backups b/scripts/verify-backups index 1d66927..5e5adf5 100755 --- a/scripts/verify-backups +++ b/scripts/verify-backups @@ -26,7 +26,7 @@ ENV_FILE="${MDB_ENV_FILE:-$ROOT/.env}" [[ -f "$ENV_FILE" ]] || ENV_FILE="$ROOT/.env.example" BACKUP_DIR="$(mdb_backup_dir "$ROOT" "$ENV_FILE")" -PREFIX="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" +PREFIX="$(mdb_image_prefix)" get() { mdb_env_get "$1" "$ENV_FILE"; } log() { printf '[verify-backups] %s\n' "$*"; } diff --git a/scripts/verify/02-pinned-versions.sh b/scripts/verify/02-pinned-versions.sh index e1dd192..240d045 100755 --- a/scripts/verify/02-pinned-versions.sh +++ b/scripts/verify/02-pinned-versions.sh @@ -48,7 +48,7 @@ for f in "${candidates[@]}"; do # published. The rule exists to pin what we consume, not what we # produce; their bases are pinned in images/bases.tsv, validated # below. - *ghcr.io/simtabi/dbpantry-*) continue ;; + *"/dbpantry-"*) continue ;; esac printf ' %s: not digest-pinned: %s\n' "${f#./}" "$line" >&2 (( violations++ )) || true diff --git a/scripts/verify/08-pg-standalone-run.sh b/scripts/verify/08-pg-standalone-run.sh index b54112c..8a381a0 100755 --- a/scripts/verify/08-pg-standalone-run.sh +++ b/scripts/verify/08-pg-standalone-run.sh @@ -7,8 +7,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" need_runtime -# SPEC section 18: "docker run ghcr.io/simtabi/dbpantry-pg with only a -# password env yields a fully configured server, no compose required." +# SPEC section 18: "docker run /dbpantry-pg with only a password env +# yields a fully configured server, no compose required." The image is the +# local build, localhost/dbpantry-pg (image_name, MDB_IMAGE_PREFIX). # This is the check that keeps layer 2 of the architecture honest. img="$(image_name pg)" diff --git a/scripts/verify/14-version-switch.sh b/scripts/verify/14-version-switch.sh index 0c442d1..d9834b3 100755 --- a/scripts/verify/14-version-switch.sh +++ b/scripts/verify/14-version-switch.sh @@ -15,7 +15,7 @@ need_runtime # 17 -> 18 are the two maintained majors (design.md D-57). 18 also moved the # data mount one level up, to /var/lib/postgresql, so each side mounts its own. -prefix="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" +prefix="$(mdb_image_prefix)" old_img="$prefix/dbpantry-pg:17" new_img="$prefix/dbpantry-pg:18" diff --git a/scripts/verify/29-upgrade-migrates-data.sh b/scripts/verify/29-upgrade-migrates-data.sh index 112290a..f44c233 100755 --- a/scripts/verify/29-upgrade-migrates-data.sh +++ b/scripts/verify/29-upgrade-migrates-data.sh @@ -17,7 +17,7 @@ cd "$MDB_ROOT" || exit 1 # PG 18's move of the data mount to /var/lib/postgresql, which `make upgrade` # has to get right on each side. -prefix="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" +prefix="$(mdb_image_prefix)" old_img="$prefix/dbpantry-pg:17" new_img="$prefix/dbpantry-pg:18" need_image "$old_img" diff --git a/scripts/verify/32-every-engine-auth.sh b/scripts/verify/32-every-engine-auth.sh index 91b6a3d..97faa5a 100755 --- a/scripts/verify/32-every-engine-auth.sh +++ b/scripts/verify/32-every-engine-auth.sh @@ -44,7 +44,7 @@ for engine in $(engine_list); do fi ver="$(engine_version "$engine")" - img="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}/dbpantry-${engine}:${ver}" + img="$(mdb_image "$engine" "$ver")" # A referenced engine is not published by us, so it runs the pinned # upstream image (SPEC section 22.5). diff --git a/scripts/verify/49-images-fully-qualified.sh b/scripts/verify/49-images-fully-qualified.sh index 11e8e3a..099726a 100755 --- a/scripts/verify/49-images-fully-qualified.sh +++ b/scripts/verify/49-images-fully-qualified.sh @@ -36,8 +36,15 @@ judge() { # image: lines, including the defaults inside ${VAR:-default}. while IFS=: read -r file _ rest; do ref="$(printf '%s' "$rest" | sed -E 's/^[[:space:]]*image:[[:space:]]*//; s/^"//; s/"$//')" - # shellcheck disable=SC2016 # a literal ${, not an expansion - while [[ "$ref" == '${'*':-'* ]]; do ref="${ref#*:-}"; ref="${ref%\}}"; done + # Every ${VAR:-default} is replaced by its default, innermost first, so a + # nested ${MDB_X_IMAGE:-${MDB_X_BASE_IMAGE:-docker.io/...}} and a prefixed + # ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-pg:${MDB_PG_VERSION:-17} are both + # judged by the name they resolve to when nothing is set. + prev="" + while [[ "$ref" != "$prev" ]]; do + prev="$ref" + ref="$(printf '%s' "$ref" | sed -E 's/\$\{[A-Za-z_][A-Za-z0-9_]*:-([^${}]*)\}/\1/g')" + done judge "$file" "$ref" done < <(grep -nE '^[[:space:]]*image:' docker-compose.yml compose.*.yml 2>/dev/null) diff --git a/scripts/verify/55-kubernetes-ha.sh b/scripts/verify/55-kubernetes-ha.sh index 3b3b35d..aea540b 100755 --- a/scripts/verify/55-kubernetes-ha.sh +++ b/scripts/verify/55-kubernetes-ha.sh @@ -95,7 +95,7 @@ ha=(--set engines.pg.enabled=true --set engines.pg.ha.enabled=true) helm template r "$chart" "${ha[@]}" >"$tmp/patroni.yaml" 2>"$tmp/err" || vfail "helm template (patroni): $(cat "$tmp/err")" n="$(grep -c '^kind: StatefulSet' "$tmp/patroni.yaml")" [[ "$n" == 1 ]] || vfail "HA should replace the standalone StatefulSet; rendered $n" -grep -q 'image: ghcr.io/simtabi/dbpantry-patroni:17' "$tmp/patroni.yaml" || vfail "the HA StatefulSet does not run dbpantry-patroni" +grep -q 'image: localhost/dbpantry-patroni:17' "$tmp/patroni.yaml" || vfail "the HA StatefulSet does not run dbpantry-patroni" grep -q '^kind: ClusterRole' "$tmp/patroni.yaml" && vfail "Patroni was given cluster-wide rights" grep -q '^kind: Role$' "$tmp/patroni.yaml" || vfail "Patroni has no Role for its DCS" awk '/^kind: Service$/{s=1} s&&/^ name: pg$/{n=1} n&&/role: primary/{print "ok"; exit}' "$tmp/patroni.yaml" | grep -q ok \ diff --git a/scripts/verify/68-pooler-tls.sh b/scripts/verify/68-pooler-tls.sh index c217be8..a5917ed 100755 --- a/scripts/verify/68-pooler-tls.sh +++ b/scripts/verify/68-pooler-tls.sh @@ -43,6 +43,7 @@ need_image "$img" pooler_img="$(grep -oE "^MDB_ENGINE_POOLER_IMAGE='[^']+'" engines/postgres/engine.conf \ | sed "s/^MDB_ENGINE_POOLER_IMAGE='//; s/'$//")" +pooler_img="$(mdb_image_ref "$pooler_img")" [[ -n "$pooler_img" ]] || vfail "engines/postgres/engine.conf declares no pooler image" need_image "$pooler_img" diff --git a/scripts/verify/73-images-local-by-default.sh b/scripts/verify/73-images-local-by-default.sh new file mode 100755 index 0000000..0ad0f55 --- /dev/null +++ b/scripts/verify/73-images-local-by-default.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# verify: our images are named localhost/ by default, never after a remote registry, and a set MDB_IMAGE_PREFIX or global.imageRegistry names every one +# tags: fast structure +# phase: 1 + +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# design.md D-78. dbpantry publishes no image (D-77), yet until 2026-11-13 every +# image was built, run and deployed as ghcr.io/simtabi/dbpantry-*: the Helm +# chart's default registry, the Compose files, the Kubernetes backend's kind +# list and a dozen scripts each carried that name. A cluster missing an image +# therefore pulled from a registry that holds nothing. The default is now +# localhost, written once in scripts/env-lib.sh (MDB_IMAGE_PREFIX_DEFAULT) and +# repeated only where a file cannot read it: Compose, the chart, the patroni +# Dockerfile and an engine descriptor. This check holds all of them to it. + +cd "$MDB_ROOT" || exit 1 +scripts/gen-compose >/dev/null 2>&1 || vfail "scripts/gen-compose failed" + +def=localhost +remote='registry.example.com/team' + +# --- 1. every default is localhost ------------------------------------------------ +grep -qx "MDB_IMAGE_PREFIX_DEFAULT=$def" scripts/env-lib.sh \ + || vfail "scripts/env-lib.sh does not set MDB_IMAGE_PREFIX_DEFAULT=$def" +grep -qx " imageRegistry: $def" deploy/helm/dbpantry/values.yaml \ + || vfail "the chart's global.imageRegistry default is not $def: $(grep -m1 'imageRegistry:' deploy/helm/dbpantry/values.yaml)" +while IFS= read -r line; do + [[ "$line" == *"ARG IMAGE_PREFIX=$def" ]] || vfail "a Dockerfile defaults IMAGE_PREFIX to something else: $line" +done < <(grep -H '^ARG IMAGE_PREFIX=' images/*/Dockerfile) + +# Compose: every image of ours is ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-*, in +# the tracked files and the ones scripts/gen-compose just wrote. +ours=0 +while IFS=: read -r file _ ref; do + ours=$(( ours + 1 )) + ref="${ref#"${ref%%[![:space:]]*}"}" + # shellcheck disable=SC2016 # a literal Compose ${...}, not an expansion + [[ "$ref" == 'image: ${MDB_IMAGE_PREFIX:-'"$def"'}/dbpantry-'* ]] \ + || vfail "$file: $ref does not take its prefix from \${MDB_IMAGE_PREFIX:-$def}" +done < <(grep -nE '^[[:space:]]*image:.*/dbpantry-' docker-compose.yml compose.*.yml) +# docker-compose.yml (4), compose.ha.yml (2) and the generated engines (5) on +# 2026-11-13; fewer means this is not reading the Compose files. +(( ours >= 8 )) || vfail "found $ours dbpantry image line(s) in the Compose files, want at least 8" +others="$(grep -hoE 'MDB_IMAGE_PREFIX:-[^}]*' docker-compose.yml compose.*.yml | sort -u)" +[[ "$others" == "MDB_IMAGE_PREFIX:-$def" ]] || vfail "a Compose default prefix is not $def: $others" + +# Engine descriptors write our images with the default prefix. +while IFS= read -r line; do + ref="$(printf '%s' "$line" | sed -E "s/.*_IMAGE='//; s/'$//")" + [[ "$ref" == "$def/"* ]] || vfail "${line%%:*}: $ref is not named $def/" +done < <(grep -HoE "_IMAGE='[^']*/dbpantry-[^']*'" engines/*/engine.conf) +vinfo "every default prefix is $def: env-lib, the chart, Compose ($ours image lines), the Dockerfiles and the descriptors" + +# --- 2. nothing names a remote registry for our images ------------------------------ +# A dotted host (or host:port) before dbpantry-, in anything but prose. The +# Markdown docs may show registry.example.com as the user's own registry. +scanned=0 +bad=0 +while IFS= read -r -d '' f; do + case "$f" in *.md|scripts/verify/73-images-local-by-default.sh) continue ;; esac + scanned=$(( scanned + 1 )) + if grep -nE '([A-Za-z0-9-]+\.)+[A-Za-z]{2,}(:[0-9]+)?/([A-Za-z0-9._-]+/)*dbpantry-[a-z]' "$f" \ + | grep -v 'ghcr.io/simtabi' >/dev/null; then + printf ' %s names a remote registry for a dbpantry image\n' "$f" >&2 + bad=$(( bad + 1 )) + fi +done < <(git ls-files -z) +(( scanned >= 300 )) || vfail "scanned $scanned tracked file(s), want at least 300: git ls-files is not listing the repository" +(( bad == 0 )) || vfail "$bad file(s) name a remote registry for our images (listed above)" + +# ghcr.io/simtabi may only describe the frozen images, or the history that +# published them. Each file is held to the number of lines measured on +# 2026-11-13, so a new mention fails here and an old one that goes stale +# fails too. +frozen_lines() { + case "$1" in + .github/workflows/release.yml) echo 1 ;; # the header: images on ghcr.io/simtabi are frozen + CHANGELOG.md) echo 3 ;; # history, and the D-77/D-78 entries + README.md) echo 1 ;; # the images are frozen + design.md) echo 4 ;; # D-02, D-23 and D-77 history, D-78 + docs/guides/kubernetes.md) echo 1 ;; # nothing falls back to the frozen images + docs/guides/licensing.md) echo 1 ;; # no registry attestation; the images are frozen + docs/guides/release.md) echo 2 ;; # the frozen tag scheme + docs/reference/spec.md) echo 4 ;; # the original SPEC, kept as written + src/internal/release/workflow_test.go) echo 4 ;; # samples of the image pushes the release guard refuses + *) echo 0 ;; + esac +} +listed=0 +while IFS= read -r -d '' f; do + [[ "$f" == scripts/verify/73-images-local-by-default.sh ]] && continue + n="$(grep -c 'ghcr.io/simtabi' "$f" || true)" + want="$(frozen_lines "$f")" + (( want > 0 )) && listed=$(( listed + 1 )) + (( n == want )) && continue + (( want == 0 )) && vfail "$f mentions ghcr.io/simtabi $n time(s); our images are localhost/ (D-78)" + vfail "$f mentions ghcr.io/simtabi $n time(s), measured $want: update the entry or the file" +done < <(git ls-files -z) +(( listed == 9 )) || vfail "found $listed of the 9 files allowed to describe the frozen images: an entry names a missing file" +vinfo "$scanned tracked file(s): no remote registry for our images; ghcr.io/simtabi only where the frozen images are described" + +# --- 3. a set prefix names every image ----------------------------------------------- +lib_out() { (. scripts/env-lib.sh && "$@"); } +[[ "$(MDB_IMAGE_PREFIX='' lib_out mdb_image pg 17)" == "$def/dbpantry-pg:17" ]] \ + || vfail "mdb_image with no prefix set is not $def/dbpantry-pg:17" +[[ "$(MDB_IMAGE_PREFIX="$remote/" lib_out mdb_image pg 17)" == "$remote/dbpantry-pg:17" ]] \ + || vfail "mdb_image does not apply MDB_IMAGE_PREFIX" +[[ "$(MDB_IMAGE_PREFIX="$remote" lib_out mdb_image_ref "$def/dbpantry-pgbouncer:dev")" == "$remote/dbpantry-pgbouncer:dev" ]] \ + || vfail "mdb_image_ref does not move the descriptor's pooler image to MDB_IMAGE_PREFIX" +[[ "$(MDB_IMAGE_PREFIX="$remote" lib_out mdb_image_ref docker.io/library/mongo:7.0)" == docker.io/library/mongo:7.0 ]] \ + || vfail "mdb_image_ref rewrote an upstream image" + +# The Makefile's own use (make shell). +grep -qE "[[:space:]]$def/dbpantry-cli:dev" < <(MDB_IMAGE_PREFIX='' make -s -n shell 2>/dev/null) \ + || vfail "make shell does not run $def/dbpantry-cli:dev by default" +grep -qE "[[:space:]]$remote/dbpantry-cli:dev" < <(MDB_IMAGE_PREFIX="$remote" make -s -n shell 2>/dev/null) \ + || vfail "make shell ignores MDB_IMAGE_PREFIX" + +# Kubernetes: helm-values hands the prefix to the chart, and the backend loads +# exactly the names the chart will run. +k8s_images() { + local values="$1" + ( + mdb_backend_register() { :; } + . scripts/env-lib.sh + . scripts/backends/kubernetes.sh + _mdb_k8s_local_images "$values" + ) +} +tmp="$(mktemp -d "${TMPDIR:-/tmp}/mdb-images.XXXXXX")" +add_cleanup "rm -rf '$tmp'" +for prefix in "$def" "$remote"; do + MDB_IMAGE_PREFIX="$( [[ "$prefix" == "$def" ]] || printf '%s' "$prefix")" MDB_PROFILES=pg,mysql,ha \ + scripts/helm-values -o "$tmp/values.yaml" 2>/dev/null || vfail "scripts/helm-values failed" + grep -qx " imageRegistry: \"$prefix\"" "$tmp/values.yaml" \ + || vfail "helm-values does not render global.imageRegistry \"$prefix\"" + n=0 + while IFS= read -r img; do + n=$(( n + 1 )) + [[ "$img" == "$prefix/dbpantry-"* ]] || vfail "the Kubernetes backend would load $img, not $prefix/dbpantry-*" + done < <(k8s_images "$tmp/values.yaml") + (( n >= 3 )) || vfail "the Kubernetes backend listed $n image(s) for pg, mysql and ha, want at least 3" + + if command -v helm >/dev/null 2>&1; then + set_args=() + [[ "$prefix" == "$def" ]] || set_args=(--set "global.imageRegistry=$prefix") + helm template r deploy/helm/dbpantry --set engines.pg.enabled=true --set engines.mysql.enabled=true \ + --set engines.pg.ha.enabled=true ${set_args[@]+"${set_args[@]}"} >"$tmp/out.yaml" 2>"$tmp/err" \ + || vfail "helm template: $(cat "$tmp/err")" + imgs="$(grep -E '^[[:space:]]+image: ' "$tmp/out.yaml" | grep '/dbpantry-' || true)" + [[ "$(printf '%s\n' "$imgs" | grep -c .)" -ge 2 ]] || vfail "helm template rendered no dbpantry image" + printf '%s\n' "$imgs" | grep -vE "image: \"?$prefix/dbpantry-" && vfail "helm template ran an image outside $prefix/ (above)" + grep -q 'ghcr.io' "$tmp/out.yaml" && vfail "helm template rendered a ghcr.io image" + grep -qE 'imagePullPolicy: IfNotPresent' "$tmp/out.yaml" || vfail "the chart no longer uses imagePullPolicy: IfNotPresent" + fi +done +if command -v helm >/dev/null 2>&1; then + vinfo "unset, every image is $def/dbpantry-*; with $remote set, every image (Make, helm-values, kind loading, the rendered chart) carries it" +else + vinfo "unset, every image is $def/dbpantry-*; with $remote set, Make, helm-values and kind loading carry it (helm is not installed, so the chart was not rendered)" +fi diff --git a/scripts/verify/lib.sh b/scripts/verify/lib.sh index 2ca4a21..7959b12 100755 --- a/scripts/verify/lib.sh +++ b/scripts/verify/lib.sh @@ -125,14 +125,14 @@ engine_selected() { image_exists() { mdb_ctr image inspect "$1" >/dev/null 2>&1; } # Fully-qualified name for one of our engine images. -# image_name pg -> ghcr.io/simtabi/dbpantry-pg:17 -# image_name mariadb -> ghcr.io/simtabi/dbpantry-mariadb:11.8 +# image_name pg -> localhost/dbpantry-pg:17 +# image_name mariadb -> localhost/dbpantry-mariadb:11.8 +# (under MDB_IMAGE_PREFIX when it is set; see mdb_image in scripts/env-lib.sh) image_name() { - local engine="$1" prefix ver - prefix="${MDB_IMAGE_PREFIX:-ghcr.io/simtabi}" + local engine="$1" ver ver="$(engine_version "$engine")" [[ -n "$ver" ]] || vfail "unknown engine: $engine" - printf '%s/dbpantry-%s:%s\n' "$prefix" "$engine" "$ver" + mdb_image "$engine" "$ver" } # The configured version for an engine: MDB__VERSION if set, otherwise From e3c1c378214602f12d743884ddca7a1d50fcafc6 Mon Sep 17 00:00:00 2001 From: Imani Manyara Date: Fri, 13 Nov 2026 14:46:06 -0500 Subject: [PATCH 2/2] Recognise localhost images in checks 42 and 72; resolve init's caddy The first CI run failed four checks, each a reader of the old image name that the literal search for ghcr.io/simtabi missed: - check 42 allowlisted our images as ghcr\.io/simtabi/[a-z-]+, and check 72 matched them by the same regex, so the pooler read as a third-party rebuild and our images as having no healthcheck. Both now name localhost/dbpantry-*; 72 under the prefix in force. - scripts/init passed the caddy service's raw image: line, now Compose syntax, to run, so make init-prod (check 26) failed. It resolves the line with the new mdb_compose_ref first. - check 73 listed one image in CI: make verify exports COMPOSE_PROFILES, which helm-values ranks above the MDB_PROFILES the check set. It sets both, counts the escaped spelling, and fails on any script that runs a Compose image: line without mdb_compose_ref. --- CHANGELOG.md | 4 ++- design.md | 2 +- scripts/env-lib.sh | 16 +++++++++++ scripts/init | 6 ++-- scripts/verify/42-image-provenance.sh | 6 ++-- scripts/verify/72-service-hardening.sh | 7 +++-- scripts/verify/73-images-local-by-default.sh | 30 ++++++++++++++++++-- 7 files changed, 61 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 588adda..ab6f9fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,9 @@ waiving them. `global.imageRegistry` when installing the chart by hand) prefixes every image. The Kubernetes guide shows `kind load docker-image`, `minikube image load` and pushing to your own registry. Check 73 fails if any default names a remote registry or a set prefix stops reaching every image, - and check 49 now resolves every `${VAR:-default}` in an image reference (design.md D-78). + and check 49 now resolves every `${VAR:-default}` in an image reference. Checks 42 and 72 recognise + our images as `localhost/dbpantry-*`, and `make init-prod` resolves the caddy image the way Compose + does (`mdb_compose_ref`) before running it (design.md D-78). - **A release runs on a pushed `vX.Y.Z` tag, or on request for a tag already on GitHub:** `gh workflow run release.yml --repo simtabi/dbpantry -f tag=vX.Y.Z`. This replaces the dispatch gated on `-f confirm=release-go` (D-76). Every job checks out the tag, and the first step refuses diff --git a/design.md b/design.md index 0d0edab..f109ed5 100644 --- a/design.md +++ b/design.md @@ -492,7 +492,7 @@ Relied on: pgBouncer configuration reference (`, because a dispatch's `GITHUB_SHA` is its branch's head, not the tag's commit: the tag-on-`main` check now tests `HEAD` after that checkout. No pre-release tag filter is declared, since dbpantry refuses pre-releases. dbpantry builds no release binaries, so it keeps its own jobs rather than calling `simtabi/.github`'s shared `go-release.yml`, which runs GoReleaser. The Kubernetes backend (`scripts/backends/kubernetes.sh`) and the Helm chart (`global.imageRegistry: ghcr.io/simtabi`) still name the frozen registry; changing them is a separate decision. The guard is `src/internal/release`'s `TestReleaseRunsOnAVersionTagOrOnRequest`, which fails on any other trigger, a branch or non-version tag filter, a missing or optional `tag` input, a first step that is not the refusal, a checkout of anything but the tag, a job that does not need the gating job, or any image publish (registry login or push, `buildx --push`, `imagetools create`, `push-to-registry: true`, a `docker/` publish action, `packages: write`); `TestReleasePolicyRejectsAWrongRelease` proves each can fail, and that a local build and scan passes. -**D-78: our images are named localhost/dbpantry-*, from one default, and nothing names a remote registry.** Follows D-77. With images no longer published, `ghcr.io/simtabi/dbpantry-*` stayed the name every image was built, run and deployed under: `scripts/build` defaulted `MDB_IMAGE_PREFIX` to it, Compose, the generated engine services, the pooler descriptor, the patroni Dockerfile, the Makefile, five scripts, six checks and both workflows wrote it out, the Helm chart's `global.imageRegistry` defaulted to it, and the Kubernetes backend loaded it into kind. A cluster without an image therefore pulled from a registry that holds nothing. The owner chose `localhost/` (2026-11-13) over an unqualified `dbpantry/`, which check 49 refuses because Podman will not resolve a short name (D-60) and which Docker would expand to Docker Hub on a pull; `localhost` is fully qualified, is the prefix Podman gives local builds, never reaches a remote registry, and loads into kind and minikube as it is. The default is written once, `MDB_IMAGE_PREFIX_DEFAULT=localhost` in `scripts/env-lib.sh`, with `mdb_image_prefix`, `mdb_image` and `mdb_image_ref` (which moves a descriptor's `localhost/dbpantry-pgbouncer:dev` to the prefix in force); the Makefile reads it from there. Files that cannot read it repeat it: Compose as `${MDB_IMAGE_PREFIX:-localhost}`, the patroni Dockerfile's `ARG IMAGE_PREFIX`, the chart's `global.imageRegistry`, the pooler descriptor, and the CI scan steps, which build with the default. `MDB_IMAGE_PREFIX` stays an environment variable, as before, not a `.env` setting. On Kubernetes, `scripts/helm-values` renders `global.imageRegistry` from it and `_mdb_k8s_local_images` reads that back, so kind loads exactly the names the chart runs; `imagePullPolicy` stays `IfNotPresent`. Check 49's Compose parser resolved only a leading `${VAR:-default}`, which misread a prefixed reference as `localhost}/...`; it now resolves every one, innermost first, and still counts 55 references. Check 73 holds every default to `localhost`, fails on a dotted registry before `dbpantry-` in any tracked file but prose, limits `ghcr.io/simtabi` to nine files that describe the frozen images or their history (each at its measured count), and proves that a set prefix reaches `mdb_image`, the pooler, `make shell`, helm-values, kind loading and the rendered chart. Images built before keep their old names: `make build` once, and the next `make up` recreates the containers; volumes are untouched. +**D-78: our images are named localhost/dbpantry-*, from one default, and nothing names a remote registry.** Follows D-77. With images no longer published, `ghcr.io/simtabi/dbpantry-*` stayed the name every image was built, run and deployed under: `scripts/build` defaulted `MDB_IMAGE_PREFIX` to it, Compose, the generated engine services, the pooler descriptor, the patroni Dockerfile, the Makefile, five scripts, six checks and both workflows wrote it out, the Helm chart's `global.imageRegistry` defaulted to it, and the Kubernetes backend loaded it into kind. A cluster without an image therefore pulled from a registry that holds nothing. The owner chose `localhost/` (2026-11-13) over an unqualified `dbpantry/`, which check 49 refuses because Podman will not resolve a short name (D-60) and which Docker would expand to Docker Hub on a pull; `localhost` is fully qualified, is the prefix Podman gives local builds, never reaches a remote registry, and loads into kind and minikube as it is. The default is written once, `MDB_IMAGE_PREFIX_DEFAULT=localhost` in `scripts/env-lib.sh`, with `mdb_image_prefix`, `mdb_image` and `mdb_image_ref` (which moves a descriptor's `localhost/dbpantry-pgbouncer:dev` to the prefix in force); the Makefile reads it from there. Files that cannot read it repeat it: Compose as `${MDB_IMAGE_PREFIX:-localhost}`, the patroni Dockerfile's `ARG IMAGE_PREFIX`, the chart's `global.imageRegistry`, the pooler descriptor, and the CI scan steps, which build with the default. `MDB_IMAGE_PREFIX` stays an environment variable, as before, not a `.env` setting. On Kubernetes, `scripts/helm-values` renders `global.imageRegistry` from it and `_mdb_k8s_local_images` reads that back, so kind loads exactly the names the chart runs; `imagePullPolicy` stays `IfNotPresent`. Check 49's Compose parser resolved only a leading `${VAR:-default}`, which misread a prefixed reference as `localhost}/...`; it now resolves every one, innermost first, and still counts 55 references. Check 73 holds every default to `localhost`, fails on a dotted registry before `dbpantry-` in any tracked file but prose, limits `ghcr.io/simtabi` to nine files that describe the frozen images or their history (each at its measured count), and proves that a set prefix reaches `mdb_image`, the pooler, `make shell`, helm-values, kind loading and the rendered chart. Images built before keep their old names: `make build` once, and the next `make up` recreates the containers; volumes are untouched. The first CI run of this change failed four checks, each a reader of the old name the literal search missed: check 42 allowlisted our images as `ghcr\.io/simtabi/[a-z-]+` and check 72 recognised them by the same regex, so both now name `localhost/dbpantry-*` (72 under the prefix in force); `scripts/init` passed the caddy service's raw `image:` line to `run`, which is now Compose syntax, so it resolves it with `mdb_compose_ref` and `make init-prod` (check 26) works again; and check 73 itself listed one image in CI because `make verify` exports `COMPOSE_PROFILES`, which `helm-values` ranks above the `MDB_PROFILES` the check set. Check 73 now sets both, counts the escaped spelling too, and fails on any script that runs a Compose `image:` line without `mdb_compose_ref`. A check run directly is not the check `make verify` runs: reproduce through `make verify CHECKS=`. ## 8. arm64 deliverability diff --git a/scripts/env-lib.sh b/scripts/env-lib.sh index 8e971d1..3497c33 100644 --- a/scripts/env-lib.sh +++ b/scripts/env-lib.sh @@ -121,3 +121,19 @@ mdb_image_ref() { *) printf '%s\n' "$1" ;; esac } + +# Resolve a Compose-style reference the way Compose does: every ${VAR:-default} +# becomes VAR from the environment, or the default when it is unset or empty, +# innermost first. For a script that reads an image: line out of a Compose file +# and runs it directly, so ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-caddy:dev +# becomes localhost/dbpantry-caddy:dev (or the prefix in force). +mdb_compose_ref() { + # shellcheck disable=SC2016 # a pattern for a literal ${...}, not an expansion + local ref="$1" re='\$\{([A-Za-z_][A-Za-z0-9_]*):-([^${}]*)\}' name fallback val + while [[ "$ref" =~ $re ]]; do + name="${BASH_REMATCH[1]}"; fallback="${BASH_REMATCH[2]}" + val="${!name:-$fallback}" + ref="${ref/"${BASH_REMATCH[0]}"/$val}" + done + printf '%s\n' "$ref" +} diff --git a/scripts/init b/scripts/init index f70cc46..057dc36 100755 --- a/scripts/init +++ b/scripts/init @@ -133,8 +133,10 @@ if (( PROD )); then # to be /image: caddy:/, which stopped matching the moment the image became # one we build -- and the only symptom was "could not compute the UI # password hash (is the container runtime running? scripts/runtime info)", which blames Docker. - caddy_image="$(awk '/^ caddy:/{f=1} f && /^[[:space:]]+image:/{print $2; exit}' \ - "$ROOT/docker-compose.yml")" + # The line is Compose syntax, ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-caddy:dev, + # so it is resolved as Compose resolves it before it is run directly. + caddy_image="$(mdb_compose_ref "$(awk '/^ caddy:/{f=1} f && /^[[:space:]]+image:/{print $2; exit}' \ + "$ROOT/docker-compose.yml")")" ui_hash="$(mdb_ctr run --rm "$caddy_image" caddy hash-password --plaintext "$ui_password" 2>/dev/null || true)" if [[ -z "$ui_hash" ]]; then printf ' could not compute the UI password hash (is docker running?)\n' >&2 diff --git a/scripts/verify/42-image-provenance.sh b/scripts/verify/42-image-provenance.sh index ceb12aa..8a2806b 100755 --- a/scripts/verify/42-image-provenance.sh +++ b/scripts/verify/42-image-provenance.sh @@ -19,8 +19,10 @@ cd "$MDB_ROOT" || exit 1 need_file "$MDB_ROOT/docs/reference/image-provenance.md" -# Namespaces belonging to the projects whose software they package. -UPSTREAM='^(pgvector/pgvector|dpage/pgadmin4|proxysql/proxysql|prom/[a-z-]+|aquasec/trivy|ghcr\.io/ferretdb/[a-z-]+|quay\.io/coreos/etcd|quay\.io/prometheuscommunity/[a-z-]+|ghcr\.io/simtabi/[a-z-]+)$' +# Namespaces belonging to the projects whose software they package, and our own +# images, which every descriptor names under the default prefix (localhost/, +# design.md D-78). +UPSTREAM='^(pgvector/pgvector|dpage/pgadmin4|proxysql/proxysql|prom/[a-z-]+|aquasec/trivy|ghcr\.io/ferretdb/[a-z-]+|quay\.io/coreos/etcd|quay\.io/prometheuscommunity/[a-z-]+|localhost/dbpantry-[a-z-]+)$' violations=0 checked=0 diff --git a/scripts/verify/72-service-hardening.sh b/scripts/verify/72-service-hardening.sh index 9ce7e4d..731f4e1 100755 --- a/scripts/verify/72-service-hardening.sh +++ b/scripts/verify/72-service-hardening.sh @@ -49,10 +49,13 @@ for df in images/*/Dockerfile; do grep -q '^HEALTHCHECK' "$df" && built_health+="$(basename "$(dirname "$df")") " done -out="$(python3 - "$s/all.json" "$built_health" <<'PY' +out="$(python3 - "$s/all.json" "$built_health" "$(mdb_image_prefix)" <<'PY' import json, re, sys svc = json.load(open(sys.argv[1]))["services"] built = set(sys.argv[2].split()) +# Our images are /dbpantry-, the prefix Compose resolved them under +# (MDB_IMAGE_PREFIX, default localhost; design.md D-78). +ours = re.compile(re.escape(sys.argv[3]) + r"/dbpantry-([a-z0-9-]+):") # rule -> {service: reason}. Each is a measured gap with its reason; the check # fails if one stops being a gap. @@ -93,7 +96,7 @@ for name, d in sorted(svc.items()): gap("privileges", name, f"{name}: missing no-new-privileges or cap_drop ALL") hc = d.get("healthcheck") or {} image = str(d.get("image", "")) - m = re.match(r"ghcr\.io/simtabi/dbpantry-([a-z0-9-]+):", image) + m = ours.match(image) health_ok = (hc and not hc.get("disable")) or (m and m.group(1) in built) if not health_ok: gap("health", name, f"{name}: no healthcheck in Compose or in its image") diff --git a/scripts/verify/73-images-local-by-default.sh b/scripts/verify/73-images-local-by-default.sh index 0ad0f55..12ffea5 100755 --- a/scripts/verify/73-images-local-by-default.sh +++ b/scripts/verify/73-images-local-by-default.sh @@ -90,7 +90,9 @@ frozen_lines() { listed=0 while IFS= read -r -d '' f; do [[ "$f" == scripts/verify/73-images-local-by-default.sh ]] && continue - n="$(grep -c 'ghcr.io/simtabi' "$f" || true)" + # Escaped too: a regex allowlist spells it ghcr\.io/simtabi, which is how + # checks 42 and 72 kept the old name past the first version of this check. + n="$(grep -cE 'ghcr\\?\.io/simtabi' "$f" || true)" want="$(frozen_lines "$f")" (( want > 0 )) && listed=$(( listed + 1 )) (( n == want )) && continue @@ -100,6 +102,19 @@ done < <(git ls-files -z) (( listed == 9 )) || vfail "found $listed of the 9 files allowed to describe the frozen images: an entry names a missing file" vinfo "$scanned tracked file(s): no remote registry for our images; ghcr.io/simtabi only where the frozen images are described" +# A script that runs an image named on a Compose image: line must resolve it the +# way Compose does first (mdb_compose_ref): the line reads +# ${MDB_IMAGE_PREFIX:-localhost}/dbpantry-..., which is not an image name. +# scripts/init's caddy lookup handed that string to `run` and failed +# `make init-prod` (check 26) in the first version of this change. +readers=0 +while IFS= read -r line; do + readers=$(( readers + 1 )) + [[ "$line" == *mdb_compose_ref* ]] \ + || vfail "a script reads a Compose image: line without mdb_compose_ref: $line" +done < <(grep -rnE "(awk|grep|sed).*image:" scripts --exclude-dir=verify | grep -vE '^[^:]+:[0-9]+:[[:space:]]*#') +vinfo "$readers script(s) read a Compose image: line, each resolving it with mdb_compose_ref" + # --- 3. a set prefix names every image ----------------------------------------------- lib_out() { (. scripts/env-lib.sh && "$@"); } [[ "$(MDB_IMAGE_PREFIX='' lib_out mdb_image pg 17)" == "$def/dbpantry-pg:17" ]] \ @@ -110,6 +125,13 @@ lib_out() { (. scripts/env-lib.sh && "$@"); } || vfail "mdb_image_ref does not move the descriptor's pooler image to MDB_IMAGE_PREFIX" [[ "$(MDB_IMAGE_PREFIX="$remote" lib_out mdb_image_ref docker.io/library/mongo:7.0)" == docker.io/library/mongo:7.0 ]] \ || vfail "mdb_image_ref rewrote an upstream image" +# shellcheck disable=SC2016 # literal Compose references +{ + [[ "$(MDB_IMAGE_PREFIX='' lib_out mdb_compose_ref '${MDB_IMAGE_PREFIX:-localhost}/dbpantry-caddy:dev')" == "$def/dbpantry-caddy:dev" ]] \ + || vfail "mdb_compose_ref does not resolve the Compose default prefix" + [[ "$(MDB_IMAGE_PREFIX="$remote" lib_out mdb_compose_ref '${MDB_IMAGE_PREFIX:-localhost}/dbpantry-caddy:dev')" == "$remote/dbpantry-caddy:dev" ]] \ + || vfail "mdb_compose_ref ignores MDB_IMAGE_PREFIX" +} # The Makefile's own use (make shell). grep -qE "[[:space:]]$def/dbpantry-cli:dev" < <(MDB_IMAGE_PREFIX='' make -s -n shell 2>/dev/null) \ @@ -131,7 +153,11 @@ k8s_images() { tmp="$(mktemp -d "${TMPDIR:-/tmp}/mdb-images.XXXXXX")" add_cleanup "rm -rf '$tmp'" for prefix in "$def" "$remote"; do - MDB_IMAGE_PREFIX="$( [[ "$prefix" == "$def" ]] || printf '%s' "$prefix")" MDB_PROFILES=pg,mysql,ha \ + # COMPOSE_PROFILES as well as MDB_PROFILES: helm-values ranks it first, and + # `make verify` exports the checkout's own (pg,ui), which enabled only pg in + # CI while a direct run of this check enabled all three. + MDB_IMAGE_PREFIX="$( [[ "$prefix" == "$def" ]] || printf '%s' "$prefix")" \ + COMPOSE_PROFILES=pg,mysql,ha MDB_PROFILES=pg,mysql,ha \ scripts/helm-values -o "$tmp/values.yaml" 2>/dev/null || vfail "scripts/helm-values failed" grep -qx " imageRegistry: \"$prefix\"" "$tmp/values.yaml" \ || vfail "helm-values does not render global.imageRegistry \"$prefix\""