Fix reproduction: read Snowflake creds from host env; add procps to base image

.github/workflows/publish-base-image.yml

@@ -0,0 +1,133 @@
+name: Publish base image
+
+# Manually-triggered, approval-gated publish of the data-eng-bench base image
+# to GHCR (ghcr.io/snowflake-labs/data-eng-bench-base).
+#
+# Auth: the workflow's built-in GITHUB_TOKEN (packages: write) — a short-lived
+# token scoped to this run. No personal access token is created or stored.
+#
+# Approval: the `build` and `merge` jobs are gated behind a single `approve`
+# job bound to the `publish-base-image` GitHub Environment. Configure required
+# reviewers on that environment (Settings -> Environments) so a publish waits
+# for a human approval before anything is pushed.
+#
+# Multi-arch: linux/amd64 and linux/arm64 are each built natively (no QEMU
+# emulation) because the image runs `dbt run` at build time, which is slow and
+# can crash under emulation. Per-arch images are pushed by digest, then a
+# manifest list is assembled in the merge job.
+#
+# Prerequisite (one-time): the GHCR package must grant this repository Write
+# access (package -> Settings -> Manage Actions access -> add data-eng-bench
+# with Write), otherwise the push 403s.
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Image version tag (e.g. 1.0.0)"
+        required: true
+        default: "1.0.0"
+      push_latest:
+        description: "Also tag and push :latest"
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  IMAGE: ghcr.io/snowflake-labs/data-eng-bench-base
+
+jobs:
+  # Single human-approval gate for the whole run. Required reviewers live on the
+  # `publish-base-image` environment; approving this unblocks build + merge.
+  approve:
+    runs-on: ubuntu-latest
+    environment: publish-base-image
+    steps:
+      - run: echo "Publish approved for ${IMAGE}:${{ inputs.version }} (latest=${{ inputs.push_latest }})"
+
+  build:
+    needs: approve
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: Checkout (with LFS for retail.duckdb)
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: base-image
+          platforms: ${{ matrix.platform }}
+          provenance: false
+          outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digest-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digest-*
+          merge-multiple: true
+
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push manifest list
+        working-directory: /tmp/digests
+        run: |
+          tags="-t ${IMAGE}:${{ inputs.version }}"
+          if [ "${{ inputs.push_latest }}" = "true" ]; then
+            tags="${tags} -t ${IMAGE}:latest"
+          fi
+          docker buildx imagetools create ${tags} \
+            $(printf "${IMAGE}@sha256:%s " *)
+
+      - name: Inspect published manifest
+        run: docker buildx imagetools inspect "${IMAGE}:${{ inputs.version }}"

.gitignore

@@ -23,3 +23,5 @@ logs/
 
 # local skill installs (npx skills add)
 .agents/
+.claude/
+skills-lock.json

README.md

@@ -143,10 +143,18 @@ harbor run --config configs/data-eng-bench-snowflake.claude-code.yaml --path tas
 ```
 
 Each task's Harbor healthcheck clones `SNOWFLAKE_SOURCE_DATABASE` into an
-isolated `retail_clone_*` database and points the agent + verifier at it, then
-drops it on completion. Password auth (above) or key-pair
-(`SNOWFLAKE_PRIVATE_KEY`, base64 PEM) both work; the role only needs
-`CREATE DATABASE` plus access to the source.
+isolated `retail_clone_*` database and points the agent + verifier at it; the
+role only needs `CREATE DATABASE` plus access to the source. The clone and
+verifier accept password auth (above) or key-pair (`SNOWFLAKE_PRIVATE_KEY`,
+base64 PEM). Note the bundled reference solutions (`solution/solve.sh`)
+authenticate dbt with key-pair, so reproducing the oracle / leaderboard on
+Snowflake requires `SNOWFLAKE_PRIVATE_KEY`.
+
+Each task drops its clone at the end of the verifier phase. A run that fails
+*before* verification (e.g. a clone timeout) can leave a `retail_clone_*`
+behind, since Harbor tasks have no always-run teardown hook. Reclaim strays
+with `base-image/sweep_snowflake_clones.py` (drops `retail_clone_*` older than
+a `--older-than-hours` cutoff; supports `--dry-run`).
 
 A `k=3` sweep over all 103 Snowflake tasks runs roughly 6 to 9 warehouse-hours
 on a free-tier account; use the fast subset for cost-bounded runs.

base-image/Dockerfile

@@ -29,6 +29,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     tmux \
     curl \
     git \
+    procps \
     && rm -rf /var/lib/apt/lists/*
 
 # Node.js 22 — required by CLI coding agents that run inside the container

base-image/cleanup_snowflake.py

@@ -98,11 +98,16 @@ def get_private_key():
 
         conn_kwargs = dict(
             account=os.environ['SNOWFLAKE_ACCOUNT'],
-            host=os.environ.get('SNOWFLAKE_HOST') or None,
             user=os.environ['SNOWFLAKE_USER'],
             warehouse=os.environ['SNOWFLAKE_WAREHOUSE'],
             role=admin_role,
         )
+        # Include host only when explicitly set. Passing host=None makes the
+        # connector fail to derive it from an org-dash account
+        # ('NoneType' has no attribute 'lower'), which silently aborted cleanup
+        # and leaked clone databases. Matches snowflake_clone.py.
+        if os.environ.get('SNOWFLAKE_HOST'):
+            conn_kwargs['host'] = os.environ['SNOWFLAKE_HOST']
         # Password when available, else key-pair (matches snowflake_clone.py).
         if os.environ.get('SNOWFLAKE_PASSWORD'):
             conn_kwargs['password'] = os.environ['SNOWFLAKE_PASSWORD']

base-image/sweep_snowflake_clones.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Drop orphaned per-task Snowflake clone databases left by DB_TYPE=snowflake runs.
+
+Each Snowflake task clones SNOWFLAKE_SOURCE_DATABASE into ``retail_clone_<hash>``
+via the Harbor healthcheck, and cleanup_snowflake.py drops it at the end of the
+task's test.sh. A run that fails *before* the verifier phase (e.g. a
+healthcheck/clone timeout or an agent crash) never reaches that cleanup, so the
+clone is orphaned. Harbor tasks have no always-run teardown hook, so run this
+sweep periodically (cron) or after a batch to reclaim leftovers.
+
+Auth mirrors snowflake_clone.py / cleanup_snowflake.py: SNOWFLAKE_PASSWORD when
+set, otherwise a base64 PKCS8 key in SNOWFLAKE_PRIVATE_KEY (+ optional
+SNOWFLAKE_PRIVATE_KEY_PASSPHRASE). SNOWFLAKE_HOST is optional and passed only
+when set (passing host=None breaks host derivation for org-dash accounts).
+
+Usage:
+  export SNOWFLAKE_ACCOUNT=... SNOWFLAKE_USER=... SNOWFLAKE_PASSWORD=...
+  export SNOWFLAKE_WAREHOUSE=COMPUTE_WH SNOWFLAKE_ROLE=SYSADMIN
+  python3 sweep_snowflake_clones.py --older-than-hours 24            # drop stale
+  python3 sweep_snowflake_clones.py --older-than-hours 0 --dry-run   # preview all
+"""
+
+import argparse
+import base64
+import datetime
+import os
+import sys
+
+
+def _connect_kwargs() -> dict:
+    kwargs = {
+        "account": os.environ["SNOWFLAKE_ACCOUNT"],
+        "user": os.environ["SNOWFLAKE_USER"],
+        "warehouse": os.environ["SNOWFLAKE_WAREHOUSE"],
+        "role": os.environ.get("SNOWFLAKE_ROLE") or None,
+    }
+    # Only pass host when explicitly set; host=None breaks org-dash derivation.
+    if os.environ.get("SNOWFLAKE_HOST"):
+        kwargs["host"] = os.environ["SNOWFLAKE_HOST"]
+
+    password = os.environ.get("SNOWFLAKE_PASSWORD")
+    if password:
+        kwargs["password"] = password
+        return kwargs
+
+    from cryptography.hazmat.backends import default_backend
+    from cryptography.hazmat.primitives import serialization
+
+    key_b64 = os.environ.get("SNOWFLAKE_PRIVATE_KEY", "")
+    if not key_b64:
+        raise SystemExit("Need SNOWFLAKE_PASSWORD or SNOWFLAKE_PRIVATE_KEY.")
+    passphrase = os.environ.get("SNOWFLAKE_PRIVATE_KEY_PASSPHRASE") or None
+    p_key = serialization.load_pem_private_key(
+        base64.b64decode(key_b64),
+        password=passphrase.encode() if passphrase else None,
+        backend=default_backend(),
+    )
+    kwargs["private_key"] = p_key.private_bytes(
+        encoding=serialization.Encoding.DER,
+        format=serialization.PrivateFormat.PKCS8,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    return kwargs
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--prefix", default="retail_clone_",
+                    help="Clone name prefix to match (default: retail_clone_)")
+    ap.add_argument("--older-than-hours", type=float, default=24.0,
+                    help="Only drop clones older than this many hours (default: 24)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="List what would be dropped without dropping")
+    args = ap.parse_args()
+
+    import snowflake.connector
+
+    conn = snowflake.connector.connect(**_connect_kwargs())
+    now = datetime.datetime.now(datetime.timezone.utc)
+    cutoff = now - datetime.timedelta(hours=args.older_than_hours)
+    prefix = args.prefix.lower()
+
+    dropped = kept = 0
+    try:
+        cur = conn.cursor()
+        # SHOW DATABASES columns: created_on(0), name(1), ...
+        cur.execute(f"SHOW DATABASES LIKE '{args.prefix}%'")
+        rows = cur.fetchall()
+        for row in rows:
+            created_on, name = row[0], row[1]
+            if not name.lower().startswith(prefix):
+                continue
+            age_h = (now - created_on).total_seconds() / 3600.0
+            if created_on > cutoff:
+                kept += 1
+                continue
+            if args.dry_run:
+                print(f"[dry-run] would drop {name} (age {age_h:.1f}h)")
+                dropped += 1
+                continue
+            try:
+                cur.execute(f'DROP DATABASE IF EXISTS "{name}"')
+                print(f"dropped {name} (age {age_h:.1f}h)")
+                dropped += 1
+            except Exception as e:
+                print(f"WARN could not drop {name}: {e}", file=sys.stderr)
+    finally:
+        conn.close()
+
+    verb = "would drop" if args.dry_run else "dropped"
+    print(f"\n{verb} {dropped} clone(s); kept {kept} younger than "
+          f"{args.older_than_hours}h.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

configs/data-eng-bench-snowflake.claude-code.yaml

@@ -25,15 +25,18 @@ quiet: true
 
 environment:
   type: docker
+  # Only DB_TYPE is set here. The SNOWFLAKE_* credentials are intentionally NOT
+  # listed: each task's environment/docker-compose.yaml already forwards
+  # ${SNOWFLAKE_*} from the host environment, so exporting them in your shell
+  # (see header) is enough — they flow straight through to the container.
+  #
+  # Do NOT re-add them here. Stock Harbor stores run-config env verbatim (it does
+  # not resolve ${VAR} templates in environment.env), so a line like
+  # `SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD}` injects the literal string
+  # "${SNOWFLAKE_PASSWORD}" into the compose env and clobbers the real exported
+  # value, breaking Snowflake auth.
   env:
     - DB_TYPE=snowflake
-    - SNOWFLAKE_ACCOUNT=${SNOWFLAKE_ACCOUNT}
-    - SNOWFLAKE_USER=${SNOWFLAKE_USER}
-    - SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD}
-    - SNOWFLAKE_WAREHOUSE=${SNOWFLAKE_WAREHOUSE:-COMPUTE_WH}
-    - SNOWFLAKE_SOURCE_DATABASE=${SNOWFLAKE_SOURCE_DATABASE:-DBT_BENCH_RETAIL}
-    - SNOWFLAKE_SCHEMA=${SNOWFLAKE_SCHEMA:-main}
-    - SNOWFLAKE_ROLE=${SNOWFLAKE_ROLE:-SYSADMIN}
 
 agents:
   - name: claude-code

configs/data-eng-bench-snowflake.codex.yaml

@@ -25,15 +25,18 @@ quiet: true
 
 environment:
   type: docker
+  # Only DB_TYPE is set here. The SNOWFLAKE_* credentials are intentionally NOT
+  # listed: each task's environment/docker-compose.yaml already forwards
+  # ${SNOWFLAKE_*} from the host environment, so exporting them in your shell
+  # (see header) is enough — they flow straight through to the container.
+  #
+  # Do NOT re-add them here. Stock Harbor stores run-config env verbatim (it does
+  # not resolve ${VAR} templates in environment.env), so a line like
+  # `SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD}` injects the literal string
+  # "${SNOWFLAKE_PASSWORD}" into the compose env and clobbers the real exported
+  # value, breaking Snowflake auth.
   env:
     - DB_TYPE=snowflake
-    - SNOWFLAKE_ACCOUNT=${SNOWFLAKE_ACCOUNT}
-    - SNOWFLAKE_USER=${SNOWFLAKE_USER}
-    - SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD}
-    - SNOWFLAKE_WAREHOUSE=${SNOWFLAKE_WAREHOUSE:-COMPUTE_WH}
-    - SNOWFLAKE_SOURCE_DATABASE=${SNOWFLAKE_SOURCE_DATABASE:-DBT_BENCH_RETAIL}
-    - SNOWFLAKE_SCHEMA=${SNOWFLAKE_SCHEMA:-main}
-    - SNOWFLAKE_ROLE=${SNOWFLAKE_ROLE:-SYSADMIN}
 
 agents:
   - name: codex

configs/data-eng-bench-snowflake.cortex-code.yaml

@@ -25,15 +25,18 @@ quiet: true
 
 environment:
   type: docker
+  # Only DB_TYPE is set here. The SNOWFLAKE_* credentials are intentionally NOT
+  # listed: each task's environment/docker-compose.yaml already forwards
+  # ${SNOWFLAKE_*} from the host environment, so exporting them in your shell
+  # (see header) is enough — they flow straight through to the container.
+  #
+  # Do NOT re-add them here. Stock Harbor stores run-config env verbatim (it does
+  # not resolve ${VAR} templates in environment.env), so a line like
+  # `SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD}` injects the literal string
+  # "${SNOWFLAKE_PASSWORD}" into the compose env and clobbers the real exported
+  # value, breaking Snowflake auth.
   env:
     - DB_TYPE=snowflake
-    - SNOWFLAKE_ACCOUNT=${SNOWFLAKE_ACCOUNT}
-    - SNOWFLAKE_USER=${SNOWFLAKE_USER}
-    - SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD}
-    - SNOWFLAKE_WAREHOUSE=${SNOWFLAKE_WAREHOUSE:-COMPUTE_WH}
-    - SNOWFLAKE_SOURCE_DATABASE=${SNOWFLAKE_SOURCE_DATABASE:-DBT_BENCH_RETAIL}
-    - SNOWFLAKE_SCHEMA=${SNOWFLAKE_SCHEMA:-main}
-    - SNOWFLAKE_ROLE=${SNOWFLAKE_ROLE:-SYSADMIN}
 
 agents:
   - name: cortex-code

tasks/cohort-retention-matrix/solution/solve.sh

@@ -24,7 +24,7 @@ pkb = p_key.private_bytes(encoding=serialization.Encoding.DER, format=serializat
 
 conn = snowflake.connector.connect(
     account=os.environ['SNOWFLAKE_ACCOUNT'],
-    host=os.environ.get('SNOWFLAKE_HOST') or None,
+    **({'host': os.environ['SNOWFLAKE_HOST']} if os.environ.get('SNOWFLAKE_HOST') else {}),
     user=os.environ['SNOWFLAKE_USER'],
     private_key=pkb,
     warehouse=os.environ['SNOWFLAKE_WAREHOUSE'],