release: gloas (glamsterdam-devnet-6)

.claude/commands/create-new-sentry-event.md

@@ -299,7 +299,18 @@ func (e *Events{EventName}) AppendServerMeta(ctx context.Context, meta *xatu.Ser
 ```
 
 ### 10. Create ClickHouse Migration
-**File**: `deploy/migrations/clickhouse/{NEXT_NUMBER}_{event_name}.up.sql`
+**File**: `deploy/migrations/clickhouse/xatu/{NEXT_NUMBER}_{event_name}.up.sql`
+
+> Sentry events are raw-event tables, so they live in the **`xatu`** migration set.
+> The `xatu` set is **database-agnostic** — it is applied to every raw pipeline
+> database (e.g. `default`, `devnets`) via the migration matrix. Therefore:
+> - **Never** qualify objects with a database name (write `beacon_api_...`, not
+>   `default.beacon_api_...`). Unqualified names resolve to the connection's
+>   `database=` at apply time.
+> - Distributed engines must target **`currentDatabase()`**, not a literal db.
+> - Keep the `{database}` macro in the Replicated path so each database gets its
+>   own ZooKeeper path.
+> - Do **not** `CREATE DATABASE` in a migration (databases are created out-of-band).
 
 ```sql
 CREATE TABLE beacon_api_eth_{api_version}_events_{event_name}_local on cluster '{cluster}' (
@@ -341,7 +352,7 @@ CREATE TABLE beacon_api_eth_{api_version}_events_{event_name}_local on cluster '
 PARTITION BY toStartOfMonth(slot_start_date_time)
 ORDER BY (slot_start_date_time, meta_network_name, meta_client_name);
 
-ALTER TABLE default.beacon_api_eth_{api_version}_events_{event_name}_local ON CLUSTER '{cluster}'
+ALTER TABLE beacon_api_eth_{api_version}_events_{event_name}_local ON CLUSTER '{cluster}'
 MODIFY COMMENT 'Contains beacon API eventstream "{event_name}" data from each sentry client attached to a beacon node.',
 COMMENT COLUMN event_date_time 'When the sentry received the event from a beacon node',
 COMMENT COLUMN slot 'Slot number in the beacon API event stream payload',
@@ -351,10 +362,10 @@ COMMENT COLUMN epoch 'The epoch number in the beacon API event stream payload',
 COMMENT COLUMN epoch_start_date_time 'The wall clock time when the epoch started';
 
 CREATE TABLE beacon_api_eth_{api_version}_events_{event_name} on cluster '{cluster}' AS beacon_api_eth_{api_version}_events_{event_name}_local
-ENGINE = Distributed('{cluster}', default, beacon_api_eth_{api_version}_events_{event_name}_local, rand());
+ENGINE = Distributed('{cluster}', currentDatabase(), beacon_api_eth_{api_version}_events_{event_name}_local, rand());
 ```
 
-**File**: `deploy/migrations/clickhouse/{NEXT_NUMBER}_{event_name}.down.sql`
+**File**: `deploy/migrations/clickhouse/xatu/{NEXT_NUMBER}_{event_name}.down.sql`
 
 ```sql
 DROP TABLE IF EXISTS beacon_api_eth_{api_version}_events_{event_name} on cluster '{cluster}';
@@ -474,8 +485,8 @@ sinks:
 - [ ] `pkg/sentry/cache/duplicate.go` - Cache field and initialization
 - [ ] `pkg/sentry/sentry.go` - Event subscription
 - [ ] `pkg/server/service/event-ingester/event/beacon/eth/{api_version}/events_{event_name}.go` - Server handler
-- [ ] `deploy/migrations/clickhouse/{number}_{event_name}.up.sql` - Schema
-- [ ] `deploy/migrations/clickhouse/{number}_{event_name}.down.sql` - Rollback
+- [ ] `deploy/migrations/clickhouse/xatu/{number}_{event_name}.up.sql` - Schema (database-agnostic: no db qualifiers, Distributed uses currentDatabase())
+- [ ] `deploy/migrations/clickhouse/xatu/{number}_{event_name}.down.sql` - Rollback
 - [ ] `deploy/local/docker-compose/vector-kafka-clickhouse.yaml` - Pipeline config
 
 ## Final Steps

.github/workflows/cannon-smoke-test.yaml

@@ -49,7 +49,12 @@ jobs:
         context: .
         load: true
         tags: ethpandaops/xatu:local
-        cache-from: type=gha
+        # Pull the cryo-builder layer from the release pipeline's registry
+        # buildcache so a cold PR branch reuses the prebuilt cryo instead of
+        # recompiling it (~10 min). Read-only: PRs keep writing only gha cache.
+        cache-from: |
+          type=gha
+          type=registry,ref=ethpandaops/xatu:cryo-buildcache
         cache-to: type=gha,mode=max
     - name: Fetch finalized epoch from beacon node
       id: beacon
@@ -162,7 +167,7 @@ jobs:
         # the target CH and fails fatal if the schema isn't applied. So
         # block until xatu-clickhouse-migrator has exited successfully.
         echo "Waiting for xatu-clickhouse-migrator to apply the schema..."
-        for i in $(seq 1 60); do
+        for i in $(seq 1 120); do
           status=$(docker inspect --format='{{.State.Status}}' xatu-clickhouse-migrator 2>/dev/null || echo "missing")
           exit_code=$(docker inspect --format='{{.State.ExitCode}}' xatu-clickhouse-migrator 2>/dev/null || echo "")
           if [ "$status" = "exited" ] && [ "$exit_code" = "0" ]; then
@@ -174,12 +179,12 @@ jobs:
             docker logs xatu-clickhouse-migrator 2>&1 | tail -100
             exit 1
           fi
-          if [ "$i" -eq 60 ]; then
-            echo "::error::xatu-clickhouse-migrator did not finish within 120s (status=$status)"
+          if [ "$i" -eq 120 ]; then
+            echo "::error::xatu-clickhouse-migrator did not finish within 240s (status=$status)"
             docker logs xatu-clickhouse-migrator 2>&1 | tail -100
             exit 1
           fi
-          echo "  migrator status=$status (attempt $i/60)..."
+          echo "  migrator status=$status (attempt $i/120)..."
           sleep 2
         done
 
@@ -232,38 +237,52 @@ jobs:
         labels:
           ethpandaops: rocks
 
-        # Only enable derivers for event types we're testing
+        # Derivers are grouped by layer (consensus/execution). Consensus
+        # derivers default to enabled:true, so every deriver this test does NOT
+        # assert must be explicitly disabled. Execution (cryo) derivers default
+        # to enabled:false and are omitted entirely — this test is consensus-only,
+        # so no ethereum.execution / EL RPC is required.
         derivers:
-          attesterSlashing:
-            enabled: false
-          blsToExecutionChange:
-            enabled: false
-          deposit:
-            enabled: false
-          withdrawal:
-            enabled: true
-          executionTransaction:
-            enabled: true
-          proposerSlashing:
-            enabled: false
-          voluntaryExit:
-            enabled: false
-          beaconBlock:
-            enabled: true
-          beaconBlobSidecar:
-            enabled: true
-          proposerDuty:
-            enabled: true
-          beaconCommittee:
-            enabled: true
+          consensus:
+            withdrawal:
+              enabled: true
+            executionTransaction:
+              enabled: true
+            beaconBlock:
+              enabled: true
+            beaconBlobSidecar:
+              enabled: true
+            proposerDuty:
+              enabled: true
+            beaconCommittee:
+              enabled: true
+            attesterSlashing:
+              enabled: false
+            proposerSlashing:
+              enabled: false
+            blsToExecutionChange:
+              enabled: false
+            deposit:
+              enabled: false
+            voluntaryExit:
+              enabled: false
+            elaboratedAttestation:
+              enabled: false
+            beaconValidators:
+              enabled: false
+            beaconSyncCommittee:
+              enabled: false
+            beaconBlockSyncAggregate:
+              enabled: false
 
         ntpServer: time.google.com
 
         ethereum:
-          beaconNodeAddress: "https://mainnet-archive-lb-bn.utility.production.platform.ethpandaops.io"
-          beaconNodeHeaders:
-            Authorization: "AUTH_HEADER"
           overrideNetworkName: "$NETWORK_NAME"
+          beacon:
+            address: "https://mainnet-archive-lb-bn.utility.production.platform.ethpandaops.io"
+            headers:
+              Authorization: "AUTH_HEADER"
         coordinator:
           address: xatu-server:8080
 

.github/workflows/clickhouse-migrations-lint.yaml

@@ -0,0 +1,49 @@
+name: ClickHouse Migrations Lint
+
+on:
+  pull_request:
+    paths:
+      - 'deploy/migrations/clickhouse/**'
+      - '.github/workflows/clickhouse-migrations-lint.yaml'
+  workflow_dispatch:
+    branches: [ '**' ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  # ClickHouse image used for the syntax check; matches docker-compose's CHVER default.
+  CHVER: latest
+
+jobs:
+  # Enforce the database-agnostic rules (no CREATE DATABASE, no db-qualified
+  # identifiers, Distributed() uses currentDatabase()). Pure bash, no dependencies.
+  database-agnostic:
+    name: database-agnostic lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - name: Lint migrations are database-agnostic
+        run: ./deploy/migrations/clickhouse/lint.sh deploy/migrations/clickhouse
+
+  # Validate every migration parses with the real ClickHouse parser. Runs in
+  # parallel with the lint job (no `needs`).
+  syntax:
+    name: clickhouse syntax check
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - name: Validate migrations parse (clickhouse format)
+        run: |
+          docker run --rm \
+            -e GITHUB_ACTIONS=true \
+            -v "$PWD":/work -w /work \
+            --entrypoint /bin/bash \
+            "clickhouse/clickhouse-server:${CHVER:-latest}" \
+            deploy/migrations/clickhouse/check-syntax.sh deploy/migrations/clickhouse

.github/workflows/goreleaser.yaml

@@ -87,3 +87,47 @@ jobs:
             -v /var/run/docker.sock:/var/run/docker.sock \
             -e RELEASE_SUFFIX=${{ env.RELEASE_SUFFIX }} \
             goreleaser/goreleaser-cross:v1.26 release --clean --config .goreleaser.yaml.new
+
+      # The xatu+cryo image is the EL-cannon variant: it bundles the cryo binary
+      # alongside xatu. cryo is built from Rust source (./Dockerfile), so it does
+      # not fit goreleaser's copy-the-prebuilt-binary docker flow — and building
+      # it inside goreleaser would force every PR's test-build to compile cryo.
+      # We build it here, release-only. The cryo ref is pinned in ./Dockerfile
+      # (ARG CRYO_GIT_REF); that layer is cache-stable, so it only recompiles
+      # when the pinned ref changes.
+      #
+      # Cache is a registry image (ethpandaops/xatu:cryo-buildcache), NOT GHA
+      # cache: GHA cache is scoped per git ref, and releases are tag-triggered,
+      # so a v1.18.2 run could not read the v1.18.1 run's cache (only its own
+      # ref + the default branch). Registry cache is ref-agnostic, so every
+      # release reuses the prior cryo-builder layer and skips the ~10 min Rust
+      # compile until CRYO_GIT_REF changes (which invalidates that layer and
+      # recompiles once, then re-warms the cache).
+      #
+      # amd64-only: EL cannon runs on amd64 servers, and compiling cryo for
+      # arm64 under QEMU emulation takes 30-60+ min (it hung a release for
+      # 35+ min). Native amd64 compiles in ~2-3 min. Add linux/arm64 back here
+      # only if cannon ever needs to run on arm64.
+      - name: Derive cryo image version
+        run: |
+          echo "CRYO_GIT_COMMIT=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
+          # Match goreleaser's {{ .Version }} (tag with a single leading 'v' stripped).
+          TAG_NAME=${GITHUB_REF#refs/tags/}
+          echo "CRYO_IMAGE_VERSION=${TAG_NAME#v}" >> "$GITHUB_ENV"
+      - name: Build and push xatu+cryo image
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
+        with:
+          context: .
+          file: Dockerfile
+          platforms: linux/amd64
+          push: true
+          provenance: false
+          sbom: false
+          build-args: |
+            VERSION=${{ env.CRYO_IMAGE_VERSION }}
+            GIT_COMMIT=${{ env.CRYO_GIT_COMMIT }}
+          tags: |
+            ethpandaops/xatu:${{ env.CRYO_IMAGE_VERSION }}-cryo
+            ethpandaops/xatu:${{ env.RELEASE_SUFFIX && format('{0}-cryo-latest', env.RELEASE_SUFFIX) || 'cryo-latest' }}
+          cache-from: type=registry,ref=ethpandaops/xatu:cryo-buildcache
+          cache-to: type=registry,ref=ethpandaops/xatu:cryo-buildcache,mode=max,ignore-error=true

.github/workflows/govulncheck.yaml

@@ -22,7 +22,13 @@ jobs:
           go-version-file: 'go.mod'
           check-latest: true
       - name: Install govulncheck
-        run: go install golang.org/x/vuln/cmd/govulncheck@latest
+        # Pinned to v1.3.0: govulncheck v1.4.0 (latest) panics with
+        # "ForEachElement called on type containing *types.TypeParam" when scanning
+        # this codebase's generics under Go 1.26 — a bug in its bundled x/tools
+        # v0.46.0 (golang/go#80055, fixed by CL 786280 but not yet tagged; tracked
+        # in golang/go#80139). v1.3.0 ships the unaffected x/tools v0.44.0. Move
+        # back to @latest once x/vuln releases a version > v1.4.0 with the fix.
+        run: go install golang.org/x/vuln/cmd/govulncheck@v1.3.0
       - name: Run govulncheck
         run: |
           set +e
@@ -45,16 +51,18 @@ jobs:
           # The two docker vulns are server-side; we only use the client (via
           # testcontainers in codegen, and via go-builder-client transitively).
           #
-          # Stdlib advisories fixed in go1.26.3 — pinned to 1.26.2 to match
-          # goreleaser-cross:v1.26 (no v1.26.3 image yet). Remove these once the
-          # container catches up and go.mod/Dockerfile move to 1.26.3:
-          #   GO-2026-4918  net/http  (x/net portion fixed via v0.53.0 bump)
-          #   GO-2026-4971  net
-          #   GO-2026-4977  net/mail
-          #   GO-2026-4980  html/template
-          #   GO-2026-4982  html/template
-          #   GO-2026-4986  net/mail
-          ALLOWED='GO-2026-4479|GO-2026-4883|GO-2026-4887|GO-2026-4918|GO-2026-4971|GO-2026-4977|GO-2026-4980|GO-2026-4982|GO-2026-4986'
+          # Stdlib advisories — pinned to 1.26.2 to match goreleaser-cross:v1.26
+          # (no v1.26.3+ image yet). Remove these once the container catches up and
+          # go.mod/Dockerfile move past the listed fix version:
+          #   GO-2026-4918  net/http       (fixed in go1.26.3; x/net portion via v0.53.0 bump)
+          #   GO-2026-4971  net            (fixed in go1.26.3)
+          #   GO-2026-4977  net/mail       (fixed in go1.26.3)
+          #   GO-2026-4980  html/template  (fixed in go1.26.3)
+          #   GO-2026-4982  html/template  (fixed in go1.26.3)
+          #   GO-2026-4986  net/mail       (fixed in go1.26.3)
+          #   GO-2026-5037  crypto/x509    (fixed in go1.26.4)
+          #   GO-2026-5039  net/textproto  (fixed in go1.26.4)
+          ALLOWED='GO-2026-4479|GO-2026-4883|GO-2026-4887|GO-2026-4918|GO-2026-4971|GO-2026-4977|GO-2026-4980|GO-2026-4982|GO-2026-4986|GO-2026-5037|GO-2026-5039'
           REPORTED=$(echo "$OUTPUT" | grep -oE 'Vulnerability #[0-9]+: GO-[0-9]+-[0-9]+' | grep -oE 'GO-[0-9]+-[0-9]+' | sort -u)
           UNEXPECTED=$(echo "$REPORTED" | grep -vE "^($ALLOWED)$" || true)
           if [ -z "$UNEXPECTED" ] && [ -n "$REPORTED" ]; then

.github/workflows/sentry-smoke-test.yaml

@@ -70,7 +70,12 @@ jobs:
         context: .
         load: true
         tags: ethpandaops/xatu:local
-        cache-from: type=gha
+        # Pull the cryo-builder layer from the release pipeline's registry
+        # buildcache so a cold PR branch reuses the prebuilt cryo instead of
+        # recompiling it (~10 min). Read-only: PRs keep writing only gha cache.
+        cache-from: |
+          type=gha
+          type=registry,ref=ethpandaops/xatu:cryo-buildcache
         cache-to: type=gha,mode=max
     - name: Build sentry-logs image
       if: steps.canary.outputs.skip != 'true'

.golangci.yml

@@ -81,6 +81,25 @@ linters:
           - gosec
           - wsl
         path: _test\.go
+      # EL cannon code (derivers + routes) repeats cryo/ClickHouse column names
+      # (a fixed external vocabulary) across many per-dataset files; goconst flags
+      # the shared names but turning column identifiers into Go constants adds no
+      # real value.
+      - linters:
+          - goconst
+        path: execution/
+      # CL cannon derivers repeat the logrus "module"/"type" field keys across
+      # many per-type files; these are a fixed convention shared by every deriver
+      # and turning them into Go constants adds no real value.
+      - linters:
+          - goconst
+        path: cannon/deriver/beacon/
+      # ClickHouse route snapshot tests repeat ClickHouse column-name assertion
+      # keys (a fixed external vocabulary) across many per-type files; turning
+      # these identifiers into Go constants adds no real value.
+      - linters:
+          - goconst
+        path: clickhouse/route/canonical/
     paths:
       - third_party$
       - builtin$

Dockerfile

@@ -3,7 +3,35 @@ WORKDIR /src
 COPY go.sum go.mod ./
 RUN go mod download
 COPY . .
-RUN go build -o /bin/app .
+# Version metadata is injected at release time (see .github/workflows/goreleaser.yaml);
+# local/CI builds fall back to dev defaults. GOOS/GOARCH come from BuildKit's
+# predefined platform args so the multi-arch release build stamps the right
+# values under QEMU emulation.
+ARG TARGETOS
+ARG TARGETARCH
+ARG VERSION=dev
+ARG GIT_COMMIT=unknown
+RUN go build \
+  -ldflags="-s -w \
+  -X github.com/ethpandaops/xatu/pkg/proto/xatu.Release=${VERSION} \
+  -X github.com/ethpandaops/xatu/pkg/proto/xatu.GitCommit=${GIT_COMMIT} \
+  -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOOS=${TARGETOS} \
+  -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOARCH=${TARGETARCH}" \
+  -o /bin/app .
+
+# cryo is the execution-layer extractor used by EL cannon. We build from git
+# (matching ethpandaops/base-images clickhouse-tools, the source of the legacy
+# pipeline's data): master's `blocks` dataset exposes gas_limit, which the
+# 0.3.2 crates.io release omits. cryo's column output is the schema contract for
+# the canonical_execution_* routes, and the state-read datasets' semantics vary
+# across cryo versions, so we PIN to a specific commit for reproducible builds
+# and stable data. This ARG line is the single place the cryo ref is pinned:
+# every build path (docker build, docker compose, `make docker`, and the
+# release cryo image) builds this Dockerfile, so bump it here to upgrade cryo
+# everywhere. Override with --build-arg CRYO_GIT_REF=<ref> for ad-hoc testing.
+FROM rust:1-bookworm AS cryo-builder
+ARG CRYO_GIT_REF=559b65455d7ef6b03e8e9e96a0e50fd4fe8a9c86
+RUN cargo install --git https://github.com/paradigmxyz/cryo --rev "${CRYO_GIT_REF}" --locked cryo_cli
 
 FROM ubuntu:latest
 RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
@@ -12,5 +40,6 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-reco
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /bin/app /xatu
+COPY --from=cryo-builder /usr/local/cargo/bin/cryo /usr/local/bin/cryo
 EXPOSE 5555
-ENTRYPOINT ["/xatu"]
+ENTRYPOINT ["/xatu"]