feat(peers): automated weekly peers.txt + addrbook.json refresh

.github/workflows/update-peers.yml

@@ -0,0 +1,106 @@
+name: Update peers
+
+on:
+  schedule:
+    - cron: "0 0 * * 1" # Mondays 00:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+env:
+  CELESTIA_APP_VERSION: v8.0.3 # bump to pull a newer addrbook generator
+  MIN_PEERS: "5"
+  CHURN_WARN_PCT: "60"
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: mainnet
+            url: https://peers.cumulo.me/peers/celestia/mainnet/peers.json
+            dir: celestia
+          - name: mocha-4
+            url: https://peers.cumulo.me/peers/celestia/testnet/peers.json
+            dir: mocha-4
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: scripts/peers/verifypeer/go.mod
+
+      - name: Ensure jq is available
+        run: command -v jq >/dev/null || { sudo apt-get update && sudo apt-get install -y jq; }
+
+      - name: Fetch + filter candidates
+        run: |
+          ./scripts/peers/fetch-filter.sh "${{ matrix.url }}" /tmp/candidates.txt
+          echo "Candidates: $(wc -l < /tmp/candidates.txt)"
+
+      - name: Verify peers (handshake + ID match)
+        working-directory: scripts/peers/verifypeer
+        run: |
+          go run . \
+            -in /tmp/candidates.txt \
+            -out "${GITHUB_WORKSPACE}/${{ matrix.dir }}/peers.txt" \
+            -report /tmp/report.json \
+            -min "${MIN_PEERS}"
+
+      - name: Generate addrbook.json
+        run: |
+          asset="celestia-app-standalone_Linux_x86_64.tar.gz"
+          curl -fsSL -o /tmp/app.tar.gz \
+            "https://github.com/celestiaorg/celestia-app/releases/download/${CELESTIA_APP_VERSION}/${asset}"
+          tar -xzf /tmp/app.tar.gz -C /tmp celestia-appd
+          /tmp/celestia-appd addrbook "${{ matrix.dir }}/peers.txt" "${{ matrix.dir }}/addrbook.json"
+
+      - name: Compute churn + build PR body
+        id: meta
+        run: |
+          mismatch_count=$(jq '[.[] | select(.status=="id_mismatch")] | length' /tmp/report.json)
+          verified=$(wc -l < "${{ matrix.dir }}/peers.txt")
+          {
+            echo "### ${{ matrix.name }} peer refresh"
+            echo ""
+            echo "- Verified peers: ${verified}"
+            echo "- ID mismatches dropped: ${mismatch_count}"
+            echo ""
+            echo "Source: ${{ matrix.url }} (filtered to tcp_verified, then independently verified via P2P handshake)."
+          } > /tmp/pr_body.md
+          if [ "${mismatch_count}" -gt 0 ]; then
+            echo "⚠️ ID mismatches were detected and excluded — see job logs." >> /tmp/pr_body.md
+            jq -r '.[] | select(.status=="id_mismatch") | "- claimed=\(.claimed_id) remote=\(.remote_id) line=\(.line)"' /tmp/report.json >> /tmp/pr_body.md
+          fi
+          base="$(git show HEAD:${{ matrix.dir }}/peers.txt 2>/dev/null || true)"
+          if [ -n "$base" ]; then
+            added=$(comm -13 <(echo "$base" | sort) <(sort "${{ matrix.dir }}/peers.txt") | grep -c . || true)
+            removed=$(comm -23 <(echo "$base" | sort) <(sort "${{ matrix.dir }}/peers.txt") | grep -c . || true)
+            total=$(echo "$base" | grep -c . || true)
+            (( total > 0 )) || total=1
+            churn=$(( (added + removed) * 100 / total ))
+            echo "churn_pct=${churn}" >> "$GITHUB_OUTPUT"
+            echo "- Churn vs current: ${churn}%" >> /tmp/pr_body.md
+            if [ "${churn}" -gt "${CHURN_WARN_PCT}" ]; then
+              echo "labels=high-churn" >> "$GITHUB_OUTPUT"
+            else
+              echo "labels=" >> "$GITHUB_OUTPUT"
+            fi
+          fi
+
+      - name: Open PR
+        uses: peter-evans/create-pull-request@v6
+        with:
+          branch: peers/auto-update-${{ matrix.name }}
+          title: "chore(peers): refresh ${{ matrix.name }} peers.txt + addrbook.json"
+          body-path: /tmp/pr_body.md
+          labels: ${{ steps.meta.outputs.labels }}
+          add-paths: |
+            ${{ matrix.dir }}/peers.txt
+            ${{ matrix.dir }}/addrbook.json
+          assignees: rootulp
+          commit-message: "chore(peers): refresh ${{ matrix.name }} peers and addrbook"

scripts/peers/fetch-filter.bats

@@ -0,0 +1,10 @@
+#!/usr/bin/env bats
+
+@test "dry-run filters tcp_verified and sorts by score desc" {
+  out="$BATS_TEST_TMPDIR/cand_out.txt"
+  run bash "$BATS_TEST_DIRNAME/fetch-filter.sh" --dry-run \
+    "$BATS_TEST_DIRNAME/testdata/peers.sample.json" "$out"
+  [ "$status" -eq 0 ]
+  run diff -u "$BATS_TEST_DIRNAME/testdata/peers.sample.golden" "$out"
+  [ "$status" -eq 0 ]
+}

scripts/peers/fetch-filter.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Fetch the Cumulo peer feed (or read a local fixture) and emit candidate
+# "id@host:port" lines: tcp_verified peers, sorted by score descending.
+#
+# Usage:
+#   fetch-filter.sh <cumulo_url> <out_file>
+#   fetch-filter.sh --dry-run <fixture_json> <out_file>
+set -euo pipefail
+
+FILTER='.peers | sort_by(.score) | reverse | .[] | select(.tcp_verified == true) | .connection_string'
+
+if [[ "${1:-}" == "--dry-run" ]]; then
+  fixture="${2:?fixture path required}"
+  out="${3:?out file required}"
+  jq -r "$FILTER" "$fixture" > "$out"
+  exit 0
+fi
+
+url="${1:?cumulo url required}"
+out="${2:?out file required}"
+curl -fsSL "$url" | jq -r "$FILTER" > "$out"

scripts/peers/testdata/peers.sample.golden

@@ -0,0 +1,2 @@
+13612522d3ce71a181881370b8f40103d4def9f1@84.32.32.148:16400
+213f44afb503be593a1c8eb27dae86a50363e500@65.109.93.17:11656

scripts/peers/testdata/peers.sample.json

@@ -0,0 +1,10 @@
+{
+  "chain_id": "celestia",
+  "network": "mainnet",
+  "peer_count": 3,
+  "peers": [
+    { "connection_string": "13612522d3ce71a181881370b8f40103d4def9f1@84.32.32.148:16400", "score": 2.1, "tcp_verified": true },
+    { "connection_string": "213f44afb503be593a1c8eb27dae86a50363e500@65.109.93.17:11656", "score": 1.5, "tcp_verified": true },
+    { "connection_string": "deadbeef00000000000000000000000000000000@10.0.0.1:26656", "score": 9.9, "tcp_verified": false }
+  ]
+}

scripts/peers/verifypeer/.gitignore

@@ -0,0 +1 @@
+/verifypeer

scripts/peers/verifypeer/go.mod

@@ -0,0 +1,91 @@
+module github.com/celestiaorg/networks/scripts/peers/verifypeer
+
+go 1.26.1
+
+replace github.com/cometbft/cometbft => github.com/celestiaorg/celestia-core v0.40.2
+
+require github.com/cometbft/cometbft v0.0.0-00010101000000-000000000000
+
+require (
+	github.com/DataDog/zstd v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.32.13 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
+	github.com/aws/smithy-go v1.24.2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/celestiaorg/go-square/v3 v3.0.2 // indirect
+	github.com/celestiaorg/nmt v0.24.2 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cockroachdb/errors v1.11.3 // indirect
+	github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
+	github.com/cockroachdb/pebble v1.1.4 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
+	github.com/cometbft/cometbft-db v1.0.4 // indirect
+	github.com/cosmos/gogoproto v1.7.2 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
+	github.com/dgraph-io/badger/v4 v4.5.1 // indirect
+	github.com/dgraph-io/ristretto/v2 v2.1.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/getsentry/sentry-go v0.31.1 // indirect
+	github.com/go-kit/kit v0.13.0 // indirect
+	github.com/go-kit/log v0.2.1 // indirect
+	github.com/go-logfmt/logfmt v0.6.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/btree v1.1.3 // indirect
+	github.com/google/flatbuffers v25.1.24+incompatible // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/jmhodges/levigo v1.0.0 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+	github.com/klauspost/reedsolomon v1.13.3 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/linxGnu/grocksdb v1.9.8 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae // indirect
+	github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_golang v1.23.2 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.67.5 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/rogpeppe/go-internal v1.14.1 // indirect
+	github.com/sasha-s/go-deadlock v0.3.9 // indirect
+	github.com/stretchr/testify v1.11.1 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
+	go.etcd.io/bbolt v1.4.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.3 // indirect
+	golang.org/x/crypto v0.49.0 // indirect
+	golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect
+	golang.org/x/net v0.52.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
+	google.golang.org/grpc v1.79.3 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)