refactor(backend): restructure config to support both networks

.env.example

@@ -4,9 +4,32 @@
 # Keep this file limited to *secrets only* - non-secret configuration lives in
 # kustomize ConfigMap patches (e.g., kustomize/overlays/local/backend-configmap-local.yaml).
 #
-# Required
-WALLET_ADDRESS=
-WALLET_PRIVATE_KEY=
+# Multi-network support
+# ---------------------
+# Dealbot now supports running against multiple Filecoin networks from a single
+# instance. The `NETWORKS` env var (set via ConfigMap, not here) selects the
+# active networks as a comma-separated list, e.g. `NETWORKS=calibration` or
+# `NETWORKS=calibration,mainnet`.
+#
+# Every network-scoped secret is prefixed with the uppercase network name:
+#   CALIBRATION_WALLET_ADDRESS, CALIBRATION_WALLET_PRIVATE_KEY
+#   MAINNET_WALLET_ADDRESS,     MAINNET_WALLET_PRIVATE_KEY
+#
+# Each active network must provide either `<NETWORK>_WALLET_PRIVATE_KEY` or
+# `<NETWORK>_SESSION_KEY_PRIVATE_KEY`. When both are set, the session key
+# takes precedence (see docs/runbooks/wallet-and-session-keys.md).
+
+# --- Required for calibration ---
+CALIBRATION_WALLET_ADDRESS=
+CALIBRATION_WALLET_PRIVATE_KEY=
+# Optional: use a session key instead of the raw wallet key.
+# CALIBRATION_SESSION_KEY_PRIVATE_KEY=
+
+# --- Required for mainnet (only if `mainnet` is in NETWORKS) ---
+# MAINNET_WALLET_ADDRESS=
+# MAINNET_WALLET_PRIVATE_KEY=
+# MAINNET_SESSION_KEY_PRIVATE_KEY=
+
 #
 # Optional (only if using an external DB or a non-default password)
-# DATABASE_PASSWORD=
+# DATABASE_PASSWORD=

apps/backend/.env.example

@@ -1,111 +1,218 @@
-# Node Environment
-NODE_ENV=development
+# =============================================================================
+# Dealbot backend — environment configuration
+# =============================================================================
+# See docs/environment-variables.md for the full reference.
+#
+# Multi-network primer
+# --------------------
+# `NETWORKS` is a comma-separated list of Filecoin networks the instance should
+# drive (supported: `calibration`, `mainnet`). Every network-scoped variable is
+# namespaced with the UPPERCASE network name:
+#
+#   NETWORKS=calibration,mainnet
+#   CALIBRATION_WALLET_PRIVATE_KEY=0x...
+#   MAINNET_WALLET_PRIVATE_KEY=0x...
+#
+# Globals (database, jobs, timeouts, HTTP ports, etc.) remain unprefixed and
+# apply to all networks. Variables for INACTIVE networks are ignored; only
+# active networks are validated at startup.
+# =============================================================================
+
+# -----------------------------------------------------------------------------
+# Application
+# -----------------------------------------------------------------------------
+NODE_ENV=production
+
+# api | worker | both (default: both)
+DEALBOT_RUN_MODE=both
 
-# Specify port for dealbot service (optional)
 DEALBOT_PORT=8080
-
-# Specify host for dealbot service (optional)
 DEALBOT_HOST=localhost
 
 # Optional public base URL for DealBot HTTP API (used to construct hosted-piece source URLs for SP pull checks)
 # DEALBOT_API_PUBLIC_URL=https://dealbot.example.com
 
+# Metrics-only HTTP server (used when DEALBOT_RUN_MODE=worker)
+DEALBOT_METRICS_PORT=9090
+DEALBOT_METRICS_HOST=0.0.0.0
+
 # Comma-separated list of allowed origins for CORS (for web dev)
 DEALBOT_ALLOWED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
 
-# Database Configuration
+# -----------------------------------------------------------------------------
+# Database
+# -----------------------------------------------------------------------------
 DATABASE_HOST=localhost
 DATABASE_PORT=5432
 DATABASE_USER=dealbot
-DATABASE_PASSWORD=dealbot_password
-DATABASE_NAME=filecoin_dealbot
-
-# Blockchain Configuration
-NETWORK=calibration # or mainnet
-WALLET_ADDRESS=0x0000000000000000000000000000000000000000
-WALLET_PRIVATE_KEY=your_private_key_here
-CHECK_DATASET_CREATION_FEES=true
-USE_ONLY_APPROVED_PROVIDERS=true
+DATABASE_PASSWORD=dealbot
+DATABASE_NAME=dealbot
+DATABASE_POOL_MAX=1
+
+# -----------------------------------------------------------------------------
+# Network selection
+# -----------------------------------------------------------------------------
+# Comma-separated. Default: calibration.
+NETWORKS=calibration,mainnet
+
+# -----------------------------------------------------------------------------
+# Per-network configuration — CALIBRATION
+# -----------------------------------------------------------------------------
+# At least one of WALLET_PRIVATE_KEY or SESSION_KEY_PRIVATE_KEY is required
+# for each ACTIVE network. Session key (if present) takes precedence.
+CALIBRATION_WALLET_ADDRESS=0x0000000000000000000000000000000000000000
+CALIBRATION_WALLET_PRIVATE_KEY=0x0000000000000000000000000000000000000000000000000000000000000000
+# CALIBRATION_SESSION_KEY_PRIVATE_KEY=
+
+# Optional: custom RPC (authenticated endpoints avoid public rate limits)
+CALIBRATION_RPC_URL=
+# Per-request RPC timeout (ms); kept above eRPC's failover budget. See #604.
+CALIBRATION_RPC_REQUEST_TIMEOUT_MS=30000
+
+CALIBRATION_CHECK_DATASET_CREATION_FEES=true
+CALIBRATION_USE_ONLY_APPROVED_PROVIDERS=true
 # Upstream pdp-explorer subgraph — drives the data-retention / overdue-periods path.
-PDP_SUBGRAPH_ENDPOINT=https://api.thegraph.com/subgraphs/filecoin/pdp
-# Dealbot-owned subgraph on Goldsky (see apps/subgraph/README.md) — drives only
-# the new sampled-retrieval candidate-piece query for now.
-SUBGRAPH_ENDPOINT=https://api.goldsky.com/api/public/<project>/subgraphs/dealbot-subgraph/<version>/gn
-
-# Minimum number of datasets per SP (default: 1). When > 1, a separate data_set_creation job provisions extra datasets.
-MIN_NUM_DATASETS_FOR_CHECKS=1
-
-# Dataset Versioning (optional)
-# Uncomment and set to enable dataset versioning (e.g., "dealbot-v1", "dealbot-v2")
-# This allows creating new logical datasets without changing wallet addresses
-# DEALBOT_DATASET_VERSION=dealbot-v1
-
-# Scheduling Configuration
-# Intervals: How often jobs run (in seconds)
-PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 # Run providers refresh every 4 hours
-DATA_RETENTION_POLL_INTERVAL_SECONDS=3600 # Run data retention poll every 60 minutes
-
-# Prometheus Metrics Configuration
-# Cache TTL for wallet balance collection (in seconds)
-PROMETHEUS_WALLET_BALANCE_TTL_SECONDS=3600 # Refresh wallet balance every 1 hour
-PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS=60 # Wait 1 minute before retry after error
-
-# Maintenance windows (UTC)
-DEALBOT_MAINTENANCE_WINDOWS_UTC=07:00,22:00
-DEALBOT_MAINTENANCE_WINDOW_MINUTES=20
-
-# pg-boss (DB-backed jobs) Configuration
-# See docs/environment-variables.md for details, limits, and fractional examples.
-DEALS_PER_SP_PER_HOUR=2
-DATASET_CREATIONS_PER_SP_PER_HOUR=1
-RETRIEVALS_PER_SP_PER_HOUR=1
-SAMPLED_RETRIEVALS_PER_SP_PER_HOUR=
-SAMPLED_RETRIEVAL_BLOCK_SAMPLE_COUNT=5
-METRICS_PER_HOUR=2
+CALIBRATION_PDP_SUBGRAPH_ENDPOINT=
+# Dealbot-owned subgraph (see apps/subgraph/README.md) — drives the sampled-retrieval
+# candidate-piece query. When unset, sampled-retrieval schedules are not created.
+CALIBRATION_SUBGRAPH_ENDPOINT=
+
+# Minimum number of datasets per SP. When > 1, a separate data_set_creation
+# job provisions extra datasets.
+CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS=1
+
+# Optional: dataset versioning (e.g. "dealbot-v1", "dealbot-v2").
+# CALIBRATION_DEALBOT_DATASET_VERSION=dealbot-v1
+
+# Per-network scheduling rates (per hour; fractional values supported)
+CALIBRATION_DEALS_PER_SP_PER_HOUR=4
+CALIBRATION_DATASET_CREATIONS_PER_SP_PER_HOUR=0.01
+CALIBRATION_RETRIEVALS_PER_SP_PER_HOUR=1
+# Sampled (anonymous) retrievals; defaults to CALIBRATION_RETRIEVALS_PER_SP_PER_HOUR when unset.
+CALIBRATION_SAMPLED_RETRIEVALS_PER_SP_PER_HOUR=
+CALIBRATION_PIECE_CLEANUP_PER_SP_PER_HOUR=0.1
 # data_set_lifecycle_check canary: creates a throwaway data set and terminates it each tick
-# (defaults: enabled on calibration, disabled on mainnet).
-# DATASET_LIFECYCLE_CHECK_ENABLED=true
-DATASET_LIFECYCLE_CHECKS_PER_SP_PER_HOUR=1
-DATA_SET_LIFECYCLE_CHECK_JOB_TIMEOUT_SECONDS=600  # 10m: create + upload + terminate + pdpEndEpoch poll
+# (default: enabled on calibration, disabled on mainnet).
+# CALIBRATION_DATASET_LIFECYCLE_CHECK_ENABLED=true
+CALIBRATION_DATASET_LIFECYCLE_CHECKS_PER_SP_PER_HOUR=1
+CALIBRATION_DEAL_JOB_TIMEOUT_SECONDS=360               # 6m: max runtime for deal jobs
+CALIBRATION_RETRIEVAL_JOB_TIMEOUT_SECONDS=60           # 1m: max runtime for retrieval jobs
+CALIBRATION_SAMPLED_RETRIEVAL_JOB_TIMEOUT_SECONDS=360  # 6m: max runtime for sampled retrieval jobs (pieces up to ~500 MiB)
+CALIBRATION_DATA_SET_CREATION_JOB_TIMEOUT_SECONDS=300  # 5m: max runtime for dataset-creation jobs
+CALIBRATION_DATA_SET_LIFECYCLE_CHECK_JOB_TIMEOUT_SECONDS=600  # 10m: create + upload + terminate + pdpEndEpoch poll
+CALIBRATION_MAX_PIECE_CLEANUP_RUNTIME_SECONDS=3000     # 50m: max runtime for piece cleanup
+
+# Per-network job intervals (seconds)
+CALIBRATION_PROVIDERS_REFRESH_INTERVAL_SECONDS=14400   # 4 hours
+CALIBRATION_DATA_RETENTION_POLL_INTERVAL_SECONDS=3600  # 1 hour
+
+# Per-network maintenance windows (UTC)
+CALIBRATION_MAINTENANCE_WINDOWS_UTC=07:00,22:00
+CALIBRATION_MAINTENANCE_WINDOW_MINUTES=20
+
+# Per-network piece cleanup configuration
+CALIBRATION_MAX_DATASET_STORAGE_SIZE_BYTES=1048576000
+CALIBRATION_TARGET_DATASET_STORAGE_SIZE_BYTES=1
+
+# Per-network pull check configuration
+CALIBRATION_PULL_CHECKS_PER_SP_PER_HOUR=1
+CALIBRATION_PULL_CHECK_JOB_TIMEOUT_SECONDS=300
+CALIBRATION_PULL_CHECK_POLL_INTERVAL_SECONDS=2
+CALIBRATION_PULL_CHECK_PIECE_SIZE_BYTES=10485760
+CALIBRATION_PULL_PIECE_CLEANUP_INTERVAL_SECONDS=604800
+
+# Per-network SP blocklists
+# CALIBRATION_BLOCKED_SP_IDS=1234,5678
+# CALIBRATION_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...
+
+# -----------------------------------------------------------------------------
+# Per-network configuration — MAINNET  (uncomment when adding to NETWORKS)
+# -----------------------------------------------------------------------------
+MAINNET_WALLET_ADDRESS=0x0000000000000000000000000000000000000000
+MAINNET_WALLET_PRIVATE_KEY=0x0000000000000000000000000000000000000000000000000000000000000000
+# MAINNET_SESSION_KEY_PRIVATE_KEY=
+# MAINNET_RPC_URL=
+MAINNET_RPC_REQUEST_TIMEOUT_MS=30000
+MAINNET_CHECK_DATASET_CREATION_FEES=false
+MAINNET_USE_ONLY_APPROVED_PROVIDERS=true
+MAINNET_PDP_SUBGRAPH_ENDPOINT=
+MAINNET_SUBGRAPH_ENDPOINT=
+MAINNET_MIN_NUM_DATASETS_FOR_CHECKS=1
+# MAINNET_DEALBOT_DATASET_VERSION=
+MAINNET_DEALS_PER_SP_PER_HOUR=1
+MAINNET_DATASET_CREATIONS_PER_SP_PER_HOUR=0.01
+MAINNET_RETRIEVALS_PER_SP_PER_HOUR=1
+MAINNET_SAMPLED_RETRIEVALS_PER_SP_PER_HOUR=
+MAINNET_PIECE_CLEANUP_PER_SP_PER_HOUR=0.05
+# data_set_lifecycle_check defaults to disabled on mainnet; set true to enable.
+# MAINNET_DATASET_LIFECYCLE_CHECK_ENABLED=false
+MAINNET_DATASET_LIFECYCLE_CHECKS_PER_SP_PER_HOUR=1
+MAINNET_DEAL_JOB_TIMEOUT_SECONDS=360
+MAINNET_RETRIEVAL_JOB_TIMEOUT_SECONDS=60
+MAINNET_SAMPLED_RETRIEVAL_JOB_TIMEOUT_SECONDS=360
+MAINNET_DATA_SET_CREATION_JOB_TIMEOUT_SECONDS=300
+MAINNET_DATA_SET_LIFECYCLE_CHECK_JOB_TIMEOUT_SECONDS=600
+MAINNET_PROVIDERS_REFRESH_INTERVAL_SECONDS=14400
+MAINNET_DATA_RETENTION_POLL_INTERVAL_SECONDS=3600
+MAINNET_MAINTENANCE_WINDOWS_UTC=07:00,22:00
+MAINNET_MAINTENANCE_WINDOW_MINUTES=20
+# MAINNET_BLOCKED_SP_IDS=1234,5678
+# MAINNET_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...
+MAINNET_MAX_DATASET_STORAGE_SIZE_BYTES=25769803776
+MAINNET_TARGET_DATASET_STORAGE_SIZE_BYTES=21474836480
+MAINNET_MAX_PIECE_CLEANUP_RUNTIME_SECONDS=3000
+MAINNET_PULL_CHECKS_PER_SP_PER_HOUR=1
+MAINNET_PULL_CHECK_JOB_TIMEOUT_SECONDS=300
+MAINNET_PULL_CHECK_POLL_INTERVAL_SECONDS=2
+MAINNET_PULL_CHECK_PIECE_SIZE_BYTES=10485760
+MAINNET_PULL_PIECE_CLEANUP_INTERVAL_SECONDS=604800
+
+# -----------------------------------------------------------------------------
+# Jobs (pg-boss)
+# -----------------------------------------------------------------------------
+# See docs/environment-variables.md for details, limits, and sizing guidance.
 PG_BOSS_LOCAL_CONCURRENCY=20
-JOB_SCHEDULER_POLL_SECONDS=300
+JOB_SCHEDULER_POLL_SECONDS=120
 JOB_WORKER_POLL_SECONDS=60
 JOB_CATCHUP_MAX_ENQUEUE=10
 JOB_SCHEDULE_PHASE_SECONDS=0
 JOB_ENQUEUE_JITTER_SECONDS=0
-DEAL_JOB_TIMEOUT_SECONDS=360          # 6m: Max runtime for deal jobs (TODO: reduce default to 3m)
-RETRIEVAL_JOB_TIMEOUT_SECONDS=60     # 1m: Max runtime for retrieval jobs (TODO: reduce default to 30s)
-SAMPLED_RETRIEVAL_JOB_TIMEOUT_SECONDS=360 # 6m: Max runtime for sampled retrieval jobs (pieces up to ~500 MiB)
-IPFS_BLOCK_FETCH_CONCURRENCY=6       # Parallel block fetches when validating IPFS DAGs
-
-# Pull Check Configuration
-PULL_CHECKS_PER_SP_PER_HOUR=1         # SP pull-pathway checks scheduled per provider per hour
-PULL_CHECK_JOB_TIMEOUT_SECONDS=300    # 5m: Max runtime for pull-check jobs
-PULL_CHECK_POLL_INTERVAL_SECONDS=2    # SP pull status polling interval
-PULL_CHECK_PIECE_SIZE_BYTES=10485760  # 10 MiB synthetic test piece size per pull check
-PULL_PIECE_MAX_CONCURRENT_STREAMS=50  # Max concurrent streams across all pieces (DoS protection)
-PULL_PIECE_MAX_STREAMS_PER_CID=3      # Max concurrent streams per pieceCid (prevents spam of single piece)
+IPFS_BLOCK_FETCH_CONCURRENCY=6                # parallel block fetches during IPFS DAG validation
+SAMPLED_RETRIEVAL_BLOCK_SAMPLE_COUNT=5         # CAR blocks sampled for IPNI / block-fetch validation
 DEALBOT_PGBOSS_POOL_MAX=1
 DEALBOT_PGBOSS_SCHEDULER_ENABLED=true
 
-# Dataset Configuration
-DEALBOT_LOCAL_DATASETS_PATH=./datasets
-KAGGLE_DATASET_TOTAL_PAGES=500
-
-# Proxy Configuration
-PROXY_LIST=http://username:password@host:port,http://username:password@host:port
-PROXY_LOCATIONS=l1,l2
-
-# Timeout Configuration (in milliseconds)
-CONNECT_TIMEOUT_MS=10000             # 10s: Connection + response-headers timeout (scoped to the header phase only)
-# HTTP_REQUEST_TIMEOUT_MS and HTTP2_REQUEST_TIMEOUT_MS default to the longest job timeout above
-# (max of DEAL_/RETRIEVAL_/SAMPLED_RETRIEVAL_/DATA_SET_CREATION_/MAX_PIECE_CLEANUP_ * 1000 ms) so the
-# HTTP-level ceiling never pre-empts a job-scoped AbortSignal. Only override when you have a non-job
-# caller of HttpClientService that needs a specific deadline.
-# HTTP_REQUEST_TIMEOUT_MS=360000
-# HTTP2_REQUEST_TIMEOUT_MS=360000
-
-# SP Blocklists configuration
-# BLOCKED_SP_IDS=1234,5678
-# BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...
+# -----------------------------------------------------------------------------
+# Pull Piece
+# -----------------------------------------------------------------------------
+PULL_PIECE_MAX_CONCURRENT_STREAMS=50  # Max concurrent streams across all pieces (DoS protection)
+PULL_PIECE_MAX_STREAMS_PER_CID=3      # Max concurrent streams per pieceCid (prevents spam of single piece)
 
+# -----------------------------------------------------------------------------
+# Clickhouse
+# -----------------------------------------------------------------------------
+CLICKHOUSE_URL=
+CLICKHOUSE_BATCH_SIZE=
+CLICKHOUSE_FLUSH_INTERVAL_MS=
+CLICKHOUSE_MAX_BUFFER_SIZE=
+
+# -----------------------------------------------------------------------------
+# Prometheus metrics
+# -----------------------------------------------------------------------------
+PROMETHEUS_WALLET_BALANCE_TTL_SECONDS=3600            # refresh cache every 1h
+PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS=60   # cooldown after fetch failure
+
+# -----------------------------------------------------------------------------
+# Dataset generation
+# -----------------------------------------------------------------------------
+DEALBOT_LOCAL_DATASETS_PATH=./datasets
+RANDOM_PIECE_SIZES=10485760                    # 10 MiB (comma-separated list supported)
+
+# -----------------------------------------------------------------------------
+# HTTP timeouts (milliseconds)
+# -----------------------------------------------------------------------------
+CONNECT_TIMEOUT_MS=10000                       # 10s: initial connection timeout
+HTTP_REQUEST_TIMEOUT_MS=240000                 # 4m:  total transfer timeout (HTTP/1.1)
+HTTP2_REQUEST_TIMEOUT_MS=240000                # 4m:  total transfer timeout (HTTP/2)
+IPNI_VERIFICATION_TIMEOUT_MS=60000             # 60s: IPNI propagation wait
+IPNI_VERIFICATION_POLLING_MS=2000              # 2s:  IPNI polling interval

apps/backend/src/app.controller.ts

@@ -1,6 +1,7 @@
 import { Controller, Get } from "@nestjs/common";
 import { ConfigService } from "@nestjs/config";
-import { type IBlockchainConfig, type IConfig, IJobsConfig } from "./config/app.config.js";
+import type { Network } from "./common/types.js";
+import type { IConfig, INetworksConfig } from "./config/index.js";
 
 @Controller("api")
 export class AppController {
@@ -21,16 +22,19 @@ export class AppController {
    */
   @Get("config")
   getConfig() {
-    const blockchain = this.configService.get<IBlockchainConfig>("blockchain");
-    const jobs = this.configService.get<IJobsConfig>("jobs");
+    const activeNetworks = this.configService.get<Network[]>("activeNetworks");
+    const networks = this.configService.get<INetworksConfig>("networks");
 
     return {
-      network: blockchain.network,
-      jobs: {
-        dealsPerSpPerHour: jobs.dealsPerSpPerHour,
-        dataSetCreationsPerSpPerHour: jobs.dataSetCreationsPerSpPerHour,
-        retrievalsPerSpPerHour: jobs.retrievalsPerSpPerHour,
-      },
+      networks: activeNetworks.map((n) => ({
+        network: n,
+        dealsPerSpPerHour: networks[n].dealsPerSpPerHour,
+        dataSetCreationsPerSpPerHour: networks[n].dataSetCreationsPerSpPerHour,
+        retrievalsPerSpPerHour: networks[n].retrievalsPerSpPerHour,
+        pullChecksPerSpPerHour: networks[n].pullChecksPerSpPerHour,
+        dataRetentionPollIntervalSeconds: networks[n].dataRetentionPollIntervalSeconds,
+        providersRefreshIntervalSeconds: networks[n].providersRefreshIntervalSeconds,
+      })),
     };
   }
 }