refactor: restructure config to support both networks

.env.example

@@ -4,9 +4,32 @@
 # Keep this file limited to *secrets only* - non-secret configuration lives in
 # kustomize ConfigMap patches (e.g., kustomize/overlays/local/backend-configmap-local.yaml).
 #
-# Required
-WALLET_ADDRESS=
-WALLET_PRIVATE_KEY=
+# Multi-network support
+# ---------------------
+# Dealbot now supports running against multiple Filecoin networks from a single
+# instance. The `NETWORKS` env var (set via ConfigMap, not here) selects the
+# active networks as a comma-separated list, e.g. `NETWORKS=calibration` or
+# `NETWORKS=calibration,mainnet`.
+#
+# Every network-scoped secret is prefixed with the uppercase network name:
+#   CALIBRATION_WALLET_ADDRESS, CALIBRATION_WALLET_PRIVATE_KEY
+#   MAINNET_WALLET_ADDRESS,     MAINNET_WALLET_PRIVATE_KEY
+#
+# Each active network must provide either `<NETWORK>_WALLET_PRIVATE_KEY` or
+# `<NETWORK>_SESSION_KEY_PRIVATE_KEY`. When both are set, the session key
+# takes precedence (see docs/runbooks/wallet-and-session-keys.md).
+
+# --- Required for calibration ---
+CALIBRATION_WALLET_ADDRESS=
+CALIBRATION_WALLET_PRIVATE_KEY=
+# Optional: use a session key instead of the raw wallet key.
+# CALIBRATION_SESSION_KEY_PRIVATE_KEY=
+
+# --- Required for mainnet (only if `mainnet` is in NETWORKS) ---
+# MAINNET_WALLET_ADDRESS=
+# MAINNET_WALLET_PRIVATE_KEY=
+# MAINNET_SESSION_KEY_PRIVATE_KEY=
+
 #
 # Optional (only if using an external DB or a non-default password)
 # DATABASE_PASSWORD=

apps/backend/.env.example

@@ -1,83 +1,164 @@
-# Node Environment
+# =============================================================================
+# Dealbot backend — environment configuration
+# =============================================================================
+# See docs/environment-variables.md for the full reference.
+#
+# Multi-network primer
+# --------------------
+# `NETWORKS` is a comma-separated list of Filecoin networks the instance should
+# drive (supported: `calibration`, `mainnet`). Every network-scoped variable is
+# namespaced with the UPPERCASE network name:
+#
+#   NETWORKS=calibration,mainnet
+#   CALIBRATION_WALLET_PRIVATE_KEY=0x...
+#   MAINNET_WALLET_PRIVATE_KEY=0x...
+#
+# Globals (database, jobs, timeouts, HTTP ports, etc.) remain unprefixed and
+# apply to all networks. Variables for INACTIVE networks are ignored; only
+# active networks are validated at startup.
+# =============================================================================
+
+# -----------------------------------------------------------------------------
+# Application
+# -----------------------------------------------------------------------------
 NODE_ENV=development
 
-# Specify port for dealbot service (optional)
-DEALBOT_PORT=8080
+# api | worker | both (default: both)
+DEALBOT_RUN_MODE=both
 
-# Specify host for dealbot service (optional)
+DEALBOT_PORT=8080
 DEALBOT_HOST=localhost
 
+# Metrics-only HTTP server (used when DEALBOT_RUN_MODE=worker)
+DEALBOT_METRICS_PORT=9090
+DEALBOT_METRICS_HOST=0.0.0.0
+
 # Comma-separated list of allowed origins for CORS (for web dev)
 DEALBOT_ALLOWED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
 
-# Database Configuration
+# Enables /api/dev/* endpoints for manual deal/retrieval testing.
+# NEVER enable in production.
+ENABLE_DEV_MODE=false
+
+# -----------------------------------------------------------------------------
+# Database
+# -----------------------------------------------------------------------------
 DATABASE_HOST=localhost
 DATABASE_PORT=5432
 DATABASE_USER=dealbot
 DATABASE_PASSWORD=dealbot_password
 DATABASE_NAME=filecoin_dealbot
-
-# Blockchain Configuration
-NETWORK=calibration # or mainnet
-WALLET_ADDRESS=0x0000000000000000000000000000000000000000
-WALLET_PRIVATE_KEY=your_private_key_here
-CHECK_DATASET_CREATION_FEES=true
-USE_ONLY_APPROVED_PROVIDERS=true
-PDP_SUBGRAPH_ENDPOINT=https://api.thegraph.com/subgraphs/filecoin/pdp
-
-# Minimum number of datasets per SP (default: 1). When > 1, a separate data_set_creation job provisions extra datasets.
-MIN_NUM_DATASETS_FOR_CHECKS=1
-
-# Dataset Versioning (optional)
-# Uncomment and set to enable dataset versioning (e.g., "dealbot-v1", "dealbot-v2")
-# This allows creating new logical datasets without changing wallet addresses
-# DEALBOT_DATASET_VERSION=dealbot-v1
-
-# Scheduling Configuration
-# Intervals: How often jobs run (in seconds)
-PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 # Run providers refresh every 4 hours
-DATA_RETENTION_POLL_INTERVAL_SECONDS=3600 # Run data retention poll every 60 minutes
-
-# Prometheus Metrics Configuration
-# Cache TTL for wallet balance collection (in seconds)
-PROMETHEUS_WALLET_BALANCE_TTL_SECONDS=3600 # Refresh wallet balance every 1 hour
-PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS=60 # Wait 1 minute before retry after error
-
-# Maintenance windows (UTC)
-DEALBOT_MAINTENANCE_WINDOWS_UTC=07:00,22:00
-DEALBOT_MAINTENANCE_WINDOW_MINUTES=20
-
-# pg-boss (DB-backed jobs) Configuration
-# See docs/environment-variables.md for details, limits, and fractional examples.
-DEALS_PER_SP_PER_HOUR=2
-DATASET_CREATIONS_PER_SP_PER_HOUR=1
-RETRIEVALS_PER_SP_PER_HOUR=1
+DATABASE_POOL_MAX=1
+
+# -----------------------------------------------------------------------------
+# Network selection
+# -----------------------------------------------------------------------------
+# Comma-separated. Default: calibration.
+NETWORKS=calibration
+
+# -----------------------------------------------------------------------------
+# Per-network configuration — CALIBRATION
+# -----------------------------------------------------------------------------
+# At least one of WALLET_PRIVATE_KEY or SESSION_KEY_PRIVATE_KEY is required
+# for each ACTIVE network. Session key (if present) takes precedence.
+CALIBRATION_WALLET_ADDRESS=0x0000000000000000000000000000000000000000
+CALIBRATION_WALLET_PRIVATE_KEY=your_calibration_private_key_here
+# CALIBRATION_SESSION_KEY_PRIVATE_KEY=
+
+# Optional: custom RPC (authenticated endpoints avoid public rate limits)
+# CALIBRATION_RPC_URL=https://filecoin.chain.love/rpc/v1?token=YOUR_API_KEY
+
+CALIBRATION_CHECK_DATASET_CREATION_FEES=true
+CALIBRATION_USE_ONLY_APPROVED_PROVIDERS=true
+CALIBRATION_PDP_SUBGRAPH_ENDPOINT=
+
+# Minimum number of datasets per SP. When > 1, a separate data_set_creation
+# job provisions extra datasets.
+CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS=1
+
+# Optional: dataset versioning (e.g. "dealbot-v1", "dealbot-v2").
+# CALIBRATION_DEALBOT_DATASET_VERSION=dealbot-v1
+
+# Per-network scheduling rates (per hour; fractional values supported)
+CALIBRATION_DEALS_PER_SP_PER_HOUR=4
+CALIBRATION_DATASET_CREATIONS_PER_SP_PER_HOUR=1
+CALIBRATION_RETRIEVALS_PER_SP_PER_HOUR=2
+CALIBRATION_PIECE_CLEANUP_PER_SP_PER_HOUR=0.05
+
+# Per-network job intervals (seconds)
+CALIBRATION_PROVIDERS_REFRESH_INTERVAL_SECONDS=14400   # 4 hours
+CALIBRATION_DATA_RETENTION_POLL_INTERVAL_SECONDS=3600  # 1 hour
+
+# Per-network maintenance windows (UTC)
+CALIBRATION_MAINTENANCE_WINDOWS_UTC=07:00,22:00
+CALIBRATION_MAINTENANCE_WINDOW_MINUTES=20
+
+# Per-network SP blocklists
+# CALIBRATION_BLOCKED_SP_IDS=1234,5678
+# CALIBRATION_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...
+
+# Per-network piece cleanup configuration
+CALIBRATION_MAX_DATASET_STORAGE_SIZE_BYTES=25769803776
+CALIBRATION_TARGET_DATASET_STORAGE_SIZE_BYTES=21474836480
+CALIBRATION_MAX_PIECE_CLEANUP_RUNTIME_SECONDS=3000
+
+# -----------------------------------------------------------------------------
+# Per-network configuration — MAINNET  (uncomment when adding to NETWORKS)
+# -----------------------------------------------------------------------------
+# MAINNET_WALLET_ADDRESS=
+# MAINNET_WALLET_PRIVATE_KEY=
+# MAINNET_SESSION_KEY_PRIVATE_KEY=
+# MAINNET_RPC_URL=
+# MAINNET_CHECK_DATASET_CREATION_FEES=true
+# MAINNET_USE_ONLY_APPROVED_PROVIDERS=true
+# MAINNET_PDP_SUBGRAPH_ENDPOINT=
+# MAINNET_MIN_NUM_DATASETS_FOR_CHECKS=1
+# MAINNET_DEALBOT_DATASET_VERSION=
+# MAINNET_DEALS_PER_SP_PER_HOUR=2
+# MAINNET_DATASET_CREATIONS_PER_SP_PER_HOUR=1
+# MAINNET_RETRIEVALS_PER_SP_PER_HOUR=1
+# MAINNET_METRICS_PER_HOUR=0.5
+# MAINNET_PROVIDERS_REFRESH_INTERVAL_SECONDS=14400
+# MAINNET_DATA_RETENTION_POLL_INTERVAL_SECONDS=3600
+# MAINNET_MAINTENANCE_WINDOWS_UTC=07:00,22:00
+# MAINNET_MAINTENANCE_WINDOW_MINUTES=20
+# MAINNET_BLOCKED_SP_IDS=1234,5678
+# MAINNET_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...
+
+# -----------------------------------------------------------------------------
+# Jobs (pg-boss)
+# -----------------------------------------------------------------------------
+# See docs/environment-variables.md for details, limits, and sizing guidance.
 PG_BOSS_LOCAL_CONCURRENCY=20
 JOB_SCHEDULER_POLL_SECONDS=300
 JOB_WORKER_POLL_SECONDS=60
 JOB_CATCHUP_MAX_ENQUEUE=10
 JOB_SCHEDULE_PHASE_SECONDS=0
 JOB_ENQUEUE_JITTER_SECONDS=0
-DEAL_JOB_TIMEOUT_SECONDS=360          # 6m: Max runtime for deal jobs (TODO: reduce default to 3m)
-RETRIEVAL_JOB_TIMEOUT_SECONDS=60     # 1m: Max runtime for retrieval jobs (TODO: reduce default to 30s)
-IPFS_BLOCK_FETCH_CONCURRENCY=6       # Parallel block fetches when validating IPFS DAGs
+DEAL_JOB_TIMEOUT_SECONDS=360                  # 6m: max runtime for deal jobs
+RETRIEVAL_JOB_TIMEOUT_SECONDS=60              # 1m: max runtime for retrieval jobs
+DATA_SET_CREATION_JOB_TIMEOUT_SECONDS=300     # 5m: max runtime for dataset-creation jobs
+IPFS_BLOCK_FETCH_CONCURRENCY=6                # parallel block fetches during IPFS DAG validation
 DEALBOT_PGBOSS_POOL_MAX=1
 DEALBOT_PGBOSS_SCHEDULER_ENABLED=true
 
-# Dataset Configuration
-DEALBOT_LOCAL_DATASETS_PATH=./datasets
-KAGGLE_DATASET_TOTAL_PAGES=500
-
-# Proxy Configuration
-PROXY_LIST=http://username:password@host:port,http://username:password@host:port
-PROXY_LOCATIONS=l1,l2
-
-# Timeout Configuration (in milliseconds)
-CONNECT_TIMEOUT_MS=10000             # 10s: Initial connection timeout
-HTTP_REQUEST_TIMEOUT_MS=240000       # 4m: Total transfer timeout for HTTP/1.1 (10MiB @ 170KB/s + overhead)
-HTTP2_REQUEST_TIMEOUT_MS=240000      # 4m: Total transfer timeout for HTTP/2 (10MiB @ 170KB/s + overhead)
-
-# SP Blocklists configuration
-# BLOCKED_SP_IDS=1234,5678
-# BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...
+# -----------------------------------------------------------------------------
+# Prometheus metrics
+# -----------------------------------------------------------------------------
+PROMETHEUS_WALLET_BALANCE_TTL_SECONDS=3600            # refresh cache every 1h
+PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS=60   # cooldown after fetch failure
 
+# -----------------------------------------------------------------------------
+# Dataset generation
+# -----------------------------------------------------------------------------
+DEALBOT_LOCAL_DATASETS_PATH=./datasets
+RANDOM_PIECE_SIZES=10485760                    # 10 MiB (comma-separated list supported)
+
+# -----------------------------------------------------------------------------
+# HTTP timeouts (milliseconds)
+# -----------------------------------------------------------------------------
+CONNECT_TIMEOUT_MS=10000                       # 10s: initial connection timeout
+HTTP_REQUEST_TIMEOUT_MS=240000                 # 4m:  total transfer timeout (HTTP/1.1)
+HTTP2_REQUEST_TIMEOUT_MS=240000                # 4m:  total transfer timeout (HTTP/2)
+IPNI_VERIFICATION_TIMEOUT_MS=60000             # 60s: IPNI propagation wait
+IPNI_VERIFICATION_POLLING_MS=2000              # 2s:  IPNI polling interval

apps/backend/src/app.controller.ts

@@ -1,6 +1,7 @@
 import { Controller, Get } from "@nestjs/common";
 import { ConfigService } from "@nestjs/config";
-import { type IBlockchainConfig, type IConfig, IJobsConfig } from "./config/app.config.js";
+import type { Network } from "./common/types.js";
+import type { IConfig, INetworksConfig } from "./config/types.js";
 
 @Controller("api")
 export class AppController {
@@ -21,16 +22,18 @@ export class AppController {
    */
   @Get("config")
   getConfig() {
-    const blockchain = this.configService.get<IBlockchainConfig>("blockchain");
-    const jobs = this.configService.get<IJobsConfig>("jobs");
+    const activeNetworks = this.configService.get<Network[]>("activeNetworks");
+    const networks = this.configService.get<INetworksConfig>("networks");
 
     return {
-      network: blockchain.network,
-      jobs: {
-        dealsPerSpPerHour: jobs.dealsPerSpPerHour,
-        dataSetCreationsPerSpPerHour: jobs.dataSetCreationsPerSpPerHour,
-        retrievalsPerSpPerHour: jobs.retrievalsPerSpPerHour,
-      },
+      networks: activeNetworks.map((n) => ({
+        network: n,
+        dealsPerSpPerHour: networks[n].dealsPerSpPerHour,
+        dataSetCreationsPerSpPerHour: networks[n].dataSetCreationsPerSpPerHour,
+        retrievalsPerSpPerHour: networks[n].retrievalsPerSpPerHour,
+        dataRetentionPollIntervalSeconds: networks[n].dataRetentionPollIntervalSeconds,
+        providersRefreshIntervalSeconds: networks[n].providersRefreshIntervalSeconds,
+      })),
     };
   }
 }

apps/backend/src/app.module.ts

@@ -3,7 +3,8 @@ import { ConfigModule } from "@nestjs/config";
 import { LoggerModule } from "nestjs-pino";
 import { AppController } from "./app.controller.js";
 import { buildLoggerModuleParams } from "./common/pino.config.js";
-import { configValidationSchema, loadConfig } from "./config/app.config.js";
+import { validateConfig } from "./config/env.schema.js";
+import { loadConfig } from "./config/loader.js";
 import { DatabaseModule } from "./database/database.module.js";
 import { DataSourceModule } from "./dataSource/dataSource.module.js";
 import { DealModule } from "./deal/deal.module.js";
@@ -18,7 +19,7 @@ import { RetrievalModule } from "./retrieval/retrieval.module.js";
     LoggerModule.forRoot(buildLoggerModuleParams()),
     ConfigModule.forRoot({
       load: [loadConfig],
-      validationSchema: configValidationSchema,
+      validate: validateConfig,
       isGlobal: true,
     }),
     DatabaseModule,

apps/backend/src/common/logging.ts

@@ -1,3 +1,5 @@
+import { Network } from "./types.js";
+
 type ErrorWithCode = Error & { code?: unknown };
 const MAX_ERROR_STACK_LENGTH = 4 * 1024;
 const ERROR_STACK_REDACTION_BUFFER_LENGTH = 512;
@@ -144,6 +146,7 @@ export type ProviderJobContext = {
   providerAddress: string;
   providerId: bigint;
   providerName: string;
+  network: Network;
 };
 
 /**
@@ -165,6 +168,7 @@ export type DataSetLogContext = {
 export type JobLogContext = {
   jobId?: string;
   providerAddress: string;
+  network: Network;
   providerId?: bigint;
   providerName?: string;
 };

apps/backend/src/common/sp-blocklist.spec.ts

@@ -1,10 +1,10 @@
 import { describe, expect, it } from "vitest";
-import { type ISpBlocklistConfig } from "../config/app.config.js";
+import type { INetworkConfig } from "../config/types.js";
 import { isSpBlocked } from "./sp-blocklist.js";
 
-const cfg = (overrides: Partial<ISpBlocklistConfig> = {}): ISpBlocklistConfig => ({
-  ids: new Set(),
-  addresses: new Set(),
+const cfg = (overrides: Partial<INetworkConfig> = {}): Pick<INetworkConfig, "blockedSpAddresses" | "blockedSpIds"> => ({
+  blockedSpIds: new Set(),
+  blockedSpAddresses: new Set(),
   ...overrides,
 });
 
@@ -15,25 +15,25 @@ describe("isSpBlocked", () => {
   });
 
   it("blocks by address", () => {
-    expect(isSpBlocked(cfg({ addresses: new Set(["0xaaa"]) }), "0xaaa")).toBe(true);
+    expect(isSpBlocked(cfg({ blockedSpAddresses: new Set(["0xaaa"]) }), "0xaaa")).toBe(true);
   });
 
   it("address matching is case-insensitive", () => {
-    const c = cfg({ addresses: new Set(["0xaaa"]) });
+    const c = cfg({ blockedSpAddresses: new Set(["0xaaa"]) });
     expect(isSpBlocked(c, "0xAAA")).toBe(true);
     expect(isSpBlocked(c, "0XAAA")).toBe(true);
   });
 
   it("blocks by numeric ID", () => {
-    expect(isSpBlocked(cfg({ ids: new Set(["42"]) }), "0xaaa", 42n)).toBe(true);
+    expect(isSpBlocked(cfg({ blockedSpIds: new Set(["42"]) }), "0xaaa", 42n)).toBe(true);
   });
 
   it("skips ID check when id is undefined", () => {
-    expect(isSpBlocked(cfg({ ids: new Set(["42"]) }), "0xaaa", undefined)).toBe(false);
+    expect(isSpBlocked(cfg({ blockedSpIds: new Set(["42"]) }), "0xaaa", undefined)).toBe(false);
   });
 
   it("does not block unrelated provider", () => {
-    const c = cfg({ ids: new Set(["5"]), addresses: new Set(["0xaaa"]) });
+    const c = cfg({ blockedSpIds: new Set(["5"]), blockedSpAddresses: new Set(["0xaaa"]) });
     expect(isSpBlocked(c, "0xbbb", 6n)).toBe(false);
   });
 });