From 61805491781ff444c1c1bf35954ea6547215ba19 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 12:47:11 +0530 Subject: [PATCH 01/23] refactor(config): introduce multi network config --- apps/backend/src/config/app.config.ts | 552 ------------------ apps/backend/src/config/constants.ts | 44 ++ apps/backend/src/config/env.helpers.ts | 24 + apps/backend/src/config/env.parsers.ts | 75 +++ apps/backend/src/config/env.schema.spec.ts | 521 +++++++++++++++++ apps/backend/src/config/env.schema.ts | 266 +++++++++ apps/backend/src/config/index.ts | 3 + .../src/config/legacy-env-compat.spec.ts | 111 ++++ apps/backend/src/config/legacy-env-compat.ts | 139 +++++ apps/backend/src/config/loader.spec.ts | 89 +++ apps/backend/src/config/loader.ts | 265 +++++++++ apps/backend/src/config/types.ts | 277 +++++++++ 12 files changed, 1814 insertions(+), 552 deletions(-) delete mode 100644 apps/backend/src/config/app.config.ts create mode 100644 apps/backend/src/config/constants.ts create mode 100644 apps/backend/src/config/env.helpers.ts create mode 100644 apps/backend/src/config/env.parsers.ts create mode 100644 apps/backend/src/config/env.schema.spec.ts create mode 100644 apps/backend/src/config/env.schema.ts create mode 100644 apps/backend/src/config/index.ts create mode 100644 apps/backend/src/config/legacy-env-compat.spec.ts create mode 100644 apps/backend/src/config/legacy-env-compat.ts create mode 100644 apps/backend/src/config/loader.spec.ts create mode 100644 apps/backend/src/config/loader.ts create mode 100644 apps/backend/src/config/types.ts diff --git a/apps/backend/src/config/app.config.ts b/apps/backend/src/config/app.config.ts deleted file mode 100644 index 1748a681..00000000 --- a/apps/backend/src/config/app.config.ts +++ /dev/null @@ -1,552 +0,0 @@ -import Joi from "joi"; -import { DEFAULT_LOCAL_DATASETS_PATH } from "../common/constants.js"; -import { parseMaintenanceWindowTimes } from "../common/maintenance-window.js"; -import type { Network } from "../common/types.js"; - -function parseIdList(value: string | undefined): Set { - if (!value || value.trim().length === 0) return new Set(); - return new Set( - value - .split(",") - .map((s) => s.trim()) - .filter((s) => s.length > 0), - ); -} - -function parseAddressList(value: string | undefined): Set { - if (!value || value.trim().length === 0) return new Set(); - return new Set( - value - .split(",") - .map((s) => s.trim().toLowerCase()) - .filter((s) => s.length > 0), - ); -} - -export const configValidationSchema = Joi.object({ - // Application - NODE_ENV: Joi.string().valid("development", "production", "test").default("development"), - DEALBOT_RUN_MODE: Joi.string().lowercase().valid("api", "worker", "both").default("both"), - DEALBOT_PORT: Joi.number().default(3000), - DEALBOT_HOST: Joi.string().default("127.0.0.1"), - DEALBOT_API_PUBLIC_URL: Joi.string().uri().optional().allow(""), - DEALBOT_METRICS_PORT: Joi.number().default(9090), - DEALBOT_METRICS_HOST: Joi.string().default("0.0.0.0"), - ENABLE_DEV_MODE: Joi.boolean().default(false), - PROMETHEUS_WALLET_BALANCE_TTL_SECONDS: Joi.number().min(60).default(3600), - PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS: Joi.number().min(1).default(60), - - // Database - DATABASE_HOST: Joi.string().required(), - DATABASE_PORT: Joi.number().default(5432), - DATABASE_POOL_MAX: Joi.number().integer().min(1).default(1), - DATABASE_USER: Joi.string().required(), - DATABASE_PASSWORD: Joi.string().required(), - DATABASE_NAME: Joi.string().required(), - - // Blockchain - NETWORK: Joi.string().valid("mainnet", "calibration").required(), - WALLET_ADDRESS: Joi.string().required(), - WALLET_PRIVATE_KEY: Joi.string().optional().empty(""), - RPC_URL: Joi.string() - .uri({ scheme: ["http", "https"] }) - .optional() - .allow(""), - SESSION_KEY_PRIVATE_KEY: Joi.string().optional().empty(""), - CHECK_DATASET_CREATION_FEES: Joi.boolean().default(true), - USE_ONLY_APPROVED_PROVIDERS: Joi.boolean().default(true), - DEALBOT_DATASET_VERSION: Joi.string().optional(), - MIN_NUM_DATASETS_FOR_CHECKS: Joi.number().integer().min(1).default(1), - PDP_SUBGRAPH_ENDPOINT: Joi.string().uri().optional().allow(""), - - // Scheduling - PROVIDERS_REFRESH_INTERVAL_SECONDS: Joi.number().default(4 * 3600), - DATA_RETENTION_POLL_INTERVAL_SECONDS: Joi.number().default(3600), - DEALBOT_MAINTENANCE_WINDOWS_UTC: Joi.string() - .default("07:00,22:00") - .custom((value, helpers) => { - try { - parseMaintenanceWindowTimes(value.split(",")); - } catch (error) { - return helpers.error("any.invalid", { - message: error instanceof Error ? error.message : "Invalid maintenance window format", - }); - } - return value; - }), - DEALBOT_MAINTENANCE_WINDOW_MINUTES: Joi.number().min(20).max(360).default(20), - - // Jobs - // Per-hour limits are guardrails to avoid excessive background load. - DEALS_PER_SP_PER_HOUR: Joi.number().min(0.001).max(20).default(4), - DATASET_CREATIONS_PER_SP_PER_HOUR: Joi.number().min(0.001).max(20).default(1), - RETRIEVALS_PER_SP_PER_HOUR: Joi.number().min(0.001).max(20).default(2), - // Polling interval for pg-boss scheduler (lower = more responsive, higher = less DB chatter). - JOB_SCHEDULER_POLL_SECONDS: Joi.number().min(60).default(300), - JOB_WORKER_POLL_SECONDS: Joi.number().min(5).default(60), - PG_BOSS_LOCAL_CONCURRENCY: Joi.number().integer().min(1).default(20), - DEALBOT_PGBOSS_SCHEDULER_ENABLED: Joi.boolean().default(true), - DEALBOT_PGBOSS_POOL_MAX: Joi.number().integer().min(1).default(1), - JOB_CATCHUP_MAX_ENQUEUE: Joi.number().min(1).default(10), - JOB_SCHEDULE_PHASE_SECONDS: Joi.number().min(0).default(0), - JOB_ENQUEUE_JITTER_SECONDS: Joi.number().min(0).default(0), - DEAL_JOB_TIMEOUT_SECONDS: Joi.number().min(120).default(360), // 6 minutes max runtime for data storage jobs (TODO: reduce default to 3 minutes) - RETRIEVAL_JOB_TIMEOUT_SECONDS: Joi.number().min(60).default(60), // 1 minute max runtime for retrieval jobs (TODO: reduce default to 30 seconds) - DATA_SET_CREATION_JOB_TIMEOUT_SECONDS: Joi.number().min(60).default(300), // 5 minutes max runtime for dataset creation jobs - // Seconds to hold the process alive after pg-boss drain completes, so Prometheus - // captures at least one scrape of the terminal counter increments emitted during - // shutdown. Default 35 covers the 30s ServiceMonitor interval plus a 5s buffer. - SHUTDOWN_FINAL_SCRAPE_DELAY_SECONDS: Joi.number().min(0).max(300).default(35), - IPFS_BLOCK_FETCH_CONCURRENCY: Joi.number().integer().min(1).max(32).default(6), - - // Pull Check - PULL_CHECKS_PER_SP_PER_HOUR: Joi.number().min(0.001).max(20).default(1), - PULL_CHECK_JOB_TIMEOUT_SECONDS: Joi.number().min(60).default(300), // 5m max runtime for pull check jobs - PULL_CHECK_POLL_INTERVAL_SECONDS: Joi.number().min(1).default(2), - PULL_CHECK_PIECE_SIZE_BYTES: Joi.number() - .integer() - .min(1024) - .default(10 * 1024 * 1024), // 10 MiB - PULL_PIECE_MAX_CONCURRENT_STREAMS: Joi.number().integer().min(1).default(50), // Max concurrent streams across all pieces - PULL_PIECE_MAX_STREAMS_PER_CID: Joi.number().integer().min(1).default(3), // Max concurrent streams per pieceCid - PULL_PIECE_CLEANUP_INTERVAL_SECONDS: Joi.number() - .integer() - .min(3600) - .default(7 * 24 * 3600), // 7 days - - // Piece Cleanup - MAX_DATASET_STORAGE_SIZE_BYTES: Joi.number() - .integer() - .min(1) - .default(24 * 1024 * 1024 * 1024), // 24 GiB per SP - TARGET_DATASET_STORAGE_SIZE_BYTES: Joi.number() - .integer() - .min(1) - .default(20 * 1024 * 1024 * 1024) // 20 GiB per SP - .custom((value, helpers) => { - const max = helpers.state.ancestors?.[0]?.MAX_DATASET_STORAGE_SIZE_BYTES; - if (max != null && value >= max) { - return helpers.error("any.invalid", { - message: `TARGET_DATASET_STORAGE_SIZE_BYTES (${value}) must be less than MAX_DATASET_STORAGE_SIZE_BYTES (${max})`, - }); - } - return value; - }, "target < max validation"), - JOB_PIECE_CLEANUP_PER_SP_PER_HOUR: Joi.number() - .min(0.001) - .max(20) - .default(1 / 24), // ~once per day - MAX_PIECE_CLEANUP_RUNTIME_SECONDS: Joi.number().min(60).default(300), // 5 minutes max runtime for cleanup jobs - - // Dataset - DEALBOT_LOCAL_DATASETS_PATH: Joi.string().default(DEFAULT_LOCAL_DATASETS_PATH), - RANDOM_PIECE_SIZES: Joi.string().default("10485760"), // 10 MiB - - // ClickHouse - CLICKHOUSE_URL: Joi.string().uri().optional(), - CLICKHOUSE_BATCH_SIZE: Joi.number().integer().min(1).default(500), - CLICKHOUSE_FLUSH_INTERVAL_MS: Joi.number().integer().min(100).default(5000), - CLICKHOUSE_MAX_BUFFER_SIZE: Joi.number().integer().min(1).default(5000), - DEALBOT_PROBE_LOCATION: Joi.string().default("unknown"), - - // Timeouts (in milliseconds) - CONNECT_TIMEOUT_MS: Joi.number().min(1000).default(10000), // 10 seconds to establish connection/receive headers - HTTP_REQUEST_TIMEOUT_MS: Joi.number().min(1000).default(240000), // 4 minutes total for HTTP requests (10MiB @ 170KB/s + overhead) - HTTP2_REQUEST_TIMEOUT_MS: Joi.number().min(1000).default(240000), // 4 minutes total for HTTP/2 requests (10MiB @ 170KB/s + overhead) - IPNI_VERIFICATION_TIMEOUT_MS: Joi.number().min(1000).default(60000), // 60 seconds max time to wait for IPNI verification - IPNI_VERIFICATION_POLLING_MS: Joi.number().min(250).default(2000), // 2 seconds between IPNI verification polls - - // SP Blocklists (comma-separated provider IDs or addresses) - BLOCKED_SP_IDS: Joi.string().optional().allow(""), - BLOCKED_SP_ADDRESSES: Joi.string().optional().allow(""), -}).or("WALLET_PRIVATE_KEY", "SESSION_KEY_PRIVATE_KEY"); - -export interface IAppConfig { - env: string; - runMode: "api" | "worker" | "both"; - port: number; - host: string; - /** - * Optional publicly reachable DealBot API base URL (e.g. `https://dealbot.example.com`). - * Used to construct hosted-piece source URLs that SPs can fetch during pull checks. - * When unset, falls back to `http://${host}:${port}`. - */ - apiPublicUrl: string | null; - metricsPort: number; - metricsHost: string; - enableDevMode: boolean; - prometheusWalletBalanceTtlSeconds: number; - prometheusWalletBalanceErrorCooldownSeconds: number; - probeLocation: string; -} - -export interface IDatabaseConfig { - host: string; - port: number; - poolMax: number; - username: string; - password: string; - database: string; -} - -export interface IBlockchainConfig { - network: Network; - rpcUrl?: string; - sessionKeyPrivateKey?: `0x${string}`; - walletAddress: string; - walletPrivateKey: `0x${string}`; - checkDatasetCreationFees: boolean; - useOnlyApprovedProviders: boolean; - dealbotDataSetVersion?: string; - minNumDataSetsForChecks: number; - pdpSubgraphEndpoint?: string; -} - -export interface ISchedulingConfig { - providersRefreshIntervalSeconds: number; - dataRetentionPollIntervalSeconds: number; - maintenanceWindowsUtc: string[]; - maintenanceWindowMinutes: number; -} - -export interface IJobsConfig { - /** - * Target number of deal creations per storage provider per hour. - * - * Increasing this increases on-chain activity and dataset uploads. - */ - dealsPerSpPerHour: number; - /** - * Target number of retrieval tests per storage provider per hour. - * - * Increasing this increases retrieval load against providers and DB writes. - */ - retrievalsPerSpPerHour: number; - /** - * Target number of dataset creation runs per storage provider per hour. - */ - dataSetCreationsPerSpPerHour: number; - /** - * How often the scheduler polls Postgres for due jobs (seconds). - * - * Lower values reduce scheduling latency but increase DB chatter. - */ - schedulerPollSeconds: number; - /** - * How often workers check for new jobs (seconds). - * - * Lower values reduce job pickup latency but increase DB chatter. - */ - workerPollSeconds: number; - /** - * Per-instance pg-boss worker concurrency for the `sp.work` queue. - */ - pgbossLocalConcurrency: number; - /** - * Enables the pg-boss scheduler loop (enqueueing due jobs). - * - * Set to false to run "worker-only" pods that only process existing jobs. - */ - pgbossSchedulerEnabled: boolean; - /** - * Maximum number of pg-boss connections per instance. - * - * Helpful when using a session-mode pooler with a low pool_size (e.g. Supabase). - */ - pgbossPoolMax: number; - /** - * Maximum number of jobs to enqueue per schedule row per poll. - * - * Prevents large backlogs from flooding workers after downtime. - */ - catchupMaxEnqueue: number; - /** - * Per-instance phase offset (seconds) applied when initializing schedules. - * - * Use this to stagger multiple dealbot deployments that are not sharing a DB. - */ - schedulePhaseSeconds: number; - /** - * Random delay (seconds) added when enqueuing jobs. - * - * Helps avoid synchronized bursts across instances. Only used with pg-boss. - */ - enqueueJitterSeconds: number; - /** - * Maximum runtime (seconds) for deal jobs before forced abort. - * - * Uses AbortController to actively cancel job execution. - */ - dealJobTimeoutSeconds: number; - /** - * Maximum runtime (seconds) for data-set creation jobs before forced abort. - * - * Uses AbortController to actively cancel job execution. - */ - dataSetCreationJobTimeoutSeconds: number; - /** - * Maximum runtime (seconds) for retrieval jobs before forced abort. - * - * Uses AbortController to actively cancel job execution. - */ - retrievalJobTimeoutSeconds: number; - /** - * Seconds to hold the process alive after pg-boss drain finishes, so Prometheus - * scrapes the terminal counter increments emitted during shutdown. - */ - shutdownFinalScrapeDelaySeconds: number; - /** - * Target number of piece cleanup runs per storage provider per hour. - * - * Increasing this makes cleanup more aggressive at the cost of more SP API calls. - * Only used when `DEALBOT_JOBS_MODE=pgboss`. - */ - pieceCleanupPerSpPerHour: number; - /** - * Maximum runtime (seconds) for piece cleanup jobs before forced abort. - * - * Uses AbortController to actively cancel job execution. - * Only used when `DEALBOT_JOBS_MODE=pgboss`. - */ - maxPieceCleanupRuntimeSeconds: number; -} - -export interface IDatasetConfig { - localDatasetsPath: string; - randomDatasetSizes: number[]; -} - -export interface ITimeoutConfig { - connectTimeoutMs: number; - httpRequestTimeoutMs: number; - http2RequestTimeoutMs: number; - ipniVerificationTimeoutMs: number; - ipniVerificationPollingMs: number; -} - -export interface IRetrievalConfig { - ipfsBlockFetchConcurrency: number; -} - -export interface IPieceCleanupConfig { - maxDatasetStorageSizeBytes: number; - targetDatasetStorageSizeBytes: number; -} - -export interface ISpBlocklistConfig { - /** Provider numeric IDs to block from all scheduled checks. */ - ids: Set; - /** Provider addresses to block from all scheduled checks (stored lowercase). */ - addresses: Set; -} - -export interface IClickhouseConfig { - /** - * ClickHouse connection URL. Must include the database in the path. - * Example: http://default:password@host:8123/dealbot - * If unset, ClickHouse emission is disabled. - */ - url: string | undefined; - batchSize: number; - flushIntervalMs: number; - maxBufferSize: number; -} - -export interface IPullPieceConfig { - /** - * Target number of pull checks per storage provider per hour. - * - * Pull checks validate the SP pull-to-park pathway by serving a temporary piece URL - * from DealBot and asking the SP to pull and park it. Independent of `deal` and `retrieval`. - */ - pullChecksPerSpPerHour: number; - /** - * Maximum runtime (seconds) for pull-check jobs before forced abort. - * - * Bounds the polling window for terminal SP pull status. - */ - pullCheckJobTimeoutSeconds: number; - /** - * Polling interval (seconds) used while waiting for a terminal SP pull status. - */ - pullCheckPollIntervalSeconds: number; - /** - * Size (bytes) of the synthetic test piece DealBot generates per pull check. - */ - pullCheckPieceSizeBytes: number; - /** - * Maximum number of concurrent piece streams across all pieceCids. - * - * Prevents DoS by limiting total server-wide streaming load. - */ - maxConcurrentStreams: number; - /** - * Maximum number of concurrent streams per pieceCid. - * - * Prevents attackers from opening many connections to the same piece. - */ - maxStreamsPerCid: number; - /** - * How often (seconds) the global `pull_piece_cleanup` job runs to delete - * expired `pull_pieces` rows (those whose `expires_at` is in the past). - * - * Defaults to 7 days (604800 s). Minimum 1 hour enforced by Joi. - */ - pullPieceCleanupIntervalSeconds: number; -} - -export interface IConfig { - app: IAppConfig; - database: IDatabaseConfig; - blockchain: IBlockchainConfig; - scheduling: ISchedulingConfig; - jobs: IJobsConfig; - dataset: IDatasetConfig; - timeouts: ITimeoutConfig; - retrieval: IRetrievalConfig; - clickhouse: IClickhouseConfig; - pieceCleanup: IPieceCleanupConfig; - spBlocklists: ISpBlocklistConfig; - pullPiece: IPullPieceConfig; -} - -export function loadConfig(): IConfig { - return { - app: { - env: process.env.NODE_ENV || "development", - runMode: (() => { - const mode = (process.env.DEALBOT_RUN_MODE || "both").toLowerCase(); - if (mode === "worker") return "worker"; - if (mode === "api") return "api"; - return "both"; - })(), - port: Number.parseInt(process.env.DEALBOT_PORT || "3000", 10), - host: process.env.DEALBOT_HOST || "127.0.0.1", - apiPublicUrl: (() => { - const raw = process.env.DEALBOT_API_PUBLIC_URL; - if (raw == null || raw.trim().length === 0) return null; - return raw.trim().replace(/\/+$/, ""); - })(), - metricsPort: Number.parseInt(process.env.DEALBOT_METRICS_PORT || "9090", 10), - metricsHost: process.env.DEALBOT_METRICS_HOST || "0.0.0.0", - enableDevMode: process.env.ENABLE_DEV_MODE === "true", - prometheusWalletBalanceTtlSeconds: Number.parseInt( - process.env.PROMETHEUS_WALLET_BALANCE_TTL_SECONDS || "3600", - 10, - ), - prometheusWalletBalanceErrorCooldownSeconds: Number.parseInt( - process.env.PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS || "60", - 10, - ), - probeLocation: process.env.DEALBOT_PROBE_LOCATION || "unknown", - }, - database: { - host: process.env.DATABASE_HOST || "localhost", - port: Number.parseInt(process.env.DATABASE_PORT || "5432", 10), - poolMax: Number.parseInt(process.env.DATABASE_POOL_MAX || "1", 10), - username: process.env.DATABASE_USER || "dealbot", - password: process.env.DATABASE_PASSWORD || "dealbot_password", - database: process.env.DATABASE_NAME || "filecoin_dealbot", - }, - blockchain: { - network: process.env.NETWORK as Network, - rpcUrl: process.env.RPC_URL || undefined, - sessionKeyPrivateKey: (process.env.SESSION_KEY_PRIVATE_KEY || undefined) as `0x${string}` | undefined, - walletAddress: process.env.WALLET_ADDRESS || "0x0000000000000000000000000000000000000000", - walletPrivateKey: (process.env.WALLET_PRIVATE_KEY || undefined) as `0x${string}`, - checkDatasetCreationFees: process.env.CHECK_DATASET_CREATION_FEES !== "false", - useOnlyApprovedProviders: process.env.USE_ONLY_APPROVED_PROVIDERS !== "false", - dealbotDataSetVersion: process.env.DEALBOT_DATASET_VERSION, - minNumDataSetsForChecks: Number.parseInt(process.env.MIN_NUM_DATASETS_FOR_CHECKS || "1", 10), - pdpSubgraphEndpoint: process.env.PDP_SUBGRAPH_ENDPOINT || "", - }, - scheduling: { - providersRefreshIntervalSeconds: Number.parseInt(process.env.PROVIDERS_REFRESH_INTERVAL_SECONDS || "14400", 10), - dataRetentionPollIntervalSeconds: Number.parseInt(process.env.DATA_RETENTION_POLL_INTERVAL_SECONDS || "3600", 10), - maintenanceWindowsUtc: (process.env.DEALBOT_MAINTENANCE_WINDOWS_UTC || "07:00,22:00") - .split(",") - .map((value) => value.trim()) - .filter((value) => value.length > 0), - maintenanceWindowMinutes: Number.parseInt(process.env.DEALBOT_MAINTENANCE_WINDOW_MINUTES || "20", 10), - }, - jobs: { - dealsPerSpPerHour: Number.parseFloat(process.env.DEALS_PER_SP_PER_HOUR || "4"), - retrievalsPerSpPerHour: Number.parseFloat(process.env.RETRIEVALS_PER_SP_PER_HOUR || "2"), - dataSetCreationsPerSpPerHour: Number.parseFloat(process.env.DATASET_CREATIONS_PER_SP_PER_HOUR || "1"), - schedulerPollSeconds: Number.parseInt(process.env.JOB_SCHEDULER_POLL_SECONDS || "300", 10), - workerPollSeconds: Number.parseInt(process.env.JOB_WORKER_POLL_SECONDS || "60", 10), - pgbossLocalConcurrency: Number.parseInt(process.env.PG_BOSS_LOCAL_CONCURRENCY || "20", 10), - pgbossSchedulerEnabled: process.env.DEALBOT_PGBOSS_SCHEDULER_ENABLED !== "false", - pgbossPoolMax: Number.parseInt(process.env.DEALBOT_PGBOSS_POOL_MAX || "1", 10), - catchupMaxEnqueue: Number.parseInt(process.env.JOB_CATCHUP_MAX_ENQUEUE || "10", 10), - schedulePhaseSeconds: Number.parseInt(process.env.JOB_SCHEDULE_PHASE_SECONDS || "0", 10), - enqueueJitterSeconds: Number.parseInt(process.env.JOB_ENQUEUE_JITTER_SECONDS || "0", 10), - dealJobTimeoutSeconds: Number.parseInt(process.env.DEAL_JOB_TIMEOUT_SECONDS || "360", 10), - retrievalJobTimeoutSeconds: Number.parseInt(process.env.RETRIEVAL_JOB_TIMEOUT_SECONDS || "60", 10), - dataSetCreationJobTimeoutSeconds: Number.parseInt(process.env.DATA_SET_CREATION_JOB_TIMEOUT_SECONDS || "300", 10), - shutdownFinalScrapeDelaySeconds: Number.parseInt(process.env.SHUTDOWN_FINAL_SCRAPE_DELAY_SECONDS || "35", 10), - pieceCleanupPerSpPerHour: Number.parseFloat(process.env.JOB_PIECE_CLEANUP_PER_SP_PER_HOUR || String(1 / 24)), - maxPieceCleanupRuntimeSeconds: Number.parseInt(process.env.MAX_PIECE_CLEANUP_RUNTIME_SECONDS || "300", 10), - }, - dataset: { - localDatasetsPath: process.env.DEALBOT_LOCAL_DATASETS_PATH || DEFAULT_LOCAL_DATASETS_PATH, - randomDatasetSizes: (() => { - const envValue = process.env.RANDOM_PIECE_SIZES; - if (envValue && envValue.trim().length > 0) { - const parsed = envValue - .split(",") - .map((s) => Number.parseInt(s.trim(), 10)) - .filter((n) => Number.isFinite(n) && !Number.isNaN(n)); - if (parsed.length > 0) { - return parsed; - } - } - return [ - 10 << 20, // 10 MiB - ]; - })(), - }, - timeouts: { - connectTimeoutMs: Number.parseInt(process.env.CONNECT_TIMEOUT_MS || "10000", 10), - httpRequestTimeoutMs: Number.parseInt(process.env.HTTP_REQUEST_TIMEOUT_MS || "240000", 10), - http2RequestTimeoutMs: Number.parseInt(process.env.HTTP2_REQUEST_TIMEOUT_MS || "240000", 10), - ipniVerificationTimeoutMs: Number.parseInt(process.env.IPNI_VERIFICATION_TIMEOUT_MS || "60000", 10), - ipniVerificationPollingMs: Number.parseInt(process.env.IPNI_VERIFICATION_POLLING_MS || "2000", 10), - }, - retrieval: { - ipfsBlockFetchConcurrency: Number.parseInt(process.env.IPFS_BLOCK_FETCH_CONCURRENCY || "6", 10), - }, - clickhouse: { - url: process.env.CLICKHOUSE_URL || undefined, - batchSize: Number.parseInt(process.env.CLICKHOUSE_BATCH_SIZE || "500", 10), - flushIntervalMs: Number.parseInt(process.env.CLICKHOUSE_FLUSH_INTERVAL_MS || "5000", 10), - maxBufferSize: Number.parseInt(process.env.CLICKHOUSE_MAX_BUFFER_SIZE || "5000", 10), - }, - pieceCleanup: { - maxDatasetStorageSizeBytes: Number.parseInt( - process.env.MAX_DATASET_STORAGE_SIZE_BYTES || String(24 * 1024 * 1024 * 1024), - 10, - ), - targetDatasetStorageSizeBytes: Number.parseInt( - process.env.TARGET_DATASET_STORAGE_SIZE_BYTES || String(20 * 1024 * 1024 * 1024), - 10, - ), - }, - spBlocklists: { - ids: parseIdList(process.env.BLOCKED_SP_IDS), - addresses: parseAddressList(process.env.BLOCKED_SP_ADDRESSES), - }, - pullPiece: { - pullChecksPerSpPerHour: Number.parseFloat(process.env.PULL_CHECKS_PER_SP_PER_HOUR || "1"), - pullCheckJobTimeoutSeconds: Number.parseInt(process.env.PULL_CHECK_JOB_TIMEOUT_SECONDS || "300", 10), - pullCheckPollIntervalSeconds: Number.parseInt(process.env.PULL_CHECK_POLL_INTERVAL_SECONDS || "2", 10), - pullCheckPieceSizeBytes: Number.parseInt(process.env.PULL_CHECK_PIECE_SIZE_BYTES || String(10 * 1024 * 1024), 10), - maxConcurrentStreams: Number.parseInt(process.env.PULL_PIECE_MAX_CONCURRENT_STREAMS || "50", 10), - maxStreamsPerCid: Number.parseInt(process.env.PULL_PIECE_MAX_STREAMS_PER_CID || "3", 10), - pullPieceCleanupIntervalSeconds: Number.parseInt( - process.env.PULL_PIECE_CLEANUP_INTERVAL_SECONDS || String(7 * 24 * 3600), - 10, - ), - }, - }; -} diff --git a/apps/backend/src/config/constants.ts b/apps/backend/src/config/constants.ts new file mode 100644 index 00000000..cb98814b --- /dev/null +++ b/apps/backend/src/config/constants.ts @@ -0,0 +1,44 @@ +import { SUPPORTED_NETWORKS } from "../common/constants.js"; +import type { Network } from "../common/types.js"; +import { NetworkDefaults } from "./types.js"; + +/** + * Default values applied to every network config when the corresponding env + * var is absent. Override per-network via `_*` env vars. + */ +export const networkDefaults = { + checkDatasetCreationFees: true, + useOnlyApprovedProviders: true, + minNumDataSetsForChecks: 1, + dealsPerSpPerHour: 4, + dealJobTimeoutSeconds: 360, + retrievalsPerSpPerHour: 2, + retrievalJobTimeoutSeconds: 60, + dataSetCreationsPerSpPerHour: 1, + dataSetCreationJobTimeoutSeconds: 300, + pieceCleanupPerSpPerHour: 1 / 24, + maxPieceCleanupRuntimeSeconds: 300, + dataRetentionPollIntervalSeconds: 3600, + providersRefreshIntervalSeconds: 4 * 3600, + maintenanceWindowsUtc: ["07:00", "22:00"], + maintenanceWindowMinutes: 20, + maxDatasetStorageSizeBytes: 24 * 1024 * 1024 * 1024, // 10GB + targetDatasetStorageSizeBytes: 20 * 1024 * 1024 * 1024, // 8GB + pullChecksPerSpPerHour: 1, + pullCheckJobTimeoutSeconds: 300, + pullCheckPollIntervalSeconds: 2, + pullCheckPieceSizeBytes: 10 * 1024 * 1024, // 10 MiB + pullPieceMaxConcurrentStreams: 50, + pullPieceMaxStreamsPerCid: 3, + pullPieceCleanupIntervalSeconds: 7 * 24 * 3600, // 7 days + + clickhouseBatchSize: 500, + clickhouseFlushIntervalMs: 5000, + clickhouseMaxBufferSize: 5000, +} satisfies NetworkDefaults; + +/** + * Uppercase env-var prefixes for every supported network, e.g. + * `["CALIBRATION", "MAINNET"]` + */ +export const NETWORK_ENV_PREFIXES = SUPPORTED_NETWORKS.map((n) => n.toUpperCase()) as Uppercase[]; diff --git a/apps/backend/src/config/env.helpers.ts b/apps/backend/src/config/env.helpers.ts new file mode 100644 index 00000000..6718de13 --- /dev/null +++ b/apps/backend/src/config/env.helpers.ts @@ -0,0 +1,24 @@ +/** + * Raw environment variable accessors. + * + * These are purely mechanical: they read a key from `process.env` (or any + * `NodeJS.ProcessEnv` map) and coerce it to the requested primitive type, + * falling back to a default when the key is absent or empty. + */ + +export const getStringEnv = (env: NodeJS.ProcessEnv, key: string, fallback: string): string => env[key] || fallback; + +export const getNumberEnv = (env: NodeJS.ProcessEnv, key: string, fallback: number): number => { + const value = env[key]; + return value ? Number.parseInt(value, 10) : fallback; +}; + +export const getFloatEnv = (env: NodeJS.ProcessEnv, key: string, fallback: number): number => { + const value = env[key]; + return value ? Number.parseFloat(value) : fallback; +}; + +export const getBooleanEnv = (env: NodeJS.ProcessEnv, key: string, fallback: boolean): boolean => { + const value = env[key]; + return value !== undefined ? value !== "false" : fallback; +}; diff --git a/apps/backend/src/config/env.parsers.ts b/apps/backend/src/config/env.parsers.ts new file mode 100644 index 00000000..6205236e --- /dev/null +++ b/apps/backend/src/config/env.parsers.ts @@ -0,0 +1,75 @@ +/** + * Domain-level environment parsers. + * + * Each function here takes a raw `NodeJS.ProcessEnv` map and produces a + * well-typed domain value. + */ + +import { SUPPORTED_NETWORKS } from "../common/constants.js"; +import type { Network } from "../common/types.js"; +import { getStringEnv } from "./env.helpers.js"; +import type { IAppConfig } from "./types.js"; + +/** + * Returns the list of active networks derived from the `NETWORKS` env var. + * Falls back to the first supported network when the variable is absent. + */ +export function parseActiveNetworks(env: NodeJS.ProcessEnv): Network[] { + const raw = getStringEnv(env, "NETWORKS", SUPPORTED_NETWORKS[0]); + return raw + .split(",") + .map((s) => s.trim()) + .filter((s): s is Network => SUPPORTED_NETWORKS.includes(s as Network)); +} + +/** + * Parses the `DEALBOT_RUN_MODE` env var into a typed run-mode string. + * Defaults to `"both"` when absent or unrecognised. + */ +export function parseRunMode(env: NodeJS.ProcessEnv): IAppConfig["runMode"] { + const mode = getStringEnv(env, "DEALBOT_RUN_MODE", "both").toLowerCase(); + if (mode === "worker") return "worker"; + if (mode === "api") return "api"; + return "both"; +} + +/** + * Parses the comma-separated `RANDOM_PIECE_SIZES` env var into an array of + * byte-lengths. Defaults to 10 MiB when absent or unparseable. + */ +export function parseRandomDatasetSizes(env: NodeJS.ProcessEnv): number[] { + const envValue = env.RANDOM_PIECE_SIZES; + + if (envValue && envValue.trim().length > 0) { + const parsed = envValue + .split(",") + .map((entry) => Number.parseInt(entry.trim(), 10)) + .filter((entry) => Number.isFinite(entry) && !Number.isNaN(entry)); + + if (parsed.length > 0) { + return parsed; + } + } + + return [10 << 20]; +} + +export function parseIdList(value: string | undefined): Set { + if (!value || value.trim().length === 0) return new Set(); + return new Set( + value + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0), + ); +} + +export function parseAddressList(value: string | undefined): Set { + if (!value || value.trim().length === 0) return new Set(); + return new Set( + value + .split(",") + .map((s) => s.trim().toLowerCase()) + .filter((s) => s.length > 0), + ); +} diff --git a/apps/backend/src/config/env.schema.spec.ts b/apps/backend/src/config/env.schema.spec.ts new file mode 100644 index 00000000..4f5e7d0e --- /dev/null +++ b/apps/backend/src/config/env.schema.spec.ts @@ -0,0 +1,521 @@ +import { zeroAddress } from "viem"; +import { describe, expect, it } from "vitest"; +import { createConfigValidationSchema } from "./env.schema.js"; + +/** + * Minimum set of env vars required for validation to pass (database block + * is always required, regardless of which networks are active). + */ +const baseEnv = { + DATABASE_HOST: "localhost", + DATABASE_USER: "test", + DATABASE_PASSWORD: "test", + DATABASE_NAME: "test", + CALIBRATION_WALLET_ADDRESS: zeroAddress, + MAINNET_WALLET_ADDRESS: zeroAddress, +}; + +/** + * Env fragment that satisfies the per-network wallet-key `.or()` constraint + * for a given network prefix. Use with `...withWalletKey("CALIBRATION")`. + */ +const withWalletKey = (prefix: string) => ({ [`${prefix}_WALLET_PRIVATE_KEY`]: "0xkey" }); + +/** + * Builds a schema where only the given networks are active. Keeps tests + * deterministic regardless of the process env that the test runner inherits. + */ +const schemaFor = (networks: string) => + createConfigValidationSchema({ ...baseEnv, NETWORKS: networks } as NodeJS.ProcessEnv); + +const validate = (schema: ReturnType, input: Record) => + schema.validate(input, { allowUnknown: true }); + +describe("createConfigValidationSchema", () => { + describe("wallet-key / session-key constraint (active networks)", () => { + it("accepts a network with WALLET_PRIVATE_KEY only", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_WALLET_PRIVATE_KEY: "0xkey", + }); + expect(error).toBeUndefined(); + }); + + it("accepts a network with SESSION_KEY_PRIVATE_KEY only", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_SESSION_KEY_PRIVATE_KEY: "0xdeadbeef", + }); + expect(error).toBeUndefined(); + }); + + it("accepts both keys being provided (loader decides precedence)", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_WALLET_PRIVATE_KEY: "0xkey", + CALIBRATION_SESSION_KEY_PRIVATE_KEY: "0xsession", + }); + expect(error).toBeUndefined(); + }); + + it("rejects an active network that has neither key", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_WALLET_PRIVATE_KEY|CALIBRATION_SESSION_KEY_PRIVATE_KEY/); + }); + + it("treats empty-string wallet key as absent (must fall back to session key)", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_WALLET_PRIVATE_KEY: "", + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_WALLET_PRIVATE_KEY|CALIBRATION_SESSION_KEY_PRIVATE_KEY/); + }); + + it("treats empty-string session key as absent when wallet key is present", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_WALLET_PRIVATE_KEY: "0xkey", + CALIBRATION_SESSION_KEY_PRIVATE_KEY: "", + }); + expect(error).toBeUndefined(); + }); + + it("rejects when every active network is missing both keys (multi-network)", () => { + const { error } = validate(schemaFor("mainnet,calibration"), { + ...baseEnv, + NETWORKS: "mainnet,calibration", + }); + expect(error).toBeDefined(); + }); + + it("requires each active network independently (multi-network)", () => { + // Only mainnet has a key; calibration is active but has neither → invalid + const { error } = validate(schemaFor("mainnet,calibration"), { + ...baseEnv, + NETWORKS: "mainnet,calibration", + ...withWalletKey("MAINNET"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_WALLET_PRIVATE_KEY|CALIBRATION_SESSION_KEY_PRIVATE_KEY/); + }); + + it("accepts multi-network when every active network provides a key", () => { + const { error } = validate(schemaFor("mainnet,calibration"), { + ...baseEnv, + NETWORKS: "mainnet,calibration", + ...withWalletKey("MAINNET"), + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + }); + }); + + describe("inactive networks", () => { + it("does not require keys for an inactive network", () => { + // Only calibration is active; mainnet has no keys → still valid + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + }); + + it("does not reject invalid-looking values on an inactive network", () => { + // MAINNET is inactive, so its fields should be optional (no strict checks). + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + MAINNET_WALLET_PRIVATE_KEY: "", + MAINNET_SESSION_KEY_PRIVATE_KEY: "", + }); + expect(error).toBeUndefined(); + }); + }); + + describe("NETWORKS env var", () => { + it("rejects an unknown network name", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "unknownnet", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/NETWORKS|Invalid network/); + }); + + it("rejects a mixed list containing an unknown network", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration,bogus", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("accepts a list with whitespace around entries", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: " calibration , mainnet ", + ...withWalletKey("CALIBRATION"), + ...withWalletKey("MAINNET"), + }); + expect(error).toBeUndefined(); + }); + + it("falls back to the default when NETWORKS is absent", () => { + const { error, value } = validate(schemaFor("calibration"), { + ...baseEnv, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + expect(value.NETWORKS).toBeTruthy(); + }); + }); + + describe("database block", () => { + it.each(["DATABASE_HOST", "DATABASE_USER", "DATABASE_PASSWORD", "DATABASE_NAME"] as const)("requires %s", (key) => { + const env: Record = { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + }; + delete env[key]; + const { error } = validate(schemaFor("calibration"), env); + expect(error).toBeDefined(); + expect(error?.message).toMatch(new RegExp(key)); + }); + + it("defaults DATABASE_PORT to 5432 when absent", () => { + const { value } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + }); + expect(value.DATABASE_PORT).toBe(5432); + }); + + it("rejects a non-numeric DATABASE_PORT", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + DATABASE_PORT: "abc", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/DATABASE_PORT/); + }); + + it("rejects DATABASE_POOL_MAX below the minimum", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + DATABASE_POOL_MAX: 0, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/DATABASE_POOL_MAX/); + }); + }); + + describe("app block", () => { + it("rejects an unknown NODE_ENV", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + NODE_ENV: "staging", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/NODE_ENV/); + }); + + it("rejects an unknown DEALBOT_RUN_MODE", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + DEALBOT_RUN_MODE: "invalid-mode", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/DEALBOT_RUN_MODE/); + }); + + it("lowercases DEALBOT_RUN_MODE before validation", () => { + const { error, value } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + DEALBOT_RUN_MODE: "WORKER", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + expect(value.DEALBOT_RUN_MODE).toBe("worker"); + }); + + it("applies sensible defaults for all optional app fields", () => { + const { error, value } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + expect(value.NODE_ENV).toBe("development"); + expect(value.DEALBOT_RUN_MODE).toBe("both"); + expect(value.DEALBOT_PORT).toBe(3000); + expect(value.ENABLE_DEV_MODE).toBe(false); + expect(value.PROMETHEUS_WALLET_BALANCE_TTL_SECONDS).toBe(3600); + }); + + it("enforces PROMETHEUS_WALLET_BALANCE_TTL_SECONDS minimum of 60", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + PROMETHEUS_WALLET_BALANCE_TTL_SECONDS: 59, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/PROMETHEUS_WALLET_BALANCE_TTL_SECONDS/); + }); + + it("coerces ENABLE_DEV_MODE boolean strings", () => { + const { error, value } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ENABLE_DEV_MODE: "true", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + expect(value.ENABLE_DEV_MODE).toBe(true); + }); + }); + + describe("per-network fields (active network)", () => { + it("rejects an RPC URL without an http(s) scheme", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_RPC_URL: "ftp://example.com", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_RPC_URL/); + }); + + it("accepts an empty RPC URL string", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_RPC_URL: "", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + }); + + it("rejects METRICS_PER_HOUR above the maximum", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_METRICS_PER_HOUR: 4, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_METRICS_PER_HOUR/); + }); + + it("rejects DEALS_PER_SP_PER_HOUR at zero (below min)", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_DEALS_PER_SP_PER_HOUR: 0, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("rejects MIN_NUM_DATASETS_FOR_CHECKS when non-integer", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS: 1.5, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS/); + }); + }); + + describe("maintenance windows", () => { + it("accepts a valid comma-separated schedule", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOWS_UTC: "06:30,18:00", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + }); + + it("rejects an invalid hour", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOWS_UTC: "25:00", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_MAINTENANCE_WINDOWS_UTC/); + }); + + it("rejects an invalid minute", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOWS_UTC: "07:70", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("rejects a malformed entry", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOWS_UTC: "7am", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("rejects MAINTENANCE_WINDOW_MINUTES below 20", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOW_MINUTES: 10, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_MAINTENANCE_WINDOW_MINUTES/); + }); + + it("rejects MAINTENANCE_WINDOW_MINUTES above 360", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOW_MINUTES: 361, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + }); + + describe("jobs block", () => { + it("applies defaults when absent", () => { + const { error, value } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + expect(value.JOB_SCHEDULER_POLL_SECONDS).toBe(300); + expect(value.JOB_WORKER_POLL_SECONDS).toBe(60); + expect(value.PG_BOSS_LOCAL_CONCURRENCY).toBe(20); + expect(value.DEALBOT_PGBOSS_SCHEDULER_ENABLED).toBe(true); + expect(value.DEAL_JOB_TIMEOUT_SECONDS).toBe(360); + }); + + it("rejects JOB_SCHEDULER_POLL_SECONDS below 60", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + JOB_SCHEDULER_POLL_SECONDS: 30, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("rejects DEAL_JOB_TIMEOUT_SECONDS below 120", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + DEAL_JOB_TIMEOUT_SECONDS: 60, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("rejects non-integer PG_BOSS_LOCAL_CONCURRENCY", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + PG_BOSS_LOCAL_CONCURRENCY: 2.5, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/PG_BOSS_LOCAL_CONCURRENCY/); + }); + }); + + describe("timeout block", () => { + it("enforces CONNECT_TIMEOUT_MS >= 1000", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CONNECT_TIMEOUT_MS: 500, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("enforces IPNI_VERIFICATION_POLLING_MS >= 250", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + IPNI_VERIFICATION_POLLING_MS: 100, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + }); + + describe("retrieval block", () => { + it("rejects IPFS_BLOCK_FETCH_CONCURRENCY above 32", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + IPFS_BLOCK_FETCH_CONCURRENCY: 33, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/IPFS_BLOCK_FETCH_CONCURRENCY/); + }); + + it("rejects IPFS_BLOCK_FETCH_CONCURRENCY when zero", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + IPFS_BLOCK_FETCH_CONCURRENCY: 0, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + }); + + describe("stability", () => { + it("returns a fresh schema instance per call", () => { + const a = schemaFor("calibration"); + const b = schemaFor("calibration"); + expect(a).not.toBe(b); + }); + }); +}); diff --git a/apps/backend/src/config/env.schema.ts b/apps/backend/src/config/env.schema.ts new file mode 100644 index 00000000..40f8affa --- /dev/null +++ b/apps/backend/src/config/env.schema.ts @@ -0,0 +1,266 @@ +/** + * Joi validation schemas for all environment variables. + * + * Structure + * --------- + * Individual schema objects are exported so tests can validate specific + * slices in isolation. `createConfigValidationSchema()` assembles the final + * Joi object schema that NestJS `ConfigModule` consumes. + * + * Per-network rules + * ----------------- + * `createPerNetworkEnvSchema(prefix)` produces the field rules for one + * network prefix (e.g. "CALIBRATION"). `createConfigValidationSchema` + * iterates all prefixes and: + * - applies rules as-is for ACTIVE networks (and queues the wallet-key + * `.or()` constraint) + * - marks every field `.optional()` for INACTIVE networks so those env + * vars are never required. + */ + +import Joi from "joi"; +import { DEFAULT_LOCAL_DATASETS_PATH, SUPPORTED_NETWORKS } from "../common/constants.js"; +import { parseMaintenanceWindowTimes } from "../common/maintenance-window.js"; +import type { Network } from "../common/types.js"; +import { NETWORK_ENV_PREFIXES } from "./constants.js"; +import { parseActiveNetworks } from "./env.parsers.js"; +import { applyLegacyEnvCompat, logLegacyEnvCompatResult } from "./legacy-env-compat.js"; + +// --------------------------------------------------------------------------- +// Custom Joi validators +// --------------------------------------------------------------------------- + +const validateNetworksEnv = (value: string, helpers: Joi.CustomHelpers) => { + const parts = value + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + + for (const part of parts) { + if (!SUPPORTED_NETWORKS.includes(part as Network)) { + return helpers.error("any.invalid", { + message: `Invalid network "${part}". Supported: ${SUPPORTED_NETWORKS.join(", ")}.`, + }); + } + } + + return parts.length === 0 ? SUPPORTED_NETWORKS[0] : value; +}; + +const validateMaintenanceWindowsEnv = (value: string, helpers: Joi.CustomHelpers) => { + try { + parseMaintenanceWindowTimes(value.split(",")); + } catch (error) { + return helpers.error("any.invalid", { + message: error instanceof Error ? error.message : "Invalid maintenance window format", + }); + } + return value; +}; + +// --------------------------------------------------------------------------- +// Static schema slices (one per config section) +// --------------------------------------------------------------------------- + +export const appEnvSchema = { + NODE_ENV: Joi.string().valid("development", "production", "test").default("development"), + DEALBOT_RUN_MODE: Joi.string().lowercase().valid("api", "worker", "both").default("both"), + DEALBOT_PORT: Joi.number().default(3000), + DEALBOT_HOST: Joi.string().default("127.0.0.1"), + DEALBOT_API_PUBLIC_URL: Joi.string().uri().optional().allow(""), + DEALBOT_METRICS_PORT: Joi.number().default(9090), + DEALBOT_METRICS_HOST: Joi.string().default("0.0.0.0"), + ENABLE_DEV_MODE: Joi.boolean().default(false), + PROMETHEUS_WALLET_BALANCE_TTL_SECONDS: Joi.number().min(60).default(3600), + PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS: Joi.number().min(1).default(60), + DEALBOT_PROBE_LOCATION: Joi.string().default("unknown"), +}; + +export const databaseEnvSchema = { + DATABASE_HOST: Joi.string().required(), + DATABASE_PORT: Joi.number().default(5432), + DATABASE_POOL_MAX: Joi.number().integer().min(1).default(1), + DATABASE_USER: Joi.string().required(), + DATABASE_PASSWORD: Joi.string().required(), + DATABASE_NAME: Joi.string().required(), +}; + +export const globalNetworkEnvSchema = { + NETWORKS: Joi.string().default(SUPPORTED_NETWORKS[0]).custom(validateNetworksEnv), +}; + +export const jobsEnvSchema = { + JOB_SCHEDULER_POLL_SECONDS: Joi.number().min(60).default(300), + JOB_WORKER_POLL_SECONDS: Joi.number().min(5).default(60), + PG_BOSS_LOCAL_CONCURRENCY: Joi.number().integer().min(1).default(20), + DEALBOT_PGBOSS_SCHEDULER_ENABLED: Joi.boolean().default(true), + DEALBOT_PGBOSS_POOL_MAX: Joi.number().integer().min(1).default(1), + JOB_CATCHUP_MAX_ENQUEUE: Joi.number().min(1).default(10), + JOB_SCHEDULE_PHASE_SECONDS: Joi.number().min(0).default(0), + JOB_ENQUEUE_JITTER_SECONDS: Joi.number().min(0).default(0), + SHUTDOWN_FINAL_SCRAPE_DELAY_SECONDS: Joi.number().min(0).max(300).default(35), +}; + +export const datasetEnvSchema = { + DEALBOT_LOCAL_DATASETS_PATH: Joi.string().default(DEFAULT_LOCAL_DATASETS_PATH), + RANDOM_PIECE_SIZES: Joi.string().default("10485760"), +}; + +export const timeoutEnvSchema = { + CONNECT_TIMEOUT_MS: Joi.number().min(1000).default(10000), + HTTP_REQUEST_TIMEOUT_MS: Joi.number().min(1000).default(240000), + HTTP2_REQUEST_TIMEOUT_MS: Joi.number().min(1000).default(240000), + IPNI_VERIFICATION_TIMEOUT_MS: Joi.number().min(1000).default(60000), + IPNI_VERIFICATION_POLLING_MS: Joi.number().min(250).default(2000), +}; + +export const retrievalEnvSchema = { + IPFS_BLOCK_FETCH_CONCURRENCY: Joi.number().integer().min(1).max(32).default(6), +}; + +// --------------------------------------------------------------------------- +// Per-network schema factory +// --------------------------------------------------------------------------- + +/** + * Returns the Joi field rules for a single network prefix (e.g. `"CALIBRATION"`). + * Fields are defined as optional here; active-network enforcement is handled + * in `createConfigValidationSchema`. + */ +export const createPerNetworkEnvSchema = (prefix: Uppercase | "") => { + const k = (key: string) => `${prefix}_${key}`; + return { + [k("WALLET_ADDRESS")]: Joi.string().required(), + [k("WALLET_PRIVATE_KEY")]: Joi.string().optional().empty(""), + [k("SESSION_KEY_PRIVATE_KEY")]: Joi.string().optional().empty(""), + [k("RPC_URL")]: Joi.string() + .uri({ scheme: ["http", "https"] }) + .optional() + .allow(""), + [k("PDP_SUBGRAPH_ENDPOINT")]: Joi.string().uri().optional().allow(""), + [k("CHECK_DATASET_CREATION_FEES")]: Joi.boolean().optional(), + [k("USE_ONLY_APPROVED_PROVIDERS")]: Joi.boolean().optional(), + [k("DEALBOT_DATASET_VERSION")]: Joi.string().optional(), + [k("MIN_NUM_DATASETS_FOR_CHECKS")]: Joi.number().integer().min(1).optional(), + [k("DEALS_PER_SP_PER_HOUR")]: Joi.number().min(0.001).max(20).optional(), + [k("RETRIEVALS_PER_SP_PER_HOUR")]: Joi.number().min(0.001).max(20).optional(), + [k("DATASET_CREATIONS_PER_SP_PER_HOUR")]: Joi.number().min(0.001).max(20).optional(), + [k("DEAL_JOB_TIMEOUT_SECONDS")]: Joi.number().min(120).default(360), + [k("RETRIEVAL_JOB_TIMEOUT_SECONDS")]: Joi.number().min(60).default(60), + [k("DATA_SET_CREATION_JOB_TIMEOUT_SECONDS")]: Joi.number().min(60).default(300), + [k("DATA_RETENTION_POLL_INTERVAL_SECONDS")]: Joi.number().optional(), + [k("PROVIDERS_REFRESH_INTERVAL_SECONDS")]: Joi.number().optional(), + [k("MAINTENANCE_WINDOWS_UTC")]: Joi.string().default("07:00,22:00").custom(validateMaintenanceWindowsEnv), + [k("MAINTENANCE_WINDOW_MINUTES")]: Joi.number().min(20).max(360).default(20), + [k("BLOCKED_SP_IDS")]: Joi.string().optional().allow(""), + [k("BLOCKED_SP_ADDRESSES")]: Joi.string().optional().allow(""), + + [k("PIECE_CLEANUP_PER_SP_PER_HOUR")]: Joi.number() + .min(0.001) + .max(20) + .default(1 / 24), + [k("MAX_PIECE_CLEANUP_RUNTIME_SECONDS")]: Joi.number().min(60).default(300), + [k("MAX_DATASET_STORAGE_SIZE_BYTES")]: Joi.number() + .integer() + .min(1) + .default(24 * 1024 * 1024 * 1024), + [k("TARGET_DATASET_STORAGE_SIZE_BYTES")]: Joi.number() + .integer() + .min(1) + .default(20 * 1024 * 1024 * 1024) // 20 GiB per SP + .custom((value, helpers) => { + const max = helpers.state.ancestors?.[0]?.MAX_DATASET_STORAGE_SIZE_BYTES; + if (max != null && value >= max) { + return helpers.error("any.invalid", { + message: `TARGET_DATASET_STORAGE_SIZE_BYTES (${value}) must be less than MAX_DATASET_STORAGE_SIZE_BYTES (${max})`, + }); + } + return value; + }, "target < max validation"), + + [k("PULL_CHECKS_PER_SP_PER_HOUR")]: Joi.number().min(0.001).max(20).default(1), + [k("PULL_CHECK_JOB_TIMEOUT_SECONDS")]: Joi.number().min(60).default(300), + [k("PULL_CHECK_POLL_INTERVAL_SECONDS")]: Joi.number().min(1).default(2), + [k("PULL_CHECK_PIECE_SIZE_BYTES")]: Joi.number() + .integer() + .min(1024) + .default(10 * 1024 * 1024), + [k("PULL_PIECE_MAX_CONCURRENT_STREAMS")]: Joi.number().integer().min(1).default(50), + [k("PULL_PIECE_MAX_STREAMS_PER_CID")]: Joi.number().integer().min(1).default(3), + [k("PULL_PIECE_CLEANUP_INTERVAL_SECONDS")]: Joi.number() + .integer() + .min(3600) + .default(7 * 24 * 3600), + + [k("CLICKHOUSE_URL")]: Joi.string().uri().optional(), + [k("CLICKHOUSE_BATCH_SIZE")]: Joi.number().integer().min(1).default(500), + [k("CLICKHOUSE_FLUSH_INTERVAL_MS")]: Joi.number().integer().min(100).default(5000), + [k("CLICKHOUSE_MAX_BUFFER_SIZE")]: Joi.number().integer().min(1).default(5000), + }; +}; + +// --------------------------------------------------------------------------- +// Dynamic schema factory +// --------------------------------------------------------------------------- + +/** + * Builds the full Joi validation schema, adapting per-network field + * requirements to which networks are present in `processEnv.NETWORKS`. + */ +export function createConfigValidationSchema(processEnv: NodeJS.ProcessEnv = process.env): Joi.ObjectSchema { + const activeNetworks = parseActiveNetworks(processEnv); + + const schemaFields: Record = { + ...appEnvSchema, + ...databaseEnvSchema, + ...globalNetworkEnvSchema, + ...jobsEnvSchema, + ...datasetEnvSchema, + ...timeoutEnvSchema, + ...retrievalEnvSchema, + }; + + const walletKeyOrConditions: [string, string][] = []; + + for (const prefix of NETWORK_ENV_PREFIXES) { + const networkRules = createPerNetworkEnvSchema(prefix); + + if (activeNetworks.includes(prefix.toLowerCase() as Network)) { + Object.assign(schemaFields, networkRules); + walletKeyOrConditions.push([`${prefix}_WALLET_PRIVATE_KEY`, `${prefix}_SESSION_KEY_PRIVATE_KEY`]); + } else { + const optionalRules = Object.fromEntries( + Object.entries(networkRules).map(([key, rule]) => [key, (rule as Joi.AnySchema).optional()]), + ); + Object.assign(schemaFields, optionalRules); + } + } + + let schema = Joi.object(schemaFields); + + for (const [walletKey, sessionKey] of walletKeyOrConditions) { + schema = schema.or(walletKey, sessionKey); + } + + return schema; +} + +// --------------------------------------------------------------------------- +// NestJS ConfigModule `validate` callback +// --------------------------------------------------------------------------- + +/** + * Entry point wired into `ConfigModule.forRoot({ validate })`. Runs AFTER + * `@nestjs/config` has merged `.env` into the env object, so the scheme + * detector sees the operator's real configuration. + */ +export function validateConfig(rawEnv: Record): Record { + logLegacyEnvCompatResult(applyLegacyEnvCompat(rawEnv as NodeJS.ProcessEnv)); + + const schema = createConfigValidationSchema(rawEnv as NodeJS.ProcessEnv); + const { error, value } = schema.validate(rawEnv, { allowUnknown: true, abortEarly: false }); + if (error) { + throw new Error(`Config validation error: ${error.message}`); + } + return value; +} diff --git a/apps/backend/src/config/index.ts b/apps/backend/src/config/index.ts new file mode 100644 index 00000000..f4dd096c --- /dev/null +++ b/apps/backend/src/config/index.ts @@ -0,0 +1,3 @@ +export { validateConfig } from "./env.schema.js"; +export { loadConfig } from "./loader.js"; +export * from "./types.js"; diff --git a/apps/backend/src/config/legacy-env-compat.spec.ts b/apps/backend/src/config/legacy-env-compat.spec.ts new file mode 100644 index 00000000..9cb759e6 --- /dev/null +++ b/apps/backend/src/config/legacy-env-compat.spec.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; +import { applyLegacyEnvCompat } from "./legacy-env-compat.js"; + +/** Build a fresh env map for every test — never mutate `process.env`. */ +const envOf = (overrides: Record = {}): NodeJS.ProcessEnv => ({ ...overrides }) as NodeJS.ProcessEnv; + +describe("applyLegacyEnvCompat", () => { + describe("skips translation", () => { + it("returns skipReason=networks_already_set when NETWORKS is set", () => { + const env = envOf({ NETWORKS: "calibration", NETWORK: "mainnet", WALLET_PRIVATE_KEY: "0xkey" }); + const before = { ...env }; + const result = applyLegacyEnvCompat(env); + expect(result.applied).toBe(false); + expect(result.skipReason).toBe("networks_already_set"); + expect(result.translatedVars).toEqual([]); + // Env left untouched. + expect(env).toEqual(before); + }); + + it("returns skipReason=no_legacy_network when NETWORK is absent", () => { + const env = envOf(); + const result = applyLegacyEnvCompat(env); + expect(result.applied).toBe(false); + expect(result.skipReason).toBe("no_legacy_network"); + }); + + it("returns skipReason=no_legacy_network when NETWORK is whitespace", () => { + const env = envOf({ NETWORK: " " }); + const result = applyLegacyEnvCompat(env); + expect(result.applied).toBe(false); + expect(result.skipReason).toBe("no_legacy_network"); + }); + + it("returns skipReason=invalid_legacy_network when NETWORK is unsupported", () => { + const env = envOf({ NETWORK: "moonbase" }); + const result = applyLegacyEnvCompat(env); + expect(result.applied).toBe(false); + expect(result.skipReason).toBe("invalid_legacy_network"); + // Must not set NETWORKS to anything bogus. + expect(env.NETWORKS).toBeUndefined(); + }); + }); + + describe("translates when legacy NETWORK is set", () => { + it("sets NETWORKS and copies each unprefixed var to its prefixed slot", () => { + const env = envOf({ + NETWORK: "calibration", + WALLET_PRIVATE_KEY: "0xkey", + WALLET_ADDRESS: "0xabc", + RPC_URL: "https://rpc.example", + DEALS_PER_SP_PER_HOUR: "3", + }); + const result = applyLegacyEnvCompat(env); + + expect(result.applied).toBe(true); + expect(result.network).toBe("calibration"); + expect(result.translatedVars).toEqual( + expect.arrayContaining(["WALLET_PRIVATE_KEY", "WALLET_ADDRESS", "RPC_URL", "DEALS_PER_SP_PER_HOUR"]), + ); + expect(env.NETWORKS).toBe("calibration"); + expect(env.CALIBRATION_WALLET_PRIVATE_KEY).toBe("0xkey"); + expect(env.CALIBRATION_WALLET_ADDRESS).toBe("0xabc"); + expect(env.CALIBRATION_RPC_URL).toBe("https://rpc.example"); + expect(env.CALIBRATION_DEALS_PER_SP_PER_HOUR).toBe("3"); + }); + + it("normalises case-variant NETWORK values", () => { + const env = envOf({ NETWORK: "MAINNET", WALLET_PRIVATE_KEY: "0xkey" }); + const result = applyLegacyEnvCompat(env); + + expect(result.applied).toBe(true); + expect(result.network).toBe("mainnet"); + expect(env.NETWORKS).toBe("mainnet"); + expect(env.MAINNET_WALLET_PRIVATE_KEY).toBe("0xkey"); + }); + + it("does not overwrite an already-set prefixed var (explicit wins)", () => { + const env = envOf({ + NETWORK: "calibration", + WALLET_PRIVATE_KEY: "0xlegacy", + CALIBRATION_WALLET_PRIVATE_KEY: "0xexplicit", + }); + const result = applyLegacyEnvCompat(env); + + expect(result.applied).toBe(true); + expect(env.CALIBRATION_WALLET_PRIVATE_KEY).toBe("0xexplicit"); + // The legacy var is still present but was not re-copied. + expect(result.translatedVars).not.toContain("WALLET_PRIVATE_KEY"); + }); + + it("skips empty legacy values", () => { + const env = envOf({ NETWORK: "calibration", WALLET_PRIVATE_KEY: "0xkey", RPC_URL: "" }); + const result = applyLegacyEnvCompat(env); + + expect(result.applied).toBe(true); + expect(result.translatedVars).not.toContain("RPC_URL"); + expect(env.CALIBRATION_RPC_URL).toBeUndefined(); + }); + + it("does not carry data between distinct env maps (no shared state)", () => { + const a = envOf({ NETWORK: "calibration", WALLET_PRIVATE_KEY: "0xa" }); + const b = envOf({ NETWORK: "mainnet", WALLET_PRIVATE_KEY: "0xb" }); + applyLegacyEnvCompat(a); + applyLegacyEnvCompat(b); + expect(a.CALIBRATION_WALLET_PRIVATE_KEY).toBe("0xa"); + expect(a.MAINNET_WALLET_PRIVATE_KEY).toBeUndefined(); + expect(b.MAINNET_WALLET_PRIVATE_KEY).toBe("0xb"); + expect(b.CALIBRATION_WALLET_PRIVATE_KEY).toBeUndefined(); + }); + }); +}); diff --git a/apps/backend/src/config/legacy-env-compat.ts b/apps/backend/src/config/legacy-env-compat.ts new file mode 100644 index 00000000..a68d25ed --- /dev/null +++ b/apps/backend/src/config/legacy-env-compat.ts @@ -0,0 +1,139 @@ +/** + * Backwards-compatibility shim for the legacy single-network env layout. + * + * Before multi-network support, a dealbot deployment was configured with + * unprefixed variables such as `NETWORK`, `WALLET_PRIVATE_KEY`, `RPC_URL`, + * `DEALS_PER_SP_PER_HOUR`, etc. Multi-network support requires a + * `NETWORKS=...` list plus per-network prefixed variables + * (`CALIBRATION_WALLET_PRIVATE_KEY`, `MAINNET_RPC_URL`, ...). + * + * This shim lets existing deployments roll forward without changing their + * ConfigMaps/Secrets in the same release as the code rollout: if `NETWORKS` + * is absent but legacy `NETWORK` is set to a supported value, it is translated + * into the new shape in place (single-network config only). The translation + * runs *before* the Joi validation schema is evaluated, so the rest of the + * code never has to branch on legacy mode. + * + * Lifecycle + * --------- + * Call `applyLegacyEnvCompat(process.env)` at the top of `validate` in + * `ConfigModule.forRoot`. The function mutates `env` in place so + * subsequent reads observe the translated values. + * + * Removal + * ------- + * Once all environments have been cut over to the new prefixed vars + * and this shim is no longer needed, delete this file and its two call sites. + */ + +import { SUPPORTED_NETWORKS } from "../common/constants.js"; +import { createPinoExitLogger } from "../common/pino.config.js"; +import type { Network } from "../common/types.js"; + +/** + * Env var names (unprefixed) that were moved into a per-network namespace. + * Each corresponds to a `_` variable in the new layout. + * + * Keep this list in sync with `createPerNetworkEnvSchema` in `env.schema.ts`. + */ +const LEGACY_PER_NETWORK_VARS = [ + "WALLET_ADDRESS", + "WALLET_PRIVATE_KEY", + "SESSION_KEY_PRIVATE_KEY", + "RPC_URL", + "PDP_SUBGRAPH_ENDPOINT", + "CHECK_DATASET_CREATION_FEES", + "USE_ONLY_APPROVED_PROVIDERS", + "DEALBOT_DATASET_VERSION", + "MIN_NUM_DATASETS_FOR_CHECKS", + "DEALS_PER_SP_PER_HOUR", + "RETRIEVALS_PER_SP_PER_HOUR", + "DATASET_CREATIONS_PER_SP_PER_HOUR", + "DATA_RETENTION_POLL_INTERVAL_SECONDS", + "PROVIDERS_REFRESH_INTERVAL_SECONDS", + "MAINTENANCE_WINDOWS_UTC", + "MAINTENANCE_WINDOW_MINUTES", + "BLOCKED_SP_IDS", + "BLOCKED_SP_ADDRESSES", + "PIECE_CLEANUP_PER_SP_PER_HOUR", + "MAX_PIECE_CLEANUP_RUNTIME_SECONDS", + "MAX_DATASET_STORAGE_SIZE_BYTES", + "TARGET_DATASET_STORAGE_SIZE_BYTES", +] as const; + +export interface LegacyEnvCompatResult { + /** True if legacy translation was applied. */ + applied: boolean; + /** The network the legacy env was resolved to (only set when applied). */ + network?: Network; + /** Legacy var names that were copied into the new prefixed slot. */ + translatedVars: string[]; + /** Reason translation was skipped, for diagnostics. */ + skipReason?: "networks_already_set" | "no_legacy_network" | "invalid_legacy_network"; +} + +/** + * Translates legacy single-network env vars into the new prefixed layout. + * Mutates `env` in place and returns a summary describing what happened. + * + * Rules: + * - If `NETWORKS` is already set, return untouched (operator has migrated). + * - Else if legacy `NETWORK` is set to a supported value, copy unprefixed + * vars into `_` slots that are currently unset. Already-set + * prefixed vars are never overwritten (explicit wins). + * - Else return untouched; downstream Joi validation will surface the + * missing-config error with its normal diagnostics. + */ +export function applyLegacyEnvCompat(env: NodeJS.ProcessEnv): LegacyEnvCompatResult { + if (typeof env.NETWORKS === "string" && env.NETWORKS.trim().length > 0) { + return { applied: false, translatedVars: [], skipReason: "networks_already_set" }; + } + + const legacyRaw = env.NETWORK; + if (typeof legacyRaw !== "string" || legacyRaw.trim().length === 0) { + return { applied: false, translatedVars: [], skipReason: "no_legacy_network" }; + } + + const legacyNetwork = legacyRaw.trim().toLowerCase(); + if (!SUPPORTED_NETWORKS.includes(legacyNetwork as Network)) { + return { applied: false, translatedVars: [], skipReason: "invalid_legacy_network" }; + } + + const network = legacyNetwork as Network; + const prefix = network.toUpperCase(); + const translatedVars: string[] = []; + + env.NETWORKS = network; + + for (const key of LEGACY_PER_NETWORK_VARS) { + const legacyValue = env[key]; + if (typeof legacyValue !== "string" || legacyValue.length === 0) continue; + + const prefixedKey = `${prefix}_${key}`; + const existing = env[prefixedKey]; + if (typeof existing === "string" && existing.length > 0) continue; + + env[prefixedKey] = legacyValue; + translatedVars.push(key); + } + + return { applied: true, network, translatedVars }; +} + +/** + * One-time console warning describing a legacy translation. + */ +export function logLegacyEnvCompatResult(result: LegacyEnvCompatResult): void { + if (!result.applied) return; + + const logger = createPinoExitLogger().child({ context: "LegacyEnvCompat" }); + logger.warn({ + level: "warn", + event: "config_legacy_env_detected", + message: + "Legacy single-network env vars detected; translated into per-network prefixed vars. " + + "Update your ConfigMap/Secrets to the prefixed names before the next release.", + network: result.network, + translatedVars: result.translatedVars, + }); +} diff --git a/apps/backend/src/config/loader.spec.ts b/apps/backend/src/config/loader.spec.ts new file mode 100644 index 00000000..bccedd91 --- /dev/null +++ b/apps/backend/src/config/loader.spec.ts @@ -0,0 +1,89 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { loadConfig } from "./loader.js"; + +/** + * `loadConfig` reads from `process.env` directly, so tests snapshot and + * restore the real env around each case. Only keys touched by a test are + * cleared on reset. + */ +const KEYS_TO_RESET = [ + "NETWORKS", + "NETWORK", + "CALIBRATION_WALLET_PRIVATE_KEY", + "CALIBRATION_WALLET_ADDRESS", + "CALIBRATION_RPC_URL", + "CALIBRATION_DEALS_PER_SP_PER_HOUR", + "CALIBRATION_BLOCKED_SP_IDS", + "MAINNET_WALLET_PRIVATE_KEY", + "MAINNET_WALLET_ADDRESS", + "MAINNET_RPC_URL", + "WALLET_PRIVATE_KEY", + "WALLET_ADDRESS", + "RPC_URL", + "DEALS_PER_SP_PER_HOUR", + "BLOCKED_SP_IDS", + "SESSION_KEY_PRIVATE_KEY", +]; + +const snapshot: Record = {}; + +beforeEach(() => { + for (const key of KEYS_TO_RESET) { + snapshot[key] = process.env[key]; + delete process.env[key]; + } + process.env.DATABASE_HOST = process.env.DATABASE_HOST ?? "localhost"; + process.env.DATABASE_USER = process.env.DATABASE_USER ?? "test"; + process.env.DATABASE_PASSWORD = process.env.DATABASE_PASSWORD ?? "test"; + process.env.DATABASE_NAME = process.env.DATABASE_NAME ?? "test"; + + return () => { + for (const key of KEYS_TO_RESET) { + if (snapshot[key] === undefined) delete process.env[key]; + else process.env[key] = snapshot[key]; + } + }; +}); + +describe("loadConfig", () => { + it("loads the active network from prefixed vars", () => { + process.env.NETWORKS = "calibration"; + process.env.CALIBRATION_WALLET_PRIVATE_KEY = "0xkey"; + process.env.CALIBRATION_RPC_URL = "https://rpc.example/calibration"; + process.env.CALIBRATION_DEALS_PER_SP_PER_HOUR = "3"; + + const cfg = loadConfig(); + + expect(cfg.activeNetworks).toEqual(["calibration"]); + expect(cfg.networks.calibration.network).toBe("calibration"); + expect(cfg.networks.calibration.rpcUrl).toBe("https://rpc.example/calibration"); + expect(cfg.networks.calibration.dealsPerSpPerHour).toBe(3); + if ("walletPrivateKey" in cfg.networks.calibration) { + expect(cfg.networks.calibration.walletPrivateKey).toBe("0xkey"); + } else { + throw new Error("calibration should be loaded as walletPrivateKey variant"); + } + }); + + it("loads both networks when both are listed in NETWORKS", () => { + process.env.NETWORKS = "calibration,mainnet"; + process.env.CALIBRATION_WALLET_PRIVATE_KEY = "0xcal"; + process.env.MAINNET_WALLET_PRIVATE_KEY = "0xmain"; + process.env.MAINNET_RPC_URL = "https://rpc.example/mainnet"; + + const cfg = loadConfig(); + + expect(cfg.activeNetworks).toEqual(["calibration", "mainnet"]); + expect(cfg.networks.mainnet.rpcUrl).toBe("https://rpc.example/mainnet"); + }); + + it("does not throw when an inactive network lacks wallet keys", () => { + // Pre-refactor, the loader iterated all SUPPORTED_NETWORKS and threw on + // missing keys for inactive networks — operators had to set keys for both. + process.env.NETWORKS = "calibration"; + process.env.CALIBRATION_WALLET_PRIVATE_KEY = "0xkey"; + // MAINNET_WALLET_PRIVATE_KEY intentionally unset. + + expect(() => loadConfig()).not.toThrow(); + }); +}); diff --git a/apps/backend/src/config/loader.ts b/apps/backend/src/config/loader.ts new file mode 100644 index 00000000..6ca912c4 --- /dev/null +++ b/apps/backend/src/config/loader.ts @@ -0,0 +1,265 @@ +/** + * Configuration loaders. + * + * Each `load*Config` function reads `process.env` (or an injected map) and + * returns a fully-typed config slice. `loadConfig` assembles all slices into + * the top-level `IConfig` object consumed by NestJS `ConfigModule`. + * + */ + +import { DEFAULT_LOCAL_DATASETS_PATH, ZERO_ADDRESS } from "../common/constants.js"; +import type { Network } from "../common/types.js"; +import { networkDefaults } from "./constants.js"; +import { getBooleanEnv, getFloatEnv, getNumberEnv, getStringEnv } from "./env.helpers.js"; +import { + parseActiveNetworks, + parseAddressList, + parseIdList, + parseRandomDatasetSizes, + parseRunMode, +} from "./env.parsers.js"; +import type { + BaseNetworkConfig, + IAppConfig, + IConfig, + IDatabaseConfig, + IDatasetConfig, + IJobsConfig, + INetworkConfig, + IRetrievalConfig, + ITimeoutConfig, +} from "./types.js"; + +// --------------------------------------------------------------------------- +// Per-section loaders +// --------------------------------------------------------------------------- + +const loadAppConfig = (env: NodeJS.ProcessEnv): IAppConfig => ({ + env: getStringEnv(env, "NODE_ENV", "development"), + runMode: parseRunMode(env), + port: getNumberEnv(env, "DEALBOT_PORT", 3000), + host: getStringEnv(env, "DEALBOT_HOST", "127.0.0.1"), + apiPublicUrl: env.DEALBOT_API_PUBLIC_URL || undefined, + metricsPort: getNumberEnv(env, "DEALBOT_METRICS_PORT", 9090), + metricsHost: getStringEnv(env, "DEALBOT_METRICS_HOST", "0.0.0.0"), + enableDevMode: env.ENABLE_DEV_MODE === "true", + prometheusWalletBalanceTtlSeconds: getNumberEnv(env, "PROMETHEUS_WALLET_BALANCE_TTL_SECONDS", 3600), + prometheusWalletBalanceErrorCooldownSeconds: getNumberEnv( + env, + "PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS", + 60, + ), + probeLocation: getStringEnv(env, "DEALBOT_PROBE_LOCATION", "unknown"), +}); + +const loadDatabaseConfig = (env: NodeJS.ProcessEnv): IDatabaseConfig => ({ + host: getStringEnv(env, "DATABASE_HOST", "localhost"), + port: getNumberEnv(env, "DATABASE_PORT", 5432), + poolMax: getNumberEnv(env, "DATABASE_POOL_MAX", 1), + username: getStringEnv(env, "DATABASE_USER", "dealbot"), + password: getStringEnv(env, "DATABASE_PASSWORD", "dealbot_password"), + database: getStringEnv(env, "DATABASE_NAME", "filecoin_dealbot"), +}); + +const loadJobsConfig = (env: NodeJS.ProcessEnv): IJobsConfig => ({ + schedulerPollSeconds: getNumberEnv(env, "JOB_SCHEDULER_POLL_SECONDS", 300), + workerPollSeconds: getNumberEnv(env, "JOB_WORKER_POLL_SECONDS", 60), + pgbossLocalConcurrency: getNumberEnv(env, "PG_BOSS_LOCAL_CONCURRENCY", 20), + pgbossSchedulerEnabled: getBooleanEnv(env, "DEALBOT_PGBOSS_SCHEDULER_ENABLED", true), + pgbossPoolMax: getNumberEnv(env, "DEALBOT_PGBOSS_POOL_MAX", 1), + catchupMaxEnqueue: getNumberEnv(env, "JOB_CATCHUP_MAX_ENQUEUE", 10), + schedulePhaseSeconds: getNumberEnv(env, "JOB_SCHEDULE_PHASE_SECONDS", 0), + enqueueJitterSeconds: getNumberEnv(env, "JOB_ENQUEUE_JITTER_SECONDS", 0), + shutdownFinalScrapeDelaySeconds: getNumberEnv(env, "SHUTDOWN_FINAL_SCRAPE_DELAY_SECONDS", 35), +}); + +const loadDatasetConfig = (env: NodeJS.ProcessEnv): IDatasetConfig => ({ + localDatasetsPath: getStringEnv(env, "DEALBOT_LOCAL_DATASETS_PATH", DEFAULT_LOCAL_DATASETS_PATH), + randomDatasetSizes: parseRandomDatasetSizes(env), +}); + +const loadTimeoutConfig = (env: NodeJS.ProcessEnv): ITimeoutConfig => ({ + connectTimeoutMs: getNumberEnv(env, "CONNECT_TIMEOUT_MS", 10000), + httpRequestTimeoutMs: getNumberEnv(env, "HTTP_REQUEST_TIMEOUT_MS", 240000), + http2RequestTimeoutMs: getNumberEnv(env, "HTTP2_REQUEST_TIMEOUT_MS", 240000), + ipniVerificationTimeoutMs: getNumberEnv(env, "IPNI_VERIFICATION_TIMEOUT_MS", 60000), + ipniVerificationPollingMs: getNumberEnv(env, "IPNI_VERIFICATION_POLLING_MS", 2000), +}); + +const loadRetrievalConfig = (env: NodeJS.ProcessEnv): IRetrievalConfig => ({ + ipfsBlockFetchConcurrency: getNumberEnv(env, "IPFS_BLOCK_FETCH_CONCURRENCY", 6), +}); + +// --------------------------------------------------------------------------- +// Per-network loader +// --------------------------------------------------------------------------- + +type DistributiveOmit = T extends any ? Omit : never; + +/** + * Reads all per-network env vars for one network's prefix + * (e.g. `"CALIBRATION"` → reads `CALIBRATION_RPC_URL`, `CALIBRATION_WALLET_ADDRESS`, ...). + * + * Legacy (unprefixed) envs are translated to this prefixed form by + * `applyLegacyEnvCompat` before `loadConfig` runs, so this function only + * needs to handle the prefixed scheme. + */ +function loadNetworkEnvPrefix( + prefix: Uppercase, + env: NodeJS.ProcessEnv, +): DistributiveOmit { + const k = (key: string) => `${prefix}_${key}`; + const get = (key: string) => env[k(key)]; + + const base = { + walletAddress: get("WALLET_ADDRESS") || ZERO_ADDRESS, + rpcUrl: get("RPC_URL") || undefined, + pdpSubgraphEndpoint: get("PDP_SUBGRAPH_ENDPOINT") || undefined, + checkDatasetCreationFees: getBooleanEnv( + env, + k("CHECK_DATASET_CREATION_FEES"), + networkDefaults.checkDatasetCreationFees, + ), + useOnlyApprovedProviders: getBooleanEnv( + env, + k("USE_ONLY_APPROVED_PROVIDERS"), + networkDefaults.useOnlyApprovedProviders, + ), + dealbotDataSetVersion: get("DEALBOT_DATASET_VERSION") || undefined, + minNumDataSetsForChecks: getNumberEnv( + env, + k("MIN_NUM_DATASETS_FOR_CHECKS"), + networkDefaults.minNumDataSetsForChecks, + ), + dealsPerSpPerHour: getFloatEnv(env, k("DEALS_PER_SP_PER_HOUR"), networkDefaults.dealsPerSpPerHour), + dealJobTimeoutSeconds: getNumberEnv(env, "DEAL_JOB_TIMEOUT_SECONDS", 360), + retrievalsPerSpPerHour: getFloatEnv(env, k("RETRIEVALS_PER_SP_PER_HOUR"), networkDefaults.retrievalsPerSpPerHour), + retrievalJobTimeoutSeconds: getNumberEnv(env, "RETRIEVAL_JOB_TIMEOUT_SECONDS", 60), + dataSetCreationsPerSpPerHour: getFloatEnv( + env, + k("DATASET_CREATIONS_PER_SP_PER_HOUR"), + networkDefaults.dataSetCreationsPerSpPerHour, + ), + dataSetCreationJobTimeoutSeconds: getNumberEnv(env, "DATA_SET_CREATION_JOB_TIMEOUT_SECONDS", 300), + dataRetentionPollIntervalSeconds: getNumberEnv( + env, + k("DATA_RETENTION_POLL_INTERVAL_SECONDS"), + networkDefaults.dataRetentionPollIntervalSeconds, + ), + providersRefreshIntervalSeconds: getNumberEnv( + env, + k("PROVIDERS_REFRESH_INTERVAL_SECONDS"), + networkDefaults.providersRefreshIntervalSeconds, + ), + pieceCleanupPerSpPerHour: getFloatEnv( + env, + k("PIECE_CLEANUP_PER_SP_PER_HOUR"), + networkDefaults.pieceCleanupPerSpPerHour, + ), + maxPieceCleanupRuntimeSeconds: getNumberEnv( + env, + k("MAX_PIECE_CLEANUP_RUNTIME_SECONDS"), + networkDefaults.maxPieceCleanupRuntimeSeconds, + ), + maxDatasetStorageSizeBytes: getNumberEnv( + env, + k("MAX_DATASET_STORAGE_SIZE_BYTES"), + networkDefaults.maxDatasetStorageSizeBytes, + ), + targetDatasetStorageSizeBytes: getNumberEnv( + env, + k("TARGET_DATASET_STORAGE_SIZE_BYTES"), + networkDefaults.targetDatasetStorageSizeBytes, + ), + + maintenanceWindowsUtc: get("MAINTENANCE_WINDOWS_UTC") + ? get("MAINTENANCE_WINDOWS_UTC")! + .split(",") + .map((v) => v.trim()) + .filter((v) => v.length > 0) + : networkDefaults.maintenanceWindowsUtc, + maintenanceWindowMinutes: getNumberEnv( + env, + k("MAINTENANCE_WINDOW_MINUTES"), + networkDefaults.maintenanceWindowMinutes, + ), + + blockedSpIds: parseIdList(get("BLOCKED_SP_IDS")), + blockedSpAddresses: parseAddressList(get("BLOCKED_SP_ADDRESSES")), + + pullChecksPerSpPerHour: getFloatEnv(env, k("PULL_CHECKS_PER_SP_PER_HOUR"), networkDefaults.pullChecksPerSpPerHour), + pullCheckJobTimeoutSeconds: getNumberEnv( + env, + "PULL_CHECK_JOB_TIMEOUT_SECONDS", + networkDefaults.pullCheckJobTimeoutSeconds, + ), + pullCheckPollIntervalSeconds: getNumberEnv( + env, + "PULL_CHECK_POLL_INTERVAL_SECONDS", + networkDefaults.pullCheckPollIntervalSeconds, + ), + pullCheckPieceSizeBytes: getNumberEnv(env, "PULL_CHECK_PIECE_SIZE_BYTES", networkDefaults.pullCheckPieceSizeBytes), + pullPieceMaxConcurrentStreams: getNumberEnv( + env, + "PULL_PIECE_MAX_CONCURRENT_STREAMS", + networkDefaults.pullPieceMaxConcurrentStreams, + ), + pullPieceMaxStreamsPerCid: getNumberEnv( + env, + "PULL_PIECE_MAX_STREAMS_PER_CID", + networkDefaults.pullPieceMaxStreamsPerCid, + ), + pullPieceCleanupIntervalSeconds: getNumberEnv( + env, + "PULL_PIECE_CLEANUP_INTERVAL_SECONDS", + networkDefaults.pullPieceCleanupIntervalSeconds, + ), + + clickhouseUrl: get("CLICKHOUSE_URL") || undefined, + clickhouseBatchSize: getNumberEnv(env, "CLICKHOUSE_BATCH_SIZE", networkDefaults.clickhouseBatchSize), + clickhouseFlushIntervalMs: getNumberEnv( + env, + "CLICKHOUSE_FLUSH_INTERVAL_MS", + networkDefaults.clickhouseFlushIntervalMs, + ), + clickhouseMaxBufferSize: getNumberEnv(env, "CLICKHOUSE_MAX_BUFFER_SIZE", networkDefaults.clickhouseMaxBufferSize), + } satisfies Omit; + + const walletPrivateKey = (get("WALLET_PRIVATE_KEY") || undefined) as `0x${string}` | undefined; + const sessionKeyPrivateKey = (get("SESSION_KEY_PRIVATE_KEY") || undefined) as `0x${string}` | undefined; + + if (sessionKeyPrivateKey) return { ...base, sessionKeyPrivateKey }; + if (walletPrivateKey) return { ...base, walletPrivateKey }; + + // Joi's .or() constraint on `${prefix}_WALLET_PRIVATE_KEY` / + // `${prefix}_SESSION_KEY_PRIVATE_KEY` ensures this branch is unreachable + throw new Error(`[config] Neither WALLET_PRIVATE_KEY nor SESSION_KEY_PRIVATE_KEY is set for ${prefix}`); +} + +const loadNetworkConfigs = (env: NodeJS.ProcessEnv): Pick => { + const activeNetworks = parseActiveNetworks(env); + const networks = {} as Record; + + for (const network of activeNetworks) { + const prefix = network.toUpperCase() as Uppercase; + networks[network] = { network, ...loadNetworkEnvPrefix(prefix, env) }; + } + + return { networks, activeNetworks }; +}; + +// --------------------------------------------------------------------------- +// Top-level loader (NestJS ConfigModule entry-point) +// --------------------------------------------------------------------------- + +export function loadConfig(): IConfig { + return { + app: loadAppConfig(process.env), + database: loadDatabaseConfig(process.env), + ...loadNetworkConfigs(process.env), + jobs: loadJobsConfig(process.env), + dataset: loadDatasetConfig(process.env), + timeouts: loadTimeoutConfig(process.env), + retrieval: loadRetrievalConfig(process.env), + }; +} diff --git a/apps/backend/src/config/types.ts b/apps/backend/src/config/types.ts new file mode 100644 index 00000000..b34d4128 --- /dev/null +++ b/apps/backend/src/config/types.ts @@ -0,0 +1,277 @@ +import { Network } from "../common/types.js"; + +export interface IAppConfig { + env: string; + runMode: "api" | "worker" | "both"; + port: number; + host: string; + /** + * Optional publicly reachable DealBot API base URL (e.g. `https://dealbot.example.com`). + * Used to construct hosted-piece source URLs that SPs can fetch during pull checks. + * When unset, falls back to `http://${host}:${port}`. + */ + apiPublicUrl: string | undefined; + metricsPort: number; + metricsHost: string; + enableDevMode: boolean; + prometheusWalletBalanceTtlSeconds: number; + prometheusWalletBalanceErrorCooldownSeconds: number; + probeLocation: string; +} + +export interface IDatabaseConfig { + host: string; + port: number; + poolMax: number; + username: string; + password: string; + database: string; +} + +export type BaseNetworkConfig = { + network: Network; + + /** Blockchain Config */ + rpcUrl?: string; + walletAddress: string; + checkDatasetCreationFees: boolean; + useOnlyApprovedProviders: boolean; + dealbotDataSetVersion?: string; + pdpSubgraphEndpoint?: string; + minNumDataSetsForChecks: number; + + /** + * Target number of deal creations per storage provider per hour. + * + * Increasing this increases on-chain activity and dataset uploads. + */ + dealsPerSpPerHour: number; + /** + * Target number of retrieval tests per storage provider per hour. + * + * Increasing this increases retrieval load against providers and DB writes. + */ + retrievalsPerSpPerHour: number; + /** + * Target number of dataset creation runs per storage provider per hour. + */ + dataSetCreationsPerSpPerHour: number; + /** + * Target number of piece cleanup runs per storage provider per hour. + * + * Increasing this makes cleanup more aggressive at the cost of more SP API calls. + */ + pieceCleanupPerSpPerHour: number; + /** + * Maximum runtime (seconds) for deal jobs before forced abort. + * + * Uses AbortController to actively cancel job execution. + */ + dealJobTimeoutSeconds: number; + /** + * Maximum runtime (seconds) for data-set creation jobs before forced abort. + * + * Uses AbortController to actively cancel job execution. + */ + dataSetCreationJobTimeoutSeconds: number; + /** + * Maximum runtime (seconds) for retrieval jobs before forced abort. + * + * Uses AbortController to actively cancel job execution. + */ + retrievalJobTimeoutSeconds: number; + maxPieceCleanupRuntimeSeconds: number; + dataRetentionPollIntervalSeconds: number; + providersRefreshIntervalSeconds: number; + + /** Maintenance Config */ + maintenanceWindowsUtc: string[]; + maintenanceWindowMinutes: number; + + /** Blocked Providers Config */ + blockedSpIds: Set; + blockedSpAddresses: Set; + + /** Piece Cleanup Config */ + maxDatasetStorageSizeBytes: number; + targetDatasetStorageSizeBytes: number; + + /** Pull Piece Config */ + /** + * Target number of pull checks per storage provider per hour. + * + * Pull checks validate the SP pull-to-park pathway by serving a temporary piece URL + * from DealBot and asking the SP to pull and park it. Independent of `deal` and `retrieval`. + */ + pullChecksPerSpPerHour: number; + /** + * Maximum runtime (seconds) for pull-check jobs before forced abort. + * + * Bounds the polling window for terminal SP pull status. + */ + pullCheckJobTimeoutSeconds: number; + /** + * Polling interval (seconds) used while waiting for a terminal SP pull status. + */ + pullCheckPollIntervalSeconds: number; + /** + * Size (bytes) of the synthetic test piece DealBot generates per pull check. + */ + pullCheckPieceSizeBytes: number; + /** + * Maximum number of concurrent piece streams across all pieceCids. + * + * Prevents DoS by limiting total server-wide streaming load. + */ + pullPieceMaxConcurrentStreams: number; + /** + * Maximum number of concurrent streams per pieceCid. + * + * Prevents attackers from opening many connections to the same piece. + */ + pullPieceMaxStreamsPerCid: number; + /** + * How often (seconds) the global `pull_piece_cleanup` job runs to delete + * expired `pull_pieces` rows (those whose `expires_at` is in the past). + * + * Defaults to 7 days (604800 s). Minimum 1 hour enforced by Joi. + */ + pullPieceCleanupIntervalSeconds: number; + + /** Clickhouse Config */ + /** + * ClickHouse connection URL. Must include the database in the path. + * Example: http://default:password@host:8123/dealbot + * If unset, ClickHouse emission is disabled. + */ + clickhouseUrl: string | undefined; + clickhouseBatchSize: number; + clickhouseFlushIntervalMs: number; + clickhouseMaxBufferSize: number; +}; + +type WalletPrivateKeyNetworkConfig = BaseNetworkConfig & { + walletPrivateKey: `0x${string}`; +}; + +type SessionKeyNetworkConfig = BaseNetworkConfig & { + sessionKeyPrivateKey: `0x${string}`; +}; + +export type INetworkConfig = WalletPrivateKeyNetworkConfig | SessionKeyNetworkConfig; + +export type INetworksConfig = Record; + +export interface IJobsConfig { + /** + * How often the scheduler polls Postgres for due jobs (seconds). + * + * Lower values reduce scheduling latency but increase DB chatter. + */ + schedulerPollSeconds: number; + /** + * How often workers check for new jobs (seconds). + * + * Lower values reduce job pickup latency but increase DB chatter. + */ + workerPollSeconds: number; + /** + * Per-instance pg-boss worker concurrency for the `sp.work` queue. + */ + pgbossLocalConcurrency: number; + /** + * Enables the pg-boss scheduler loop (enqueueing due jobs). + * + * Set to false to run "worker-only" pods that only process existing jobs. + */ + pgbossSchedulerEnabled: boolean; + /** + * Maximum number of pg-boss connections per instance. + * + * Helpful when using a session-mode pooler with a low pool_size (e.g. Supabase). + */ + pgbossPoolMax: number; + /** + * Maximum number of jobs to enqueue per schedule row per poll. + * + * Prevents large backlogs from flooding workers after downtime. + */ + catchupMaxEnqueue: number; + /** + * Per-instance phase offset (seconds) applied when initializing schedules. + * + * Use this to stagger multiple dealbot deployments that are not sharing a DB. + */ + schedulePhaseSeconds: number; + /** + * Random delay (seconds) added when enqueuing jobs. + * + * Helps avoid synchronized bursts across instances. Only used with pg-boss. + */ + enqueueJitterSeconds: number; + /** + * Seconds to hold the process alive after pg-boss drain finishes, so Prometheus + * scrapes the terminal counter increments emitted during shutdown. + */ + shutdownFinalScrapeDelaySeconds: number; +} + +export interface IDatasetConfig { + localDatasetsPath: string; + randomDatasetSizes: number[]; +} + +export interface ITimeoutConfig { + connectTimeoutMs: number; + httpRequestTimeoutMs: number; + http2RequestTimeoutMs: number; + ipniVerificationTimeoutMs: number; + ipniVerificationPollingMs: number; +} + +export interface IRetrievalConfig { + ipfsBlockFetchConcurrency: number; +} + +export interface IConfig { + app: IAppConfig; + database: IDatabaseConfig; + networks: INetworksConfig; + activeNetworks: Network[]; + jobs: IJobsConfig; + dataset: IDatasetConfig; + timeouts: ITimeoutConfig; + retrieval: IRetrievalConfig; +} + +export type NetworkDefaults = Pick< + INetworkConfig, + | "dealbotDataSetVersion" + | "checkDatasetCreationFees" + | "useOnlyApprovedProviders" + | "minNumDataSetsForChecks" + | "dealsPerSpPerHour" + | "dealJobTimeoutSeconds" + | "retrievalsPerSpPerHour" + | "retrievalJobTimeoutSeconds" + | "dataSetCreationsPerSpPerHour" + | "dataSetCreationJobTimeoutSeconds" + | "pieceCleanupPerSpPerHour" + | "maxPieceCleanupRuntimeSeconds" + | "dataRetentionPollIntervalSeconds" + | "providersRefreshIntervalSeconds" + | "maintenanceWindowsUtc" + | "maintenanceWindowMinutes" + | "maxDatasetStorageSizeBytes" + | "targetDatasetStorageSizeBytes" + | "pullChecksPerSpPerHour" + | "pullCheckJobTimeoutSeconds" + | "pullCheckPollIntervalSeconds" + | "pullCheckPieceSizeBytes" + | "pullPieceMaxConcurrentStreams" + | "pullPieceMaxStreamsPerCid" + | "pullPieceCleanupIntervalSeconds" + | "clickhouseBatchSize" + | "clickhouseFlushIntervalMs" + | "clickhouseMaxBufferSize" +>; From 274a932140db48d4c635e3ae82a0d43a356bd946 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 12:47:54 +0530 Subject: [PATCH 02/23] chore: migrate to multi network --- apps/backend/src/app.controller.ts | 22 +- apps/backend/src/app.module.ts | 6 +- apps/backend/src/common/synapse-factory.ts | 8 +- .../src/wallet-sdk/wallet-sdk.service.spec.ts | 149 +++++------ .../src/wallet-sdk/wallet-sdk.service.ts | 240 +++++++++++------- apps/backend/src/worker.module.ts | 6 +- 6 files changed, 245 insertions(+), 186 deletions(-) diff --git a/apps/backend/src/app.controller.ts b/apps/backend/src/app.controller.ts index ae9f337a..36508e48 100644 --- a/apps/backend/src/app.controller.ts +++ b/apps/backend/src/app.controller.ts @@ -1,6 +1,7 @@ import { Controller, Get } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; -import { type IBlockchainConfig, type IConfig, IJobsConfig } from "./config/app.config.js"; +import type { Network } from "./common/types.js"; +import type { IConfig, INetworksConfig } from "./config/index.js"; @Controller("api") export class AppController { @@ -21,16 +22,19 @@ export class AppController { */ @Get("config") getConfig() { - const blockchain = this.configService.get("blockchain"); - const jobs = this.configService.get("jobs"); + const activeNetworks = this.configService.get("activeNetworks"); + const networks = this.configService.get("networks"); return { - network: blockchain.network, - jobs: { - dealsPerSpPerHour: jobs.dealsPerSpPerHour, - dataSetCreationsPerSpPerHour: jobs.dataSetCreationsPerSpPerHour, - retrievalsPerSpPerHour: jobs.retrievalsPerSpPerHour, - }, + networks: activeNetworks.map((n) => ({ + network: n, + dealsPerSpPerHour: networks[n].dealsPerSpPerHour, + dataSetCreationsPerSpPerHour: networks[n].dataSetCreationsPerSpPerHour, + retrievalsPerSpPerHour: networks[n].retrievalsPerSpPerHour, + pullChecksPerSpPerHour: networks[n].pullChecksPerSpPerHour, + dataRetentionPollIntervalSeconds: networks[n].dataRetentionPollIntervalSeconds, + providersRefreshIntervalSeconds: networks[n].providersRefreshIntervalSeconds, + })), }; } } diff --git a/apps/backend/src/app.module.ts b/apps/backend/src/app.module.ts index ed29e2ce..57e4bfc1 100644 --- a/apps/backend/src/app.module.ts +++ b/apps/backend/src/app.module.ts @@ -4,7 +4,7 @@ import { LoggerModule } from "nestjs-pino"; import { AppController } from "./app.controller.js"; import { ClickhouseModule } from "./clickhouse/clickhouse.module.js"; import { buildLoggerModuleParams } from "./common/pino.config.js"; -import { configValidationSchema, loadConfig } from "./config/app.config.js"; +import { loadConfig, validateConfig } from "./config/index.js"; import { DatabaseModule } from "./database/database.module.js"; import { DataSourceModule } from "./dataSource/dataSource.module.js"; import { DealModule } from "./deal/deal.module.js"; @@ -19,9 +19,9 @@ import { RetrievalModule } from "./retrieval/retrieval.module.js"; imports: [ LoggerModule.forRoot(buildLoggerModuleParams()), ConfigModule.forRoot({ - load: [loadConfig], - validationSchema: configValidationSchema, isGlobal: true, + load: [loadConfig], + validate: validateConfig, }), DatabaseModule, MetricsPrometheusModule, diff --git a/apps/backend/src/common/synapse-factory.ts b/apps/backend/src/common/synapse-factory.ts index d939c44d..35fe1595 100644 --- a/apps/backend/src/common/synapse-factory.ts +++ b/apps/backend/src/common/synapse-factory.ts @@ -28,7 +28,7 @@ import * as SessionKey from "@filoz/synapse-core/session-key"; import { calibration, mainnet, Synapse } from "@filoz/synapse-sdk"; import { createClient, custom, http } from "viem"; import { privateKeyToAccount } from "viem/accounts"; -import type { IBlockchainConfig } from "../config/app.config.js"; +import type { INetworkConfig } from "../config/index.js"; export interface SynapseInstanceResult { synapse: Synapse; @@ -36,13 +36,13 @@ export interface SynapseInstanceResult { } /** Create a Synapse instance from blockchain config. See file-level docs. */ -export async function createSynapseFromConfig(config: IBlockchainConfig): Promise { +export async function createSynapseFromConfig(config: INetworkConfig): Promise { const chain = config.network === "mainnet" ? mainnet : calibration; const rpcUrl = config.rpcUrl; const transport = rpcUrl ? http(rpcUrl) : http(); - const sessionKeyPK = config.sessionKeyPrivateKey; - if (sessionKeyPK) { + if ("sessionKeyPrivateKey" in config) { + const sessionKeyPK = config.sessionKeyPrivateKey; const walletAddress = config.walletAddress as `0x${string}`; const sessionKey = SessionKey.fromSecp256k1({ privateKey: sessionKeyPK, diff --git a/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts b/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts index ed114f53..bf5a50b5 100644 --- a/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts +++ b/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts @@ -1,7 +1,7 @@ import type { ConfigService } from "@nestjs/config"; import { stringToHex } from "viem"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { configValidationSchema, type IBlockchainConfig, type IConfig } from "../config/app.config.js"; +import type { IConfig } from "../config/index.js"; import { WalletSdkService } from "./wallet-sdk.service.js"; import type { PDPProviderEx } from "./wallet-sdk.types.js"; @@ -11,15 +11,42 @@ type LoggerLike = { log: (message: string) => void; }; -const baseConfig: IBlockchainConfig = { - network: "calibration", +const baseNetworkConfig = { + network: "calibration" as const, walletAddress: "0x0000000000000000000000000000000000000000", - walletPrivateKey: "0xtest", + walletPrivateKey: "0xtest" as `0x${string}`, checkDatasetCreationFees: false, useOnlyApprovedProviders: false, minNumDataSetsForChecks: 1, pdpSubgraphEndpoint: "https://api.thegraph.com/subgraphs/filecoin/pdp", -}; + dealsPerSpPerHour: 4, + retrievalsPerSpPerHour: 2, + dataSetCreationsPerSpPerHour: 1, + dataRetentionPollIntervalSeconds: 3600, + providersRefreshIntervalSeconds: 14400, + maintenanceWindowsUtc: ["07:00", "22:00"], + maintenanceWindowMinutes: 20, + pieceCleanupPerSpPerHour: 1, + maxPieceCleanupRuntimeSeconds: 300, + maxDatasetStorageSizeBytes: 24 * 1024 * 1024 * 1024, + targetDatasetStorageSizeBytes: 20 * 1024 * 1024 * 1024, + blockedSpIds: new Set(), + blockedSpAddresses: new Set(), + dealJobTimeoutSeconds: 300, + dataSetCreationJobTimeoutSeconds: 300, + retrievalJobTimeoutSeconds: 300, + pullChecksPerSpPerHour: 1, + pullCheckJobTimeoutSeconds: 300, + pullCheckPollIntervalSeconds: 2, + pullCheckPieceSizeBytes: 10 * 1024 * 1024, // 10 MiB + pullPieceMaxConcurrentStreams: 50, + pullPieceMaxStreamsPerCid: 3, + pullPieceCleanupIntervalSeconds: 7 * 24 * 3600, // 7 days + clickhouseUrl: "http://localhost:8123", + clickhouseBatchSize: 500, + clickhouseFlushIntervalMs: 5000, + clickhouseMaxBufferSize: 5000, +} satisfies IConfig["networks"]["calibration"]; const makeProvider = (overrides: Partial): PDPProviderEx => ({ @@ -37,57 +64,6 @@ const makeProvider = (overrides: Partial): PDPProviderEx => ...overrides, }) as PDPProviderEx; -describe("config validation", () => { - const requiredEnv = { - DATABASE_HOST: "localhost", - DATABASE_USER: "test", - DATABASE_PASSWORD: "test", - DATABASE_NAME: "test", - WALLET_ADDRESS: "0x1234567890123456789012345678901234567890", - NETWORK: "calibration", - }; - - it("requires WALLET_PRIVATE_KEY when SESSION_KEY_PRIVATE_KEY is absent", () => { - const { error } = configValidationSchema.validate(requiredEnv, { allowUnknown: true }); - expect(error).toBeDefined(); - expect(error?.message).toMatch(/WALLET_PRIVATE_KEY/); - }); - - it("accepts missing WALLET_PRIVATE_KEY when SESSION_KEY_PRIVATE_KEY is set", () => { - const { error } = configValidationSchema.validate( - { ...requiredEnv, SESSION_KEY_PRIVATE_KEY: "0xdeadbeef" }, - { allowUnknown: true }, - ); - expect(error).toBeUndefined(); - }); - - it("accepts both WALLET_PRIVATE_KEY and SESSION_KEY_PRIVATE_KEY (session key takes precedence)", () => { - const { error } = configValidationSchema.validate( - { ...requiredEnv, WALLET_PRIVATE_KEY: "0xkey", SESSION_KEY_PRIVATE_KEY: "0xsession" }, - { allowUnknown: true }, - ); - expect(error).toBeUndefined(); - }); - - it("treats empty string WALLET_PRIVATE_KEY as absent", () => { - const { error } = configValidationSchema.validate( - { ...requiredEnv, WALLET_PRIVATE_KEY: "" }, - { allowUnknown: true }, - ); - // Empty string is normalized to undefined by .empty(""), so .or() treats it as absent - expect(error).toBeDefined(); - expect(error?.message).toMatch(/WALLET_PRIVATE_KEY|SESSION_KEY_PRIVATE_KEY/); - }); - - it("treats empty string SESSION_KEY_PRIVATE_KEY as absent", () => { - const { error } = configValidationSchema.validate( - { ...requiredEnv, WALLET_PRIVATE_KEY: "0xkey", SESSION_KEY_PRIVATE_KEY: "" }, - { allowUnknown: true }, - ); - expect(error).toBeUndefined(); - }); -}); - describe("WalletSdkService", () => { let service: WalletSdkService; let repoMock: { create: ReturnType; upsert: ReturnType }; @@ -100,7 +76,11 @@ describe("WalletSdkService", () => { }; const configService = { - get: vi.fn((key: keyof IConfig) => (key === "blockchain" ? baseConfig : undefined)), + get: vi.fn((key: keyof IConfig) => { + if (key === "activeNetworks") return ["calibration"]; + if (key === "networks") return { calibration: baseNetworkConfig }; + return undefined; + }), } as unknown as ConfigService; service = new WalletSdkService(configService, repoMock as any); @@ -151,8 +131,8 @@ describe("WalletSdkService", () => { expect(options).toEqual(expect.objectContaining({ conflictPaths: ["address", "network"] })); expect(entities).toEqual( expect.arrayContaining([ - expect.objectContaining({ network: "calibration", address: "0xdup", providerId: 21n, name: "new" }), - expect.objectContaining({ network: "calibration", address: "0xother", providerId: 22n }), + expect.objectContaining({ address: "0xdup", network: "calibration", providerId: 21n, name: "new" }), + expect.objectContaining({ address: "0xother", network: "calibration", providerId: 22n }), ]), ); }); @@ -186,7 +166,9 @@ describe("WalletSdkService", () => { const [entities] = repoMock.upsert.mock.calls[0]; expect(entities).toEqual( - expect.arrayContaining([expect.objectContaining({ address: "0xdup2", providerId: 30n, name: "active" })]), + expect.arrayContaining([ + expect.objectContaining({ address: "0xdup2", network: "calibration", providerId: 30n, name: "active" }), + ]), ); }); @@ -216,7 +198,9 @@ describe("WalletSdkService", () => { const [entities] = repoMock.upsert.mock.calls[0]; expect(entities).toEqual( - expect.arrayContaining([expect.objectContaining({ address: "0xdup3", providerId: 41n, name: "second" })]), + expect.arrayContaining([ + expect.objectContaining({ address: "0xdup3", network: "calibration", providerId: 41n, name: "second" }), + ]), ); }); @@ -226,10 +210,16 @@ describe("WalletSdkService", () => { resolveLoad = resolve; }); const loadProvidersInternal = vi.fn(() => loadPromise); + // Inject a mock network state for calibration + const mockState = { + providersLoadedOnce: false, + providersLoadPromise: null, + }; + (service as any).networkStates.set("calibration", mockState); (service as any).loadProvidersInternal = loadProvidersInternal; - const first = service.ensureProvidersLoaded(); - const second = service.ensureProvidersLoaded(); + const first = service.ensureProvidersLoaded("calibration"); + const second = service.ensureProvidersLoaded("calibration"); expect(loadProvidersInternal).toHaveBeenCalledTimes(1); @@ -237,34 +227,49 @@ describe("WalletSdkService", () => { await Promise.all([first, second]); expect(loadProvidersInternal).toHaveBeenCalledTimes(1); - expect((service as any).providersLoadedOnce).toBe(true); + expect(mockState.providersLoadedOnce).toBe(true); }); it("retries ensureProvidersLoaded after a failed load", async () => { const loadProvidersInternal = vi.fn().mockResolvedValueOnce(false).mockResolvedValueOnce(true); + const mockState = { + providersLoadedOnce: false, + providersLoadPromise: null, + }; + (service as any).networkStates.set("calibration", mockState); (service as any).loadProvidersInternal = loadProvidersInternal; - await service.ensureProvidersLoaded(); - await service.ensureProvidersLoaded(); + await service.ensureProvidersLoaded("calibration"); + await service.ensureProvidersLoaded("calibration"); expect(loadProvidersInternal).toHaveBeenCalledTimes(2); - expect((service as any).providersLoadedOnce).toBe(true); + expect(mockState.providersLoadedOnce).toBe(true); }); describe("ensureWalletAllowances", () => { it("performs read-only check in session key mode", async () => { - (service as any)._isSessionKeyMode = true; - // getUploadCosts needs _synapseClient but will fail without a real RPC + const mockState = { + isSessionKeyMode: true, + synapseClient: null, + config: baseNetworkConfig, + }; + (service as any).networkStates.set("calibration", mockState); + // getUploadCosts needs synapseClient but will fail without a real RPC // Verify it doesn't fall through to the storageManager.prepare path (service as any)._synapseClient = null; - await expect(service.ensureWalletAllowances()).rejects.toThrow(); + await expect(service.ensureWalletAllowances("calibration")).rejects.toThrow(); // storageManager.prepare was never called (it would also throw, but differently) }); it("attempts allowances in direct key mode", async () => { - (service as any)._isSessionKeyMode = false; + const mockState = { + isSessionKeyMode: false, + storageManager: undefined, + config: baseNetworkConfig, + }; + (service as any).networkStates.set("calibration", mockState); // storageManager is not initialized so prepare() will throw - await expect(service.ensureWalletAllowances()).rejects.toThrow(); + await expect(service.ensureWalletAllowances("calibration")).rejects.toThrow(); }); }); diff --git a/apps/backend/src/wallet-sdk/wallet-sdk.service.ts b/apps/backend/src/wallet-sdk/wallet-sdk.service.ts index e5d8af45..c6d2e9fd 100644 --- a/apps/backend/src/wallet-sdk/wallet-sdk.service.ts +++ b/apps/backend/src/wallet-sdk/wallet-sdk.service.ts @@ -1,4 +1,4 @@ -import { PDPProvider } from "@filoz/synapse-sdk"; +import { PDPProvider, Synapse } from "@filoz/synapse-sdk"; import type { PaymentsService } from "@filoz/synapse-sdk/payments"; import { SPRegistryService } from "@filoz/synapse-sdk/sp-registry"; import { StorageManager } from "@filoz/synapse-sdk/storage"; @@ -12,36 +12,39 @@ import { type Hex } from "viem"; import { DEV_TAG } from "../common/constants.js"; import { toStructuredError } from "../common/logging.js"; import { createSynapseFromConfig } from "../common/synapse-factory.js"; -import { Network } from "../common/types.js"; -import type { IBlockchainConfig, IConfig } from "../config/app.config.js"; +import type { Network } from "../common/types.js"; +import type { IConfig, INetworkConfig } from "../config/index.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; import type { PDPProviderEx, WalletServices } from "./wallet-sdk.types.js"; export type SynapseViemClient = Client; +interface NetworkState { + config: INetworkConfig; + synapse: Synapse; + paymentsService: PaymentsService; + warmStorageService: WarmStorageService; + spRegistry: SPRegistryService; + storageManager: StorageManager; + synapseClient: SynapseViemClient; + isSessionKeyMode: boolean; + providerCache: Map; + activeProviderAddresses: Set; + approvedProviderAddresses: Set; + providersLoadPromise: Promise | null; + providersLoadedOnce: boolean; +} + @Injectable() export class WalletSdkService implements OnModuleInit { private readonly logger = new Logger(WalletSdkService.name); - private readonly blockchainConfig: IBlockchainConfig; - private paymentsService: PaymentsService; - private warmStorageService: WarmStorageService; - private spRegistry: SPRegistryService; - private storageManager: StorageManager; - private providerCache: Map = new Map(); - private activeProviderAddresses: Set = new Set(); - private approvedProviderAddresses: Set = new Set(); - private providersLoadPromise: Promise | null = null; - private providersLoadedOnce = false; - private _isSessionKeyMode = false; - private _synapseClient: any; + private readonly networkStates: Map = new Map(); constructor( private readonly configService: ConfigService, @InjectRepository(StorageProvider) private readonly spRepository: Repository, - ) { - this.blockchainConfig = this.configService.get("blockchain"); - } + ) {} async onModuleInit() { if (process.env.DEALBOT_DISABLE_CHAIN === "true") { @@ -52,35 +55,53 @@ export class WalletSdkService implements OnModuleInit { }); return; } - await this.initializeServices(); - await this.ensureProvidersLoaded(); + const activeNetworks = this.configService.get("activeNetworks"); + for (const network of activeNetworks) { + await this.initializeServicesForNetwork(network); + await this.ensureProvidersLoaded(network); + } } /** - * Initialize wallet services with provider and signer + * Initialize wallet services for a specific network. */ - private async initializeServices(): Promise { - const { synapse, isSessionKeyMode } = await createSynapseFromConfig(this.blockchainConfig); + private async initializeServicesForNetwork(network: Network): Promise { + const networkConfig = this.configService.get("networks")[network]; + const { synapse, isSessionKeyMode } = await createSynapseFromConfig(networkConfig); this.logger.log({ event: "wallet_sdk_initialized", message: isSessionKeyMode ? "Initialized wallet SDK services (session key mode)" : "Initialized wallet SDK services", - network: this.blockchainConfig.network, - walletAddress: this.blockchainConfig.walletAddress, + network, + walletAddress: networkConfig.walletAddress, }); - this.warmStorageService = new WarmStorageService({ - client: synapse.client, - }); - this.spRegistry = new SPRegistryService({ - client: synapse.client, + this.networkStates.set(network, { + synapse, + isSessionKeyMode, + config: networkConfig, + paymentsService: synapse.payments, + warmStorageService: new WarmStorageService({ client: synapse.client }), + spRegistry: new SPRegistryService({ client: synapse.client }), + storageManager: synapse.storage, + synapseClient: synapse.client, + providerCache: new Map(), + activeProviderAddresses: new Set(), + approvedProviderAddresses: new Set(), + providersLoadPromise: null, + providersLoadedOnce: false, }); - this.paymentsService = synapse.payments; - this.storageManager = synapse.storage; - this._synapseClient = synapse.client; - this._isSessionKeyMode = isSessionKeyMode; + } + + private getNetworkState(network: Network): NetworkState { + const target = network; + const state = this.networkStates.get(target); + if (!state) { + throw new Error(`No initialized state for network "${target}". Ensure NETWORKS includes this network.`); + } + return state; } /** @@ -88,42 +109,45 @@ export class WalletSdkService implements OnModuleInit { * This allows dealbot to test all FWSS SPs, even those not yet approved * Only loads active, approved providers that support the PDP product */ - async loadProviders(): Promise { - if (this.providersLoadPromise) { - await this.providersLoadPromise; + async loadProviders(network: Network): Promise { + const state = this.getNetworkState(network); + if (state.providersLoadPromise) { + await state.providersLoadPromise; return; } - this.providersLoadPromise = this.loadProvidersInternal(); + state.providersLoadPromise = this.loadProvidersInternal(network); try { - const success = await this.providersLoadPromise; + const success = await state.providersLoadPromise; if (success) { - this.providersLoadedOnce = true; + state.providersLoadedOnce = true; } } finally { - this.providersLoadPromise = null; + state.providersLoadPromise = null; } } - async ensureProvidersLoaded(): Promise { - if (this.providersLoadedOnce) { + async ensureProvidersLoaded(network: Network): Promise { + const state = this.getNetworkState(network); + if (state.providersLoadedOnce) { return; } - await this.loadProviders(); + await this.loadProviders(network); } - private async loadProvidersInternal(): Promise { + private async loadProvidersInternal(network: Network): Promise { + const state = this.getNetworkState(network); try { this.logger.log({ event: "providers_load_started", message: "Loading all service providers from sp-registry", }); - const approvedIds = await this.warmStorageService.getApprovedProviderIds(); + const approvedIds = await state.warmStorageService.getApprovedProviderIds(); - const totalProviders = await this.spRegistry.getProviderCount(); + const totalProviders = await state.spRegistry.getProviderCount(); - const activeProviders = await this.spRegistry.getAllActiveProviders(); + const activeProviders = await state.spRegistry.getAllActiveProviders(); const activeProviderIds = new Set(activeProviders.map((info) => info.id)); const allProviderIds = Array.from({ length: Number(totalProviders) }, (_, i) => BigInt(i + 1)); const inactiveProviderIds = allProviderIds.filter((id) => !activeProviderIds.has(id)); @@ -134,7 +158,7 @@ export class WalletSdkService implements OnModuleInit { // (empty capabilities), which causes getPDPProvidersByIds to throw. for (const id of inactiveProviderIds) { try { - const provider = await this.spRegistry.getProvider({ providerId: id }); + const provider = await state.spRegistry.getProvider({ providerId: id }); if (provider) { providerInfos.push(provider); } @@ -156,15 +180,16 @@ export class WalletSdkService implements OnModuleInit { message: "Skipping dev provider", providerId: info.id, providerName: info.name, + network, }); return false; } return true; }); - this.providerCache.clear(); - this.activeProviderAddresses.clear(); - this.approvedProviderAddresses.clear(); + state.providerCache.clear(); + state.activeProviderAddresses.clear(); + state.approvedProviderAddresses.clear(); const extendedProviders = validProviders.map((info) => { const supportsIpniIpfs = !!info.pdp.ipniIpfs; const isApproved = approvedIds.includes(info.id); @@ -177,13 +202,14 @@ export class WalletSdkService implements OnModuleInit { providerId: info.id, providerName: info.name, providerAddress: info.serviceProvider, + network, }); } // select approved, active providers - if (info.isActive) this.activeProviderAddresses.add(info.serviceProvider); - if (isApproved && info.isActive) this.approvedProviderAddresses.add(info.serviceProvider); - this.providerCache.set(info.serviceProvider, { + if (info.isActive) state.activeProviderAddresses.add(info.serviceProvider); + if (isApproved && info.isActive) state.approvedProviderAddresses.add(info.serviceProvider); + state.providerCache.set(info.serviceProvider, { ...info, isApproved, }); @@ -194,7 +220,7 @@ export class WalletSdkService implements OnModuleInit { }; }); - this.syncProvidersToDatabase(extendedProviders, this.blockchainConfig.network).catch((err) => + this.syncProvidersToDatabase(extendedProviders, network).catch((err) => this.logger.error({ event: "providers_sync_to_db_failed", message: "Failed to sync providers to DB", @@ -205,9 +231,10 @@ export class WalletSdkService implements OnModuleInit { this.logger.log({ event: "providers_load_completed", message: "Loaded providers from on-chain", - totalProviders: this.providerCache.size, - testingProviders: this.activeProviderAddresses.size, - approvedProviders: this.approvedProviderAddresses.size, + network, + totalProviders: state.providerCache.size, + testingProviders: state.activeProviderAddresses.size, + approvedProviders: state.approvedProviderAddresses.size, }); return true; } catch (error) { @@ -215,11 +242,12 @@ export class WalletSdkService implements OnModuleInit { event: "providers_load_failed", message: "Failed to load registered providers from on-chain", error: toStructuredError(error), + network, }); // Fallback to empty array, let the application handle this gracefully - this.providerCache.clear(); - this.activeProviderAddresses.clear(); - this.approvedProviderAddresses.clear(); + state.providerCache.clear(); + state.activeProviderAddresses.clear(); + state.approvedProviderAddresses.clear(); return false; } } @@ -227,34 +255,36 @@ export class WalletSdkService implements OnModuleInit { /** * Get count of approved providers */ - getApprovedProvidersCount(): number { - return this.approvedProviderAddresses.size; + getApprovedProvidersCount(network: Network): number { + return this.getNetworkState(network).approvedProviderAddresses.size; } /** * Get count of all active providers supporting ipniIpfs */ - getAllActiveProvidersCount(): number { - return this.activeProviderAddresses.size; + getAllActiveProvidersCount(network: Network): number { + return this.getNetworkState(network).activeProviderAddresses.size; } /** * Get count of testing providers */ - getTestingProvidersCount(): number { - return this.blockchainConfig.useOnlyApprovedProviders - ? this.getApprovedProvidersCount() - : this.getAllActiveProvidersCount(); + getTestingProvidersCount(network: Network): number { + const state = this.getNetworkState(network); + return state.config.useOnlyApprovedProviders + ? state.approvedProviderAddresses.size + : state.activeProviderAddresses.size; } /** * Get approved providers */ - getApprovedProviders(): PDPProviderEx[] { + getApprovedProviders(network: Network): PDPProviderEx[] { + const state = this.getNetworkState(network); const approvedProviders: PDPProviderEx[] = []; - for (const address of this.approvedProviderAddresses) { - const provider = this.providerCache.get(address); + for (const address of state.approvedProviderAddresses) { + const provider = state.providerCache.get(address); if (provider) approvedProviders.push(provider); } @@ -264,11 +294,12 @@ export class WalletSdkService implements OnModuleInit { /** * Get all active providers */ - getAllActiveProviders(): PDPProviderEx[] { + getAllActiveProviders(network: Network): PDPProviderEx[] { + const state = this.getNetworkState(network); const activeProviders: PDPProviderEx[] = []; - for (const address of this.activeProviderAddresses) { - const provider = this.providerCache.get(address); + for (const address of state.activeProviderAddresses) { + const provider = state.providerCache.get(address); if (provider) activeProviders.push(provider); } @@ -278,17 +309,21 @@ export class WalletSdkService implements OnModuleInit { /** * Get testing providers */ - getTestingProviders(): PDPProviderEx[] { - return this.blockchainConfig.useOnlyApprovedProviders ? this.getApprovedProviders() : this.getAllActiveProviders(); + getTestingProviders(network: Network): PDPProviderEx[] { + const state = this.getNetworkState(network); + return state.config.useOnlyApprovedProviders + ? this.getApprovedProviders(network) + : this.getAllActiveProviders(network); } /** * Get wallet services (now returns instance variables) */ - getWalletServices(): WalletServices { + getWalletServices(network: Network): WalletServices { + const state = this.getNetworkState(network); return { - paymentsService: this.paymentsService, - warmStorageService: this.warmStorageService, + paymentsService: state.paymentsService, + warmStorageService: state.warmStorageService, }; } @@ -296,9 +331,10 @@ export class WalletSdkService implements OnModuleInit { * Get wallet balances in base units. * USDFC is the available balance in the Filecoin Pay contract (funds minus lockups). */ - async getWalletBalances(): Promise<{ usdfc: bigint; fil: bigint }> { - const accountInfo = await this.paymentsService.accountInfo(); - const filBalance = await this.paymentsService.walletBalance(); + async getWalletBalances(network: Network): Promise<{ usdfc: bigint; fil: bigint }> { + const state = this.getNetworkState(network); + const accountInfo = await state.paymentsService.accountInfo(); + const filBalance = await state.paymentsService.walletBalance(); return { usdfc: accountInfo.availableFunds, fil: filBalance, @@ -306,10 +342,10 @@ export class WalletSdkService implements OnModuleInit { } /** - * Get approved provider info by address + * Get provider info by address for a specific network. */ - getProviderInfo(address: string): PDPProviderEx | undefined { - return this.providerCache.get(address); + getProviderInfo(address: string, network: Network): PDPProviderEx | undefined { + return this.getNetworkState(network).providerCache.get(address); } /** @@ -320,8 +356,12 @@ export class WalletSdkService implements OnModuleInit { * Returns `null` when chain integration is disabled or the client has not been * initialized yet. */ - getSynapseClient(): SynapseViemClient | null { - return (this._synapseClient as SynapseViemClient | null) ?? null; + getSynapseClient(network: Network): SynapseViemClient | null { + return (this.getNetworkState(network).synapseClient as SynapseViemClient | null) ?? null; + } + + getSynapse(network: Network): Synapse { + return this.getNetworkState(network).synapse; } /** @@ -329,11 +369,12 @@ export class WalletSdkService implements OnModuleInit { * Skipped in session key mode, deposits and operator approvals must be * done separately via the Safe multisig UI. */ - async ensureWalletAllowances(): Promise { - if (this._isSessionKeyMode) { + async ensureWalletAllowances(network: Network): Promise { + const state = this.getNetworkState(network); + if (state.isSessionKeyMode) { const { getUploadCosts } = await import("@filoz/synapse-core/warm-storage"); - const costs = await getUploadCosts(this._synapseClient, { - clientAddress: this.blockchainConfig.walletAddress as `0x${string}`, + const costs = await getUploadCosts(state.synapseClient, { + clientAddress: state.config.walletAddress as `0x${string}`, dataSize: 100n * 1024n * 1024n * 1024n, }); @@ -342,6 +383,7 @@ export class WalletSdkService implements OnModuleInit { event: "wallet_status_check_completed", message: "Session key mode: account is funded and approved", costs: this.serializeBigInt(costs), + network, }); } else { this.logger.error({ @@ -351,6 +393,7 @@ export class WalletSdkService implements OnModuleInit { depositNeeded: costs.depositNeeded.toString(), needsApproval: costs.needsFwssMaxApproval, costs: this.serializeBigInt(costs), + network, }); throw new Error( `Session key mode: wallet not ready (depositNeeded=${costs.depositNeeded.toString()}, needsFwssMaxApproval=${costs.needsFwssMaxApproval})`, @@ -359,7 +402,7 @@ export class WalletSdkService implements OnModuleInit { return; } const STORAGE_SIZE_GB = 100n; - const { costs, transaction } = await this.storageManager.prepare({ + const { costs, transaction } = await state.storageManager.prepare({ dataSize: STORAGE_SIZE_GB * 1024n * 1024n * 1024n, }); @@ -368,6 +411,7 @@ export class WalletSdkService implements OnModuleInit { depositAmount: transaction?.depositAmount, includesApproval: transaction?.includesApproval, costs, + network, }); if (transaction) { @@ -376,6 +420,7 @@ export class WalletSdkService implements OnModuleInit { depositAmount: transaction.depositAmount.toString(), includesApproval: transaction?.includesApproval, costs, + network, }); const { hash } = await transaction.execute(); @@ -386,6 +431,7 @@ export class WalletSdkService implements OnModuleInit { depositAmount: transaction.depositAmount.toString(), includesApproval: transaction.includesApproval, costs, + network, }); } } @@ -430,7 +476,7 @@ export class WalletSdkService implements OnModuleInit { } /** - * Create or update provider in database + * Create or update provider in database with network scoping */ async syncProvidersToDatabase(providerInfos: PDPProviderEx[], network: Network): Promise { try { @@ -447,6 +493,7 @@ export class WalletSdkService implements OnModuleInit { event: "duplicate_provider_address", message: "Duplicate provider address detected", address, + network, existingProviderId: existing.id, newProviderId: info.id, }); @@ -492,6 +539,7 @@ export class WalletSdkService implements OnModuleInit { event: "duplicate_provider_addresses_unresolved", message: "Duplicate provider addresses without active/inactive resolution; keeping highest providerId entries", + network, details: formatDetails(conflictAddresses), }); } @@ -501,6 +549,7 @@ export class WalletSdkService implements OnModuleInit { this.logger.warn({ event: "duplicate_provider_addresses_resolved", message: "Duplicate provider addresses detected; replaced inactive entries with active ones", + network, details: formatDetails(resolvedOnly), }); } @@ -531,6 +580,7 @@ export class WalletSdkService implements OnModuleInit { event: "track_providers_failed", message: "Failed to track providers", error: toStructuredError(error), + network, }); throw error; } diff --git a/apps/backend/src/worker.module.ts b/apps/backend/src/worker.module.ts index 9e5cdd62..7e7beaa6 100644 --- a/apps/backend/src/worker.module.ts +++ b/apps/backend/src/worker.module.ts @@ -3,7 +3,7 @@ import { ConfigModule } from "@nestjs/config"; import { LoggerModule } from "nestjs-pino"; import { ClickhouseModule } from "./clickhouse/clickhouse.module.js"; import { buildLoggerModuleParams } from "./common/pino.config.js"; -import { configValidationSchema, loadConfig } from "./config/app.config.js"; +import { loadConfig, validateConfig } from "./config/index.js"; import { DatabaseModule } from "./database/database.module.js"; import { JobsModule } from "./jobs/jobs.module.js"; import { MetricsPrometheusModule } from "./metrics-prometheus/metrics-prometheus.module.js"; @@ -13,9 +13,9 @@ import { PullCheckModule } from "./pull-check/pull-check.module.js"; imports: [ LoggerModule.forRoot(buildLoggerModuleParams()), ConfigModule.forRoot({ - load: [loadConfig], - validationSchema: configValidationSchema, isGlobal: true, + load: [loadConfig], + validate: validateConfig, }), DatabaseModule, MetricsPrometheusModule, From 407c76dee198a1d239aded7fb0d6fe08b979bbc5 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 12:48:23 +0530 Subject: [PATCH 03/23] migrate to multi-network --- .../src/dataSource/dataSource.service.spec.ts | 2 +- .../src/dataSource/dataSource.service.ts | 2 +- apps/backend/src/database/database.module.ts | 2 +- .../deal-addons/strategies/ipni.strategy.ts | 2 +- .../src/http-client/http-client.service.ts | 2 +- .../wallet-balance.collector.spec.ts | 22 ++- .../wallet-balance.collector.ts | 14 +- .../piece-cleanup.service.spec.ts | 128 +++++++++--------- .../piece-cleanup/piece-cleanup.service.ts | 84 +++++------- .../strategies/ipfs-block.strategy.ts | 4 +- .../src/retrieval/retrieval.service.ts | 11 +- 11 files changed, 137 insertions(+), 136 deletions(-) diff --git a/apps/backend/src/dataSource/dataSource.service.spec.ts b/apps/backend/src/dataSource/dataSource.service.spec.ts index 0c850509..b8e656d1 100644 --- a/apps/backend/src/dataSource/dataSource.service.spec.ts +++ b/apps/backend/src/dataSource/dataSource.service.spec.ts @@ -2,7 +2,7 @@ import { ConfigService } from "@nestjs/config"; import * as fs from "fs"; import * as path from "path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { IConfig } from "../config/app.config.js"; +import { IConfig } from "../config/index.js"; import { DataSourceService } from "./dataSource.service.js"; describe("DataSourceService", () => { diff --git a/apps/backend/src/dataSource/dataSource.service.ts b/apps/backend/src/dataSource/dataSource.service.ts index 87fb3964..c2d39b09 100644 --- a/apps/backend/src/dataSource/dataSource.service.ts +++ b/apps/backend/src/dataSource/dataSource.service.ts @@ -7,7 +7,7 @@ import * as path from "path"; import { toStructuredError } from "../common/logging.js"; import { writeWithBackpressure } from "../common/stream-utils.js"; import type { DataFile } from "../common/types.js"; -import type { IConfig, IDatasetConfig } from "../config/app.config.js"; +import type { IConfig, IDatasetConfig } from "../config/index.js"; export interface DeterministicBytesOptions { /** Arbitrary namespace/key to scope the output (e.g. "nonce", "seed:round-1") */ diff --git a/apps/backend/src/database/database.module.ts b/apps/backend/src/database/database.module.ts index 18f67b3c..0a9b6386 100644 --- a/apps/backend/src/database/database.module.ts +++ b/apps/backend/src/database/database.module.ts @@ -6,7 +6,7 @@ import { DataSource, type DataSourceOptions } from "typeorm"; import { fileURLToPath } from "url"; import { toStructuredError } from "../common/logging.js"; import { createPinoExitLogger } from "../common/pino.config.js"; -import type { IAppConfig, IConfig, IDatabaseConfig } from "../config/app.config.js"; +import type { IAppConfig, IConfig, IDatabaseConfig } from "../config/index.js"; import { DataRetentionBaseline } from "./entities/data-retention-baseline.entity.js"; import { Deal } from "./entities/deal.entity.js"; import { JobScheduleState } from "./entities/job-schedule-state.entity.js"; diff --git a/apps/backend/src/deal-addons/strategies/ipni.strategy.ts b/apps/backend/src/deal-addons/strategies/ipni.strategy.ts index f6651145..fdafc210 100644 --- a/apps/backend/src/deal-addons/strategies/ipni.strategy.ts +++ b/apps/backend/src/deal-addons/strategies/ipni.strategy.ts @@ -8,7 +8,7 @@ import type { Repository } from "typeorm"; import { delay } from "../../common/abort-utils.js"; import { buildUnixfsCar } from "../../common/car-utils.js"; import { type DealLogContext, getErrorMessage, toStructuredError } from "../../common/logging.js"; -import type { IConfig } from "../../config/app.config.js"; +import type { IConfig } from "../../config/index.js"; import { Deal } from "../../database/entities/deal.entity.js"; import type { DealMetadata, IpniMetadata } from "../../database/types.js"; import { IpniStatus, ServiceType } from "../../database/types.js"; diff --git a/apps/backend/src/http-client/http-client.service.ts b/apps/backend/src/http-client/http-client.service.ts index 47e9e4ee..9659c4a6 100644 --- a/apps/backend/src/http-client/http-client.service.ts +++ b/apps/backend/src/http-client/http-client.service.ts @@ -7,7 +7,7 @@ import type { AxiosRequestConfig } from "axios"; import { firstValueFrom } from "rxjs"; import { request as undiciRequest } from "undici"; import { toStructuredError } from "../common/logging.js"; -import type { IConfig } from "../config/app.config.js"; +import type { IConfig } from "../config/index.js"; import type { HttpVersion, RequestMetrics, RequestWithMetrics } from "./types.js"; @Injectable() diff --git a/apps/backend/src/metrics-prometheus/wallet-balance.collector.spec.ts b/apps/backend/src/metrics-prometheus/wallet-balance.collector.spec.ts index e61d42a2..9442b0fa 100644 --- a/apps/backend/src/metrics-prometheus/wallet-balance.collector.spec.ts +++ b/apps/backend/src/metrics-prometheus/wallet-balance.collector.spec.ts @@ -1,14 +1,14 @@ import type { ConfigService } from "@nestjs/config"; import type { Gauge } from "prom-client"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { IConfig } from "../config/app.config.js"; +import type { IConfig } from "../config/index.js"; import type { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; import { WalletBalanceCollector } from "./wallet-balance.collector.js"; describe("WalletBalanceCollector", () => { let collector: WalletBalanceCollector; let gaugeMock: { set: ReturnType; collect?: () => Promise }; - let walletSdkMock: { getWalletBalances: ReturnType }; + let walletSdkMock: { getWalletBalances: ReturnType; getNetworkConfig: ReturnType }; let configMock: { get: ReturnType }; let collectFn: () => Promise; @@ -16,6 +16,7 @@ describe("WalletBalanceCollector", () => { gaugeMock = { set: vi.fn() }; walletSdkMock = { getWalletBalances: vi.fn(async () => ({ usdfc: 50_000_000n, fil: 1_000_000_000n })), + getNetworkConfig: vi.fn(() => ({ walletAddress: "0xABCDEF1234567890" })), }; configMock = { get: vi.fn((key: string) => { @@ -25,8 +26,11 @@ describe("WalletBalanceCollector", () => { prometheusWalletBalanceErrorCooldownSeconds: 60, }; } - if (key === "blockchain") { - return { walletAddress: "0xABCDEF1234567890" }; + if (key === "activeNetworks") { + return ["calibration"]; + } + if (key === "networks") { + return { calibration: { walletAddress: "0xABCDEF1234567890", network: "calibration" } }; } return {}; }), @@ -55,8 +59,14 @@ describe("WalletBalanceCollector", () => { await collectFn(); expect(walletSdkMock.getWalletBalances).toHaveBeenCalledOnce(); - expect(gaugeMock.set).toHaveBeenCalledWith({ currency: "USDFC", wallet: "0xABCDEF" }, 50_000_000); - expect(gaugeMock.set).toHaveBeenCalledWith({ currency: "FIL", wallet: "0xABCDEF" }, 1_000_000_000); + expect(gaugeMock.set).toHaveBeenCalledWith( + { currency: "USDFC", wallet: "0xABCDEF", network: "calibration" }, + 50_000_000, + ); + expect(gaugeMock.set).toHaveBeenCalledWith( + { currency: "FIL", wallet: "0xABCDEF", network: "calibration" }, + 1_000_000_000, + ); }); it("returns cached values without fetching again within the TTL window", async () => { diff --git a/apps/backend/src/metrics-prometheus/wallet-balance.collector.ts b/apps/backend/src/metrics-prometheus/wallet-balance.collector.ts index be01ab15..8f536c15 100644 --- a/apps/backend/src/metrics-prometheus/wallet-balance.collector.ts +++ b/apps/backend/src/metrics-prometheus/wallet-balance.collector.ts @@ -3,7 +3,7 @@ import { ConfigService } from "@nestjs/config"; import { InjectMetric } from "@willsoto/nestjs-prometheus"; import type { Gauge } from "prom-client"; import { toStructuredError } from "../common/logging.js"; -import type { IConfig } from "../config/app.config.js"; +import type { IConfig, INetworksConfig } from "../config/index.js"; import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; @Injectable() @@ -42,11 +42,13 @@ export class WalletBalanceCollector implements OnModuleInit { this.refreshPromise = (async () => { try { - const { usdfc, fil } = await this.walletSdkService.getWalletBalances(); - const { network, walletAddress } = this.configService.get("blockchain", { infer: true }); - const walletShort = walletAddress.slice(0, 8); - this.walletBalanceGauge.set({ currency: "USDFC", wallet: walletShort, network }, Number(usdfc)); - this.walletBalanceGauge.set({ currency: "FIL", wallet: walletShort, network }, Number(fil)); + const activeNetworks = this.configService.get("activeNetworks"); + for (const network of activeNetworks) { + const { usdfc, fil } = await this.walletSdkService.getWalletBalances(network); + const walletShort = this.configService.get("networks")[network].walletAddress.slice(0, 8); + this.walletBalanceGauge.set({ currency: "USDFC", wallet: walletShort, network }, Number(usdfc)); + this.walletBalanceGauge.set({ currency: "FIL", wallet: walletShort, network }, Number(fil)); + } this.cachedAt = Date.now(); } catch (error) { this.logger.warn({ diff --git a/apps/backend/src/piece-cleanup/piece-cleanup.service.spec.ts b/apps/backend/src/piece-cleanup/piece-cleanup.service.spec.ts index b2ef7c48..154fb1c6 100644 --- a/apps/backend/src/piece-cleanup/piece-cleanup.service.spec.ts +++ b/apps/backend/src/piece-cleanup/piece-cleanup.service.spec.ts @@ -3,7 +3,7 @@ import { Test, TestingModule } from "@nestjs/testing"; import { getRepositoryToken } from "@nestjs/typeorm"; import { calculateActualStorage, listDataSets } from "filecoin-pin/core/data-set"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { IConfig } from "../config/app.config.js"; +import type { IConfig } from "../config/index.js"; import { Deal } from "../database/entities/deal.entity.js"; import { DealStatus } from "../database/types.js"; import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; @@ -71,6 +71,13 @@ describe("PieceCleanupService", () => { function createWalletSdkMock() { return { getProviderInfo: vi.fn().mockReturnValue({ id: 9, name: "Test SP" }), + getSynapse: vi.fn().mockReturnValue({ + storage: { + createContext: vi.fn().mockResolvedValue({ + deletePiece: vi.fn(), + }), + }, + }), }; } @@ -79,17 +86,15 @@ describe("PieceCleanupService", () => { function createConfigMock() { return { get: vi.fn((key: keyof IConfig) => { - if (key === "pieceCleanup") { + if (key === "networks") { return { - maxDatasetStorageSizeBytes: THRESHOLD_BYTES, - targetDatasetStorageSizeBytes: TARGET_BYTES, - }; - } - if (key === "blockchain") { - return { - walletPrivateKey: "0x1234567890123456789012345678901234567890123456789012345678901234", - network: "calibration", - walletAddress: "0x123", + calibration: { + walletPrivateKey: "0x1234567890123456789012345678901234567890123456789012345678901234", + network: "calibration", + walletAddress: "0x123", + maxDatasetStorageSizeBytes: THRESHOLD_BYTES, + targetDatasetStorageSizeBytes: TARGET_BYTES, + }, }; } return undefined; @@ -149,7 +154,7 @@ describe("PieceCleanupService", () => { it("returns total bytes from the query builder", async () => { mockQueryBuilder(50 * MiB); - const result = await service.getStoredBytesForProvider("0xProvider"); + const result = await service.getStoredBytesForProvider("0xProvider", "calibration"); expect(result).toBe(50 * MiB); expect(dealRepoMock.createQueryBuilder).toHaveBeenCalledWith("deal"); @@ -158,7 +163,7 @@ describe("PieceCleanupService", () => { it("returns 0 when no deals exist", async () => { mockQueryBuilder(0); - const result = await service.getStoredBytesForProvider("0xProvider"); + const result = await service.getStoredBytesForProvider("0xProvider", "calibration"); expect(result).toBe(0); }); @@ -172,7 +177,7 @@ describe("PieceCleanupService", () => { }; dealRepoMock.createQueryBuilder.mockReturnValue(qb); - const result = await service.getStoredBytesForProvider("0xProvider"); + const result = await service.getStoredBytesForProvider("0xProvider", "calibration"); expect(result).toBe(0); }); @@ -183,7 +188,7 @@ describe("PieceCleanupService", () => { const abortController = new AbortController(); vi.mocked(listDataSets).mockReturnValueOnce(new Promise(() => {}) as any); - const result = service.getLiveStoredBytesForProvider("0xProvider", abortController.signal); + const result = service.getLiveStoredBytesForProvider("0xProvider", "calibration", abortController.signal); abortController.abort(new Error("listing timed out")); await expect(result).rejects.toThrow("listing timed out"); @@ -210,7 +215,7 @@ describe("PieceCleanupService", () => { }, ] as any); - await service.getLiveStoredBytesForProvider("0xProvider", signal); + await service.getLiveStoredBytesForProvider("0xProvider", "calibration", signal); expect(calculateActualStorage).toHaveBeenCalledWith( expect.anything(), @@ -246,7 +251,9 @@ describe("PieceCleanupService", () => { timedOut: true, }); - await expect(service.getLiveStoredBytesForProvider("0xProvider")).rejects.toThrow("Live storage query timed out"); + await expect(service.getLiveStoredBytesForProvider("0xProvider", "calibration")).rejects.toThrow( + "Live storage query timed out", + ); }); }); @@ -255,7 +262,7 @@ describe("PieceCleanupService", () => { const deals = [makeDeal({ createdAt: new Date("2024-01-01") }), makeDeal({ createdAt: new Date("2024-01-02") })]; dealRepoMock.find.mockResolvedValue(deals); - const result = await service.getCleanupCandidates("0xProvider", 10); + const result = await service.getCleanupCandidates("0xProvider", "calibration", 10); expect(result).toEqual(deals); expect(dealRepoMock.find).toHaveBeenCalledWith( @@ -274,7 +281,7 @@ describe("PieceCleanupService", () => { it("respects the limit parameter", async () => { dealRepoMock.find.mockResolvedValue([]); - await service.getCleanupCandidates("0xProvider", 5); + await service.getCleanupCandidates("0xProvider", "calibration", 5); expect(dealRepoMock.find).toHaveBeenCalledWith( expect.objectContaining({ @@ -288,7 +295,7 @@ describe("PieceCleanupService", () => { it("skips cleanup when stored bytes are below threshold", async () => { vi.spyOn(service, "getLiveStoredBytesForProvider").mockResolvedValue(50 * MiB); // 50 MiB < 100 MiB threshold - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.skipped).toBe(true); expect(result.deleted).toBe(0); @@ -300,7 +307,7 @@ describe("PieceCleanupService", () => { it("skips cleanup when stored bytes equal threshold", async () => { vi.spyOn(service, "getLiveStoredBytesForProvider").mockResolvedValue(THRESHOLD_BYTES); // exactly at threshold - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.skipped).toBe(true); expect(result.deleted).toBe(0); @@ -310,7 +317,7 @@ describe("PieceCleanupService", () => { vi.spyOn(service, "getLiveStoredBytesForProvider").mockRejectedValue(new Error("network error")); mockQueryBuilder(50 * MiB); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.skipped).toBe(true); expect(result.storedBytes).toBe(50 * MiB); @@ -329,7 +336,7 @@ describe("PieceCleanupService", () => { dealRepoMock.find.mockResolvedValue([deal1, deal2, deal3, deal4, deal5]); const deletePieceSpy = vi.spyOn(service, "deletePiece").mockResolvedValue(undefined); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.skipped).toBe(false); expect(result.deleted).toBe(5); @@ -343,9 +350,9 @@ describe("PieceCleanupService", () => { vi.spyOn(service, "getLiveStoredBytesForProvider").mockRejectedValue(new Error("cleanup timeout")); mockQueryBuilder(THRESHOLD_BYTES + 1); - await expect(service.cleanupPiecesForProvider("0xProvider", abortController.signal)).rejects.toThrow( - "cleanup timeout", - ); + await expect( + service.cleanupPiecesForProvider("0xProvider", "calibration", abortController.signal), + ).rejects.toThrow("cleanup timeout"); expect(dealRepoMock.find).not.toHaveBeenCalled(); expect(dealRepoMock.createQueryBuilder).not.toHaveBeenCalled(); @@ -379,7 +386,9 @@ describe("PieceCleanupService", () => { }); mockQueryBuilder(THRESHOLD_BYTES + 1); - await expect(service.cleanupPiecesForProvider("0xProvider")).rejects.toThrow("Live storage query timed out"); + await expect(service.cleanupPiecesForProvider("0xProvider", "calibration")).rejects.toThrow( + "Live storage query timed out", + ); expect(dealRepoMock.find).not.toHaveBeenCalled(); expect(dealRepoMock.createQueryBuilder).not.toHaveBeenCalled(); @@ -389,7 +398,7 @@ describe("PieceCleanupService", () => { vi.spyOn(service, "getLiveStoredBytesForProvider").mockResolvedValue(200 * MiB); // above threshold dealRepoMock.find.mockResolvedValue([]); // no candidates - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.skipped).toBe(false); expect(result.deleted).toBe(0); @@ -410,7 +419,7 @@ describe("PieceCleanupService", () => { const deletePieceSpy = vi.spyOn(service, "deletePiece").mockResolvedValue(undefined); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.deleted).toBe(5); // 50 MiB = 5 × 10 MiB expect(result.failed).toBe(0); @@ -428,7 +437,7 @@ describe("PieceCleanupService", () => { vi.spyOn(service, "deletePiece").mockRejectedValueOnce(new Error("SDK error")).mockResolvedValueOnce(undefined); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.deleted).toBe(1); expect(result.failed).toBe(1); @@ -443,7 +452,9 @@ describe("PieceCleanupService", () => { const abortController = new AbortController(); abortController.abort(new Error("aborted")); - await expect(service.cleanupPiecesForProvider("0xProvider", abortController.signal)).rejects.toThrow("aborted"); + await expect( + service.cleanupPiecesForProvider("0xProvider", "calibration", abortController.signal), + ).rejects.toThrow("aborted"); }); it("bails out when all deletions in a batch fail", async () => { @@ -455,7 +466,7 @@ describe("PieceCleanupService", () => { vi.spyOn(service, "deletePiece").mockRejectedValue(new Error("persistent failure")); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.deleted).toBe(0); expect(result.failed).toBe(2); @@ -472,7 +483,7 @@ describe("PieceCleanupService", () => { const deletePieceSpy = vi.spyOn(service, "deletePiece").mockResolvedValue(undefined); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); // Piece is still deleted expect(result.deleted).toBe(1); @@ -495,7 +506,7 @@ describe("PieceCleanupService", () => { const deletePieceSpy = vi.spyOn(service, "deletePiece").mockResolvedValue(undefined); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.deleted).toBe(4); expect(deletePieceSpy).toHaveBeenCalledTimes(4); @@ -508,34 +519,30 @@ describe("PieceCleanupService", () => { it("throws when deal is missing pieceId", async () => { const deal = makeDeal({ pieceId: undefined }); - await expect(service.deletePiece(deal)).rejects.toThrow("missing pieceId"); + await expect(service.deletePiece(deal, "calibration")).rejects.toThrow("missing pieceId"); }); it("throws when deal is missing dataSetId", async () => { const deal = makeDeal({ dataSetId: undefined }); - await expect(service.deletePiece(deal)).rejects.toThrow("missing dataSetId"); + await expect(service.deletePiece(deal, "calibration")).rejects.toThrow("missing dataSetId"); }); it("calls Synapse SDK to delete piece and marks deal as cleaned up", async () => { - const { createSynapseFromConfig } = await import("../common/synapse-factory.js"); const deletePieceMock = vi.fn().mockResolvedValue(undefined); const createContextMock = vi.fn().mockResolvedValue({ deletePiece: deletePieceMock, }); - (createSynapseFromConfig as ReturnType).mockResolvedValue({ - synapse: { - storage: { - createContext: createContextMock, - }, + walletSdkMock.getSynapse.mockReturnValue({ + storage: { + createContext: createContextMock, }, - isSessionKeyMode: false, }); const deal = makeDeal({ pieceId: 42, dataSetId: 1n, spAddress: "0xProvider" }); dealRepoMock.save.mockResolvedValue(deal); - await service.deletePiece(deal); + await service.deletePiece(deal, "calibration"); expect(createContextMock).toHaveBeenCalledWith({ providerId: 9, @@ -553,6 +560,7 @@ describe("PieceCleanupService", () => { const createContextMock = vi.fn().mockResolvedValue({ deletePiece: deletePieceMock, }); + walletSdkMock.getSynapse.mockReturnValue(null); (createSynapseFromConfig as ReturnType).mockResolvedValue({ synapse: { storage: { @@ -565,55 +573,51 @@ describe("PieceCleanupService", () => { const deal = makeDeal({ pieceId: 42, dataSetId: 1n, spAddress: "0xProvider" }); dealRepoMock.save.mockResolvedValue(deal); - await service.deletePiece(deal); + await service.deletePiece(deal, "calibration"); + expect(createContextMock).toHaveBeenCalledWith({ + providerId: 9, + dataSetId: 1n, + }); expect(deal.cleanedUp).toBe(true); expect(deal.cleanedUpAt).toBeInstanceOf(Date); expect(dealRepoMock.save).toHaveBeenCalledWith(deal); }); it("treats 'Piece ID already scheduled for removal' revert as idempotent success", async () => { - const { createSynapseFromConfig } = await import("../common/synapse-factory.js"); const deletePieceMock = vi.fn().mockRejectedValue(new Error("Piece ID already scheduled for removal")); const createContextMock = vi.fn().mockResolvedValue({ deletePiece: deletePieceMock, }); - (createSynapseFromConfig as ReturnType).mockResolvedValue({ - synapse: { - storage: { - createContext: createContextMock, - }, + walletSdkMock.getSynapse.mockReturnValue({ + storage: { + createContext: createContextMock, }, - isSessionKeyMode: false, }); const deal = makeDeal({ pieceId: 42, dataSetId: 1n, spAddress: "0xProvider" }); dealRepoMock.save.mockResolvedValue(deal); - await service.deletePiece(deal); + await service.deletePiece(deal, "calibration"); expect(deal.cleanedUp).toBe(true); expect(dealRepoMock.save).toHaveBeenCalledWith(deal); }); it("rethrows non-idempotent errors", async () => { - const { createSynapseFromConfig } = await import("../common/synapse-factory.js"); const deletePieceMock = vi.fn().mockRejectedValue(new Error("network timeout")); const createContextMock = vi.fn().mockResolvedValue({ deletePiece: deletePieceMock, }); - (createSynapseFromConfig as ReturnType).mockResolvedValue({ - synapse: { - storage: { - createContext: createContextMock, - }, + walletSdkMock.getSynapse.mockReturnValue({ + storage: { + createContext: createContextMock, }, - isSessionKeyMode: false, }); const deal = makeDeal({ pieceId: 42, dataSetId: 1n, spAddress: "0xProvider" }); - await expect(service.deletePiece(deal)).rejects.toThrow("network timeout"); + await expect(service.deletePiece(deal, "calibration")).rejects.toThrow("network timeout"); }); it("respects abort signal before SDK call", async () => { @@ -621,7 +625,7 @@ describe("PieceCleanupService", () => { const abortController = new AbortController(); abortController.abort(new Error("cancelled")); - await expect(service.deletePiece(deal, abortController.signal)).rejects.toThrow("cancelled"); + await expect(service.deletePiece(deal, "calibration", abortController.signal)).rejects.toThrow("cancelled"); }); }); }); diff --git a/apps/backend/src/piece-cleanup/piece-cleanup.service.ts b/apps/backend/src/piece-cleanup/piece-cleanup.service.ts index 990793a3..fda6e6b9 100644 --- a/apps/backend/src/piece-cleanup/piece-cleanup.service.ts +++ b/apps/backend/src/piece-cleanup/piece-cleanup.service.ts @@ -1,12 +1,13 @@ import { Synapse } from "@filoz/synapse-sdk"; -import { Injectable, Logger, type OnModuleDestroy, type OnModuleInit } from "@nestjs/common"; +import { Injectable, Logger } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import { InjectRepository } from "@nestjs/typeorm"; import { calculateActualStorage, listDataSets } from "filecoin-pin/core/data-set"; import { IsNull, Not, Repository } from "typeorm"; import { type PieceCleanupLogContext, type ProviderJobContext, toStructuredError } from "../common/logging.js"; import { createSynapseFromConfig } from "../common/synapse-factory.js"; -import type { IBlockchainConfig, IConfig } from "../config/app.config.js"; +import { Network } from "../common/types.js"; +import type { IConfig, INetworkConfig } from "../config/index.js"; import { Deal } from "../database/entities/deal.entity.js"; import { DealStatus } from "../database/types.js"; import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; @@ -27,52 +28,30 @@ export interface CleanupResult { class LiveStorageQueryTimedOutError extends Error {} @Injectable() -export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { +export class PieceCleanupService { private readonly logger = new Logger(PieceCleanupService.name); - private readonly blockchainConfig: IBlockchainConfig; - private sharedSynapse?: Synapse; constructor( private readonly configService: ConfigService, @InjectRepository(Deal) private readonly dealRepository: Repository, private readonly walletSdkService: WalletSdkService, - ) { - this.blockchainConfig = this.configService.get("blockchain"); - } - - async onModuleInit(): Promise { - if (process.env.DEALBOT_DISABLE_CHAIN === "true") { - this.logger.warn("Chain integration disabled; skipping Synapse initialization for piece cleanup."); - return; - } - try { - this.logger.log("Initializing shared Synapse instance for piece cleanup."); - const { synapse } = await createSynapseFromConfig(this.blockchainConfig); - this.sharedSynapse = synapse; - } catch (error) { - this.logger.error({ - event: "piece_cleanup_synapse_init_failed", - message: "Failed to initialize shared Synapse instance for piece cleanup; will create on demand", - error: toStructuredError(error), - }); - } - } + ) {} - async onModuleDestroy(): Promise { - if (this.sharedSynapse) { - this.sharedSynapse = undefined; - } + private getNetworkConfig(network: Network): INetworkConfig { + return this.configService.get("networks")[network]; } - private async createSynapseInstance(): Promise { + private async createSynapseInstance(network: Network): Promise { + const networkConfig = this.getNetworkConfig(network); try { - const { synapse } = await createSynapseFromConfig(this.blockchainConfig); + const { synapse } = await createSynapseFromConfig(networkConfig); return synapse; } catch (error) { this.logger.error({ event: "synapse_init_failed", message: "Failed to initialize Synapse for piece cleanup", + network, error: toStructuredError(error), }); throw error; @@ -113,13 +92,14 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { */ private async checkProviderQuota( spAddress: string, + network: Network, signal?: AbortSignal, logContext?: ProviderJobContext, ): Promise<{ isOverQuota: boolean; storedBytes: number; thresholdBytes: number }> { - const thresholdBytes = this.configService.get("pieceCleanup").maxDatasetStorageSizeBytes; + const thresholdBytes = this.getNetworkConfig(network).maxDatasetStorageSizeBytes; let storedBytes: number; try { - storedBytes = await this.getLiveStoredBytesForProvider(spAddress, signal); + storedBytes = await this.getLiveStoredBytesForProvider(spAddress, network, signal); } catch (error) { if (signal?.aborted || error instanceof LiveStorageQueryTimedOutError) { throw error; @@ -131,7 +111,7 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { providerAddress: spAddress, error: toStructuredError(error), }); - storedBytes = await this.getStoredBytesForProvider(spAddress); + storedBytes = await this.getStoredBytesForProvider(spAddress, network); } return { isOverQuota: storedBytes > thresholdBytes, storedBytes, thresholdBytes }; } @@ -147,13 +127,14 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { */ async cleanupPiecesForProvider( spAddress: string, + network: Network, signal?: AbortSignal, logContext?: ProviderJobContext, ): Promise { const { maxDatasetStorageSizeBytes: thresholdBytes, targetDatasetStorageSizeBytes: targetBytes } = - this.configService.get("pieceCleanup"); + this.getNetworkConfig(network); - const { storedBytes, isOverQuota } = await this.checkProviderQuota(spAddress, signal, logContext); + const { storedBytes, isOverQuota } = await this.checkProviderQuota(spAddress, network, signal, logContext); const cleanupLogContext: PieceCleanupLogContext = { ...logContext, @@ -184,13 +165,13 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { let failed = 0; let bytesRemoved = 0; - const synapse = this.sharedSynapse ?? (await this.createSynapseInstance()); + const synapse = this.walletSdkService.getSynapse(network) ?? (await this.createSynapseInstance(network)); // Fetch candidates in batches. Keep deleting until back under quota or runtime cap. while (bytesRemoved < excessBytes) { signal?.throwIfAborted(); - const candidates = await this.getCleanupCandidates(spAddress, 50); + const candidates = await this.getCleanupCandidates(spAddress, network, 50); if (candidates.length === 0) { this.logger.warn({ @@ -218,7 +199,7 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { } try { - await this.deletePiece(deal, signal, synapse, cleanupLogContext); + await this.deletePiece(deal, network, signal, synapse, cleanupLogContext); deleted++; batchDeletedCount++; bytesRemoved += Number(deal.pieceSize || 0); @@ -274,8 +255,8 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { /** * Query the provider's actual storage via filecoin-pin. */ - async getLiveStoredBytesForProvider(spAddress: string, signal?: AbortSignal): Promise { - const synapse = this.sharedSynapse ?? (await this.createSynapseInstance()); + async getLiveStoredBytesForProvider(spAddress: string, network: Network, signal?: AbortSignal): Promise { + const synapse = this.walletSdkService.getSynapse(network) ?? (await this.createSynapseInstance(network)); const datasets = await this.awaitWithAbort( listDataSets(synapse, { @@ -319,9 +300,8 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { * Calculate total stored bytes for a provider from the deals table. * Only counts completed deals that have not already been cleaned up. */ - async getStoredBytesForProvider(spAddress: string): Promise { - const walletAddress = this.blockchainConfig.walletAddress; - const network = this.blockchainConfig.network; + async getStoredBytesForProvider(spAddress: string, network: Network): Promise { + const walletAddress = this.getNetworkConfig(network).walletAddress; const result = await this.dealRepository .createQueryBuilder("deal") @@ -341,9 +321,9 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { /** * Get the oldest completed deals (candidates for cleanup). */ - async getCleanupCandidates(spAddress: string, limit: number): Promise { - const walletAddress = this.blockchainConfig.walletAddress; - const network = this.blockchainConfig.network; + async getCleanupCandidates(spAddress: string, network: Network, limit: number): Promise { + const walletAddress = this.getNetworkConfig(network).walletAddress; + return this.dealRepository.find({ where: { spAddress, @@ -364,6 +344,7 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { */ async deletePiece( deal: Deal, + network: Network, signal?: AbortSignal, existingSynapse?: Synapse, logContext?: PieceCleanupLogContext, @@ -377,11 +358,14 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { signal?.throwIfAborted(); - const providerId = this.walletSdkService.getProviderInfo(deal.spAddress)?.id; + const providerId = this.walletSdkService.getProviderInfo(deal.spAddress, network)?.id; if (providerId === undefined) { throw new Error(`Provider ID not found for SP address ${deal.spAddress}`); } - const synapse = existingSynapse ?? this.sharedSynapse ?? (await this.createSynapseInstance()); + const synapse = + existingSynapse ?? this.walletSdkService.getSynapse(network) ?? (await this.createSynapseInstance(network)); + + console.log(synapse); const storage = await synapse.storage.createContext({ providerId, dataSetId: deal.dataSetId, diff --git a/apps/backend/src/retrieval-addons/strategies/ipfs-block.strategy.ts b/apps/backend/src/retrieval-addons/strategies/ipfs-block.strategy.ts index d2676897..d55d16be 100644 --- a/apps/backend/src/retrieval-addons/strategies/ipfs-block.strategy.ts +++ b/apps/backend/src/retrieval-addons/strategies/ipfs-block.strategy.ts @@ -6,7 +6,7 @@ import { CID } from "multiformats/cid"; import * as raw from "multiformats/codecs/raw"; import { sha256 } from "multiformats/hashes/sha2"; import { toStructuredError } from "../../common/logging.js"; -import type { IConfig } from "../../config/app.config.js"; +import type { IConfig } from "../../config/index.js"; import { ServiceType } from "../../database/types.js"; import { HttpClientService } from "../../http-client/http-client.service.js"; import { WalletSdkService } from "../../wallet-sdk/wallet-sdk.service.js"; @@ -89,7 +89,7 @@ export class IpfsBlockRetrievalStrategy implements IRetrievalAddon { } private getSpEndpoint(config: RetrievalConfiguration): string { - const providerInfo = this.walletSdkService.getProviderInfo(config.storageProvider); + const providerInfo = this.walletSdkService.getProviderInfo(config.storageProvider, config.deal.network); if (!providerInfo) { throw new Error(`Provider ${config.storageProvider} not found in approved providers`); diff --git a/apps/backend/src/retrieval/retrieval.service.ts b/apps/backend/src/retrieval/retrieval.service.ts index 5671c932..7774d81e 100644 --- a/apps/backend/src/retrieval/retrieval.service.ts +++ b/apps/backend/src/retrieval/retrieval.service.ts @@ -5,8 +5,8 @@ import { CID } from "multiformats/cid"; import type { Repository } from "typeorm"; import { ClickhouseService } from "../clickhouse/clickhouse.service.js"; import { type ProviderJobContext, type RetrievalLogContext, toStructuredError } from "../common/logging.js"; -import type { Hex } from "../common/types.js"; -import type { IConfig } from "../config/app.config.js"; +import type { Hex, Network } from "../common/types.js"; +import type { IConfig } from "../config/index.js"; import { Deal } from "../database/entities/deal.entity.js"; import { Retrieval } from "../database/entities/retrieval.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; @@ -57,10 +57,11 @@ export class RetrievalService { async performRandomRetrievalForProvider( spAddress: string, + network: Network, signal?: AbortSignal, logContext?: ProviderJobContext, ): Promise { - const deal = await this.selectRandomSuccessfulDealForProvider(spAddress); + const deal = await this.selectRandomSuccessfulDealForProvider(spAddress, network); if (!deal) { this.logger.warn({ ...logContext, @@ -503,8 +504,8 @@ export class RetrievalService { * We select a random successful deal (DEAL_CREATED only) for a given provider. * Uses Postgres ORDER BY RANDOM() since Dealbot is Postgres-only. */ - private async selectRandomSuccessfulDealForProvider(spAddress: string): Promise { - const { network, walletAddress } = this.configService.get("blockchain", { infer: true }); + private async selectRandomSuccessfulDealForProvider(spAddress: string, network: Network): Promise { + const walletAddress = this.configService.get("networks", { infer: true })[network].walletAddress; const randomDatasetSizes = this.getRandomDatasetSizes(); const query = this.dealRepository From d379d342c91d510322625e67c4adaeccac635c1e Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 12:50:33 +0530 Subject: [PATCH 04/23] migrate data-retention service --- apps/backend/src/common/sp-blocklist.spec.ts | 18 +- apps/backend/src/common/sp-blocklist.ts | 12 +- .../data-retention.service.spec.ts | 181 ++++++++++-------- .../data-retention/data-retention.service.ts | 15 +- 4 files changed, 127 insertions(+), 99 deletions(-) diff --git a/apps/backend/src/common/sp-blocklist.spec.ts b/apps/backend/src/common/sp-blocklist.spec.ts index daf6a789..ac020fcf 100644 --- a/apps/backend/src/common/sp-blocklist.spec.ts +++ b/apps/backend/src/common/sp-blocklist.spec.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from "vitest"; -import { type ISpBlocklistConfig } from "../config/app.config.js"; +import type { INetworkConfig } from "../config/types.js"; import { isSpBlocked } from "./sp-blocklist.js"; -const cfg = (overrides: Partial = {}): ISpBlocklistConfig => ({ - ids: new Set(), - addresses: new Set(), +const cfg = (overrides: Partial = {}): Pick => ({ + blockedSpIds: new Set(), + blockedSpAddresses: new Set(), ...overrides, }); @@ -15,25 +15,25 @@ describe("isSpBlocked", () => { }); it("blocks by address", () => { - expect(isSpBlocked(cfg({ addresses: new Set(["0xaaa"]) }), "0xaaa")).toBe(true); + expect(isSpBlocked(cfg({ blockedSpAddresses: new Set(["0xaaa"]) }), "0xaaa")).toBe(true); }); it("address matching is case-insensitive", () => { - const c = cfg({ addresses: new Set(["0xaaa"]) }); + const c = cfg({ blockedSpAddresses: new Set(["0xaaa"]) }); expect(isSpBlocked(c, "0xAAA")).toBe(true); expect(isSpBlocked(c, "0XAAA")).toBe(true); }); it("blocks by numeric ID", () => { - expect(isSpBlocked(cfg({ ids: new Set(["42"]) }), "0xaaa", 42n)).toBe(true); + expect(isSpBlocked(cfg({ blockedSpIds: new Set(["42"]) }), "0xaaa", 42n)).toBe(true); }); it("skips ID check when id is undefined", () => { - expect(isSpBlocked(cfg({ ids: new Set(["42"]) }), "0xaaa", undefined)).toBe(false); + expect(isSpBlocked(cfg({ blockedSpIds: new Set(["42"]) }), "0xaaa", undefined)).toBe(false); }); it("does not block unrelated provider", () => { - const c = cfg({ ids: new Set(["5"]), addresses: new Set(["0xaaa"]) }); + const c = cfg({ blockedSpIds: new Set(["5"]), blockedSpAddresses: new Set(["0xaaa"]) }); expect(isSpBlocked(c, "0xbbb", 6n)).toBe(false); }); }); diff --git a/apps/backend/src/common/sp-blocklist.ts b/apps/backend/src/common/sp-blocklist.ts index 8152d99f..084d6612 100644 --- a/apps/backend/src/common/sp-blocklist.ts +++ b/apps/backend/src/common/sp-blocklist.ts @@ -1,11 +1,15 @@ -import type { ISpBlocklistConfig } from "../config/app.config.js"; +import { INetworkConfig } from "src/config/types.js"; /** * Returns true if the provider is in the SP blocklist. * Checks by address (case-insensitive) first, then by numeric ID if provided. */ -export function isSpBlocked(cfg: ISpBlocklistConfig, address: string, id?: bigint | null): boolean { - if (cfg.addresses.has(address.toLowerCase())) return true; - if (id != null && cfg.ids.has(String(id))) return true; +export function isSpBlocked( + cfg: Pick, + address: string, + id?: bigint | null, +): boolean { + if (cfg.blockedSpAddresses.has(address.toLowerCase())) return true; + if (id != null && cfg.blockedSpIds.has(String(id))) return true; return false; } diff --git a/apps/backend/src/data-retention/data-retention.service.spec.ts b/apps/backend/src/data-retention/data-retention.service.spec.ts index 933c5cbe..72916c42 100644 --- a/apps/backend/src/data-retention/data-retention.service.spec.ts +++ b/apps/backend/src/data-retention/data-retention.service.spec.ts @@ -3,7 +3,7 @@ import type { Counter, Gauge } from "prom-client"; import { Repository } from "typeorm"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { ClickhouseService } from "../clickhouse/clickhouse.service.js"; -import type { IConfig } from "../config/app.config.js"; +import type { IConfig } from "../config/index.js"; import type { DataRetentionBaseline } from "../database/entities/data-retention-baseline.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; import { buildCheckMetricLabels } from "../metrics-prometheus/check-metric-labels.js"; @@ -68,11 +68,18 @@ describe("DataRetentionService", () => { beforeEach(() => { configServiceMock = { get: vi.fn((key: keyof IConfig) => { - if (key === "blockchain") { - return { pdpSubgraphEndpoint: "https://example.com/subgraph", network: "calibration" }; + if (key === "activeNetworks") { + return ["calibration"]; } - if (key === "spBlocklists") { - return { ids: new Set(), addresses: new Set() }; + if (key === "networks") { + return { + calibration: { + pdpSubgraphEndpoint: "https://example.com/subgraph", + network: "calibration", + blockedSpIds: new Set(), + blockedSpAddresses: new Set(), + }, + }; } return undefined; }), @@ -156,11 +163,13 @@ describe("DataRetentionService", () => { }); it("returns early when pdpSubgraphEndpoint is empty", async () => { - (configServiceMock.get as ReturnType).mockReturnValue({ - pdpSubgraphEndpoint: "", + (configServiceMock.get as ReturnType).mockImplementation((key: keyof IConfig) => { + if (key === "activeNetworks") return ["calibration"]; + if (key === "networks") return { calibration: { pdpSubgraphEndpoint: "", network: "calibration" } }; + return undefined; }); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(pdpSubgraphServiceMock.fetchSubgraphMeta).not.toHaveBeenCalled(); expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).not.toHaveBeenCalled(); @@ -169,30 +178,44 @@ describe("DataRetentionService", () => { it("returns early when no testing providers configured", async () => { walletSdkServiceMock.getTestingProviders.mockReturnValueOnce(null); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).not.toHaveBeenCalled(); }); it("returns early when all providers are blocked for data-retention", async () => { (configServiceMock.get as ReturnType).mockImplementation((key: string) => { - if (key === "blockchain") return { pdpSubgraphEndpoint: "https://example.com/subgraph", network: "calibration" }; - if (key === "spBlocklists") return { ids: new Set(), addresses: new Set([PROVIDER_A, PROVIDER_B]) }; + if (key === "networks") + return { + calibration: { + pdpSubgraphEndpoint: "https://example.com/subgraph", + network: "calibration", + blockedSpIds: new Set(), + blockedSpAddresses: new Set([PROVIDER_A, PROVIDER_B]), + }, + }; }); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).not.toHaveBeenCalled(); }); it("excludes blocked providers from data-retention polling while retaining unblocked ones", async () => { (configServiceMock.get as ReturnType).mockImplementation((key: string) => { - if (key === "blockchain") return { pdpSubgraphEndpoint: "https://example.com/subgraph", network: "calibration" }; - if (key === "spBlocklists") return { ids: new Set(), addresses: new Set([PROVIDER_A]) }; + if (key === "networks") + return { + calibration: { + pdpSubgraphEndpoint: "https://example.com/subgraph", + network: "calibration", + blockedSpIds: new Set(), + blockedSpAddresses: new Set([PROVIDER_A]), + }, + }; }); pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); const allAddressesPolled: string[] = ( pdpSubgraphServiceMock.fetchProvidersWithDatasets.mock.calls as [string, { addresses: string[] }][] @@ -204,7 +227,7 @@ describe("DataRetentionService", () => { it("returns early when testing providers array is empty", async () => { walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).not.toHaveBeenCalled(); }); @@ -212,7 +235,7 @@ describe("DataRetentionService", () => { it("sets baseline on first poll without emitting counters (fresh deploy / new provider)", async () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(pdpSubgraphServiceMock.fetchSubgraphMeta).toHaveBeenCalled(); expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).toHaveBeenCalledWith("https://example.com/subgraph", { @@ -241,7 +264,7 @@ describe("DataRetentionService", () => { it("computes deltas correctly on consecutive polls", async () => { // First poll: blockNumber=1200 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); const firstCallCount = counterMock.labels.mock.calls.length; @@ -260,7 +283,7 @@ describe("DataRetentionService", () => { }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll should have incremented counters with the delta expect(counterMock.labels.mock.calls.length).toBeGreaterThan(firstCallCount); @@ -270,11 +293,11 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValue([makeProvider()]); // First poll - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); counterMock.labels.mockClear(); // Second poll with same data and same block number - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // No new increments since deltas are zero expect(counterMock.labels).not.toHaveBeenCalled(); @@ -303,7 +326,7 @@ describe("DataRetentionService", () => { const providerB = makeProvider({ address: PROVIDER_B, totalFaultedPeriods: 20n }); pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([providerA, providerB]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); const labelCalls = counterMock.labels.mock.calls; const providerAFaulted = labelCalls.some( @@ -331,7 +354,7 @@ describe("DataRetentionService", () => { const provider = makeProvider(); pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([provider]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // totalFaultedPeriods = 10, totalProvingPeriods = 100 // confirmedTotalSuccess = 100 - 10 = 90 @@ -356,7 +379,7 @@ describe("DataRetentionService", () => { it("handles empty providers array without errors", async () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(counterMock.labels).not.toHaveBeenCalled(); }); @@ -376,7 +399,7 @@ describe("DataRetentionService", () => { const provider = makeProvider(); pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([provider]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // totalFaultedPeriods = 10, totalProvingPeriods = 100 // confirmedTotalSuccess = 100 - 10 = 90 @@ -402,7 +425,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockRejectedValueOnce(new Error("subgraph down")); // Should not throw - await expect(service.pollDataRetention()).resolves.toBeUndefined(); + await expect(service.pollDataRetention("calibration")).resolves.toBeUndefined(); }); it("resets baseline on negative deltas without incrementing counters", async () => { @@ -410,14 +433,14 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 100n, totalProvingPeriods: 200n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); counterMock.labels.mockClear(); // Second poll: lower values (e.g., chain reorg or subgraph correction) pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 50n, totalProvingPeriods: 100n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Both deltas are negative, so counters should not be incremented expect(counterMock.labels).not.toHaveBeenCalled(); @@ -426,7 +449,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 52n, totalProvingPeriods: 105n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should now increment based on new baseline (52-50=2 faulted, 55-50=5 success) expect(counterMock.labels).toHaveBeenCalled(); @@ -451,7 +474,7 @@ describe("DataRetentionService", () => { makeProvider({ totalFaultedPeriods: largeValue, totalProvingPeriods: largeValue * 2n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should have been called multiple times (chunked increments) expect(counterMock.inc).toHaveBeenCalled(); @@ -481,7 +504,7 @@ describe("DataRetentionService", () => { makeProvider({ totalFaultedPeriods: maxSafeInt, totalProvingPeriods: maxSafeInt * 2n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should increment without chunking since it's exactly at the boundary expect(counterMock.inc).toHaveBeenCalled(); @@ -505,7 +528,7 @@ describe("DataRetentionService", () => { }); pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([provider]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Uses subgraph totals directly: faulted=5*5=25, success=45*5=225 const incCalls = counterMock.inc.mock.calls; @@ -524,7 +547,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValue([]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should be called twice: once for first 50, once for remaining 25 expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).toHaveBeenCalledTimes(2); @@ -554,7 +577,7 @@ describe("DataRetentionService", () => { .mockRejectedValueOnce(new Error("Subgraph timeout")) .mockResolvedValueOnce([]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Both batches should be attempted expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).toHaveBeenCalledTimes(2); @@ -565,7 +588,7 @@ describe("DataRetentionService", () => { const PROVIDER_C = "0x1234567890123456789012345678901234567890"; pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_C })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should not increment counters for missing provider expect(counterMock.labels).not.toHaveBeenCalled(); @@ -579,7 +602,7 @@ describe("DataRetentionService", () => { makeProvider({ address: PROVIDER_B }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Repository should not be queried since no stale providers expect(mockSPRepository.find).not.toHaveBeenCalled(); @@ -588,7 +611,7 @@ describe("DataRetentionService", () => { it("successfully cleans up stale provider with valid database entry", async () => { // First poll: establish baseline for PROVIDER_A pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: PROVIDER_A removed from active list, only PROVIDER_B active walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -611,7 +634,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should fetch stale provider info from database (network-scoped) expect(mockSPRepository.find).toHaveBeenCalledWith({ @@ -643,7 +666,7 @@ describe("DataRetentionService", () => { it("skips cleanup entirely when database fetch fails", async () => { // First poll: establish baseline pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed, but DB fails walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -659,7 +682,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should attempt to fetch from database expect(mockSPRepository.find).toHaveBeenCalled(); @@ -682,7 +705,7 @@ describe("DataRetentionService", () => { ]); counterMock.labels.mockClear(); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should compute delta from original baseline, not from zero expect(counterMock.labels).toHaveBeenCalled(); @@ -691,7 +714,7 @@ describe("DataRetentionService", () => { it("retains baseline when provider not found in database", async () => { // First poll: establish baseline pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed from active list walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -708,7 +731,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should NOT remove counters (provider not in DB) expect(counterMock.remove).not.toHaveBeenCalled(); @@ -728,7 +751,7 @@ describe("DataRetentionService", () => { ]); counterMock.labels.mockClear(); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should use old baseline (delta from 10 to 12 = 2) expect(counterMock.labels).toHaveBeenCalled(); @@ -737,7 +760,7 @@ describe("DataRetentionService", () => { it("retains baseline when provider has null providerId", async () => { // First poll: establish baseline pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -761,7 +784,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should NOT remove counters (missing providerId) expect(counterMock.remove).not.toHaveBeenCalled(); @@ -770,7 +793,7 @@ describe("DataRetentionService", () => { it("retains baseline when counter removal throws error", async () => { // First poll: establish baseline pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -798,7 +821,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should attempt removal expect(counterMock.remove).toHaveBeenCalled(); @@ -818,7 +841,7 @@ describe("DataRetentionService", () => { ]); counterMock.labels.mockClear(); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should compute delta from original baseline expect(counterMock.labels).toHaveBeenCalled(); @@ -840,7 +863,7 @@ describe("DataRetentionService", () => { makeProvider({ address: PROVIDER_C }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: only PROVIDER_A remains active walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -854,7 +877,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should fetch both stale providers in one query (network-scoped) expect(mockSPRepository.find).toHaveBeenCalledWith({ @@ -869,7 +892,7 @@ describe("DataRetentionService", () => { it("skips cleanup when processing errors occurred", async () => { // First poll: establish baseline pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed, but processing has errors walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -879,7 +902,7 @@ describe("DataRetentionService", () => { // Simulate processing error pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockRejectedValueOnce(new Error("Processing failed")); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should NOT attempt cleanup due to processing errors expect(mockSPRepository.find).not.toHaveBeenCalled(); @@ -898,7 +921,7 @@ describe("DataRetentionService", () => { makeProvider({ address: PROVIDER_MIXED_CASE.toLowerCase() as `0x${string}` }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -916,7 +939,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should successfully find and clean up provider despite case difference expect(counterMock.remove).toHaveBeenCalled(); @@ -941,7 +964,7 @@ describe("DataRetentionService", () => { // With DB baseline: faultedDelta = 10 - 10 = 0, successDelta = 90 - 90 = 0 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Key assertion: counters should NOT be incremented because deltas are zero expect(counterMock.labels).not.toHaveBeenCalled(); @@ -964,7 +987,7 @@ describe("DataRetentionService", () => { // faultedDelta = 10 - 8 = 2, successDelta = 90 - 85 = 5 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should increment by only the delta, not the full cumulative values expect(counterMock.labels).toHaveBeenCalledWith(expect.objectContaining({ value: "failure" })); @@ -979,16 +1002,16 @@ describe("DataRetentionService", () => { it("reloads baselines from DB on every poll", async () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValue([makeProvider()]); - await service.pollDataRetention(); - await service.pollDataRetention(); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); + await service.pollDataRetention("calibration"); + await service.pollDataRetention("calibration"); expect(mockBaselineRepository.find).toHaveBeenCalledTimes(3); }); it("does not double-count when poll ownership alternates across worker pods", async () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); const secondPod = new DataRetentionService( configServiceMock, @@ -1005,7 +1028,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 11n, totalProvingPeriods: 102n }), ]); - await secondPod.pollDataRetention(); + await secondPod.pollDataRetention("calibration"); counterMock.labels.mockClear(); counterMock.inc.mockClear(); @@ -1014,7 +1037,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 12n, totalProvingPeriods: 104n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Third poll must use the second pod's persisted baseline: failure +5, success +5. // A stale first-pod baseline would emit +10 and +10 here. @@ -1032,7 +1055,7 @@ describe("DataRetentionService", () => { makeProvider({ totalFaultedPeriods: 12n, totalProvingPeriods: 105n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(counterMock.labels).not.toHaveBeenCalled(); @@ -1041,7 +1064,7 @@ describe("DataRetentionService", () => { makeProvider({ totalFaultedPeriods: 12n, totalProvingPeriods: 105n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Since the failed write did not advance the DB or local baseline, the next // successful poll emits the original persisted-baseline delta once. @@ -1062,13 +1085,13 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValue([makeProvider()]); // First poll: DB load fails, poll bails out to avoid emitting bloated values - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(mockBaselineRepository.find).toHaveBeenCalledTimes(1); expect(pdpSubgraphServiceMock.fetchSubgraphMeta).not.toHaveBeenCalled(); expect(counterMock.labels).not.toHaveBeenCalled(); // Second poll: DB load succeeds, baselines restored, normal delta computation - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(mockBaselineRepository.find).toHaveBeenCalledTimes(2); // Deltas from DB baseline: faultedDelta = 10 - 10 = 0, successDelta = 90 - 90 = 0 expect(counterMock.labels).not.toHaveBeenCalled(); @@ -1078,7 +1101,7 @@ describe("DataRetentionService", () => { // First poll: fresh deploy, no baselines in DB // Baseline set to: faultedPeriods=10, successPeriods=90 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); counterMock.labels.mockClear(); counterMock.inc.mockClear(); @@ -1090,7 +1113,7 @@ describe("DataRetentionService", () => { makeProvider({ totalFaultedPeriods: 12n, totalProvingPeriods: 105n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // faultedDelta = (12 - 10) * 5 = 10, successDelta = ((105 - 12) - 90) * 5 = 15 expect(counterMock.labels).toHaveBeenCalled(); @@ -1101,7 +1124,7 @@ describe("DataRetentionService", () => { it("deletes baseline from DB when stale provider is cleaned up", async () => { // First poll: establish baseline for PROVIDER_A pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: PROVIDER_A removed from active list walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -1114,7 +1137,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should delete the baseline from DB (network-scoped) expect(mockBaselineRepository.delete).toHaveBeenCalledWith({ @@ -1130,7 +1153,7 @@ describe("DataRetentionService", () => { // estimatedOverduePeriods = (1200 - 901) / 100 = 2.99 -> 2 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(gaugeMock.labels).toHaveBeenCalledWith( expect.objectContaining({ @@ -1147,7 +1170,7 @@ describe("DataRetentionService", () => { // nextDeadline=2000 > currentBlock=1200 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ proofSets: [] })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(gaugeMock.set).toHaveBeenCalledWith(0); }); @@ -1157,7 +1180,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 100n, totalProvingPeriods: 200n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); gaugeMock.labels.mockClear(); gaugeMock.set.mockClear(); @@ -1165,7 +1188,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 50n, totalProvingPeriods: 100n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Gauge should still be emitted despite negative deltas on counters expect(gaugeMock.labels).toHaveBeenCalled(); @@ -1175,7 +1198,7 @@ describe("DataRetentionService", () => { it("naturally resets gauge to 0 when subgraph catches up", async () => { // First poll: provider is overdue (currentBlock=1200, nextDeadline=1000) pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(gaugeMock.set).toHaveBeenCalledWith(2); @@ -1191,7 +1214,7 @@ describe("DataRetentionService", () => { }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Gauge should reset to 0 because nextDeadline (1300) > currentBlock (1200) expect(gaugeMock.set).toHaveBeenCalledWith(0); @@ -1200,7 +1223,7 @@ describe("DataRetentionService", () => { it("removes overdue gauge when stale provider is cleaned up", async () => { // First poll: establish baseline for PROVIDER_A pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: PROVIDER_A removed from active list walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -1213,7 +1236,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should remove overdue gauge for stale provider (both approved and unapproved labels) const approvedLabels = buildCheckMetricLabels({ diff --git a/apps/backend/src/data-retention/data-retention.service.ts b/apps/backend/src/data-retention/data-retention.service.ts index 59e01a8a..7992cd7f 100644 --- a/apps/backend/src/data-retention/data-retention.service.ts +++ b/apps/backend/src/data-retention/data-retention.service.ts @@ -8,7 +8,7 @@ import { ClickhouseService } from "../clickhouse/clickhouse.service.js"; import { toStructuredError } from "../common/logging.js"; import { isSpBlocked } from "../common/sp-blocklist.js"; import type { Network } from "../common/types.js"; -import { IConfig } from "../config/app.config.js"; +import { IConfig, INetworksConfig } from "../config/index.js"; import { DataRetentionBaseline } from "../database/entities/data-retention-baseline.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; import { buildCheckMetricLabels, CheckMetricLabels } from "../metrics-prometheus/check-metric-labels.js"; @@ -59,8 +59,9 @@ export class DataRetentionService { * converts them to challenge counts, and increments Prometheus counters with the * challenge delta since the last poll. */ - async pollDataRetention(): Promise { - const { network, pdpSubgraphEndpoint } = this.configService.get("blockchain", { infer: true }); + async pollDataRetention(network: Network): Promise { + const networkConfig = this.configService.get("networks")[network]; + const pdpSubgraphEndpoint = networkConfig.pdpSubgraphEndpoint; if (!pdpSubgraphEndpoint) { this.logger.warn({ event: "pdp_subgraph_endpoint_not_configured", @@ -78,8 +79,8 @@ export class DataRetentionService { try { const subgraphMeta = await this.pdpSubgraphService.fetchSubgraphMeta(pdpSubgraphEndpoint); - const allProviderInfos = this.walletSdkService.getTestingProviders(); - const spBlocklists = this.configService.get("spBlocklists"); + const allProviderInfos = this.walletSdkService.getTestingProviders(network); + const spBlocklists = this.configService.get("networks", { infer: true })[network]; const providerInfos = allProviderInfos?.filter((p) => !isSpBlocked(spBlocklists, p.serviceProvider, p.id)); if (!providerInfos || providerInfos.length === 0) { @@ -122,7 +123,7 @@ export class DataRetentionService { ), ); } - return this.processProvider(provider, providerInfo, blockNumberBigInt, baselines); + return this.processProvider(provider, providerInfo, blockNumberBigInt, baselines, network); }), ); @@ -344,6 +345,7 @@ export class DataRetentionService { pdpProvider: PDPProviderEx, currentBlock: bigint, baselines: Map, + network: Network, ): Promise { const { address, totalFaultedPeriods, totalProvingPeriods, proofSets } = provider; // Note: Query filters proofSets with nextDeadline_lt: $blockNumber, so all deadlines are in the past @@ -376,7 +378,6 @@ export class DataRetentionService { successPeriods: confirmedTotalSuccess, }; - const network = this.configService.get("blockchain", { infer: true }).network; const providerLabels = buildCheckMetricLabels({ checkType: "dataRetention", providerId: pdpProvider.id, From a60a542d531c28cda19054c37f13cc70ea02018c Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 14:00:25 +0530 Subject: [PATCH 05/23] migrate deal service --- apps/backend/src/deal/deal.service.spec.ts | 242 ++++++++++++++------- apps/backend/src/deal/deal.service.ts | 113 +++++----- 2 files changed, 222 insertions(+), 133 deletions(-) diff --git a/apps/backend/src/deal/deal.service.spec.ts b/apps/backend/src/deal/deal.service.spec.ts index 156643d5..2218eef9 100644 --- a/apps/backend/src/deal/deal.service.spec.ts +++ b/apps/backend/src/deal/deal.service.spec.ts @@ -8,6 +8,7 @@ import { generatePrivateKey } from "viem/accounts"; import { afterEach, beforeEach, describe, expect, it, Mock, vi } from "vitest"; import { ClickhouseService } from "../clickhouse/clickhouse.service.js"; import { DealJobTerminatedDataSetError } from "../common/errors.js"; +import { IConfig } from "../config/index.js"; import { Deal } from "../database/entities/deal.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; import { DealStatus, IpniStatus } from "../database/types.js"; @@ -25,6 +26,22 @@ import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; import type { PDPProviderEx } from "../wallet-sdk/wallet-sdk.types.js"; import { DealService } from "./deal.service.js"; +const DEFAULT_NETWORK = "calibration"; + +const calibDefaults = { + walletPrivateKey: generatePrivateKey(), + network: DEFAULT_NETWORK, + walletAddress: "0x123", + checkDatasetCreationFees: false, + useOnlyApprovedProviders: false, + minNumDataSetsForChecks: 1, + dealsPerSpPerHour: 4, + retrievalsPerSpPerHour: 2, + dataSetCreationsPerSpPerHour: 1, + dataRetentionPollIntervalSeconds: 3600, + providersRefreshIntervalSeconds: 14400, +}; + vi.mock("@filoz/synapse-sdk", async (importOriginal) => { const actual = await importOriginal(); return { @@ -98,20 +115,12 @@ describe("DealService", () => { const mockConfigService = { get: vi.fn().mockImplementation((key: string) => { - if (key === "scheduling") { - return { - dealIntervalSeconds: 30, - dealStartOffsetSeconds: 0, - retrievalIntervalSeconds: 60, - retrievalStartOffsetSeconds: 600, - metricsStartOffsetSeconds: 900, - }; + if (key === "activeNetworks") { + return [DEFAULT_NETWORK]; } - if (key === "blockchain") { + if (key === "networks") { return { - walletPrivateKey: generatePrivateKey(), - network: "calibration", - walletAddress: "0x123", + calibration: calibDefaults, }; } return undefined; @@ -136,6 +145,13 @@ describe("DealService", () => { paymentsService: {}, warmStorageService: mockWarmStorageService, }), + getSynapse: vi.fn().mockReturnValue({ + storage: { + createContext: vi.fn().mockResolvedValue({ + deletePiece: vi.fn(), + }), + }, + }), }; const mockDealAddonsService = { @@ -293,6 +309,7 @@ describe("DealService", () => { mockProviderInfo, mockDealInput, uploadPayload, + DEFAULT_NETWORK, )) as Deal; expect(createContextMock).toHaveBeenCalledWith( @@ -388,11 +405,12 @@ describe("DealService", () => { providerInfo, mockDealInput, uploadPayload, + DEFAULT_NETWORK, )) as Deal; const labels = { checkType: "dataStorage", - network: "calibration", + network: DEFAULT_NETWORK, providerId: "42", providerName: "Test Provider", providerStatus: "approved", @@ -454,7 +472,7 @@ describe("DealService", () => { }); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, DEFAULT_NETWORK), ).rejects.toThrow("Dealbot did not receive onProgress events during upload"); expect(mockDataStorageMetrics.observeIngestMs).not.toHaveBeenCalled(); @@ -499,6 +517,7 @@ describe("DealService", () => { mockProviderInfo, mockDealInput, uploadPayload, + DEFAULT_NETWORK, )) as Deal; expect(deal.ingestLatencyMs).toBeNull(); @@ -556,6 +575,7 @@ describe("DealService", () => { mockProviderInfo, zeroSizeDealInput, uploadPayload, + DEFAULT_NETWORK, )) as Deal; expect(deal.ingestLatencyMs).toBe(1000); @@ -583,13 +603,13 @@ describe("DealService", () => { (executeUpload as Mock).mockRejectedValue(new Error("timed out waiting for upload")); - await expect(service.createDeal(mockSynapseInstance, providerInfo, mockDealInput, uploadPayload)).rejects.toThrow( - "timed out", - ); + await expect( + service.createDeal(mockSynapseInstance, providerInfo, mockDealInput, uploadPayload, DEFAULT_NETWORK), + ).rejects.toThrow("timed out"); const labels = { checkType: "dataStorage", - network: "calibration", + network: DEFAULT_NETWORK, providerId: "7", providerName: "Test Provider", providerStatus: "unapproved", @@ -618,13 +638,13 @@ describe("DealService", () => { (executeUpload as Mock).mockRejectedValue(new Error("connection refused")); - await expect(service.createDeal(mockSynapseInstance, providerInfo, mockDealInput, uploadPayload)).rejects.toThrow( - "connection refused", - ); + await expect( + service.createDeal(mockSynapseInstance, providerInfo, mockDealInput, uploadPayload, DEFAULT_NETWORK), + ).rejects.toThrow("connection refused"); const labels = { checkType: "dataStorage", - network: "calibration", + network: DEFAULT_NETWORK, providerId: "7", providerName: "Test Provider", providerStatus: "unapproved", @@ -675,6 +695,7 @@ describe("DealService", () => { mockProviderInfo, mockDealInput, uploadPayload, + DEFAULT_NETWORK, undefined, abortController.signal, ), @@ -707,7 +728,7 @@ describe("DealService", () => { (executeUpload as Mock).mockRejectedValue(error); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, DEFAULT_NETWORK), ).rejects.toThrow("Upload failed"); expect(mockDeal.status).toBe(DealStatus.FAILED); @@ -725,7 +746,7 @@ describe("DealService", () => { createContextMock.mockRejectedValue(error); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, DEFAULT_NETWORK), ).rejects.toThrow("Storage creation failed"); expect(mockDeal.status).toBe(DealStatus.FAILED); @@ -743,7 +764,7 @@ describe("DealService", () => { createContextMock.mockRejectedValue(error); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, DEFAULT_NETWORK), ).rejects.toThrow("secret-token"); expect(mockDeal.status).toBe(DealStatus.FAILED); @@ -767,6 +788,7 @@ describe("DealService", () => { mockProviderInfo, mockDealInput, uploadPayload, + DEFAULT_NETWORK, undefined, abortController.signal, ), @@ -801,7 +823,7 @@ describe("DealService", () => { dealAddonsMock.handleStored.mockRejectedValueOnce(ipniError); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, DEFAULT_NETWORK), ).rejects.toThrow("IPNI verification failed"); expect(mockDeal.status).toBe(DealStatus.FAILED); @@ -839,7 +861,7 @@ describe("DealService", () => { }); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, DEFAULT_NETWORK), ).rejects.toBe(commitError); expect(capturedAddonSignal?.aborted).toBe(true); @@ -877,7 +899,7 @@ describe("DealService", () => { }); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, DEFAULT_NETWORK), ).rejects.toThrow("Retrieval gate failed"); expect(mockDeal.status).toBe(DealStatus.FAILED); @@ -909,12 +931,12 @@ describe("DealService", () => { ); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, DEFAULT_NETWORK), ).rejects.toThrow("timed out"); const labels = { checkType: "dataStorage", - network: "calibration", + network: DEFAULT_NETWORK, providerId: "42", providerName: "Test Provider", providerStatus: "approved", @@ -959,12 +981,12 @@ describe("DealService", () => { retrievalAddonsMock.testAllRetrievalMethods.mockRejectedValue(new Error("retrieval timed out")); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, DEFAULT_NETWORK), ).rejects.toThrow("retrieval timed out"); const labels = { checkType: "dataStorage", - network: "calibration", + network: DEFAULT_NETWORK, providerId: "42", providerName: "Test Provider", providerStatus: "approved", @@ -1013,6 +1035,7 @@ describe("DealService", () => { mockProviderInfo, mockDealInput, uploadPayload, + DEFAULT_NETWORK, )) as Deal; expect(deal.dealLatencyMs).toBeGreaterThanOrEqual(0); @@ -1053,11 +1076,24 @@ describe("DealService", () => { }); const createServiceWithVersion = async (dealbotDataSetVersion: string | undefined) => { - mockConfigService.get.mockReturnValue({ + const versionNetworkConfig = { walletPrivateKey: generatePrivateKey(), network: "calibration", walletAddress: "0x123", + checkDatasetCreationFees: false, + useOnlyApprovedProviders: false, + minNumDataSetsForChecks: 1, + dealsPerSpPerHour: 4, + retrievalsPerSpPerHour: 2, + dataSetCreationsPerSpPerHour: 1, + dataRetentionPollIntervalSeconds: 3600, + providersRefreshIntervalSeconds: 14400, dealbotDataSetVersion, + }; + mockConfigService.get.mockImplementation((key: string) => { + if (key === "activeNetworks") return ["calibration"]; + if (key === "networks") return { calibration: versionNetworkConfig }; + return undefined; }); const module: TestingModule = await Test.createTestingModule({ @@ -1090,7 +1126,13 @@ describe("DealService", () => { carData: Uint8Array.from([1, 2, 3]), rootCid: CID.parse(mockRootCid), }; - await testService.createDeal(mockSynapseInstance, mockProviderInfo, dealInputWithMetadata, uploadPayload); + await testService.createDeal( + mockSynapseInstance, + mockProviderInfo, + dealInputWithMetadata, + uploadPayload, + DEFAULT_NETWORK, + ); expect(createContextMock).toHaveBeenCalledWith({ providerId: 101n, @@ -1107,7 +1149,13 @@ describe("DealService", () => { carData: Uint8Array.from([1, 2, 3]), rootCid: CID.parse(mockRootCid), }; - await testService.createDeal(mockSynapseInstance, mockProviderInfo, dealInputWithMetadata, uploadPayload); + await testService.createDeal( + mockSynapseInstance, + mockProviderInfo, + dealInputWithMetadata, + uploadPayload, + DEFAULT_NETWORK, + ); expect(createContextMock).toHaveBeenCalledWith({ providerId: 101n, @@ -1123,7 +1171,13 @@ describe("DealService", () => { carData: Uint8Array.from([1, 2, 3]), rootCid: CID.parse(mockRootCid), }; - await testService.createDeal(mockSynapseInstance, mockProviderInfo, dealInputWithMetadata, uploadPayload); + await testService.createDeal( + mockSynapseInstance, + mockProviderInfo, + dealInputWithMetadata, + uploadPayload, + DEFAULT_NETWORK, + ); expect(createContextMock).toHaveBeenCalledWith({ providerId: 101n, @@ -1152,7 +1206,13 @@ describe("DealService", () => { }, }; - await testService.createDeal(mockSynapseInstance, mockProviderInfo, dealInputWithConflict, uploadPayload); + await testService.createDeal( + mockSynapseInstance, + mockProviderInfo, + dealInputWithConflict, + uploadPayload, + DEFAULT_NETWORK, + ); // Verify config value overwrites dealInput value expect(createContextMock).toHaveBeenCalledWith({ @@ -1197,6 +1257,7 @@ describe("DealService", () => { mockProviderInfo, mockDealInput, uploadPayload, + DEFAULT_NETWORK, )) as Deal; // Verify that pieceId from piecesConfirmed (123) is preserved and not overwritten by undefined @@ -1239,7 +1300,7 @@ describe("DealService", () => { }; vi.spyOn(service as any, "createSynapseInstance").mockImplementation(() => synapseMock as unknown as Synapse); - const result = await service.getDataSetProvisioningStatus("0xprovider", { dealbotDS: "1" }); + const result = await service.getDataSetProvisioningStatus("0xprovider", { dealbotDS: "1" }, DEFAULT_NETWORK); expect(result).toEqual({ status: "missing" }); }); @@ -1249,10 +1310,10 @@ describe("DealService", () => { createContext: vi.fn().mockResolvedValue({ dataSetId: 7n }), }, }; - vi.spyOn(service as any, "createSynapseInstance").mockImplementation(() => synapseMock as unknown as Synapse); + mockWalletSdkService.getSynapse.mockReturnValue(synapseMock as unknown as Synapse); mockWarmStorageService.validateDataSet.mockResolvedValueOnce(undefined); - const result = await service.getDataSetProvisioningStatus("0xprovider", { dealbotDS: "1" }); + const result = await service.getDataSetProvisioningStatus("0xprovider", { dealbotDS: "1" }, DEFAULT_NETWORK); expect(result).toEqual({ status: "live", dataSetId: 7n }); }); @@ -1262,10 +1323,10 @@ describe("DealService", () => { createContext: vi.fn().mockResolvedValue({ dataSetId: 9n }), }, }; - vi.spyOn(service as any, "createSynapseInstance").mockImplementation(() => synapseMock as unknown as Synapse); + mockWalletSdkService.getSynapse.mockReturnValue(synapseMock as unknown as Synapse); mockDatasetLivenessService.isDataSetLive.mockResolvedValueOnce(false); - const result = await service.getDataSetProvisioningStatus("0xprovider", { dealbotDS: "1" }); + const result = await service.getDataSetProvisioningStatus("0xprovider", { dealbotDS: "1" }, DEFAULT_NETWORK); expect(result).toEqual({ status: "terminated", dataSetId: 9n }); }); }); @@ -1305,10 +1366,9 @@ describe("DealService", () => { }); it("returns undefined when minDataSets=1 and baseline is live", async () => { - (service as any).blockchainConfig.minNumDataSetsForChecks = 1; probeSpy.mockResolvedValueOnce({ status: "live", dataSetId: 1n }); - const result = await service.resolveDataSetMetadataForDeal("0xprovider"); + const result = await service.resolveDataSetMetadataForDeal("0xprovider", DEFAULT_NETWORK); expect(result).toBeUndefined(); expect(probeSpy).toHaveBeenCalledTimes(1); const [, metadata] = probeSpy.mock.calls[0] ?? []; @@ -1316,20 +1376,24 @@ describe("DealService", () => { }); it("throws DealJobTerminatedDataSetError when baseline is terminated", async () => { - (service as any).blockchainConfig.minNumDataSetsForChecks = 1; probeSpy.mockResolvedValueOnce({ status: "terminated", dataSetId: 42n }); - await expect(service.resolveDataSetMetadataForDeal("0xprovider")).rejects.toBeInstanceOf( + await expect(service.resolveDataSetMetadataForDeal("0xprovider", DEFAULT_NETWORK)).rejects.toBeInstanceOf( DealJobTerminatedDataSetError, ); }); it("uses indexed slot when live; does not probe baseline", async () => { - (service as any).blockchainConfig.minNumDataSetsForChecks = 3; + mockConfigService.get.mockImplementation((key: keyof IConfig) => { + if (key === "networks") { + return { calibration: { ...calibDefaults, minNumDataSetsForChecks: 3 } }; + } + return undefined; + }); vi.spyOn(Math, "random").mockReturnValue(0.5); // → index 1 probeSpy.mockResolvedValueOnce({ status: "live", dataSetId: 7n }); - const result = await service.resolveDataSetMetadataForDeal("0xprovider"); + const result = await service.resolveDataSetMetadataForDeal("0xprovider", DEFAULT_NETWORK); expect(result).toEqual({ dealbotDS: "1" }); expect(probeSpy).toHaveBeenCalledTimes(1); const [, metadata] = probeSpy.mock.calls[0] ?? []; @@ -1341,17 +1405,27 @@ describe("DealService", () => { ["terminated", () => probeSpy.mockResolvedValueOnce({ status: "terminated", dataSetId: 99n })], ["probe throws", () => probeSpy.mockRejectedValueOnce(new Error("rpc failure"))], ])("falls back to baseline when indexed slot is %s", async (_label, setupIndexedProbe) => { - (service as any).blockchainConfig.minNumDataSetsForChecks = 3; + mockConfigService.get.mockImplementation((key: keyof IConfig) => { + if (key === "networks") { + return { calibration: { ...calibDefaults, minNumDataSetsForChecks: 3 } }; + } + return undefined; + }); vi.spyOn(Math, "random").mockReturnValue(0.5); setupIndexedProbe(); probeSpy.mockResolvedValueOnce({ status: "live", dataSetId: 1n }); - const result = await service.resolveDataSetMetadataForDeal("0xprovider"); + const result = await service.resolveDataSetMetadataForDeal("0xprovider", DEFAULT_NETWORK); expect(result).toBeUndefined(); }); it("propagates abort from indexed probe", async () => { - (service as any).blockchainConfig.minNumDataSetsForChecks = 3; + mockConfigService.get.mockImplementation((key: keyof IConfig) => { + if (key === "networks") { + return { calibration: { ...calibDefaults, minNumDataSetsForChecks: 3 } }; + } + return undefined; + }); vi.spyOn(Math, "random").mockReturnValue(0.5); const controller = new AbortController(); probeSpy.mockImplementationOnce(async () => { @@ -1359,7 +1433,9 @@ describe("DealService", () => { throw new Error("aborted"); }); - await expect(service.resolveDataSetMetadataForDeal("0xprovider", controller.signal)).rejects.toThrow(); + await expect( + service.resolveDataSetMetadataForDeal("0xprovider", DEFAULT_NETWORK, controller.signal), + ).rejects.toThrow(); expect(probeSpy).toHaveBeenCalledTimes(1); }); }); @@ -1376,7 +1452,7 @@ describe("DealService", () => { storage: { terminateDataSet: terminateMock }, client: { waitForTransactionReceipt: waitForReceiptMock }, }; - vi.spyOn(service as any, "createSynapseInstance").mockImplementation(() => synapseMock as unknown as Synapse); + mockWalletSdkService.getSynapse.mockReturnValue(synapseMock); mockWarmStorageService.getDataSet.mockResolvedValueOnce({ pdpEndEpoch: 0n }); mockWarmStorageService.getDataSet.mockResolvedValueOnce({ pdpEndEpoch: 12345n }); @@ -1388,7 +1464,7 @@ describe("DealService", () => { value: { transaction: transactionMock }, }); - const result = await service.repairTerminatedDataSet("0xaaa", 9n, undefined, 5_000); + const result = await service.repairTerminatedDataSet("0xaaa", 9n, DEFAULT_NETWORK, undefined, 5_000); expect(terminateMock).toHaveBeenCalledWith({ dataSetId: 9n }); expect(waitForReceiptMock).toHaveBeenCalledWith({ hash: "0xhash" }); @@ -1406,8 +1482,7 @@ describe("DealService", () => { storage: { terminateDataSet: terminateMock }, client: { waitForTransactionReceipt: vi.fn() }, }; - vi.spyOn(service as any, "createSynapseInstance").mockImplementation(() => synapseMock as unknown as Synapse); - + mockWalletSdkService.getSynapse.mockReturnValue(synapseMock); mockWarmStorageService.getDataSet.mockResolvedValueOnce({ pdpEndEpoch: 999n }); const updateFn = vi.fn().mockResolvedValue({ affected: 1 }); @@ -1417,7 +1492,7 @@ describe("DealService", () => { value: { transaction: transactionMock }, }); - const result = await service.repairTerminatedDataSet("0xaaa", 9n, undefined, 5_000); + const result = await service.repairTerminatedDataSet("0xaaa", 9n, DEFAULT_NETWORK, undefined, 5_000); expect(terminateMock).not.toHaveBeenCalled(); expect(updateFn).toHaveBeenCalled(); @@ -1430,7 +1505,7 @@ describe("DealService", () => { storage: { terminateDataSet: terminateMock }, client: { waitForTransactionReceipt: vi.fn() }, }; - vi.spyOn(service as any, "createSynapseInstance").mockImplementation(() => synapseMock as unknown as Synapse); + mockWalletSdkService.getSynapse.mockReturnValue(synapseMock); mockWarmStorageService.getDataSet.mockResolvedValueOnce({ pdpEndEpoch: 0n }); mockWarmStorageService.getDataSet.mockResolvedValueOnce({ pdpEndEpoch: 7n }); @@ -1442,7 +1517,7 @@ describe("DealService", () => { value: { transaction: transactionMock }, }); - const result = await service.repairTerminatedDataSet("0xaaa", 9n, undefined, 5_000); + const result = await service.repairTerminatedDataSet("0xaaa", 9n, DEFAULT_NETWORK, undefined, 5_000); expect(terminateMock).toHaveBeenCalled(); expect(updateFn).toHaveBeenCalled(); @@ -1492,9 +1567,9 @@ describe("DealService", () => { vi.spyOn(service as any, "createSynapseInstance").mockResolvedValue(synapseMock); mockDatasetLivenessService.isDataSetLive.mockResolvedValueOnce(false); - await expect(service.createDeal(synapseMock, providerInfo, dealInput, uploadPayload)).rejects.toBeInstanceOf( - DealJobTerminatedDataSetError, - ); + await expect( + service.createDeal(synapseMock, providerInfo, dealInput, uploadPayload, DEFAULT_NETWORK), + ).rejects.toBeInstanceOf(DealJobTerminatedDataSetError); expect(executeUpload as Mock).not.toHaveBeenCalled(); expect(mockDataStorageMetrics.recordUploadStatus).not.toHaveBeenCalled(); expect(mockDataStorageMetrics.recordDataStorageStatus).not.toHaveBeenCalled(); @@ -1504,14 +1579,19 @@ describe("DealService", () => { describe("getBaseDataSetMetadata", () => { it("always includes IPNI metadata key", () => { - const metadata = service.getBaseDataSetMetadata(); + const metadata = service.getBaseDataSetMetadata(DEFAULT_NETWORK); expect(metadata).toEqual(expect.objectContaining({ [METADATA_KEYS.WITH_IPFS_INDEXING]: "" })); }); it("includes dataset version when configured along with IPNI metadata", () => { - (service as any).blockchainConfig.dealbotDataSetVersion = "v1"; + mockConfigService.get.mockImplementation((key: keyof IConfig) => { + if (key === "networks") { + return { calibration: { ...calibDefaults, dealbotDataSetVersion: "v1" } }; + } + return undefined; + }); - const metadata = service.getBaseDataSetMetadata(); + const metadata = service.getBaseDataSetMetadata(DEFAULT_NETWORK); expect(metadata).toEqual({ [METADATA_KEYS.WITH_IPFS_INDEXING]: "", @@ -1545,7 +1625,7 @@ describe("DealService", () => { it("throws when provider is not found in registry", async () => { vi.spyOn(mockWalletSdkService, "getProviderInfo").mockReturnValue(undefined); - await expect(service.createDataSetWithPiece("0xunknown", { dealbotDS: "1" })).rejects.toThrow( + await expect(service.createDataSetWithPiece("0xunknown", { dealbotDS: "1" }, DEFAULT_NETWORK)).rejects.toThrow( "Provider 0xunknown not found in registry", ); }); @@ -1557,14 +1637,14 @@ describe("DealService", () => { const synapseMock = { storage: { createContext: createContextMock }, } as unknown as Synapse; - vi.spyOn(service as any, "createSynapseInstance").mockImplementation(() => synapseMock); + mockWalletSdkService.getSynapse.mockReturnValue(synapseMock); (executeUpload as Mock).mockImplementation(async (_service, _data, _rootCid, options) => { await triggerUploadProgress(options?.onProgress); return { pieceCid: "bafk-seed", pieceId: 1, transactionHash: "0xhash" }; }); - await service.createDataSetWithPiece("0xprovider", { dealbotDS: "1" }); + await service.createDataSetWithPiece("0xprovider", { dealbotDS: "1" }, DEFAULT_NETWORK); expect(createContextMock).toHaveBeenCalledWith({ providerId: 101n, @@ -1596,18 +1676,15 @@ describe("DealService", () => { it("does not invoke data-storage-check metrics or Deal persistence", async () => { vi.spyOn(mockWalletSdkService, "getProviderInfo").mockReturnValue(mockProviderInfo); const createContextMock = vi.fn().mockResolvedValue({ dataSetId: 1 }); - vi.spyOn(service as any, "createSynapseInstance").mockImplementation( - () => - ({ - storage: { createContext: createContextMock }, - }) as unknown as Synapse, - ); + mockWalletSdkService.getSynapse.mockReturnValue({ + storage: { createContext: createContextMock }, + }); (executeUpload as Mock).mockImplementation(async (_s, _d, _r, opts) => { await triggerUploadProgress(opts?.onProgress); return { pieceCid: "bafk-seed" }; }); - await service.createDataSetWithPiece("0xprovider", {}); + await service.createDataSetWithPiece("0xprovider", {}, DEFAULT_NETWORK); expect(mockDataStorageMetrics.observeIngestMs).not.toHaveBeenCalled(); expect(mockDataStorageMetrics.recordUploadStatus).not.toHaveBeenCalled(); @@ -1618,16 +1695,13 @@ describe("DealService", () => { it("fails when upload completes without a pieceCid", async () => { vi.spyOn(mockWalletSdkService, "getProviderInfo").mockReturnValue(mockProviderInfo); - vi.spyOn(service as any, "createSynapseInstance").mockImplementation( - () => - ({ - storage: { createContext: vi.fn().mockResolvedValue({ dataSetId: 1 }) }, - }) as unknown as Synapse, - ); + mockWalletSdkService.getSynapse.mockReturnValue({ + storage: { createContext: vi.fn().mockResolvedValue({ dataSetId: 1 }) }, + }); (executeUpload as Mock).mockResolvedValue({}); - await expect(service.createDataSetWithPiece("0xprovider", {})).rejects.toThrow( + await expect(service.createDataSetWithPiece("0xprovider", {}, DEFAULT_NETWORK)).rejects.toThrow( "Data-set creation upload completed without producing a pieceCid", ); expect(mockDataSetCreationMetrics.recordStatus).not.toHaveBeenCalledWith( @@ -1653,7 +1727,7 @@ describe("DealService", () => { await opts?.onProgress?.({ type: "stored", data: { pieceCid: "bafk" } }); }); - await expect(service.createDataSetWithPiece("0xprovider", {})).resolves.toBeUndefined(); + await expect(service.createDataSetWithPiece("0xprovider", {}, DEFAULT_NETWORK)).resolves.toBeUndefined(); expect(mockDataSetCreationMetrics.recordStatus).toHaveBeenCalledWith( expect.objectContaining({ checkType: "dataSetCreation" }), "success", @@ -1674,7 +1748,7 @@ describe("DealService", () => { }); const controller = new AbortController(); - const resultPromise = service.createDataSetWithPiece("0xprovider", {}, controller.signal); + const resultPromise = service.createDataSetWithPiece("0xprovider", {}, DEFAULT_NETWORK, controller.signal); controller.abort(new Error("Job aborted")); await expect(resultPromise).rejects.toThrow("Job aborted"); diff --git a/apps/backend/src/deal/deal.service.ts b/apps/backend/src/deal/deal.service.ts index 5bb91bf9..2478ac99 100644 --- a/apps/backend/src/deal/deal.service.ts +++ b/apps/backend/src/deal/deal.service.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { setTimeout as setTimeoutAsync } from "node:timers/promises"; import { METADATA_KEYS, SIZE_CONSTANTS, Synapse } from "@filoz/synapse-sdk"; -import { Injectable, Logger, type OnModuleDestroy, type OnModuleInit } from "@nestjs/common"; +import { Injectable, Logger } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import { InjectRepository } from "@nestjs/typeorm"; import { executeUpload } from "filecoin-pin"; @@ -19,8 +19,8 @@ import { toStructuredError, } from "../common/logging.js"; import { createSynapseFromConfig } from "../common/synapse-factory.js"; -import type { DataFile, Hex } from "../common/types.js"; -import type { IBlockchainConfig, IConfig } from "../config/app.config.js"; +import type { DataFile, Hex, Network } from "../common/types.js"; +import type { IConfig, INetworkConfig } from "../config/types.js"; import { Deal } from "../database/entities/deal.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; import { DealStatus, IpniStatus, ServiceType } from "../database/types.js"; @@ -51,10 +51,8 @@ type UploadResultSummary = { }; @Injectable() -export class DealService implements OnModuleInit, OnModuleDestroy { +export class DealService { private readonly logger = new Logger(DealService.name); - private readonly blockchainConfig: IBlockchainConfig; - private sharedSynapse?: Synapse; constructor( private readonly dataSourceService: DataSourceService, @@ -71,27 +69,16 @@ export class DealService implements OnModuleInit, OnModuleDestroy { private readonly dataSetCreationMetrics: DataSetCreationCheckMetrics, private readonly clickhouseService: ClickhouseService, private readonly datasetLivenessService: DatasetLivenessService, - ) { - this.blockchainConfig = this.configService.get("blockchain"); - } - - async onModuleInit() { - this.logger.log({ - event: "synapse_initialization", - message: "Creating shared Synapse instance", - }); - this.sharedSynapse = await this.createSynapseInstance(); - } + ) {} - async onModuleDestroy(): Promise { - if (this.sharedSynapse) { - this.sharedSynapse = undefined; - } + private getNetworkConfig(network: Network): INetworkConfig { + return this.configService.get("networks")[network]; } async createDealForProvider( pdpProvider: PDPProviderEx, options: { + network: Network; existingDealId?: string; signal?: AbortSignal; logContext?: ProviderJobContext; @@ -101,6 +88,7 @@ export class DealService implements OnModuleInit, OnModuleDestroy { const extraDataSetMetadata = await this.resolveDataSetMetadataForDeal( pdpProvider.serviceProvider, + options.network, options.signal, options.logContext, ); @@ -108,13 +96,15 @@ export class DealService implements OnModuleInit, OnModuleDestroy { const { preprocessed, cleanup } = await this.prepareDealInput(options.signal, options.logContext); try { - const synapse = this.sharedSynapse ?? (await this.createSynapseInstance()); + const synapse = + this.walletSdkService.getSynapse(options.network) ?? (await this.createSynapseInstance(options.network)); const uploadPayload = await this.prepareUploadPayload(preprocessed, options.signal); return await this.createDeal( synapse, pdpProvider, preprocessed, uploadPayload, + options.network, options.existingDealId, options.signal, extraDataSetMetadata, @@ -142,17 +132,29 @@ export class DealService implements OnModuleInit, OnModuleDestroy { */ async resolveDataSetMetadataForDeal( providerAddress: string, + network: Network, signal?: AbortSignal, logContext?: ProviderJobContext, ): Promise | undefined> { signal?.throwIfAborted(); - const baseDataSetMetadata = this.getBaseDataSetMetadata(); + const baseDataSetMetadata = this.getBaseDataSetMetadata(network); - const indexedMetadata = await this.tryIndexedDataSetSlot(providerAddress, baseDataSetMetadata, signal, logContext); + const indexedMetadata = await this.tryIndexedDataSetSlot( + providerAddress, + baseDataSetMetadata, + network, + signal, + logContext, + ); if (indexedMetadata !== undefined) return indexedMetadata; try { - const baselineStatus = await this.getDataSetProvisioningStatus(providerAddress, baseDataSetMetadata, signal); + const baselineStatus = await this.getDataSetProvisioningStatus( + providerAddress, + baseDataSetMetadata, + network, + signal, + ); if (baselineStatus.status === "terminated") { throw new DealJobTerminatedDataSetError(baselineStatus.dataSetId); } @@ -173,10 +175,11 @@ export class DealService implements OnModuleInit, OnModuleDestroy { private async tryIndexedDataSetSlot( providerAddress: string, baseDataSetMetadata: Record, + network: Network, signal: AbortSignal | undefined, logContext: ProviderJobContext | undefined, ): Promise | undefined> { - const minDataSets = this.blockchainConfig.minNumDataSetsForChecks; + const minDataSets = this.getNetworkConfig(network).minNumDataSetsForChecks; if (minDataSets <= 1) return undefined; const dsIndex = Math.floor(Math.random() * minDataSets); if (dsIndex === 0) return undefined; @@ -186,6 +189,7 @@ export class DealService implements OnModuleInit, OnModuleDestroy { const status = await this.getDataSetProvisioningStatus( providerAddress, { ...baseDataSetMetadata, ...dsIndexMetadata }, + network, signal, ); if (status.status === "live") return dsIndexMetadata; @@ -242,19 +246,20 @@ export class DealService implements OnModuleInit, OnModuleDestroy { return { preprocessed, cleanup }; } - getBaseDataSetMetadata(): Record { + getBaseDataSetMetadata(network: Network): Record { // IPNI is always enabled for all deals const metadata: Record = { [METADATA_KEYS.WITH_IPFS_INDEXING]: "", }; - if (this.blockchainConfig.dealbotDataSetVersion) { - metadata.dealbotDataSetVersion = this.blockchainConfig.dealbotDataSetVersion; + const networkConfig = this.getNetworkConfig(network); + if (networkConfig.dealbotDataSetVersion) { + metadata.dealbotDataSetVersion = networkConfig.dealbotDataSetVersion; } return metadata; } - getWalletAddress(): string { - return this.blockchainConfig.walletAddress; + getWalletAddress(network: Network): string { + return this.getNetworkConfig(network).walletAddress; } async createDeal( @@ -262,12 +267,12 @@ export class DealService implements OnModuleInit, OnModuleDestroy { pdpProvider: PDPProviderEx, dealInput: DealPreprocessingResult, uploadPayload: UploadPayload, + network: Network, existingDealId?: string, signal?: AbortSignal, extraDataSetMetadata?: Record, logContext?: ProviderJobContext, ): Promise { - const network = this.blockchainConfig.network; const providerAddress = pdpProvider.serviceProvider; const checkType = "dataStorage" as const; let providerLabels = buildCheckMetricLabels({ @@ -311,12 +316,13 @@ export class DealService implements OnModuleInit, OnModuleDestroy { deal.id = randomUUID(); } + const networkCfg = this.getNetworkConfig(network); deal.fileName = dealInput.processedData.name; deal.fileSize = dealInput.processedData.size; deal.spAddress = providerAddress; deal.network = network; deal.status = DealStatus.PENDING; - deal.walletAddress = this.blockchainConfig.walletAddress; + deal.walletAddress = networkCfg.walletAddress; deal.metadata = dealInput.metadata; deal.serviceTypes = dealInput.appliedAddons; @@ -353,8 +359,8 @@ export class DealService implements OnModuleInit, OnModuleDestroy { const dataSetMetadata = { ...dealInput.synapseConfig.dataSetMetadata, ...extraDataSetMetadata }; - if (this.blockchainConfig.dealbotDataSetVersion) { - dataSetMetadata.dealbotDataSetVersion = this.blockchainConfig.dealbotDataSetVersion; + if (networkCfg.dealbotDataSetVersion) { + dataSetMetadata.dealbotDataSetVersion = networkCfg.dealbotDataSetVersion; } const filecoinPinLogger = createFilecoinPinLogger(this.logger, dealLogContext); @@ -700,13 +706,14 @@ export class DealService implements OnModuleInit, OnModuleDestroy { async getDataSetProvisioningStatus( providerAddress: string, metadata: Record, + network: Network, signal?: AbortSignal, ): Promise< { status: "missing" } | { status: "live"; dataSetId: bigint } | { status: "terminated"; dataSetId: bigint } > { signal?.throwIfAborted(); - const synapse = this.sharedSynapse ?? (await this.createSynapseInstance()); - const providerInfo = this.walletSdkService.getProviderInfo(providerAddress); + const synapse = this.walletSdkService.getSynapse(network) ?? (await this.createSynapseInstance(network)); + const providerInfo = this.walletSdkService.getProviderInfo(providerAddress, network); if (!providerInfo) { throw new Error(`Provider ${providerAddress} not found in registry`); } @@ -750,13 +757,14 @@ export class DealService implements OnModuleInit, OnModuleDestroy { async repairTerminatedDataSet( providerAddress: string, dataSetId: bigint, + network: Network, signal?: AbortSignal, pollTimeoutMs = 60_000, ): Promise<{ dealsAffected: number; pdpEndEpoch: bigint }> { signal?.throwIfAborted(); - const synapse = this.sharedSynapse ?? (await this.createSynapseInstance()); - const providerInfo = this.walletSdkService.getProviderInfo(providerAddress); - const { warmStorageService } = this.walletSdkService.getWalletServices(); + const synapse = this.walletSdkService.getSynapse(network) ?? (await this.createSynapseInstance(network)); + const providerInfo = this.walletSdkService.getProviderInfo(providerAddress, network); + const { warmStorageService } = this.walletSdkService.getWalletServices(network); let pdpEndEpoch: bigint; const existing = await awaitWithAbort(warmStorageService.getDataSet({ dataSetId }), signal); @@ -803,7 +811,7 @@ export class DealService implements OnModuleInit, OnModuleDestroy { }); } } - pdpEndEpoch = await this.waitForPdpEndEpoch(dataSetId, pollTimeoutMs, signal); + pdpEndEpoch = await this.waitForPdpEndEpoch(dataSetId, pollTimeoutMs, network, signal); } const result = await this.dealRepository.manager.transaction(async (manager) => { @@ -830,8 +838,13 @@ export class DealService implements OnModuleInit, OnModuleDestroy { * Poll FWSS getDataSet({dataSetId}).pdpEndEpoch until non-zero. Exponential * backoff capped at 8s. Throws on timeout. */ - private async waitForPdpEndEpoch(dataSetId: bigint, timeoutMs: number, signal?: AbortSignal): Promise { - const { warmStorageService } = this.walletSdkService.getWalletServices(); + private async waitForPdpEndEpoch( + dataSetId: bigint, + timeoutMs: number, + network: Network, + signal?: AbortSignal, + ): Promise { + const { warmStorageService } = this.walletSdkService.getWalletServices(network); const start = Date.now(); let delay = 1_000; while (Date.now() - start < timeoutMs) { @@ -859,16 +872,17 @@ export class DealService implements OnModuleInit, OnModuleDestroy { async createDataSetWithPiece( providerAddress: string, metadata: Record, + network: Network, signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); - const providerInfo = this.walletSdkService.getProviderInfo(providerAddress); + const providerInfo = this.walletSdkService.getProviderInfo(providerAddress, network); if (!providerInfo) { throw new Error(`Provider ${providerAddress} not found in registry`); } const labels = buildCheckMetricLabels({ + network, checkType: "dataSetCreation", - network: this.blockchainConfig.network, providerId: providerInfo.id, providerName: providerInfo.name, providerIsApproved: providerInfo.isApproved, @@ -892,7 +906,7 @@ export class DealService implements OnModuleInit, OnModuleDestroy { let transactionHash: string | undefined; try { - const synapse = this.sharedSynapse ?? (await this.createSynapseInstance()); + const synapse = this.walletSdkService.getSynapse(network) ?? (await this.createSynapseInstance(network)); signal?.throwIfAborted(); const DATA_SET_CREATION_PIECE_SIZE = 200 * 1024; // 200 KiB @@ -1031,14 +1045,15 @@ export class DealService implements OnModuleInit, OnModuleDestroy { // Deal Creation Helpers // ============================================================================ - private async createSynapseInstance(): Promise { + private async createSynapseInstance(network: Network): Promise { try { - const { synapse, isSessionKeyMode } = await createSynapseFromConfig(this.blockchainConfig); + const networkCfg = this.getNetworkConfig(network); + const { synapse, isSessionKeyMode } = await createSynapseFromConfig(networkCfg); if (isSessionKeyMode) { this.logger.log({ event: "synapse_session_key_init", message: "Initializing Synapse with session key", - walletAddress: this.blockchainConfig.walletAddress, + walletAddress: networkCfg.walletAddress, }); } return synapse; From 5bfe206ddeabb82e71c675a4c096142877b69302 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 14:13:21 +0530 Subject: [PATCH 06/23] migrate dev-tools --- apps/backend/src/common/logging.ts | 4 ++ .../src/dev-tools/dev-tools.controller.ts | 38 +++++++++++++--- .../src/dev-tools/dev-tools.service.ts | 45 ++++++++++++------- .../src/dev-tools/dto/trigger-deal.dto.ts | 15 ++++++- .../dev-tools/dto/trigger-retrieval.dto.ts | 15 ++++++- 5 files changed, 93 insertions(+), 24 deletions(-) diff --git a/apps/backend/src/common/logging.ts b/apps/backend/src/common/logging.ts index 8cc31559..5a541ae5 100644 --- a/apps/backend/src/common/logging.ts +++ b/apps/backend/src/common/logging.ts @@ -1,3 +1,5 @@ +import { Network } from "./types.js"; + type ErrorWithCode = Error & { code?: unknown }; const MAX_ERROR_STACK_LENGTH = 4 * 1024; const ERROR_STACK_REDACTION_BUFFER_LENGTH = 512; @@ -160,6 +162,7 @@ export type ProviderJobContext = { providerAddress: string; providerId: bigint; providerName: string; + network: Network; }; /** @@ -181,6 +184,7 @@ export type DataSetLogContext = { export type JobLogContext = { jobId?: string; providerAddress: string; + network: Network; providerId?: bigint; providerName?: string; }; diff --git a/apps/backend/src/dev-tools/dev-tools.controller.ts b/apps/backend/src/dev-tools/dev-tools.controller.ts index 7ae09d0e..55ccd2a3 100644 --- a/apps/backend/src/dev-tools/dev-tools.controller.ts +++ b/apps/backend/src/dev-tools/dev-tools.controller.ts @@ -1,6 +1,6 @@ import { Controller, Get, Logger, Param, Query, UsePipes, ValidationPipe } from "@nestjs/common"; import { ApiOperation, ApiQuery, ApiResponse, ApiTags } from "@nestjs/swagger"; -import { DevToolsService } from "./dev-tools.service.js"; +import { DEFAULT_NETWORK, DevToolsService } from "./dev-tools.service.js"; import { TriggerDealQueryDto, TriggerDealResponseDto } from "./dto/trigger-deal.dto.js"; import { TriggerRetrievalQueryDto, TriggerRetrievalResponseDto } from "./dto/trigger-retrieval.dto.js"; @@ -13,18 +13,28 @@ export class DevToolsController { @Get("providers") @ApiOperation({ summary: "List available storage providers" }) + @ApiQuery({ + name: "network", + default: DEFAULT_NETWORK, + required: false, + description: "Network to query (mainnet or calibration, default: calibration)", + example: "calibration", + enum: ["mainnet", "calibration"], + }) @ApiResponse({ status: 200, description: "List of available storage providers for testing", }) - listProviders() { + @UsePipes(new ValidationPipe({ transform: true })) + listProviders(@Query("network") network?: "mainnet" | "calibration") { this.logger.log({ event: "api_request", message: "GET /api/dev/providers", endpoint: "/api/dev/providers", method: "GET", + network, }); - return this.devToolsService.listProviders(); + return this.devToolsService.listProviders(network); } @Get("deal") @@ -35,6 +45,14 @@ export class DevToolsController { description: "Storage provider address", example: "0x1234567890abcdef1234567890abcdef12345678", }) + @ApiQuery({ + name: "network", + default: DEFAULT_NETWORK, + required: false, + description: "Network to use (mainnet or calibration, default: calibration)", + example: "calibration", + enum: ["mainnet", "calibration"], + }) @ApiResponse({ status: 200, description: "Deal accepted - use /api/dev/deals/:dealId to check progress", @@ -56,8 +74,9 @@ export class DevToolsController { endpoint: "/api/dev/deal", method: "GET", spAddress: query.spAddress, + network: query.network, }); - return this.devToolsService.triggerDeal(query.spAddress); + return this.devToolsService.triggerDeal(query.spAddress, query.network); } @Get("deals/:dealId") @@ -96,6 +115,14 @@ export class DevToolsController { description: "Storage provider address (uses most recent deal for this SP)", example: "0x1234567890abcdef1234567890abcdef12345678", }) + @ApiQuery({ + name: "network", + default: DEFAULT_NETWORK, + required: false, + description: "Network to query (mainnet or calibration, default: calibration)", + example: "calibration", + enum: ["mainnet", "calibration"], + }) @ApiResponse({ status: 200, description: "Test results", @@ -118,7 +145,8 @@ export class DevToolsController { method: "GET", dealId: query.dealId, spAddress: query.spAddress, + network: query.network, }); - return this.devToolsService.triggerRetrieval(query.dealId, query.spAddress); + return this.devToolsService.triggerRetrieval(query.dealId, query.spAddress, query.network); } } diff --git a/apps/backend/src/dev-tools/dev-tools.service.ts b/apps/backend/src/dev-tools/dev-tools.service.ts index 87522b0f..e859618f 100644 --- a/apps/backend/src/dev-tools/dev-tools.service.ts +++ b/apps/backend/src/dev-tools/dev-tools.service.ts @@ -1,10 +1,10 @@ import { BadRequestException, Injectable, Logger, NotFoundException } from "@nestjs/common"; -import { ConfigService } from "@nestjs/config"; import { InjectRepository } from "@nestjs/typeorm"; import type { Repository } from "typeorm"; +import { SUPPORTED_NETWORKS } from "../common/constants.js"; import { DealJobTerminatedDataSetError } from "../common/errors.js"; import { type DealLogContext, toStructuredError } from "../common/logging.js"; -import type { IBlockchainConfig, IConfig } from "../config/app.config.js"; +import type { Network } from "../common/types.js"; import { Deal } from "../database/entities/deal.entity.js"; import { DealStatus, RetrievalStatus } from "../database/types.js"; import { DealService } from "../deal/deal.service.js"; @@ -13,6 +13,8 @@ import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; import type { TriggerDealResponseDto } from "./dto/trigger-deal.dto.js"; import type { RetrievalMethodResultDto, TriggerRetrievalResponseDto } from "./dto/trigger-retrieval.dto.js"; +export const DEFAULT_NETWORK: Network = SUPPORTED_NETWORKS[0]; + @Injectable() export class DevToolsService { private readonly logger = new Logger(DevToolsService.name); @@ -21,7 +23,6 @@ export class DevToolsService { private readonly walletSdkService: WalletSdkService, private readonly dealService: DealService, private readonly retrievalService: RetrievalService, - private readonly configService: ConfigService, @InjectRepository(Deal) private readonly dealRepository: Repository, ) {} @@ -29,12 +30,13 @@ export class DevToolsService { /** * List all available storage providers for testing */ - listProviders(): unknown[] { - const providers = this.walletSdkService.getTestingProviders(); + listProviders(network: Network = DEFAULT_NETWORK): unknown[] { + const providers = this.walletSdkService.getTestingProviders(network); this.logger.log({ event: "providers_listed", message: "Listing available providers", count: providers.length, + network, }); // Serialize BigInt values to strings for JSON response return providers.map((p) => this.serializeBigInt(p)); @@ -73,15 +75,16 @@ export class DevToolsService { * Trigger a deal for a specific storage provider. * Returns immediately with deal ID - processing happens in background. */ - async triggerDeal(spAddress: string): Promise { + async triggerDeal(spAddress: string, network: Network = DEFAULT_NETWORK): Promise { this.logger.log({ event: "deal_trigger_requested", message: "Triggering deal for storage provider", spAddress, + network, }); // Validate SP exists - const providerInfo = this.walletSdkService.getProviderInfo(spAddress); + const providerInfo = this.walletSdkService.getProviderInfo(spAddress, network); if (!providerInfo) { throw new NotFoundException(`Storage provider not found: ${spAddress}`); } @@ -96,8 +99,8 @@ export class DevToolsService { // Create a pending deal record first so we can return the ID immediately const pendingDeal = this.dealRepository.create({ spAddress, - network: this.configService.get("blockchain").network, - walletAddress: this.dealService.getWalletAddress(), + network, + walletAddress: this.dealService.getWalletAddress(network), fileName: "pending", fileSize: 0, status: DealStatus.PENDING, @@ -119,10 +122,11 @@ export class DevToolsService { message: "Created pending deal, starting background processing", dealId, spAddress, + network, }); // Fire off the deal creation in the background (don't await) - this.processDealInBackground(dealId, providerInfo, dealLogContext).catch((err) => { + this.processDealInBackground(dealId, providerInfo, network, dealLogContext).catch((err) => { this.logger.error({ ...dealLogContext, event: "background_deal_processing_failed", @@ -149,6 +153,7 @@ export class DevToolsService { private async processDealInBackground( dealId: string, providerInfo: ReturnType, + network: Network, dealLogContext: DealLogContext, ): Promise { if (!providerInfo || providerInfo.id == null) { @@ -156,8 +161,10 @@ export class DevToolsService { } try { const deal = await this.dealService.createDealForProvider(providerInfo, { + network, existingDealId: dealId, logContext: { + network, jobId: "dev_tools_manual_deal", providerAddress: providerInfo.serviceProvider, providerId: providerInfo.id, @@ -228,13 +235,17 @@ export class DevToolsService { /** * Trigger data fetch for a deal by ID or most recent deal for an SP */ - async triggerRetrieval(dealId?: string, spAddress?: string): Promise { + async triggerRetrieval( + dealId?: string, + spAddress?: string, + network: Network = DEFAULT_NETWORK, + ): Promise { if (!dealId && !spAddress) { throw new BadRequestException("Either dealId or spAddress must be provided"); } // Find the deal - const deal = await this.findDeal(dealId, spAddress); + const deal = await this.findDeal(dealId, spAddress, network); const pieceCid = deal.pieceCid; if (!pieceCid) { throw new BadRequestException(`Deal ${deal.id} has no piece CID - cannot perform data fetch`); @@ -304,7 +315,7 @@ export class DevToolsService { /** * Find a deal by ID or most recent deal for an SP */ - private async findDeal(dealId?: string, spAddress?: string): Promise { + private async findDeal(dealId?: string, spAddress?: string, network: Network = DEFAULT_NETWORK): Promise { let deal: Deal | null = null; if (dealId) { @@ -316,17 +327,17 @@ export class DevToolsService { throw new NotFoundException(`Deal not found: ${dealId}`); } } else if (spAddress) { - // Find most recent successful deal for this SP + // Find most recent successful deal for this SP on the given network deal = await this.dealRepository.findOne({ where: [ - { spAddress, status: DealStatus.DEAL_CREATED }, - { spAddress, status: DealStatus.PIECE_ADDED }, + { spAddress, network, status: DealStatus.DEAL_CREATED }, + { spAddress, network, status: DealStatus.PIECE_ADDED }, ], order: { createdAt: "DESC" }, }); if (!deal) { - throw new NotFoundException(`No successful deals found for SP: ${spAddress}`); + throw new NotFoundException(`No successful deals found for SP: ${spAddress} on network: ${network}`); } } diff --git a/apps/backend/src/dev-tools/dto/trigger-deal.dto.ts b/apps/backend/src/dev-tools/dto/trigger-deal.dto.ts index 297af800..59c6f9cb 100644 --- a/apps/backend/src/dev-tools/dto/trigger-deal.dto.ts +++ b/apps/backend/src/dev-tools/dto/trigger-deal.dto.ts @@ -1,5 +1,6 @@ import { ApiProperty } from "@nestjs/swagger"; -import { IsNotEmpty, IsString } from "class-validator"; +import { IsIn, IsNotEmpty, IsOptional, IsString } from "class-validator"; +import type { Network } from "../../common/types.js"; export class TriggerDealQueryDto { @ApiProperty({ @@ -9,6 +10,18 @@ export class TriggerDealQueryDto { @IsString() @IsNotEmpty() spAddress: string; + + @ApiProperty({ + description: "Network to use", + example: "calibration", + required: false, + enum: ["mainnet", "calibration"], + default: "calibration", + }) + @IsString() + @IsIn(["mainnet", "calibration"]) + @IsOptional() + network?: Network; } export class TriggerDealResponseDto { diff --git a/apps/backend/src/dev-tools/dto/trigger-retrieval.dto.ts b/apps/backend/src/dev-tools/dto/trigger-retrieval.dto.ts index 6911ff58..76387f32 100644 --- a/apps/backend/src/dev-tools/dto/trigger-retrieval.dto.ts +++ b/apps/backend/src/dev-tools/dto/trigger-retrieval.dto.ts @@ -1,5 +1,6 @@ import { ApiProperty } from "@nestjs/swagger"; -import { IsOptional, IsString, IsUUID, ValidateIf } from "class-validator"; +import { IsIn, IsOptional, IsString, IsUUID, ValidateIf } from "class-validator"; +import type { Network } from "../../common/types.js"; export class TriggerRetrievalQueryDto { @ApiProperty({ @@ -21,6 +22,18 @@ export class TriggerRetrievalQueryDto { @IsOptional() @ValidateIf((o) => !o.dealId) spAddress?: string; + + @ApiProperty({ + description: "Network to use", + example: "calibration", + required: false, + enum: ["mainnet", "calibration"], + default: "calibration", + }) + @IsString() + @IsIn(["mainnet", "calibration"]) + @IsOptional() + network?: Network; } export class RetrievalMethodResultDto { From 9681d737c9339837a5ecd71316bf260d2f5e5390 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 16:01:07 +0530 Subject: [PATCH 07/23] test: fix retrieval service specs --- .../src/retrieval/retrieval.service.spec.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/backend/src/retrieval/retrieval.service.spec.ts b/apps/backend/src/retrieval/retrieval.service.spec.ts index a66711c0..83f2626c 100644 --- a/apps/backend/src/retrieval/retrieval.service.spec.ts +++ b/apps/backend/src/retrieval/retrieval.service.spec.ts @@ -3,6 +3,7 @@ import { Test, TestingModule } from "@nestjs/testing"; import { getRepositoryToken } from "@nestjs/typeorm"; import { afterEach, describe, expect, it, vi } from "vitest"; import { ClickhouseService } from "../clickhouse/clickhouse.service.js"; +import { Network } from "../common/types.js"; import { Deal } from "../database/entities/deal.entity.js"; import { Retrieval } from "../database/entities/retrieval.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; @@ -34,7 +35,7 @@ describe("RetrievalService timeouts", () => { get: vi.fn((key: string) => { if (key === "app") return { runMode: "api" }; if (key === "jobs") return { pgbossSchedulerEnabled: false }; - if (key === "blockchain") return { network: "calibration" }; + if (key === "networks") return { calibration: { walletAddress: "0x123" } }; if (key === "dataset") return { randomDatasetSizes: [10] }; if (key === "timeouts") return { ipniVerificationTimeoutMs: 10_000, ipniVerificationPollingMs: 2_000 }; return undefined; @@ -78,6 +79,7 @@ describe("RetrievalService timeouts", () => { const buildDeal = (overrides: Partial = {}): Deal => ({ id: "deal-1", + network: "calibration", spAddress: "0xsp", walletAddress: "0xwallet", pieceCid: "bafy-piece", @@ -174,6 +176,7 @@ describe("RetrievalService timeouts", () => { mockSpRepository.findOne.mockResolvedValue({ address: "0xsp", + network: "calibration", providerId: 7, isApproved: false, name: "Test SP", @@ -222,6 +225,7 @@ describe("RetrievalService timeouts", () => { await service.performAllRetrievals(buildDeal()); const labels = { + network: "calibration", checkType: "retrieval", providerId: "7", providerName: "Test SP", @@ -254,6 +258,7 @@ describe("RetrievalService timeouts", () => { await expect(service.performAllRetrievals(buildDeal())).rejects.toThrow("timeout"); const labels = { + network: "calibration", checkType: "retrieval", providerId: "7", providerName: "Test SP", @@ -320,6 +325,7 @@ describe("RetrievalService timeouts", () => { const retrievals = await service.performAllRetrievals(buildDeal(), abortController.signal); const labels = { + network: "calibration", checkType: "retrieval", providerId: "7", providerName: "Test SP", @@ -593,7 +599,7 @@ describe("RetrievalService DB/provider drift", () => { const mockConfigService = { get: vi.fn((key: string) => { if (key === "jobs") return { mode: "cron" }; - if (key === "blockchain") return { useOnlyApprovedProviders: false, walletAddress: "0x123" }; + if (key === "networks") return { calibration: { walletAddress: "0x123" } }; if (key === "dataset") return { randomDatasetSizes: [10] }; if (key === "timeouts") return { ipniVerificationTimeoutMs: 10_000, ipniVerificationPollingMs: 2_000 }; return undefined; @@ -646,10 +652,10 @@ describe("RetrievalService DB/provider drift", () => { it("selectRandomSuccessfulDealForProvider excludes cleaned-up deals", async () => { const { qb, calls } = createMockQueryBuilder(); const svc = (await createServiceWithQb(qb)) as unknown as { - selectRandomSuccessfulDealForProvider: (spAddress: string) => Promise; + selectRandomSuccessfulDealForProvider: (spAddress: string, network: Network) => Promise; }; - await svc.selectRandomSuccessfulDealForProvider("0xSP"); + await svc.selectRandomSuccessfulDealForProvider("0xSP", "calibration"); const cleanedUpCall = calls.find((c) => c.clause.includes("cleaned_up")); expect(cleanedUpCall).toBeDefined(); From 5bdbd42aef2bfbba82ddef463f45e78171ed9bf0 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 16:01:22 +0530 Subject: [PATCH 08/23] chore: remove log --- apps/backend/src/piece-cleanup/piece-cleanup.service.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/backend/src/piece-cleanup/piece-cleanup.service.ts b/apps/backend/src/piece-cleanup/piece-cleanup.service.ts index fda6e6b9..999dd8d9 100644 --- a/apps/backend/src/piece-cleanup/piece-cleanup.service.ts +++ b/apps/backend/src/piece-cleanup/piece-cleanup.service.ts @@ -364,8 +364,6 @@ export class PieceCleanupService { } const synapse = existingSynapse ?? this.walletSdkService.getSynapse(network) ?? (await this.createSynapseInstance(network)); - - console.log(synapse); const storage = await synapse.storage.createContext({ providerId, dataSetId: deal.dataSetId, From ebed65ea5b84585cc294c89933f7169d35f44a8b Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 16:09:17 +0530 Subject: [PATCH 09/23] migrate job service to multi network --- .../src/jobs/data-set-creation.handler.ts | 8 +- apps/backend/src/jobs/jobs.service.spec.ts | 406 +++++++++++------- apps/backend/src/jobs/jobs.service.ts | 337 ++++++++------- 3 files changed, 445 insertions(+), 306 deletions(-) diff --git a/apps/backend/src/jobs/data-set-creation.handler.ts b/apps/backend/src/jobs/data-set-creation.handler.ts index 1a1f29b6..26081a6a 100644 --- a/apps/backend/src/jobs/data-set-creation.handler.ts +++ b/apps/backend/src/jobs/data-set-creation.handler.ts @@ -1,5 +1,6 @@ import type { Logger } from "@nestjs/common"; import type { DataSetLogContext, ProviderJobContext } from "../common/logging.js"; +import { Network } from "../common/types.js"; import type { DealService } from "../deal/deal.service.js"; export interface DataSetCreationDeps { @@ -23,6 +24,7 @@ export interface DataSetCreationDeps { export async function provisionNextMissingDataSet( deps: DataSetCreationDeps, spAddress: string, + network: Network, minDataSets: number, baseDataSetMetadata: Record, dataSetLogContext: ProviderJobContext, @@ -45,7 +47,7 @@ export async function provisionNextMissingDataSet( dataSetIndex: i, }; - const status = await dealService.getDataSetProvisioningStatus(spAddress, metadata, signal); + const status = await dealService.getDataSetProvisioningStatus(spAddress, metadata, network, signal); if (status.status === "live") { existingCount++; @@ -59,7 +61,7 @@ export async function provisionNextMissingDataSet( message: "Detected PDP-terminated dataset; running repair", dataSetId: status.dataSetId.toString(), }); - const result = await dealService.repairTerminatedDataSet(spAddress, status.dataSetId, signal); + const result = await dealService.repairTerminatedDataSet(spAddress, status.dataSetId, network, signal); logger.log({ ...logContext, event: "data_set_repair_completed", @@ -75,7 +77,7 @@ export async function provisionNextMissingDataSet( event: "creating_provisioned_data_set", message: "Creating provisioned data-set", }); - await dealService.createDataSetWithPiece(spAddress, metadata, signal); + await dealService.createDataSetWithPiece(spAddress, metadata, network, signal); logger.log({ ...logContext, event: "data_set_provisioning_progress", diff --git a/apps/backend/src/jobs/jobs.service.spec.ts b/apps/backend/src/jobs/jobs.service.spec.ts index b75a5131..f46c537c 100644 --- a/apps/backend/src/jobs/jobs.service.spec.ts +++ b/apps/backend/src/jobs/jobs.service.spec.ts @@ -1,6 +1,7 @@ +import { Network } from "src/common/types.js"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DealJobTerminatedDataSetError } from "../common/errors.js"; -import type { IConfig, ISpBlocklistConfig } from "../config/app.config.js"; +import type { IConfig, INetworkConfig, INetworksConfig } from "../config/types.js"; import { DATA_RETENTION_POLL_QUEUE, PROVIDERS_REFRESH_QUEUE, @@ -9,6 +10,8 @@ import { } from "./job-queues.js"; import { JobsService } from "./jobs.service.js"; +const DEFAULT_NETWORK = "calibration"; + type JobsServiceDeps = ConstructorParameters; const callPrivate = (target: T, key: string, ...args: unknown[]) => { @@ -116,46 +119,54 @@ describe("JobsService schedule rows", () => { storageProvidersTested: { set: vi.fn() } as unknown as JobsServiceDeps[20], }; - const emptySpBlocklists: ISpBlocklistConfig = { - ids: new Set(), - addresses: new Set(), - }; + const baseNetworkConfig = { + walletPrivateKey: "0x", + network: DEFAULT_NETWORK, + useOnlyApprovedProviders: false, + minNumDataSetsForChecks: 1, + dealsPerSpPerHour: 4, + retrievalsPerSpPerHour: 2, + dataSetCreationsPerSpPerHour: 1, + dataRetentionPollIntervalSeconds: 3600, + providersRefreshIntervalSeconds: 14400, + walletAddress: "0x0000000000000000000000000000000000000000", + checkDatasetCreationFees: true, + maintenanceWindowsUtc: ["07:00", "22:00"], + maintenanceWindowMinutes: 20, + blockedSpIds: new Set(), + blockedSpAddresses: new Set(), + pieceCleanupPerSpPerHour: 1, + maxPieceCleanupRuntimeSeconds: 300, + maxDatasetStorageSizeBytes: 24 * 1024 * 1024 * 1024, + targetDatasetStorageSizeBytes: 20 * 1024 * 1024 * 1024, + dealJobTimeoutSeconds: 360, + dataSetCreationJobTimeoutSeconds: 300, + retrievalJobTimeoutSeconds: 60, + pullChecksPerSpPerHour: 1, + pullCheckJobTimeoutSeconds: 300, + pullCheckPollIntervalSeconds: 2, + pullCheckPieceSizeBytes: 10 * 1024 * 1024, + pullPieceMaxConcurrentStreams: 50, + pullPieceMaxStreamsPerCid: 3, + pullPieceCleanupIntervalSeconds: 7 * 24 * 3600, + clickhouseUrl: "http://localhost:8123", + clickhouseBatchSize: 500, + clickhouseFlushIntervalMs: 5000, + clickhouseMaxBufferSize: 5000, + } satisfies IConfig["networks"]["calibration"]; baseConfigValues = { app: { runMode: "both" } as IConfig["app"], - blockchain: { - useOnlyApprovedProviders: false, - minNumDataSetsForChecks: 1, - network: "calibration", - } as IConfig["blockchain"], - scheduling: { - providersRefreshIntervalSeconds: 4 * 3600, - dataRetentionPollIntervalSeconds: 3600, - maintenanceWindowsUtc: ["07:00", "22:00"], - maintenanceWindowMinutes: 20, - } as IConfig["scheduling"], + activeNetworks: [DEFAULT_NETWORK] as IConfig["activeNetworks"], + networks: { calibration: baseNetworkConfig } as unknown as IConfig["networks"], jobs: { schedulePhaseSeconds: 0, catchupMaxEnqueue: 10, pgbossLocalConcurrency: 9, pgbossSchedulerEnabled: true, workerPollSeconds: 60, - dealJobTimeoutSeconds: 360, - retrievalJobTimeoutSeconds: 60, - dataSetCreationJobTimeoutSeconds: 300, shutdownFinalScrapeDelaySeconds: 35, - pieceCleanupPerSpPerHour: 1, - maxPieceCleanupRuntimeSeconds: 300, } as IConfig["jobs"], - pullPiece: { - pullChecksPerSpPerHour: 1, - pullCheckJobTimeoutSeconds: 300, - pullCheckPollIntervalSeconds: 2, - pullCheckPieceSizeBytes: 10 * 1024 * 1024, - maxConcurrentStreams: 50, - maxStreamsPerCid: 3, - pullPieceCleanupIntervalSeconds: 7 * 24 * 3600, - }, database: { host: "localhost", port: 5432, @@ -163,10 +174,6 @@ describe("JobsService schedule rows", () => { password: "pass", database: "dealbot", } as IConfig["database"], - pieceCleanup: { - maxDatasetStorageSizeBytes: 24 * 1024 * 1024 * 1024, - } as IConfig["pieceCleanup"], - spBlocklists: emptySpBlocklists, }; configService = { @@ -221,16 +228,16 @@ describe("JobsService schedule rows", () => { return "success"; }); - await callPrivate(service, "recordJobExecution", "deal", run); + await callPrivate(service, "recordJobExecution", "deal", DEFAULT_NETWORK, run); expect(run).toHaveBeenCalled(); - expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }); + expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", network: DEFAULT_NETWORK }); expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", handler_result: "success", - network: "calibration", + network: DEFAULT_NETWORK, }); - expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }, 5); + expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal", network: DEFAULT_NETWORK }, 5); }); it("records metrics for failed job execution", async () => { @@ -247,15 +254,15 @@ describe("JobsService schedule rows", () => { throw new Error("boom"); }); - await expect(callPrivate(service, "recordJobExecution", "deal", run)).rejects.toThrow("boom"); + await expect(callPrivate(service, "recordJobExecution", "deal", DEFAULT_NETWORK, run)).rejects.toThrow("boom"); - expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }); + expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", network: DEFAULT_NETWORK }); expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", handler_result: "error", - network: "calibration", + network: DEFAULT_NETWORK, }); - expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }, 2); + expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal", network: DEFAULT_NETWORK }, 2); }); it("records metrics for aborted job execution", async () => { @@ -272,15 +279,15 @@ describe("JobsService schedule rows", () => { return "aborted" as const; }); - await callPrivate(service, "recordJobExecution", "deal", run); + await callPrivate(service, "recordJobExecution", "deal", DEFAULT_NETWORK, run); - expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }); + expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", network: DEFAULT_NETWORK }); expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", handler_result: "aborted", - network: "calibration", + network: DEFAULT_NETWORK, }); - expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }, 3); + expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal", network: DEFAULT_NETWORK }, 3); }); it("deal job records aborted when abort signal fires", async () => { @@ -330,9 +337,9 @@ describe("JobsService schedule rows", () => { const jobPromise = callPrivate(service, "handleDealJob", { id: "job-123", data: { + network: DEFAULT_NETWORK, jobType: "deal", spAddress: "0xaaa", - network: "calibration", intervalSeconds: 60, }, }); @@ -344,7 +351,7 @@ describe("JobsService schedule rows", () => { expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", handler_result: "aborted", - network: "calibration", + network: DEFAULT_NETWORK, }); }); @@ -367,7 +374,7 @@ describe("JobsService schedule rows", () => { } as unknown as JobsServiceDeps[0]; const retrievalService = { - performRandomRetrievalForProvider: vi.fn(async (_sp: string, signal: AbortSignal) => { + performRandomRetrievalForProvider: vi.fn(async (_sp: string, _network: Network, signal: AbortSignal) => { await new Promise((resolve) => { if (signal.aborted) return resolve(); signal.addEventListener("abort", () => resolve(), { once: true }); @@ -395,7 +402,7 @@ describe("JobsService schedule rows", () => { data: { jobType: "retrieval", spAddress: "0xaaa", - network: "calibration", + network: DEFAULT_NETWORK, intervalSeconds: 60, }, }); @@ -406,7 +413,7 @@ describe("JobsService schedule rows", () => { expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "retrieval", handler_result: "aborted", - network: "calibration", + network: DEFAULT_NETWORK, }); }); @@ -435,13 +442,14 @@ describe("JobsService schedule rows", () => { data: { jobType: "retrieval", spAddress: "0xaaa", - network: "calibration", + network: DEFAULT_NETWORK, intervalSeconds: 60, }, }); expect(retrievalService.performRandomRetrievalForProvider).toHaveBeenCalledWith( "0xaaa", + DEFAULT_NETWORK, expect.any(AbortSignal), expect.objectContaining({ jobId: "job-retrieval-provider-fallback", @@ -477,7 +485,7 @@ describe("JobsService schedule rows", () => { data: { jobType: "retrieval", spAddress: "0xaaa", - network: "calibration", + network: DEFAULT_NETWORK, intervalSeconds: 60, }, }), @@ -487,7 +495,7 @@ describe("JobsService schedule rows", () => { expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "retrieval", handler_result: "error", - network: "calibration", + network: DEFAULT_NETWORK, }); }); @@ -507,17 +515,17 @@ describe("JobsService schedule rows", () => { .mockResolvedValueOnce([{ job_type: "deal", min_age_seconds: 12 }]) .mockResolvedValueOnce([{ job_type: "retrieval", min_age_seconds: 34 }]); - await callPrivate(service, "updateQueueMetrics"); + await callPrivate(service, "updateQueueMetrics", DEFAULT_NETWORK); - expect(jobsQueuedGauge.set).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }, 0); - expect(jobsQueuedGauge.set).toHaveBeenCalledWith({ job_type: "retrieval", network: "calibration" }, 0); - expect(jobsQueuedGauge.set).toHaveBeenCalledWith({ job_type: "data_retention_poll", network: "calibration" }, 0); - expect(jobsInFlightGauge.set).toHaveBeenCalledWith({ job_type: "retrieval", network: "calibration" }, 1); - expect(jobsQueuedGauge.set).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }, 2); + expect(jobsQueuedGauge.set).toHaveBeenCalledWith({ job_type: "deal", network: DEFAULT_NETWORK }, 0); + expect(jobsQueuedGauge.set).toHaveBeenCalledWith({ job_type: "retrieval", network: DEFAULT_NETWORK }, 0); + expect(jobsQueuedGauge.set).toHaveBeenCalledWith({ job_type: "data_retention_poll", network: DEFAULT_NETWORK }, 0); + expect(jobsInFlightGauge.set).toHaveBeenCalledWith({ job_type: "retrieval", network: DEFAULT_NETWORK }, 1); + expect(jobsQueuedGauge.set).toHaveBeenCalledWith({ job_type: "deal", network: DEFAULT_NETWORK }, 2); - expect(oldestQueuedGauge.set).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }, 12); - expect(oldestInFlightGauge.set).toHaveBeenCalledWith({ job_type: "retrieval", network: "calibration" }, 34); - expect(jobsPausedGauge.set).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }, 0); + expect(oldestQueuedGauge.set).toHaveBeenCalledWith({ job_type: "deal", network: DEFAULT_NETWORK }, 12); + expect(oldestInFlightGauge.set).toHaveBeenCalledWith({ job_type: "retrieval", network: DEFAULT_NETWORK }, 34); + expect(jobsPausedGauge.set).toHaveBeenCalledWith({ job_type: "deal", network: DEFAULT_NETWORK }, 0); }); it("registers pg-boss workers with per-queue batch sizes", async () => { @@ -645,15 +653,17 @@ describe("JobsService schedule rows", () => { jobScheduleRepositoryMock.countPausedSchedules.mockResolvedValueOnce([{ job_type: "deal", count: 2 }]); jobScheduleRepositoryMock.minBossJobAgeSecondsByState.mockResolvedValueOnce([]).mockResolvedValueOnce([]); - await callPrivate(service, "updateQueueMetrics"); + await callPrivate(service, "updateQueueMetrics", DEFAULT_NETWORK); - expect(jobsPausedGauge.set).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }, 2); + expect(jobsPausedGauge.set).toHaveBeenCalledWith({ job_type: "deal", network: DEFAULT_NETWORK }, 2); }); it("adds schedule rows for newly seen providers", async () => { baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -666,8 +676,8 @@ describe("JobsService schedule rows", () => { storageProviderRepositoryMock.find.mockResolvedValueOnce([providerA]).mockResolvedValueOnce([providerA, providerB]); - await callPrivate(service, "ensureScheduleRows"); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", DEFAULT_NETWORK); + await callPrivate(service, "ensureScheduleRows", DEFAULT_NETWORK); // Check upserts for providerB const upsertCalls = jobScheduleRepositoryMock.upsertSchedule.mock.calls; @@ -686,18 +696,18 @@ describe("JobsService schedule rows", () => { const providerA = { address: "0xaaa" }; storageProviderRepositoryMock.find.mockResolvedValueOnce([providerA]); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", DEFAULT_NETWORK); expect(jobScheduleRepositoryMock.deleteSchedulesForInactiveProviders).toHaveBeenCalledWith( [providerA.address], - "calibration", + DEFAULT_NETWORK, ); }); it("does not delete schedule rows when no active providers exist", async () => { storageProviderRepositoryMock.find.mockResolvedValueOnce([]); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", DEFAULT_NETWORK); expect(jobScheduleRepositoryMock.deleteSchedulesForInactiveProviders).not.toHaveBeenCalled(); }); @@ -705,7 +715,9 @@ describe("JobsService schedule rows", () => { it("uses approved-only filter when configured", async () => { baseConfigValues = { ...baseConfigValues, - blockchain: { useOnlyApprovedProviders: true, network: "calibration" } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, useOnlyApprovedProviders: true }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -714,30 +726,30 @@ describe("JobsService schedule rows", () => { service = buildService({ configService }); storageProviderRepositoryMock.find.mockResolvedValueOnce([]); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", DEFAULT_NETWORK); expect(storageProviderRepositoryMock.find).toHaveBeenCalledWith({ select: { address: true, providerId: true }, - where: { isActive: true, isApproved: true, network: "calibration" }, + where: { isActive: true, isApproved: true, network: DEFAULT_NETWORK }, }); }); it("always inserts global data_retention_poll and providers_refresh schedules", async () => { storageProviderRepositoryMock.find.mockResolvedValueOnce([]); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", DEFAULT_NETWORK); expect(jobScheduleRepositoryMock.upsertSchedule).toHaveBeenCalledWith( "providers_refresh", "", - "calibration", + DEFAULT_NETWORK, expect.any(Number), expect.any(Date), ); expect(jobScheduleRepositoryMock.upsertSchedule).toHaveBeenCalledWith( "data_retention_poll", "", - "calibration", + DEFAULT_NETWORK, expect.any(Number), expect.any(Date), ); @@ -769,19 +781,19 @@ describe("JobsService schedule rows", () => { id: 1, job_type: "deal", sp_address: "0xaaa", - network: "calibration", + network: DEFAULT_NETWORK, interval_seconds: 1, next_run_at: "2024-01-01T00:00:00Z", }, ]); - await callPrivate(service, "enqueueDueJobs"); + await callPrivate(service, "enqueueDueJobs", DEFAULT_NETWORK); expect(send).toHaveBeenCalledTimes(3); for (const call of send.mock.calls) { expect(call[0]).toBe("sp.work"); - expect(call[1]).toMatchObject({ jobType: "deal", spAddress: "0xaaa", network: "calibration" }); - expect(call[2]).toMatchObject({ singletonKey: "calibration:0xaaa", retryLimit: 0 }); + expect(call[1]).toMatchObject({ jobType: "deal", spAddress: "0xaaa", network: DEFAULT_NETWORK }); + expect(call[2]).toMatchObject({ singletonKey: `${DEFAULT_NETWORK}:0xaaa`, retryLimit: 0 }); expect(call[2]?.startAfter).toBeUndefined(); } @@ -805,12 +817,13 @@ describe("JobsService schedule rows", () => { id: 10, job_type: "providers_refresh", sp_address: "", + network: DEFAULT_NETWORK, interval_seconds: 1800, next_run_at: "2024-01-01T00:00:00Z", }, ]); - await callPrivate(service, "enqueueDueJobs"); + await callPrivate(service, "enqueueDueJobs", DEFAULT_NETWORK); // Should only enqueue once despite being 8 intervals overdue expect(send).toHaveBeenCalledTimes(1); @@ -837,13 +850,13 @@ describe("JobsService schedule rows", () => { id: 11, job_type: "providers_refresh", sp_address: "", - network: "calibration", + network: DEFAULT_NETWORK, interval_seconds: 14400, next_run_at: "2024-01-01T00:00:00Z", }, ]); - await callPrivate(service, "enqueueDueJobs"); + await callPrivate(service, "enqueueDueJobs", DEFAULT_NETWORK); expect(send).toHaveBeenCalledTimes(1); expect(send.mock.calls[0][2]).toMatchObject({ @@ -855,11 +868,13 @@ describe("JobsService schedule rows", () => { it("global jobs are skipped during maintenance windows", async () => { baseConfigValues = { ...baseConfigValues, - scheduling: { - ...baseConfigValues.scheduling, - maintenanceWindowsUtc: ["03:00"], - maintenanceWindowMinutes: 60, - } as IConfig["scheduling"], + networks: { + calibration: { + ...baseConfigValues.networks?.calibration, + maintenanceWindowsUtc: ["03:00"], + maintenanceWindowMinutes: 60, + } as INetworkConfig, + } as INetworksConfig, }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -878,13 +893,14 @@ describe("JobsService schedule rows", () => { { id: 20, job_type: "providers_refresh", + network: DEFAULT_NETWORK, sp_address: "", interval_seconds: 1800, next_run_at: "2024-01-01T03:00:00Z", }, ]); - await callPrivate(service, "enqueueDueJobs"); + await callPrivate(service, "enqueueDueJobs", DEFAULT_NETWORK); // Global job should not be enqueued during maintenance expect(send).not.toHaveBeenCalled(); @@ -898,11 +914,13 @@ describe("JobsService schedule rows", () => { it("defers jobs until maintenance window ends (same-day)", async () => { baseConfigValues = { ...baseConfigValues, - scheduling: { - ...baseConfigValues.scheduling, - maintenanceWindowsUtc: ["07:00"], - maintenanceWindowMinutes: 20, - } as IConfig["scheduling"], + networks: { + calibration: { + ...baseConfigValues.networks?.calibration, + maintenanceWindowsUtc: ["07:00"], + maintenanceWindowMinutes: 20, + } as INetworkConfig, + } as INetworksConfig, }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -914,13 +932,13 @@ describe("JobsService schedule rows", () => { (service as unknown as { safeSend: typeof safeSend }).safeSend = safeSend; const now = new Date("2024-01-01T07:05:00Z"); - const maintenance = callPrivate(service, "getMaintenanceWindowStatus", now) as any; + const maintenance = callPrivate(service, "getMaintenanceWindowStatus", now, DEFAULT_NETWORK) as any; await callPrivate( service, "deferJobForMaintenance", "deal", - { jobType: "deal", spAddress: "0xaaa", network: "calibration", intervalSeconds: 60 }, + { jobType: "deal", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 60 }, maintenance, now, ); @@ -929,7 +947,7 @@ describe("JobsService schedule rows", () => { expect(safeSend).toHaveBeenCalledWith( "deal", "sp.work", - { jobType: "deal", spAddress: "0xaaa", network: "calibration", intervalSeconds: 60 }, + { jobType: "deal", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 60 }, { startAfter: expectedResumeAt }, ); }); @@ -937,11 +955,13 @@ describe("JobsService schedule rows", () => { it("defers jobs until maintenance window ends (wraps midnight)", async () => { baseConfigValues = { ...baseConfigValues, - scheduling: { - ...baseConfigValues.scheduling, - maintenanceWindowsUtc: ["23:50"], - maintenanceWindowMinutes: 20, - } as IConfig["scheduling"], + networks: { + calibration: { + ...baseConfigValues.networks?.calibration, + maintenanceWindowsUtc: ["23:50"], + maintenanceWindowMinutes: 20, + } as INetworkConfig, + } as INetworksConfig, }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -953,13 +973,13 @@ describe("JobsService schedule rows", () => { (service as unknown as { safeSend: typeof safeSend }).safeSend = safeSend; const now = new Date("2024-01-01T23:55:00Z"); - const maintenance = callPrivate(service, "getMaintenanceWindowStatus", now) as any; + const maintenance = callPrivate(service, "getMaintenanceWindowStatus", now, DEFAULT_NETWORK) as any; await callPrivate( service, "deferJobForMaintenance", "retrieval", - { jobType: "retrieval", spAddress: "0xbbb", network: "calibration", intervalSeconds: 60 }, + { jobType: "retrieval", spAddress: "0xbbb", network: DEFAULT_NETWORK, intervalSeconds: 60 }, maintenance, now, ); @@ -968,7 +988,7 @@ describe("JobsService schedule rows", () => { expect(safeSend).toHaveBeenCalledWith( "retrieval", "sp.work", - { jobType: "retrieval", spAddress: "0xbbb", network: "calibration", intervalSeconds: 60 }, + { jobType: "retrieval", spAddress: "0xbbb", network: DEFAULT_NETWORK, intervalSeconds: 60 }, { startAfter: expectedResumeAt }, ); }); @@ -994,7 +1014,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDealJob", { id: "job-deal-1", - data: { jobType: "deal", spAddress: "0xaaa", network: "calibration", intervalSeconds: 60 }, + data: { jobType: "deal", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 60 }, }); expect(dealService.createDealForProvider).toHaveBeenCalledTimes(1); @@ -1034,7 +1054,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDealJob", { id: "job-deal-no-quota-gate", - data: { jobType: "deal", spAddress: "0xaaa", network: "calibration", intervalSeconds: 60 }, + data: { jobType: "deal", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 60 }, }); expect(pieceCleanupService.cleanupPiecesForProvider).not.toHaveBeenCalled(); @@ -1066,14 +1086,14 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDealJob", { id: "job-deal-terminated", - data: { jobType: "deal", spAddress: "0xaaa", network: "calibration", intervalSeconds: 60 }, + data: { jobType: "deal", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 60 }, }); expect(dealService.createDealForProvider).toHaveBeenCalledTimes(1); expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", handler_result: "error", - network: "calibration", + network: DEFAULT_NETWORK, }); }); @@ -1099,13 +1119,14 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDataSetCreationJob", { id: "job-ds-1", - data: { jobType: "data_set_creation", spAddress: "0xaaa", network: "calibration", intervalSeconds: 3600 }, + data: { jobType: "data_set_creation", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 3600 }, }); expect(dealService.createDataSetWithPiece).toHaveBeenCalledTimes(1); expect(dealService.createDataSetWithPiece).toHaveBeenCalledWith( "0xaaa", { withIpniIndexing: "" }, + DEFAULT_NETWORK, expect.any(AbortSignal), ); }); @@ -1116,7 +1137,9 @@ describe("JobsService schedule rows", () => { baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -1141,13 +1164,14 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDataSetCreationJob", { id: "job-ds-2", - data: { jobType: "data_set_creation", spAddress: "0xaaa", network: "calibration", intervalSeconds: 3600 }, + data: { jobType: "data_set_creation", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 3600 }, }); expect(dealService.createDataSetWithPiece).not.toHaveBeenCalled(); expect(dealService.getDataSetProvisioningStatus).toHaveBeenCalledWith( "0xaaa", { dealbotDataSetVersion: "v1" }, + DEFAULT_NETWORK, expect.any(AbortSignal), ); }); @@ -1157,7 +1181,9 @@ describe("JobsService schedule rows", () => { vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -1182,7 +1208,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDataSetCreationJob", { id: "job-ds-3", - data: { jobType: "data_set_creation", spAddress: "0xaaa", network: "calibration", intervalSeconds: 3600 }, + data: { jobType: "data_set_creation", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 3600 }, }); // Only the first missing data set (index 0) should be created @@ -1190,6 +1216,7 @@ describe("JobsService schedule rows", () => { expect(dealService.createDataSetWithPiece).toHaveBeenCalledWith( "0xaaa", { dealbotDataSetVersion: "v1" }, + DEFAULT_NETWORK, expect.any(AbortSignal), ); }); @@ -1199,7 +1226,9 @@ describe("JobsService schedule rows", () => { vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -1227,7 +1256,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDataSetCreationJob", { id: "job-ds-3b", - data: { jobType: "data_set_creation", spAddress: "0xaaa", network: "calibration", intervalSeconds: 3600 }, + data: { jobType: "data_set_creation", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 3600 }, }); // Should skip index 0 (exists) and create only index 1 @@ -1235,6 +1264,7 @@ describe("JobsService schedule rows", () => { expect(dealService.createDataSetWithPiece).toHaveBeenCalledWith( "0xaaa", { dealbotDataSetVersion: "v1", dealbotDS: "1" }, + DEFAULT_NETWORK, expect.any(AbortSignal), ); }); @@ -1258,9 +1288,16 @@ describe("JobsService schedule rows", () => { provisionNextMissingDataSet( { dealService, logger }, "0xaaa", + DEFAULT_NETWORK, 5, {}, - { providerAddress: "0xaaa", jobId: "job-ds-4", providerId: 1n, providerName: "test-provider" }, + { + providerAddress: "0xaaa", + jobId: "job-ds-4", + providerId: 1n, + providerName: "test-provider", + network: DEFAULT_NETWORK, + }, controller.signal, ), ).rejects.toThrow("Job timed out"); @@ -1284,12 +1321,13 @@ describe("JobsService schedule rows", () => { await provisionNextMissingDataSet( { dealService, logger }, "0xaaa", + DEFAULT_NETWORK, 3, {}, - { providerAddress: "0xaaa", jobId: "job-ds-term", providerId: 1n, providerName: "sp" }, + { providerAddress: "0xaaa", jobId: "job-ds-term", providerId: 1n, providerName: "sp", network: DEFAULT_NETWORK }, ); - expect(dealService.repairTerminatedDataSet).toHaveBeenCalledWith("0xaaa", 7n, undefined); + expect(dealService.repairTerminatedDataSet).toHaveBeenCalledWith("0xaaa", 7n, DEFAULT_NETWORK, undefined); expect(dealService.createDataSetWithPiece).not.toHaveBeenCalled(); }); @@ -1302,33 +1340,41 @@ describe("JobsService schedule rows", () => { const activeGauge = metricsMocks.storageProvidersActive as unknown as { set: ReturnType }; const testedGauge = metricsMocks.storageProvidersTested as unknown as { set: ReturnType }; - await callPrivate(service, "updateStorageProviderGauges"); + await callPrivate(service, "updateStorageProviderGauges", DEFAULT_NETWORK); - expect(activeGauge.set).toHaveBeenCalledWith({ status: "active", network: "calibration" }, 7); - expect(activeGauge.set).toHaveBeenCalledWith({ status: "inactive", network: "calibration" }, 3); - expect(testedGauge.set).toHaveBeenCalledWith({ network: "calibration" }, 7); + expect(activeGauge.set).toHaveBeenCalledWith({ status: "active", network: DEFAULT_NETWORK }, 7); + expect(activeGauge.set).toHaveBeenCalledWith({ status: "inactive", network: DEFAULT_NETWORK }, 3); + expect(testedGauge.set).toHaveBeenCalledWith({ network: DEFAULT_NETWORK }, 7); }); it("filters tested providers by isApproved when useOnlyApprovedProviders is enabled", async () => { - baseConfigValues.blockchain = { - useOnlyApprovedProviders: true, - minNumDataSetsForChecks: 1, - } as IConfig["blockchain"]; + baseConfigValues = { + ...baseConfigValues, + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, useOnlyApprovedProviders: true }, + } as unknown as IConfig["networks"], + }; service = buildService(); storageProviderRepositoryMock.count.mockResolvedValueOnce(10).mockResolvedValueOnce(7).mockResolvedValueOnce(5); // testedCount (only approved) - await callPrivate(service, "updateStorageProviderGauges"); + await callPrivate(service, "updateStorageProviderGauges", DEFAULT_NETWORK); expect(storageProviderRepositoryMock.count).toHaveBeenNthCalledWith(3, { - where: { isActive: true, isApproved: true }, + where: { isActive: true, isApproved: true, network: DEFAULT_NETWORK }, }); }); it("subtracts globally blocked providers from tested gauge when global blocklist is non-empty", async () => { - baseConfigValues.spBlocklists = { - ids: new Set(), - addresses: new Set(["0xblocked"]), + baseConfigValues = { + ...baseConfigValues, + networks: { + calibration: { + ...(baseConfigValues.networks as any).calibration, + blockedSpIds: new Set(), + blockedSpAddresses: new Set(["0xblocked"]), + }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -1348,9 +1394,9 @@ describe("JobsService schedule rows", () => { const testedGauge = metricsMocks.storageProvidersTested as unknown as { set: ReturnType }; - await callPrivate(service, "updateStorageProviderGauges"); + await callPrivate(service, "updateStorageProviderGauges", DEFAULT_NETWORK); - expect(testedGauge.set).toHaveBeenCalledWith({ network: "calibration" }, 2); // 3 providers minus 1 globally blocked + expect(testedGauge.set).toHaveBeenCalledWith({ network: DEFAULT_NETWORK }, 2); // 3 providers minus 1 globally blocked }); it("catches storage provider gauge errors without rethrowing", async () => { @@ -1362,10 +1408,19 @@ describe("JobsService schedule rows", () => { const providerA = { address: "0xaaa", providerId: 1n }; storageProviderRepositoryMock.find.mockResolvedValueOnce([providerA]); - baseConfigValues.spBlocklists = { ids: new Set(["1"]), addresses: new Set() }; + baseConfigValues = { + ...baseConfigValues, + networks: { + calibration: { + ...(baseConfigValues.networks as any).calibration, + blockedSpIds: new Set(["1"]), + blockedSpAddresses: new Set(), + }, + } as unknown as IConfig["networks"], + }; service = buildService(); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", DEFAULT_NETWORK); const upsertCalls = jobScheduleRepositoryMock.upsertSchedule.mock.calls; const jobTypes = upsertCalls.filter((c) => c[1] === providerA.address).map((c) => c[0]); @@ -1374,14 +1429,23 @@ describe("JobsService schedule rows", () => { expect(jobTypes).not.toContain("retrieval"); // Blocked provider is excluded from the active-address list passed to cleanup, // so its existing schedule rows will be deleted. - expect(jobScheduleRepositoryMock.deleteSchedulesForInactiveProviders).toHaveBeenCalledWith([], "calibration"); + expect(jobScheduleRepositoryMock.deleteSchedulesForInactiveProviders).toHaveBeenCalledWith([], DEFAULT_NETWORK); }); it("deal job is skipped at runtime when provider is blocked", async () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); - baseConfigValues.spBlocklists = { ids: new Set(["1"]), addresses: new Set() }; + baseConfigValues = { + ...baseConfigValues, + networks: { + calibration: { + ...(baseConfigValues.networks as any).calibration, + blockedSpIds: new Set(["1"]), + blockedSpAddresses: new Set(), + }, + } as unknown as IConfig["networks"], + }; const dealService = { createDealForProvider: vi.fn(), @@ -1400,7 +1464,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDealJob", { id: "job-blocked-deal", - data: { jobType: "deal", spAddress: "0xaaa", network: "calibration", intervalSeconds: 60 }, + data: { jobType: "deal", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 60 }, }); expect(dealService.createDealForProvider).not.toHaveBeenCalled(); @@ -1410,7 +1474,16 @@ describe("JobsService schedule rows", () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); - baseConfigValues.spBlocklists = { ids: new Set(["2"]), addresses: new Set() }; + baseConfigValues = { + ...baseConfigValues, + networks: { + calibration: { + ...(baseConfigValues.networks as any).calibration, + blockedSpIds: new Set(["2"]), + blockedSpAddresses: new Set(), + }, + } as unknown as IConfig["networks"], + }; const retrievalService = { performRandomRetrievalForProvider: vi.fn() }; const walletSdkService = { @@ -1424,7 +1497,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleRetrievalJob", { id: "job-blocked-retrieval", - data: { jobType: "retrieval", spAddress: "0xaaa", network: "calibration", intervalSeconds: 60 }, + data: { jobType: "retrieval", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 60 }, }); expect(retrievalService.performRandomRetrievalForProvider).not.toHaveBeenCalled(); @@ -1434,7 +1507,16 @@ describe("JobsService schedule rows", () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); - baseConfigValues.spBlocklists = { ids: new Set(["3"]), addresses: new Set() }; + baseConfigValues = { + ...baseConfigValues, + networks: { + calibration: { + ...(baseConfigValues.networks as any).calibration, + blockedSpIds: new Set(["3"]), + blockedSpAddresses: new Set(), + }, + } as unknown as IConfig["networks"], + }; const dealService = { getBaseDataSetMetadata: vi.fn(() => ({})), @@ -1453,7 +1535,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDataSetCreationJob", { id: "job-blocked-ds", - data: { jobType: "data_set_creation", spAddress: "0xaaa", network: "calibration", intervalSeconds: 3600 }, + data: { jobType: "data_set_creation", spAddress: "0xaaa", network: DEFAULT_NETWORK, intervalSeconds: 3600 }, }); expect(dealService.createDataSetWithPiece).not.toHaveBeenCalled(); @@ -1463,7 +1545,16 @@ describe("JobsService schedule rows", () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); - baseConfigValues.spBlocklists = { ids: new Set(), addresses: new Set(["0xaaa"]) }; + baseConfigValues = { + ...baseConfigValues, + networks: { + calibration: { + ...(baseConfigValues.networks as any).calibration, + blockedSpIds: new Set(), + blockedSpAddresses: new Set(["0xaaa"]), + }, + } as unknown as IConfig["networks"], + }; const completedCounter = metricsMocks.jobsCompletedCounter as unknown as { inc: ReturnType }; const dealService = { @@ -1522,7 +1613,7 @@ describe("JobsService schedule rows", () => { data: { jobType: testCase.jobType, spAddress: "0xaaa", - network: "calibration", + network: DEFAULT_NETWORK, intervalSeconds: testCase.intervalSeconds, }, }); @@ -1531,7 +1622,7 @@ describe("JobsService schedule rows", () => { expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: testCase.jobType, handler_result: "success", - network: "calibration", + network: DEFAULT_NETWORK, }); } @@ -1568,20 +1659,17 @@ describe("JobsService schedule rows", () => { it("picks the longest timeout across all job types, including pullCheck under pullPiece", async () => { vi.useFakeTimers(); - const overriddenJobs = { - ...baseConfigValues.jobs, - dealJobTimeoutSeconds: 120, - retrievalJobTimeoutSeconds: 60, - dataSetCreationJobTimeoutSeconds: 120, - } as IConfig["jobs"]; - const overriddenPullPiece = { - ...baseConfigValues.pullPiece, - pullCheckJobTimeoutSeconds: 600, - } as IConfig["pullPiece"]; baseConfigValues = { ...baseConfigValues, - jobs: overriddenJobs, - pullPiece: overriddenPullPiece, + networks: { + calibration: { + ...(baseConfigValues.networks as any).calibration, + dealJobTimeoutSeconds: 120, + retrievalJobTimeoutSeconds: 60, + dataSetCreationJobTimeoutSeconds: 120, + pullCheckJobTimeoutSeconds: 600, + }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), diff --git a/apps/backend/src/jobs/jobs.service.ts b/apps/backend/src/jobs/jobs.service.ts index 1b4866d5..fd33c31c 100644 --- a/apps/backend/src/jobs/jobs.service.ts +++ b/apps/backend/src/jobs/jobs.service.ts @@ -10,7 +10,7 @@ import { type JobLogContext, type ProviderJobContext, toStructuredError } from " import { getMaintenanceWindowStatus } from "../common/maintenance-window.js"; import { isSpBlocked } from "../common/sp-blocklist.js"; import type { Network } from "../common/types.js"; -import type { IConfig, ISpBlocklistConfig } from "../config/app.config.js"; +import type { IConfig } from "../config/index.js"; import { DataRetentionService } from "../data-retention/data-retention.service.js"; import type { JobType } from "../database/entities/job-schedule-state.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; @@ -122,8 +122,11 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { const workersEnabled = runMode !== "api"; if (process.env.DEALBOT_DISABLE_CHAIN !== "true") { - await this.walletSdkService.ensureWalletAllowances(); - await this.walletSdkService.ensureProvidersLoaded(); + const activeNetworks = this.configService.get("activeNetworks", { infer: true }); + for (const network of activeNetworks) { + await this.walletSdkService.ensureWalletAllowances(network); + await this.walletSdkService.ensureProvidersLoaded(network); + } } await this.startBoss(); if (!this.boss) { @@ -200,14 +203,26 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { * can finish (or hit their per-job AbortController) before pg-boss force-fails * them via `failWip()`. */ - const jobs = this.configService.get("jobs"); - const pullPiece = this.configService.get("pullPiece"); - const longestJobTimeoutSec = Math.max( - jobs.dealJobTimeoutSeconds, - jobs.retrievalJobTimeoutSeconds, - jobs.dataSetCreationJobTimeoutSeconds, - pullPiece.pullCheckJobTimeoutSeconds, - ); + const activeNetworks = this.configService.get("activeNetworks", { infer: true }); + const networksConfig = this.configService.get("networks", { infer: true }); + + let longestJobTimeoutSec = 0; + + for (const network of activeNetworks) { + const cfg = networksConfig[network]; + + const maxNetworkTimeout = Math.max( + cfg.dealJobTimeoutSeconds, + cfg.retrievalJobTimeoutSeconds, + cfg.dataSetCreationJobTimeoutSeconds, + cfg.pullCheckJobTimeoutSeconds, + ); + + if (maxNetworkTimeout > longestJobTimeoutSec) { + longestJobTimeoutSec = maxNetworkTimeout; + } + } + const stopTimeoutMs = (longestJobTimeoutSec + 60) * 1000; await this.boss.stop({ graceful: true, timeout: stopTimeoutMs }); this.boss = null; @@ -218,12 +233,15 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { * Without this delay, the pod exits before its next scrape and the in-memory * counter deltas die with it, leaving `pending` rows without matching terminals. */ - const finalScrapeDelayMs = jobs.shutdownFinalScrapeDelaySeconds * 1000; + const { shutdownFinalScrapeDelaySeconds } = this.configService.get("jobs", { + infer: true, + }); + const finalScrapeDelayMs = shutdownFinalScrapeDelaySeconds * 1000; if (finalScrapeDelayMs > 0) { this.logger.log({ event: "pgboss_post_drain_scrape_hold", message: "Holding process for final Prometheus scrape after drain", - delaySeconds: jobs.shutdownFinalScrapeDelaySeconds, + delaySeconds: shutdownFinalScrapeDelaySeconds, }); await new Promise((resolve) => setTimeout(resolve, finalScrapeDelayMs)); } @@ -348,8 +366,8 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { message: "Skipping unknown SP job type", jobType: job.data.jobType, providerAddress: job.data.spAddress, - providerId: this.walletSdkService.getProviderInfo(job.data.spAddress)?.id, - providerName: this.walletSdkService.getProviderInfo(job.data.spAddress)?.name, + providerId: this.walletSdkService.getProviderInfo(job.data.spAddress, job.data.network)?.id, + providerName: this.walletSdkService.getProviderInfo(job.data.spAddress, job.data.network)?.name, }); }, ) @@ -403,17 +421,21 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { ); } - private getMaintenanceWindowStatus(now: Date = new Date()) { - const scheduling = this.configService.get("scheduling"); - return getMaintenanceWindowStatus(now, scheduling.maintenanceWindowsUtc, scheduling.maintenanceWindowMinutes); + private getMaintenanceWindowStatus(now: Date = new Date(), network: Network) { + const networkConfig = this.configService.get("networks", { infer: true })[network]; + return getMaintenanceWindowStatus(now, networkConfig.maintenanceWindowsUtc, networkConfig.maintenanceWindowMinutes); } - private async resolveProviderJobContext(spAddress: string, jobId: string): Promise { - let providerInfo = this.walletSdkService.getProviderInfo(spAddress); + private async resolveProviderJobContext( + spAddress: string, + jobId: string, + network: Network, + ): Promise { + let providerInfo = this.walletSdkService.getProviderInfo(spAddress, network); if (providerInfo == null && process.env.DEALBOT_DISABLE_CHAIN !== "true") { - await this.walletSdkService.loadProviders(); - providerInfo = this.walletSdkService.getProviderInfo(spAddress); + await this.walletSdkService.loadProviders(network); + providerInfo = this.walletSdkService.getProviderInfo(spAddress, network); } let providerId = providerInfo?.id; @@ -422,7 +444,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { // Fall back to DB if either providerId or providerName is missing if (providerId == null || !providerName) { const provider = await this.storageProviderRepository.findOne({ - where: { address: spAddress }, + where: { address: spAddress, network }, select: { providerId: true, name: true }, }); providerId = providerId ?? provider?.providerId ?? undefined; @@ -442,6 +464,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { providerAddress: spAddress, providerId, providerName, + network, }; } @@ -450,9 +473,10 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { spAddress: string, jobId: string, message: string, + network: Network, ): Promise { - const spBlocklists = this.configService.get("spBlocklists"); - if (isSpBlocked(spBlocklists, spAddress)) { + const networkCfg = this.configService.get("networks", { infer: true })[network]; + if (isSpBlocked(networkCfg, spAddress)) { this.logger.log({ jobId, providerAddress: spAddress, @@ -462,8 +486,8 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { return null; } - const logContext = await this.resolveProviderJobContext(spAddress, jobId); - if (isSpBlocked(spBlocklists, spAddress, logContext.providerId)) { + const logContext = await this.resolveProviderJobContext(spAddress, jobId, network); + if (isSpBlocked(networkCfg, spAddress, logContext.providerId)) { this.logger.log({ ...logContext, event: `${jobType}_job_blocked`, @@ -475,27 +499,32 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { return logContext; } - private logMaintenanceSkip(taskLabel: string, windowLabel?: string, logContext?: Partial) { - const scheduling = this.configService.get("scheduling"); + private logMaintenanceSkip( + taskLabel: string, + network: Network, + windowLabel?: string, + logContext?: Partial, + ) { + const networkCfg = this.configService.get("networks", { infer: true })[network]; const label = windowLabel ?? "unknown"; this.logger.log({ ...logContext, event: "maintenance_window_active", - message: `Maintenance window active (${label} UTC, ${scheduling.maintenanceWindowMinutes}m); deferring ${taskLabel}`, + message: `Maintenance window active (${label} UTC, ${networkCfg.maintenanceWindowMinutes}m); deferring ${taskLabel}`, }); } private async handleDealJob(job: SpJob): Promise { const data = job.data; - const spAddress = data.spAddress; + const { spAddress, network } = data; const now = new Date(); - const maintenance = this.getMaintenanceWindowStatus(now); + const maintenance = this.getMaintenanceWindowStatus(now, network); if (maintenance.active) { - this.logMaintenanceSkip(`deal job for ${spAddress}`, maintenance.window?.label, { + this.logMaintenanceSkip(`deal job for ${spAddress}`, network, maintenance.window?.label, { jobId: job.id, providerAddress: spAddress, - providerId: this.walletSdkService.getProviderInfo(spAddress)?.id, - providerName: this.walletSdkService.getProviderInfo(spAddress)?.name, + providerId: this.walletSdkService.getProviderInfo(spAddress, network)?.id, + providerName: this.walletSdkService.getProviderInfo(spAddress, network)?.name, }); await this.deferJobForMaintenance("deal", data, maintenance, now); return; @@ -511,24 +540,25 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { abortController.abort(abortReason); }, timeoutMs); - await this.recordJobExecution("deal", async () => { + await this.recordJobExecution("deal", network, async () => { const logContext = await this.resolveRunnableProviderJobContext( "deal", spAddress, job.id, "Deal job skipped: provider is blocked for scheduled data-storage checks", + network, ); if (logContext == null) { clearTimeout(timeoutId); return "success"; } try { - let provider = this.walletSdkService.getTestingProviders().find((p) => p.serviceProvider === spAddress); + let provider = this.walletSdkService.getTestingProviders(network).find((p) => p.serviceProvider === spAddress); if (!provider) { if (process.env.DEALBOT_DISABLE_CHAIN !== "true") { - await this.walletSdkService.loadProviders(); + await this.walletSdkService.loadProviders(network); } - provider = this.walletSdkService.getTestingProviders().find((p) => p.serviceProvider === spAddress); + provider = this.walletSdkService.getTestingProviders(network).find((p) => p.serviceProvider === spAddress); if (!provider) { this.logger.warn({ ...logContext, @@ -541,8 +571,10 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { abortController.signal.throwIfAborted(); await this.dealService.createDealForProvider(provider, { + network, signal: abortController.signal, logContext: { + network, jobId: logContext.jobId, providerAddress: logContext.providerAddress, providerId: provider.id ?? logContext.providerId, @@ -588,15 +620,15 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { private async handleRetrievalJob(job: SpJob): Promise { const data = job.data; - const spAddress = data.spAddress; + const { spAddress, network } = data; const now = new Date(); - const maintenance = this.getMaintenanceWindowStatus(now); + const maintenance = this.getMaintenanceWindowStatus(now, network); if (maintenance.active) { - this.logMaintenanceSkip(`retrieval job for ${spAddress}`, maintenance.window?.label, { + this.logMaintenanceSkip(`retrieval job for ${spAddress}`, network, maintenance.window?.label, { jobId: job.id, providerAddress: spAddress, - providerId: this.walletSdkService.getProviderInfo(spAddress)?.id, - providerName: this.walletSdkService.getProviderInfo(spAddress)?.name, + providerId: this.walletSdkService.getProviderInfo(spAddress, network)?.id, + providerName: this.walletSdkService.getProviderInfo(spAddress, network)?.name, }); await this.deferJobForMaintenance("retrieval", data, maintenance, now); return; @@ -612,19 +644,25 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { abortController.abort(abortReason); }, timeoutMs); - await this.recordJobExecution("retrieval", async () => { + await this.recordJobExecution("retrieval", network, async () => { const logContext = await this.resolveRunnableProviderJobContext( "retrieval", spAddress, job.id, "Retrieval job skipped: provider is blocked for scheduled retrieval checks", + network, ); if (logContext == null) { clearTimeout(timeoutId); return "success"; } try { - await this.retrievalService.performRandomRetrievalForProvider(spAddress, abortController.signal, logContext); + await this.retrievalService.performRandomRetrievalForProvider( + spAddress, + network, + abortController.signal, + logContext, + ); return "success"; } catch (error) { if (abortController.signal.aborted) { @@ -654,37 +692,37 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { } private async handleDataRetentionJob(data: DataRetentionJobData): Promise { - void data; - await this.recordJobExecution("data_retention_poll", async () => { - await this.dataRetentionService.pollDataRetention(); + await this.recordJobExecution("data_retention_poll", data.network, async () => { + await this.dataRetentionService.pollDataRetention(data.network); return "success"; }); } private async handleProvidersRefreshJob(data: ProvidersRefreshJobData): Promise { - void data; - await this.recordJobExecution("providers_refresh", async () => { + await this.recordJobExecution("providers_refresh", data.network, async () => { if (process.env.DEALBOT_DISABLE_CHAIN === "true") { this.logger.warn({ event: "chain_integration_disabled", message: "Chain integration disabled; skipping provider refresh job.", + network: data.network, }); } else { - await this.walletSdkService.loadProviders(); + await this.walletSdkService.loadProviders(data.network); } - await this.updateStorageProviderGauges(); + await this.updateStorageProviderGauges(data.network); return "success"; }); } private async handlePullPieceCleanupJob(data: PullPieceCleanupJobData): Promise { void data; - await this.recordJobExecution("pull_piece_cleanup", async () => { + await this.recordJobExecution("pull_piece_cleanup", data.network, async () => { const deletedCount = await this.pullCheckService.deleteExpiredPullPieces(); this.logger.log({ event: "pull_piece_cleanup_completed", message: "Deleted expired pull piece registrations", deletedCount, + network: data.network, }); return "success"; }); @@ -692,22 +730,22 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { private async handlePullCheckJob(job: SpJob): Promise { const data = job.data; - const spAddress = data.spAddress; + const { spAddress, network } = data; const now = new Date(); - const maintenance = this.getMaintenanceWindowStatus(now); + const maintenance = this.getMaintenanceWindowStatus(now, network); if (maintenance.active) { - this.logMaintenanceSkip(`pull_check job for ${spAddress}`, maintenance.window?.label, { + this.logMaintenanceSkip(`pull_check job for ${spAddress}`, network, maintenance.window?.label, { jobId: job.id, providerAddress: spAddress, - providerId: this.walletSdkService.getProviderInfo(spAddress)?.id, - providerName: this.walletSdkService.getProviderInfo(spAddress)?.name, + providerId: this.walletSdkService.getProviderInfo(spAddress, network)?.id, + providerName: this.walletSdkService.getProviderInfo(spAddress, network)?.name, }); await this.deferJobForMaintenance("pull_check", data, maintenance, now); return; } const abortController = new AbortController(); - const timeoutSeconds = this.configService.get("pullPiece", { infer: true }).pullCheckJobTimeoutSeconds; + const timeoutSeconds = this.configService.get("networks", { infer: true })[network].pullCheckJobTimeoutSeconds; const timeoutMs = Math.max(60000, timeoutSeconds * 1000); const effectiveTimeoutSeconds = Math.round(timeoutMs / 1000); const abortReason = new Error(`Pull check job timeout (${effectiveTimeoutSeconds}s) for ${spAddress}`); @@ -715,12 +753,13 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { abortController.abort(abortReason); }, timeoutMs); - await this.recordJobExecution("pull_check", async () => { + await this.recordJobExecution("pull_check", network, async () => { const logContext = await this.resolveRunnableProviderJobContext( "pull_check", spAddress, job.id, "Pull check job skipped: provider is blocked for scheduled pull checks", + network, ); if (logContext == null) { clearTimeout(timeoutId); @@ -758,14 +797,14 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { private async handlePieceCleanupJob(job: SpJob): Promise { const data = job.data; - const spAddress = data.spAddress; + const { spAddress, network } = data; const now = new Date(); - const maintenance = this.getMaintenanceWindowStatus(now); + const maintenance = this.getMaintenanceWindowStatus(now, network); if (maintenance.active) { - this.logMaintenanceSkip(`piece_cleanup job for ${spAddress}`, maintenance.window?.label, { + this.logMaintenanceSkip(`piece_cleanup job for ${spAddress}`, network, maintenance.window?.label, { jobId: job.id, providerAddress: spAddress, - providerId: this.walletSdkService.getProviderInfo(spAddress)?.id, + providerId: this.walletSdkService.getProviderInfo(spAddress, network)?.id, }); await this.deferJobForMaintenance("piece_cleanup", data, maintenance, now); return; @@ -781,10 +820,10 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { abortController.abort(abortReason); }, timeoutMs); - await this.recordJobExecution("piece_cleanup", async () => { - const logContext = await this.resolveProviderJobContext(spAddress, job.id); + await this.recordJobExecution("piece_cleanup", network, async () => { + const logContext = await this.resolveProviderJobContext(spAddress, job.id, network); try { - await this.pieceCleanupService.cleanupPiecesForProvider(spAddress, abortController.signal, logContext); + await this.pieceCleanupService.cleanupPiecesForProvider(spAddress, network, abortController.signal, logContext); return "success"; } catch (error) { if (abortController.signal.aborted) { @@ -812,30 +851,33 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { }); } - private async updateStorageProviderGauges(): Promise { + private async updateStorageProviderGauges(network: Network): Promise { try { - const totalProviders = await this.storageProviderRepository.count(); - const activeCount = await this.storageProviderRepository.count({ where: { isActive: true } }); + const networkFilter = { network }; + const totalProviders = await this.storageProviderRepository.count({ where: networkFilter }); + const activeCount = await this.storageProviderRepository.count({ where: { ...networkFilter, isActive: true } }); const inactiveCount = Math.max(0, totalProviders - activeCount); - const { network, useOnlyApprovedProviders } = this.configService.get("blockchain", { infer: true }); - this.storageProvidersActive.set({ status: "active", network }, activeCount); this.storageProvidersActive.set({ status: "inactive", network }, inactiveCount); - const testedWhere = useOnlyApprovedProviders ? { isActive: true, isApproved: true } : { isActive: true }; - const spBlocklists = this.configService.get("spBlocklists"); - const hasGlobalBlocklist = spBlocklists.addresses.size > 0 || spBlocklists.ids.size > 0; + const networkCfg = this.configService.get("networks", { infer: true })[network]; + + const testedWhere = networkCfg.useOnlyApprovedProviders + ? { network, isActive: true, isApproved: true } + : { network, isActive: true }; + const hasGlobalBlocklist = networkCfg.blockedSpIds.size > 0 || networkCfg.blockedSpAddresses.size > 0; let testedCount: number; if (hasGlobalBlocklist) { const testedProviders = await this.storageProviderRepository.find({ select: { address: true, providerId: true }, where: testedWhere, }); - testedCount = testedProviders.filter((p) => !isSpBlocked(spBlocklists, p.address, p.providerId)).length; + testedCount = testedProviders.filter((p) => !isSpBlocked(networkCfg, p.address, p.providerId)).length; } else { testedCount = await this.storageProviderRepository.count({ where: testedWhere }); } + this.storageProvidersTested.set({ network }, testedCount); } catch (error) { this.logger.warn({ @@ -848,22 +890,22 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { private async handleDataSetCreationJob(job: SpJob): Promise { const data = job.data; - const spAddress = data.spAddress; + const { spAddress, network } = data; const now = new Date(); - const maintenance = this.getMaintenanceWindowStatus(now); + const maintenance = this.getMaintenanceWindowStatus(now, network); if (maintenance.active) { - this.logMaintenanceSkip(`data_set_creation job for ${spAddress}`, maintenance.window?.label, { + this.logMaintenanceSkip(`data_set_creation job for ${spAddress}`, network, maintenance.window?.label, { jobId: job.id, providerAddress: spAddress, - providerId: this.walletSdkService.getProviderInfo(spAddress)?.id, - providerName: this.walletSdkService.getProviderInfo(spAddress)?.name, + providerId: this.walletSdkService.getProviderInfo(spAddress, network)?.id, + providerName: this.walletSdkService.getProviderInfo(spAddress, network)?.name, }); await this.deferJobForMaintenance("data_set_creation", data, maintenance, now); return; } - const minDataSets = this.configService.get("blockchain").minNumDataSetsForChecks; - const baseDataSetMetadata = this.dealService.getBaseDataSetMetadata(); + const minDataSets = this.configService.get("networks", { infer: true })[network].minNumDataSetsForChecks; + const baseDataSetMetadata = this.dealService.getBaseDataSetMetadata(network); // Create AbortController for job timeout enforcement const abortController = new AbortController(); @@ -875,12 +917,13 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { abortController.abort(abortReason); }, timeoutMs); - await this.recordJobExecution("data_set_creation", async () => { + await this.recordJobExecution("data_set_creation", network, async () => { const dataSetLogContext = await this.resolveRunnableProviderJobContext( "data_set_creation", spAddress, job.id, "Data set creation job skipped: provider is blocked for scheduled data-storage checks", + network, ); if (dataSetLogContext == null) { clearTimeout(timeoutId); @@ -890,6 +933,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { await provisionNextMissingDataSet( { dealService: this.dealService, logger: this.logger }, spAddress, + network, minDataSets, baseDataSetMetadata, dataSetLogContext, @@ -922,12 +966,16 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { }); } - private maintenanceResumeAt(now: Date, maintenance: ReturnType): Date | null { + private maintenanceResumeAt( + now: Date, + network: Network, + maintenance: ReturnType, + ): Date | null { if (!maintenance.active || !maintenance.window) { return null; } - const scheduling = this.configService.get("scheduling"); - const durationMinutes = scheduling.maintenanceWindowMinutes; + const networkConfig = this.configService.get("networks", { infer: true })[network]; + const durationMinutes = networkConfig.maintenanceWindowMinutes; if (durationMinutes <= 0) { return null; } @@ -954,7 +1002,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { maintenance: ReturnType, now: Date, ): Promise { - const resumeAt = this.maintenanceResumeAt(now, maintenance); + const resumeAt = this.maintenanceResumeAt(now, data.network, maintenance); if (resumeAt == null) { return; } @@ -986,29 +1034,32 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { * Runs one scheduler tick and updates queue metrics. */ private async runTick(): Promise { - try { - await this.ensureScheduleRows(); - await this.enqueueDueJobs(); - } catch (error) { - this.logger.error({ - event: "pgboss_scheduler_tick_failed", - message: "pg-boss scheduler core tick failed", - error: toStructuredError(error), - }); - } + const activeNetworks = this.configService.get("activeNetworks"); + for (const network of activeNetworks) { + try { + await this.ensureScheduleRows(network); + await this.enqueueDueJobs(network); + } catch (error) { + this.logger.error({ + event: "pgboss_scheduler_tick_failed", + message: "pg-boss scheduler core tick failed", + error: toStructuredError(error), + }); + } - try { - await this.updateQueueMetrics(); - } catch (error) { - this.logger.error({ - event: "pgboss_scheduler_metrics_update_failed", - message: "pg-boss scheduler metrics update failed", - error: toStructuredError(error), - }); + try { + await this.updateQueueMetrics(network); + } catch (error) { + this.logger.error({ + event: "pgboss_scheduler_metrics_update_failed", + message: "pg-boss scheduler metrics update failed", + error: toStructuredError(error), + }); + } } } - private getIntervalSecondsForRates(): { + private getIntervalSecondsForRates(network: Network): { dealIntervalSeconds: number; retrievalIntervalSeconds: number; dataSetCreationIntervalSeconds: number; @@ -1018,24 +1069,24 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { pullCheckIntervalSeconds: number; pullPieceCleanupIntervalSeconds: number; } { - const jobsConfig = this.configService.get("jobs", { infer: true }); - const scheduling = this.configService.get("scheduling", { infer: true }); - const pullPieceConfig = this.configService.get("pullPiece", { infer: true }); - - const dealsPerHour = jobsConfig.dealsPerSpPerHour; - const retrievalsPerHour = jobsConfig.retrievalsPerSpPerHour; - const dataSetCreationsPerHour = jobsConfig.dataSetCreationsPerSpPerHour; - const pieceCleanupPerHour = jobsConfig.pieceCleanupPerSpPerHour; - const pullChecksPerHour = pullPieceConfig.pullChecksPerSpPerHour; - - const dealIntervalSeconds = Math.max(1, Math.round(3600 / dealsPerHour)); - const retrievalIntervalSeconds = Math.max(1, Math.round(3600 / retrievalsPerHour)); - const dataSetCreationIntervalSeconds = Math.max(1, Math.round(3600 / dataSetCreationsPerHour)); - const pieceCleanupIntervalSeconds = Math.max(1, Math.round(3600 / pieceCleanupPerHour)); - const pullCheckIntervalSeconds = Math.max(1, Math.round(3600 / pullChecksPerHour)); - const dataRetentionPollIntervalSeconds = scheduling.dataRetentionPollIntervalSeconds; - const providersRefreshIntervalSeconds = scheduling.providersRefreshIntervalSeconds; - const pullPieceCleanupIntervalSeconds = pullPieceConfig.pullPieceCleanupIntervalSeconds; + const networkCfg = this.configService.get("networks", { infer: true })[network]; + + const { + dealsPerSpPerHour, + retrievalsPerSpPerHour, + dataSetCreationsPerSpPerHour, + pieceCleanupPerSpPerHour, + pullChecksPerSpPerHour, + dataRetentionPollIntervalSeconds, + providersRefreshIntervalSeconds, + pullPieceCleanupIntervalSeconds, + } = networkCfg; + + const dealIntervalSeconds = Math.max(1, Math.round(3600 / dealsPerSpPerHour)); + const retrievalIntervalSeconds = Math.max(1, Math.round(3600 / retrievalsPerSpPerHour)); + const dataSetCreationIntervalSeconds = Math.max(1, Math.round(3600 / dataSetCreationsPerSpPerHour)); + const pieceCleanupIntervalSeconds = Math.max(1, Math.round(3600 / pieceCleanupPerSpPerHour)); + const pullCheckIntervalSeconds = Math.max(1, Math.round(3600 / pullChecksPerSpPerHour)); return { dealIntervalSeconds, @@ -1056,7 +1107,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { * - Pauses rows for providers that are no longer active. * - Ensures global data_retention_poll, providers_refresh, and pull_piece_cleanup jobs exist. */ - private async ensureScheduleRows(): Promise { + private async ensureScheduleRows(network: Network): Promise { const now = new Date(); const { dealIntervalSeconds, @@ -1067,12 +1118,10 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { pieceCleanupIntervalSeconds, pullCheckIntervalSeconds, pullPieceCleanupIntervalSeconds, - } = this.getIntervalSecondsForRates(); - - const blockchainCfg = this.configService.get("blockchain", { infer: true }); - const network = blockchainCfg.network; + } = this.getIntervalSecondsForRates(network); - const useOnlyApprovedProviders = blockchainCfg.useOnlyApprovedProviders; + const networkCfg = this.configService.get("networks")[network]; + const useOnlyApprovedProviders = networkCfg.useOnlyApprovedProviders; // Active providers are guaranteed to support ipniIpfs // as validated by WalletSdkService.loadProvidersInternal() const providers = await this.storageProviderRepository.find({ @@ -1087,13 +1136,12 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { const dataRetentionPollStartAt = new Date(now.getTime() + phaseMs); const providersRefreshStartAt = new Date(now.getTime() + phaseMs); - const minDataSets = blockchainCfg.minNumDataSetsForChecks; + const minDataSets = networkCfg.minNumDataSetsForChecks; const cleanupStartAt = new Date(now.getTime() + phaseMs); const pullCheckStartAt = new Date(now.getTime() + phaseMs); - const spBlocklistsCfg = this.configService.get("spBlocklists"); const unblockedAddresses = providers - .filter(({ address, providerId }) => !isSpBlocked(spBlocklistsCfg, address, providerId)) + .filter(({ address, providerId }) => !isSpBlocked(networkCfg, address, providerId)) .map(({ address }) => address); const blockedCount = providers.length - unblockedAddresses.length; if (blockedCount > 0) { @@ -1158,7 +1206,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { }); } - // Global job schedules (sp_address = '') + // Per-network global job schedules (sp_address = '') await this.jobScheduleRepository.upsertSchedule( "data_retention_poll", "", @@ -1208,16 +1256,15 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { return { intervalMs, nextRunAt, runsDue }; } - private async enqueueDueJobs(): Promise { + private async enqueueDueJobs(network: Network): Promise { if (!this.boss) return; - const network = this.configService.get("blockchain", { infer: true }).network; const now = new Date(); - const maintenance = this.getMaintenanceWindowStatus(now); + const maintenance = this.getMaintenanceWindowStatus(now, network); const catchupMax = this.catchupMaxEnqueue(); if (maintenance.active) { - this.logMaintenanceSkip("Global job enqueues", maintenance.window?.label); + this.logMaintenanceSkip("Global job enqueues", network, maintenance.window?.label); } await this.jobScheduleRepository.runTransaction(async (manager) => { @@ -1350,8 +1397,11 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { /** * Records handler start/end metrics around a job execution. */ - private async recordJobExecution(jobType: JobType, run: () => Promise): Promise { - const network = this.configService.get("blockchain", { infer: true }).network; + private async recordJobExecution( + jobType: JobType, + network: Network, + run: () => Promise, + ): Promise { const startedAt = Date.now(); this.jobsStartedCounter.inc({ job_type: jobType, network }); try { @@ -1370,8 +1420,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { /** * Refreshes queue depth and age gauges from pg-boss tables. */ - private async updateQueueMetrics(): Promise { - const network = this.configService.get("blockchain", { infer: true }).network; + private async updateQueueMetrics(network: Network): Promise { const jobTypes: JobType[] = [ "deal", "retrieval", From 490abf5b67a9de1b6a68c94a39930f313fa80cc2 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 16:13:20 +0530 Subject: [PATCH 10/23] test: fix env schema tests --- apps/backend/src/config/env.schema.spec.ts | 26 ++++++---------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/apps/backend/src/config/env.schema.spec.ts b/apps/backend/src/config/env.schema.spec.ts index 4f5e7d0e..7c276b99 100644 --- a/apps/backend/src/config/env.schema.spec.ts +++ b/apps/backend/src/config/env.schema.spec.ts @@ -322,36 +322,35 @@ describe("createConfigValidationSchema", () => { expect(error).toBeUndefined(); }); - it("rejects METRICS_PER_HOUR above the maximum", () => { + it("rejects DEALS_PER_SP_PER_HOUR at zero (below min)", () => { const { error } = validate(schemaFor("calibration"), { ...baseEnv, NETWORKS: "calibration", - CALIBRATION_METRICS_PER_HOUR: 4, + CALIBRATION_DEALS_PER_SP_PER_HOUR: 0, ...withWalletKey("CALIBRATION"), }); expect(error).toBeDefined(); - expect(error?.message).toMatch(/CALIBRATION_METRICS_PER_HOUR/); }); - it("rejects DEALS_PER_SP_PER_HOUR at zero (below min)", () => { + it("rejects MIN_NUM_DATASETS_FOR_CHECKS when non-integer", () => { const { error } = validate(schemaFor("calibration"), { ...baseEnv, NETWORKS: "calibration", - CALIBRATION_DEALS_PER_SP_PER_HOUR: 0, + CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS: 1.5, ...withWalletKey("CALIBRATION"), }); expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS/); }); - it("rejects MIN_NUM_DATASETS_FOR_CHECKS when non-integer", () => { + it("rejects DEAL_JOB_TIMEOUT_SECONDS below 120", () => { const { error } = validate(schemaFor("calibration"), { ...baseEnv, NETWORKS: "calibration", - CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS: 1.5, + CALIBRATION_DEAL_JOB_TIMEOUT_SECONDS: 60, ...withWalletKey("CALIBRATION"), }); expect(error).toBeDefined(); - expect(error?.message).toMatch(/CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS/); }); }); @@ -431,7 +430,6 @@ describe("createConfigValidationSchema", () => { expect(value.JOB_WORKER_POLL_SECONDS).toBe(60); expect(value.PG_BOSS_LOCAL_CONCURRENCY).toBe(20); expect(value.DEALBOT_PGBOSS_SCHEDULER_ENABLED).toBe(true); - expect(value.DEAL_JOB_TIMEOUT_SECONDS).toBe(360); }); it("rejects JOB_SCHEDULER_POLL_SECONDS below 60", () => { @@ -444,16 +442,6 @@ describe("createConfigValidationSchema", () => { expect(error).toBeDefined(); }); - it("rejects DEAL_JOB_TIMEOUT_SECONDS below 120", () => { - const { error } = validate(schemaFor("calibration"), { - ...baseEnv, - NETWORKS: "calibration", - DEAL_JOB_TIMEOUT_SECONDS: 60, - ...withWalletKey("CALIBRATION"), - }); - expect(error).toBeDefined(); - }); - it("rejects non-integer PG_BOSS_LOCAL_CONCURRENCY", () => { const { error } = validate(schemaFor("calibration"), { ...baseEnv, From 60dc67e778efe94e90b1749bae07f92fd5a09d84 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 16:17:50 +0530 Subject: [PATCH 11/23] chore: add missing legacy per-network environment variables --- apps/backend/src/config/legacy-env-compat.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apps/backend/src/config/legacy-env-compat.ts b/apps/backend/src/config/legacy-env-compat.ts index a68d25ed..ea8c1cbd 100644 --- a/apps/backend/src/config/legacy-env-compat.ts +++ b/apps/backend/src/config/legacy-env-compat.ts @@ -49,6 +49,9 @@ const LEGACY_PER_NETWORK_VARS = [ "DEALS_PER_SP_PER_HOUR", "RETRIEVALS_PER_SP_PER_HOUR", "DATASET_CREATIONS_PER_SP_PER_HOUR", + "DEAL_JOB_TIMEOUT_SECONDS", + "RETRIEVAL_JOB_TIMEOUT_SECONDS", + "DATA_SET_CREATION_JOB_TIMEOUT_SECONDS", "DATA_RETENTION_POLL_INTERVAL_SECONDS", "PROVIDERS_REFRESH_INTERVAL_SECONDS", "MAINTENANCE_WINDOWS_UTC", @@ -59,6 +62,17 @@ const LEGACY_PER_NETWORK_VARS = [ "MAX_PIECE_CLEANUP_RUNTIME_SECONDS", "MAX_DATASET_STORAGE_SIZE_BYTES", "TARGET_DATASET_STORAGE_SIZE_BYTES", + "PULL_CHECKS_PER_SP_PER_HOUR", + "PULL_CHECK_JOB_TIMEOUT_SECONDS", + "PULL_CHECK_POLL_INTERVAL_SECONDS", + "PULL_CHECK_PIECE_SIZE_BYTES", + "PULL_PIECE_MAX_CONCURRENT_STREAMS", + "PULL_PIECE_MAX_STREAMS_PER_CID", + "PULL_PIECE_CLEANUP_INTERVAL_SECONDS", + "CLICKHOUSE_URL", + "CLICKHOUSE_BATCH_SIZE", + "CLICKHOUSE_FLUSH_INTERVAL_MS", + "CLICKHOUSE_MAX_BUFFER_SIZE", ] as const; export interface LegacyEnvCompatResult { From 50f4fdeb93d08901c12e3ebac7a30a501aff7a1d Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 16:24:36 +0530 Subject: [PATCH 12/23] add network to dataset liveness service --- .../dataset-liveness.service.spec.ts | 32 ++++++++++--------- .../dataset-liveness.service.ts | 23 ++++++++----- apps/backend/src/deal/deal.service.ts | 13 +++++--- .../src/retrieval/retrieval.service.ts | 11 +++++-- 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/apps/backend/src/dataset-liveness/dataset-liveness.service.spec.ts b/apps/backend/src/dataset-liveness/dataset-liveness.service.spec.ts index 7a246d55..2e9c7775 100644 --- a/apps/backend/src/dataset-liveness/dataset-liveness.service.spec.ts +++ b/apps/backend/src/dataset-liveness/dataset-liveness.service.spec.ts @@ -57,41 +57,41 @@ describe("DatasetLivenessService", () => { describe("isDataSetLive", () => { it("returns true when both probes report live", async () => { - await expect(service.isDataSetLive("0xprovider", 1n)).resolves.toBe(true); + await expect(service.isDataSetLive("0xprovider", 1n, "calibration")).resolves.toBe(true); }); it("returns false when FWSS validateDataSet reports not live", async () => { mockWarmStorageService.validateDataSet.mockRejectedValueOnce( new Error("Data set 1 does not exist or is not live"), ); - await expect(service.isDataSetLive("0xprovider", 1n)).resolves.toBe(false); + await expect(service.isDataSetLive("0xprovider", 1n, "calibration")).resolves.toBe(false); }); it("returns false when SP HTTP probe returns 409 with the terminated body", async () => { fetchMock.mockResolvedValueOnce( new Response("Data set has been terminated due to unrecoverable proving failure", { status: 409 }), ); - await expect(service.isDataSetLive("0xprovider", 1n)).resolves.toBe(false); + await expect(service.isDataSetLive("0xprovider", 1n, "calibration")).resolves.toBe(false); }); it("treats SP HTTP 409 with a different body as live", async () => { fetchMock.mockResolvedValueOnce(new Response("piece already exists", { status: 409 })); - await expect(service.isDataSetLive("0xprovider", 1n)).resolves.toBe(true); + await expect(service.isDataSetLive("0xprovider", 1n, "calibration")).resolves.toBe(true); }); it("treats SP HTTP non-409 responses as live", async () => { fetchMock.mockResolvedValueOnce(new Response("At least one piece must be provided", { status: 400 })); - await expect(service.isDataSetLive("0xprovider", 1n)).resolves.toBe(true); + await expect(service.isDataSetLive("0xprovider", 1n, "calibration")).resolves.toBe(true); }); it("treats SP HTTP network errors as live", async () => { fetchMock.mockRejectedValueOnce(new Error("ECONNREFUSED")); - await expect(service.isDataSetLive("0xprovider", 1n)).resolves.toBe(true); + await expect(service.isDataSetLive("0xprovider", 1n, "calibration")).resolves.toBe(true); }); it("rethrows FWSS validateDataSet errors that do not match the terminal message", async () => { mockWarmStorageService.validateDataSet.mockRejectedValueOnce(new Error("ECONNREFUSED 127.0.0.1:8545")); - await expect(service.isDataSetLive("0xprovider", 1n)).rejects.toThrow("ECONNREFUSED"); + await expect(service.isDataSetLive("0xprovider", 1n, "calibration")).rejects.toThrow("ECONNREFUSED"); }); it("returns false when SP reports terminated even if FWSS RPC throws transiently", async () => { @@ -99,11 +99,11 @@ describe("DatasetLivenessService", () => { fetchMock.mockResolvedValueOnce( new Response("Data set has been terminated due to unrecoverable proving failure", { status: 409 }), ); - await expect(service.isDataSetLive("0xprovider", 1n)).resolves.toBe(false); + await expect(service.isDataSetLive("0xprovider", 1n, "calibration")).resolves.toBe(false); }); it("posts an empty JSON body to the SP addPieces endpoint", async () => { - await service.isDataSetLive("0xprovider", 42n); + await service.isDataSetLive("0xprovider", 42n, "calibration"); expect(fetchMock).toHaveBeenCalledTimes(1); const [calledUrl, init] = fetchMock.mock.calls[0] as unknown as [URL, RequestInit]; expect(String(calledUrl)).toBe("https://sp.example/pdp/data-sets/42/pieces"); @@ -114,14 +114,14 @@ describe("DatasetLivenessService", () => { it("aborts when outer signal is already aborted", async () => { const ac = new AbortController(); ac.abort(); - await expect(service.isDataSetLive("0xprovider", 1n, ac.signal)).rejects.toThrow(); + await expect(service.isDataSetLive("0xprovider", 1n, "calibration", ac.signal)).rejects.toThrow(); }); }); describe("isPieceLive", () => { it("returns true when PDPVerifier.pieceLive returns true", async () => { readContractMock.mockResolvedValueOnce(true); - await expect(service.isPieceLive(1n, 42n)).resolves.toBe(true); + await expect(service.isPieceLive(1n, 42n, "calibration")).resolves.toBe(true); expect(readContractMock).toHaveBeenCalledWith( expect.objectContaining({ chain: { id: 314 } }), expect.objectContaining({ @@ -134,24 +134,26 @@ describe("DatasetLivenessService", () => { it("returns false when PDPVerifier.pieceLive returns false", async () => { readContractMock.mockResolvedValueOnce(false); - await expect(service.isPieceLive(1n, 42n)).resolves.toBe(false); + await expect(service.isPieceLive(1n, 42n, "calibration")).resolves.toBe(false); }); it("throws when synapse client is not available", async () => { mockWalletSdkService.getSynapseClient.mockReturnValueOnce(null); - await expect(service.isPieceLive(1n, 42n)).rejects.toThrow("Synapse client not available for pieceLive read"); + await expect(service.isPieceLive(1n, 42n, "calibration")).rejects.toThrow( + "Synapse client not available for pieceLive read", + ); }); it("propagates RPC errors", async () => { readContractMock.mockRejectedValueOnce(new Error("RPC down")); - await expect(service.isPieceLive(1n, 42n)).rejects.toThrow("RPC down"); + await expect(service.isPieceLive(1n, 42n, "calibration")).rejects.toThrow("RPC down"); }); it("aborts when outer signal is already aborted", async () => { const ac = new AbortController(); ac.abort(); readContractMock.mockResolvedValueOnce(true); - await expect(service.isPieceLive(1n, 42n, ac.signal)).rejects.toThrow(); + await expect(service.isPieceLive(1n, 42n, "calibration", ac.signal)).rejects.toThrow(); }); }); }); diff --git a/apps/backend/src/dataset-liveness/dataset-liveness.service.ts b/apps/backend/src/dataset-liveness/dataset-liveness.service.ts index 56d7a389..ef4d4891 100644 --- a/apps/backend/src/dataset-liveness/dataset-liveness.service.ts +++ b/apps/backend/src/dataset-liveness/dataset-liveness.service.ts @@ -3,6 +3,7 @@ import { Injectable, Logger } from "@nestjs/common"; import { readContract } from "viem/actions"; import { awaitWithAbort } from "../common/abort-utils.js"; import { toStructuredError } from "../common/logging.js"; +import { Network } from "../common/types.js"; import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; const PDP_LIVENESS_PROBE_TIMEOUT_MS = 10_000; @@ -28,11 +29,16 @@ export class DatasetLivenessService { constructor(private readonly walletSdkService: WalletSdkService) {} - async isDataSetLive(providerAddress: string, dataSetId: bigint, signal?: AbortSignal): Promise { + async isDataSetLive( + providerAddress: string, + dataSetId: bigint, + network: Network, + signal?: AbortSignal, + ): Promise { signal?.throwIfAborted(); const settled = await Promise.allSettled([ - this.probeFwssDataSetLive(dataSetId, signal), - this.probeSpHttpDataSetLive(providerAddress, dataSetId, signal), + this.probeFwssDataSetLive(dataSetId, network, signal), + this.probeSpHttpDataSetLive(providerAddress, dataSetId, network, signal), ]); if (settled.some((r) => r.status === "fulfilled" && r.value === false)) { return false; @@ -60,9 +66,9 @@ export class DatasetLivenessService { * * Source: `FilOzone/pdp` PDPVerifier.sol `pieceLive`. */ - async isPieceLive(dataSetId: bigint, pieceId: bigint, signal?: AbortSignal): Promise { + async isPieceLive(dataSetId: bigint, pieceId: bigint, network: Network, signal?: AbortSignal): Promise { signal?.throwIfAborted(); - const client = this.walletSdkService.getSynapseClient(); + const client = this.walletSdkService.getSynapseClient(network); if (!client) { throw new Error("Synapse client not available for pieceLive read"); } @@ -79,9 +85,9 @@ export class DatasetLivenessService { return Boolean(result); } - protected async probeFwssDataSetLive(dataSetId: bigint, signal?: AbortSignal): Promise { + protected async probeFwssDataSetLive(dataSetId: bigint, network: Network, signal?: AbortSignal): Promise { signal?.throwIfAborted(); - const { warmStorageService } = this.walletSdkService.getWalletServices(); + const { warmStorageService } = this.walletSdkService.getWalletServices(network); try { await awaitWithAbort(warmStorageService.validateDataSet({ dataSetId }), signal); return true; @@ -98,10 +104,11 @@ export class DatasetLivenessService { protected async probeSpHttpDataSetLive( providerAddress: string, dataSetId: bigint, + network: Network, signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); - const providerInfo = this.walletSdkService.getProviderInfo(providerAddress); + const providerInfo = this.walletSdkService.getProviderInfo(providerAddress, network); if (!providerInfo) { throw new Error(`Provider ${providerAddress} not found in registry`); } diff --git a/apps/backend/src/deal/deal.service.ts b/apps/backend/src/deal/deal.service.ts index 2478ac99..a14ac833 100644 --- a/apps/backend/src/deal/deal.service.ts +++ b/apps/backend/src/deal/deal.service.ts @@ -376,7 +376,7 @@ export class DealService { // pdpEndEpoch=0; createContext returns it and the next add-pieces path // would fail. See #379. if (storage.dataSetId !== undefined) { - const live = await this.isDataSetLive(providerAddress, storage.dataSetId, signal); + const live = await this.isDataSetLive(providerAddress, storage.dataSetId, network, signal); if (!live) { preUploadTerminated = true; throw new DealJobTerminatedDataSetError(storage.dataSetId); @@ -729,7 +729,7 @@ export class DealService { return { status: "missing" }; } const dataSetId = context.dataSetId; - const isLive = await this.isDataSetLive(providerAddress, dataSetId, signal); + const isLive = await this.isDataSetLive(providerAddress, dataSetId, network, signal); return isLive ? { status: "live", dataSetId } : { status: "terminated", dataSetId }; } @@ -738,8 +738,13 @@ export class DealService { * probe rationale. Kept on DealService to preserve existing call sites * (`getDataSetProvisioningStatus`, `createDeal` post-context guard). */ - async isDataSetLive(providerAddress: string, dataSetId: bigint, signal?: AbortSignal): Promise { - return this.datasetLivenessService.isDataSetLive(providerAddress, dataSetId, signal); + async isDataSetLive( + providerAddress: string, + dataSetId: bigint, + network: Network, + signal?: AbortSignal, + ): Promise { + return this.datasetLivenessService.isDataSetLive(providerAddress, dataSetId, network, signal); } /** diff --git a/apps/backend/src/retrieval/retrieval.service.ts b/apps/backend/src/retrieval/retrieval.service.ts index 7774d81e..226ac4bc 100644 --- a/apps/backend/src/retrieval/retrieval.service.ts +++ b/apps/backend/src/retrieval/retrieval.service.ts @@ -159,7 +159,13 @@ export class RetrievalService { return []; } - const pieceLive = await this.checkPieceLive(deal.dataSetId, BigInt(deal.pieceId), signal, retrievalLogContext); + const pieceLive = await this.checkPieceLive( + deal.dataSetId, + BigInt(deal.pieceId), + deal.network, + signal, + retrievalLogContext, + ); signal?.throwIfAborted(); if (!pieceLive) { const updateResult = await this.dealRepository.update( @@ -657,11 +663,12 @@ export class RetrievalService { private async checkPieceLive( dataSetId: bigint, pieceId: bigint, + network: Network, signal: AbortSignal | undefined, logContext: RetrievalLogContext, ): Promise { try { - return await this.datasetLivenessService.isPieceLive(dataSetId, pieceId, signal); + return await this.datasetLivenessService.isPieceLive(dataSetId, pieceId, network, signal); } catch (error) { if (signal?.aborted) throw error; this.logger.warn({ From a0b9beb027ebb77f37af693944a778c76d6ffb7e Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 16:50:33 +0530 Subject: [PATCH 13/23] migrate pull-check service --- apps/backend/src/config/constants.ts | 2 - apps/backend/src/config/env.schema.ts | 8 +- apps/backend/src/config/legacy-env-compat.ts | 2 - apps/backend/src/config/loader.ts | 17 ++--- apps/backend/src/config/types.ts | 30 ++++---- .../dataset-liveness.service.ts | 2 +- .../src/pull-check/pull-check.service.spec.ts | 74 ++++++++++++------- .../src/pull-check/pull-check.service.ts | 42 ++++++----- .../pull-piece-stream-tracker.service.spec.ts | 7 +- .../pull-piece-stream-tracker.service.ts | 4 +- 10 files changed, 102 insertions(+), 86 deletions(-) diff --git a/apps/backend/src/config/constants.ts b/apps/backend/src/config/constants.ts index cb98814b..0eda35ec 100644 --- a/apps/backend/src/config/constants.ts +++ b/apps/backend/src/config/constants.ts @@ -28,8 +28,6 @@ export const networkDefaults = { pullCheckJobTimeoutSeconds: 300, pullCheckPollIntervalSeconds: 2, pullCheckPieceSizeBytes: 10 * 1024 * 1024, // 10 MiB - pullPieceMaxConcurrentStreams: 50, - pullPieceMaxStreamsPerCid: 3, pullPieceCleanupIntervalSeconds: 7 * 24 * 3600, // 7 days clickhouseBatchSize: 500, diff --git a/apps/backend/src/config/env.schema.ts b/apps/backend/src/config/env.schema.ts index 40f8affa..846f5a9b 100644 --- a/apps/backend/src/config/env.schema.ts +++ b/apps/backend/src/config/env.schema.ts @@ -101,6 +101,11 @@ export const jobsEnvSchema = { SHUTDOWN_FINAL_SCRAPE_DELAY_SECONDS: Joi.number().min(0).max(300).default(35), }; +export const pullPieceEnvSchema = { + PULL_PIECE_MAX_CONCURRENT_STREAMS: Joi.number().integer().min(1).default(50), + PULL_PIECE_MAX_STREAMS_PER_CID: Joi.number().integer().min(1).default(3), +}; + export const datasetEnvSchema = { DEALBOT_LOCAL_DATASETS_PATH: Joi.string().default(DEFAULT_LOCAL_DATASETS_PATH), RANDOM_PIECE_SIZES: Joi.string().default("10485760"), @@ -185,8 +190,6 @@ export const createPerNetworkEnvSchema = (prefix: Uppercase | "") => { .integer() .min(1024) .default(10 * 1024 * 1024), - [k("PULL_PIECE_MAX_CONCURRENT_STREAMS")]: Joi.number().integer().min(1).default(50), - [k("PULL_PIECE_MAX_STREAMS_PER_CID")]: Joi.number().integer().min(1).default(3), [k("PULL_PIECE_CLEANUP_INTERVAL_SECONDS")]: Joi.number() .integer() .min(3600) @@ -215,6 +218,7 @@ export function createConfigValidationSchema(processEnv: NodeJS.ProcessEnv = pro ...databaseEnvSchema, ...globalNetworkEnvSchema, ...jobsEnvSchema, + ...pullPieceEnvSchema, ...datasetEnvSchema, ...timeoutEnvSchema, ...retrievalEnvSchema, diff --git a/apps/backend/src/config/legacy-env-compat.ts b/apps/backend/src/config/legacy-env-compat.ts index ea8c1cbd..d6ca54e3 100644 --- a/apps/backend/src/config/legacy-env-compat.ts +++ b/apps/backend/src/config/legacy-env-compat.ts @@ -66,8 +66,6 @@ const LEGACY_PER_NETWORK_VARS = [ "PULL_CHECK_JOB_TIMEOUT_SECONDS", "PULL_CHECK_POLL_INTERVAL_SECONDS", "PULL_CHECK_PIECE_SIZE_BYTES", - "PULL_PIECE_MAX_CONCURRENT_STREAMS", - "PULL_PIECE_MAX_STREAMS_PER_CID", "PULL_PIECE_CLEANUP_INTERVAL_SECONDS", "CLICKHOUSE_URL", "CLICKHOUSE_BATCH_SIZE", diff --git a/apps/backend/src/config/loader.ts b/apps/backend/src/config/loader.ts index 6ca912c4..f21581d0 100644 --- a/apps/backend/src/config/loader.ts +++ b/apps/backend/src/config/loader.ts @@ -26,6 +26,7 @@ import type { IDatasetConfig, IJobsConfig, INetworkConfig, + IPullPieceConfig, IRetrievalConfig, ITimeoutConfig, } from "./types.js"; @@ -73,6 +74,11 @@ const loadJobsConfig = (env: NodeJS.ProcessEnv): IJobsConfig => ({ shutdownFinalScrapeDelaySeconds: getNumberEnv(env, "SHUTDOWN_FINAL_SCRAPE_DELAY_SECONDS", 35), }); +const loadPullPieceConfig = (env: NodeJS.ProcessEnv): IPullPieceConfig => ({ + maxConcurrentStreams: getNumberEnv(env, "PULL_PIECE_MAX_CONCURRENT_STREAMS", 5), + maxStreamsPerCid: getNumberEnv(env, "PULL_PIECE_MAX_STREAMS_PER_CID", 3), +}); + const loadDatasetConfig = (env: NodeJS.ProcessEnv): IDatasetConfig => ({ localDatasetsPath: getStringEnv(env, "DEALBOT_LOCAL_DATASETS_PATH", DEFAULT_LOCAL_DATASETS_PATH), randomDatasetSizes: parseRandomDatasetSizes(env), @@ -199,16 +205,6 @@ function loadNetworkEnvPrefix( networkDefaults.pullCheckPollIntervalSeconds, ), pullCheckPieceSizeBytes: getNumberEnv(env, "PULL_CHECK_PIECE_SIZE_BYTES", networkDefaults.pullCheckPieceSizeBytes), - pullPieceMaxConcurrentStreams: getNumberEnv( - env, - "PULL_PIECE_MAX_CONCURRENT_STREAMS", - networkDefaults.pullPieceMaxConcurrentStreams, - ), - pullPieceMaxStreamsPerCid: getNumberEnv( - env, - "PULL_PIECE_MAX_STREAMS_PER_CID", - networkDefaults.pullPieceMaxStreamsPerCid, - ), pullPieceCleanupIntervalSeconds: getNumberEnv( env, "PULL_PIECE_CLEANUP_INTERVAL_SECONDS", @@ -258,6 +254,7 @@ export function loadConfig(): IConfig { database: loadDatabaseConfig(process.env), ...loadNetworkConfigs(process.env), jobs: loadJobsConfig(process.env), + pullPiece: loadPullPieceConfig(process.env), dataset: loadDatasetConfig(process.env), timeouts: loadTimeoutConfig(process.env), retrieval: loadRetrievalConfig(process.env), diff --git a/apps/backend/src/config/types.ts b/apps/backend/src/config/types.ts index b34d4128..6c088ad5 100644 --- a/apps/backend/src/config/types.ts +++ b/apps/backend/src/config/types.ts @@ -118,18 +118,6 @@ export type BaseNetworkConfig = { * Size (bytes) of the synthetic test piece DealBot generates per pull check. */ pullCheckPieceSizeBytes: number; - /** - * Maximum number of concurrent piece streams across all pieceCids. - * - * Prevents DoS by limiting total server-wide streaming load. - */ - pullPieceMaxConcurrentStreams: number; - /** - * Maximum number of concurrent streams per pieceCid. - * - * Prevents attackers from opening many connections to the same piece. - */ - pullPieceMaxStreamsPerCid: number; /** * How often (seconds) the global `pull_piece_cleanup` job runs to delete * expired `pull_pieces` rows (those whose `expires_at` is in the past). @@ -216,6 +204,21 @@ export interface IJobsConfig { shutdownFinalScrapeDelaySeconds: number; } +export interface IPullPieceConfig { + /** + * Maximum number of concurrent piece streams across all pieceCids. + * + * Prevents DoS by limiting total server-wide streaming load. + */ + maxConcurrentStreams: number; + /** + * Maximum number of concurrent streams per pieceCid. + * + * Prevents attackers from opening many connections to the same piece. + */ + maxStreamsPerCid: number; +} + export interface IDatasetConfig { localDatasetsPath: string; randomDatasetSizes: number[]; @@ -239,6 +242,7 @@ export interface IConfig { networks: INetworksConfig; activeNetworks: Network[]; jobs: IJobsConfig; + pullPiece: IPullPieceConfig; dataset: IDatasetConfig; timeouts: ITimeoutConfig; retrieval: IRetrievalConfig; @@ -268,8 +272,6 @@ export type NetworkDefaults = Pick< | "pullCheckJobTimeoutSeconds" | "pullCheckPollIntervalSeconds" | "pullCheckPieceSizeBytes" - | "pullPieceMaxConcurrentStreams" - | "pullPieceMaxStreamsPerCid" | "pullPieceCleanupIntervalSeconds" | "clickhouseBatchSize" | "clickhouseFlushIntervalMs" diff --git a/apps/backend/src/dataset-liveness/dataset-liveness.service.ts b/apps/backend/src/dataset-liveness/dataset-liveness.service.ts index ef4d4891..5eb5f5c1 100644 --- a/apps/backend/src/dataset-liveness/dataset-liveness.service.ts +++ b/apps/backend/src/dataset-liveness/dataset-liveness.service.ts @@ -3,7 +3,7 @@ import { Injectable, Logger } from "@nestjs/common"; import { readContract } from "viem/actions"; import { awaitWithAbort } from "../common/abort-utils.js"; import { toStructuredError } from "../common/logging.js"; -import { Network } from "../common/types.js"; +import type { Network } from "../common/types.js"; import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; const PDP_LIVENESS_PROBE_TIMEOUT_MS = 10_000; diff --git a/apps/backend/src/pull-check/pull-check.service.spec.ts b/apps/backend/src/pull-check/pull-check.service.spec.ts index 86bf4e0c..60d72014 100644 --- a/apps/backend/src/pull-check/pull-check.service.spec.ts +++ b/apps/backend/src/pull-check/pull-check.service.spec.ts @@ -2,7 +2,8 @@ import { Readable } from "node:stream"; import { ConfigService } from "@nestjs/config"; import { Test, type TestingModule } from "@nestjs/testing"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { IConfig } from "../config/app.config.js"; +import type { ProviderJobContext } from "../common/logging.js"; +import type { IConfig } from "../config/index.js"; import { DataSourceService } from "../dataSource/dataSource.service.js"; import { HttpClientService } from "../http-client/http-client.service.js"; import { PullCheckCheckMetrics } from "../metrics-prometheus/check-metrics.service.js"; @@ -11,6 +12,8 @@ import type { PDPProviderEx } from "../wallet-sdk/wallet-sdk.types.js"; import { PullCheckService } from "./pull-check.service.js"; import { PullPieceRepository } from "./pull-piece.repository.js"; +const DEFAULT_NETWORK = "calibration"; + // `@filoz/synapse-core/piece` is mocked so that piece CIDs are deterministic // strings rather than real CID objects, keeping the tests fast and isolated // from the SDK's internal hashing. @@ -99,16 +102,17 @@ describe("PullCheckService", () => { configValues = { app: { host: "localhost", port: 3000, apiPublicUrl: "https://dealbot.example" } as IConfig["app"], - blockchain: { network: "calibration", walletAddress: "0xwallet" } as IConfig["blockchain"], - pullPiece: { - pullChecksPerSpPerHour: 1, - pullCheckJobTimeoutSeconds: 300, - pullCheckPollIntervalSeconds: 5, - pullCheckPieceSizeBytes: 1024, - maxConcurrentStreams: 50, - maxStreamsPerCid: 3, - pullPieceCleanupIntervalSeconds: 7 * 24 * 3600, - }, + networks: { + [DEFAULT_NETWORK]: { + network: DEFAULT_NETWORK, + walletAddress: "0xwallet", + pullChecksPerSpPerHour: 1, + pullCheckJobTimeoutSeconds: 300, + pullCheckPollIntervalSeconds: 5, + pullCheckPieceSizeBytes: 1024, + pullPieceCleanupIntervalSeconds: 7 * 24 * 3600, + }, + } as IConfig["networks"], dataset: { localDatasetsPath: "/tmp/datasets" } as IConfig["dataset"], }; @@ -140,35 +144,35 @@ describe("PullCheckService", () => { const provider = makeProvider(); walletSdkServiceMock.getProviderInfo.mockReturnValue(provider); - expect(service.validateProviderInfo("0xsp")).toBe(provider); + expect(service.validateProviderInfo("0xsp", DEFAULT_NETWORK)).toBe(provider); }); it("throws when the provider is unknown", () => { walletSdkServiceMock.getProviderInfo.mockReturnValue(undefined); - expect(() => service.validateProviderInfo("0xsp")).toThrow(/not found/); + expect(() => service.validateProviderInfo("0xsp", DEFAULT_NETWORK)).toThrow(/not found/); }); it("throws when the provider is inactive", () => { walletSdkServiceMock.getProviderInfo.mockReturnValue(makeProvider({ isActive: false })); - expect(() => service.validateProviderInfo("0xsp")).toThrow(/not active/); + expect(() => service.validateProviderInfo("0xsp", DEFAULT_NETWORK)).toThrow(/not active/); }); it("throws when the provider is missing a numeric id", () => { walletSdkServiceMock.getProviderInfo.mockReturnValue(makeProvider({ id: undefined as unknown as bigint })); - expect(() => service.validateProviderInfo("0xsp")).toThrow(/missing providerId/); + expect(() => service.validateProviderInfo("0xsp", DEFAULT_NETWORK)).toThrow(/missing providerId/); }); it("throws when the provider is missing a PDP serviceURL", () => { walletSdkServiceMock.getProviderInfo.mockReturnValue( makeProvider({ pdp: { serviceURL: "" } as PDPProviderEx["pdp"] }), ); - expect(() => service.validateProviderInfo("0xsp")).toThrow(/missing serviceURL/); + expect(() => service.validateProviderInfo("0xsp", DEFAULT_NETWORK)).toThrow(/missing serviceURL/); }); }); describe("preparePullPiece", () => { it("generates deterministic bytes, computes the piece CID, and registers the pull piece", async () => { - const prepared = await service.preparePullPiece("0xsp"); + const prepared = await service.preparePullPiece("0xsp", DEFAULT_NETWORK); expect(dataSourceServiceMock.generateBytesStream).toHaveBeenCalledWith({ providerAddress: "0xsp", @@ -185,14 +189,20 @@ describe("PullCheckService", () => { it("falls back to host:port when apiPublicUrl is not configured", async () => { configValues.app = { host: "localhost", port: 3000 } as IConfig["app"]; - const prepared = await service.preparePullPiece("0xsp"); + const prepared = await service.preparePullPiece("0xsp", DEFAULT_NETWORK); expect(prepared.sourceUrl).toBe("http://localhost:3000/api/piece/bafk-test-piece"); }); }); describe("validateByDirectPieceFetch", () => { const provider = makeProvider(); - const logContext = { jobId: "job-1", providerAddress: "0xsp", providerId: 42n, providerName: "test-sp" }; + const logContext: ProviderJobContext = { + jobId: "job-1", + providerAddress: "0xsp", + providerId: 42n, + providerName: "test-sp", + network: DEFAULT_NETWORK, + }; function makeStreamResponse( overrides: { statusCode?: number; headers?: Record; cidResult?: string } = {}, @@ -271,7 +281,13 @@ describe("PullCheckService", () => { }); describe("runPullCheck", () => { - const logContext = { jobId: "job-1", providerAddress: "0xsp", providerId: 42n, providerName: "test-sp" }; + const logContext: ProviderJobContext = { + jobId: "job-1", + providerAddress: "0xsp", + providerId: 42n, + providerName: "test-sp", + network: DEFAULT_NETWORK, + }; function arrangeHappyPath() { // Pre-stage a registration that preparePullPiece will install. @@ -311,7 +327,7 @@ describe("PullCheckService", () => { it("runs the full lifecycle, observes all metrics, and records success", async () => { const { registration } = arrangeHappyPath(); - await service.runPullCheck("0xsp", undefined, logContext); + await service.runPullCheck("0xsp", DEFAULT_NETWORK, undefined, logContext); // Submit timestamp is stamped on the registration. expect(registryMock.markPullSubmitted).toHaveBeenCalledWith(registration.pieceCid, expect.any(Date)); @@ -339,7 +355,7 @@ describe("PullCheckService", () => { // Simulate a cached pull: SP never fetched from us. registryMock.resolve.mockResolvedValue({ ...registration, firstByteAt: undefined }); - await service.runPullCheck("0xsp", undefined, logContext); + await service.runPullCheck("0xsp", DEFAULT_NETWORK, undefined, logContext); expect(metricsMock.observeStartedMs).not.toHaveBeenCalled(); expect(metricsMock.observeThroughputBps).toHaveBeenCalledTimes(1); @@ -353,7 +369,7 @@ describe("PullCheckService", () => { pieces: [], } as unknown as Awaited>); - await expect(service.runPullCheck("0xsp", undefined, logContext)).rejects.toThrow( + await expect(service.runPullCheck("0xsp", DEFAULT_NETWORK, undefined, logContext)).rejects.toThrow( /Storage provider failed to pull piece/, ); @@ -366,7 +382,7 @@ describe("PullCheckService", () => { arrangeHappyPath(); vi.mocked(waitForPullPieces).mockRejectedValue(new Error("polling timed out after 300s")); - await expect(service.runPullCheck("0xsp", undefined, logContext)).rejects.toThrow(); + await expect(service.runPullCheck("0xsp", DEFAULT_NETWORK, undefined, logContext)).rejects.toThrow(); expect(metricsMock.recordStatus).toHaveBeenLastCalledWith(expect.any(Object), "failure.timedout"); }); @@ -380,7 +396,9 @@ describe("PullCheckService", () => { .mockResolvedValueOnce("bafk-test-piece" as any) .mockResolvedValueOnce("bafk-mismatch" as any); - await expect(service.runPullCheck("0xsp", undefined, logContext)).rejects.toThrow(/validation failed/); + await expect(service.runPullCheck("0xsp", DEFAULT_NETWORK, undefined, logContext)).rejects.toThrow( + /validation failed/, + ); expect(metricsMock.recordStatus).toHaveBeenLastCalledWith(expect.any(Object), "failure.other"); expect(registryMock.forget).not.toHaveBeenCalled(); }); @@ -390,7 +408,7 @@ describe("PullCheckService", () => { const controller = new AbortController(); controller.abort(new Error("Pull check job timeout (300s) for 0xsp")); - await expect(service.runPullCheck("0xsp", controller.signal, logContext)).rejects.toThrow(); + await expect(service.runPullCheck("0xsp", DEFAULT_NETWORK, controller.signal, logContext)).rejects.toThrow(); // No SP-side calls were issued. expect(pullPieces).not.toHaveBeenCalled(); expect(waitForPullPieces).not.toHaveBeenCalled(); @@ -402,7 +420,9 @@ describe("PullCheckService", () => { arrangeHappyPath(); walletSdkServiceMock.getSynapseClient.mockReturnValue(null); - await expect(service.runPullCheck("0xsp", undefined, logContext)).rejects.toThrow(/Synapse client unavailable/); + await expect(service.runPullCheck("0xsp", DEFAULT_NETWORK, undefined, logContext)).rejects.toThrow( + /Synapse client unavailable/, + ); expect(metricsMock.recordStatus).toHaveBeenLastCalledWith(expect.any(Object), "failure.other"); }); }); diff --git a/apps/backend/src/pull-check/pull-check.service.ts b/apps/backend/src/pull-check/pull-check.service.ts index 39e38cdb..ca08c942 100644 --- a/apps/backend/src/pull-check/pull-check.service.ts +++ b/apps/backend/src/pull-check/pull-check.service.ts @@ -5,7 +5,8 @@ import { Injectable, Logger } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import type { Address } from "viem"; import { type ProviderJobContext, toStructuredError } from "../common/logging.js"; -import type { IAppConfig, IConfig, IPullPieceConfig } from "../config/app.config.js"; +import type { Network } from "../common/types.js"; +import type { IAppConfig, IConfig, INetworkConfig } from "../config/index.js"; import { DataSourceService } from "../dataSource/dataSource.service.js"; import { HttpClientService } from "../http-client/http-client.service.js"; import { buildCheckMetricLabels, classifyFailureStatus } from "../metrics-prometheus/check-metric-labels.js"; @@ -33,13 +34,13 @@ export class PullCheckService { * the provider is unknown, inactive, missing a numeric provider id, or * missing a PDP serviceURL. Returns the enriched provider info on success. */ - validateProviderInfo(spAddress: string): PDPProviderEx { - const providerInfo = this.walletSdkService.getProviderInfo(spAddress); + validateProviderInfo(spAddress: string, network: Network): PDPProviderEx { + const providerInfo = this.walletSdkService.getProviderInfo(spAddress, network); if (!providerInfo) { - throw new Error(`Storage provider not found: ${spAddress}`); + throw new Error(`Storage provider not found: ${spAddress} on ${network}`); } if (!providerInfo.isActive) { - throw new Error(`Storage provider is not active: ${spAddress}`); + throw new Error(`Storage provider is not active: ${spAddress} on ${network}`); } if (providerInfo.id == null) { throw new Error(`Storage provider is missing providerId: ${spAddress}`); @@ -62,13 +63,14 @@ export class PullCheckService { */ async runPullCheck( spAddress: string, + network: Network, signal: AbortSignal | undefined, logContext: ProviderJobContext, ): Promise { - const providerInfo = this.validateProviderInfo(spAddress); + const providerInfo = this.validateProviderInfo(spAddress, network); const labels = buildCheckMetricLabels({ + network: network, checkType: "pullCheck", - network: this.configService.get("blockchain.network", { infer: true }), providerId: providerInfo.id, providerName: providerInfo.name, providerIsApproved: providerInfo.isApproved, @@ -79,11 +81,11 @@ export class PullCheckService { try { signal?.throwIfAborted(); - prepared = await this.preparePullPiece(spAddress); + prepared = await this.preparePullPiece(spAddress, network); const pieceCidStr = prepared.registration.pieceCid; const pieceCidParsed = parsePieceCid(pieceCidStr); - const synapseClient = this.requireSynapseClient(); + const synapseClient = this.requireSynapseClient(network); // Resolve pull options for either the existing-dataset or new-dataset SP // pull pathway. `pullPieces` requires both dataSetId and clientDataSetId @@ -113,12 +115,12 @@ export class PullCheckService { requestLatencyMs, }); - const pullPieceConfig = this.getPullPieceConfig(); + const networkCfg = this.getNetworkConfig(network); // `waitForPullPieces` polls the SP repeatedly until a terminal pull status is reported const finalResponse = await waitForPullPieces(synapseClient, { ...pullPiecesOptions, - timeout: pullPieceConfig.pullCheckJobTimeoutSeconds * 1000, - pollInterval: pullPieceConfig.pullCheckPollIntervalSeconds * 1000, + timeout: networkCfg.pullCheckJobTimeoutSeconds * 1000, + pollInterval: networkCfg.pullCheckPollIntervalSeconds * 1000, }); signal?.throwIfAborted(); const completionLatencyMs = Date.now() - requestSubmittedAt.getTime(); @@ -259,9 +261,9 @@ export class PullCheckService { * Generate a synthetic test piece, compute its piece CID, register it for * `/api/piece/:pieceCid` serving, and return the source URL plus registration. */ - async preparePullPiece(providerAddress: string): Promise { - const pullPieceConfig = this.getPullPieceConfig(); - const targetSize = pullPieceConfig.pullCheckPieceSizeBytes; + async preparePullPiece(providerAddress: string, network: Network): Promise { + const networkCfg = this.getNetworkConfig(network); + const targetSize = networkCfg.pullCheckPieceSizeBytes; const key = crypto.randomBytes(16).toString("hex"); const dataStream = this.dataSourceService.generateBytesStream({ @@ -280,15 +282,15 @@ export class PullCheckService { providerAddress, key, size: targetSize, - expiresAt: new Date(Date.now() + pullPieceConfig.pullCheckJobTimeoutSeconds * 2 * 1000), + expiresAt: new Date(Date.now() + networkCfg.pullCheckJobTimeoutSeconds * 2 * 1000), }; await this.pullPieceRepository.register(registration); return { registration, sourceUrl }; } - private getPullPieceConfig(): IPullPieceConfig { - return this.configService.get("pullPiece", { infer: true }); + private getNetworkConfig(network: Network): INetworkConfig { + return this.configService.get("networks", { infer: true })[network]; } private resolvePublicBaseUrl(): string { @@ -297,8 +299,8 @@ export class PullCheckService { return `http://${appConfig.host}:${appConfig.port}`; } - private requireSynapseClient(): SynapseViemClient { - const client = this.walletSdkService.getSynapseClient(); + private requireSynapseClient(network: Network): SynapseViemClient { + const client = this.walletSdkService.getSynapseClient(network); if (client == null) { throw new Error("Synapse client unavailable: chain integration must be enabled for pull checks"); } diff --git a/apps/backend/src/pull-check/pull-piece-stream-tracker.service.spec.ts b/apps/backend/src/pull-check/pull-piece-stream-tracker.service.spec.ts index db1659f1..d1bab8e2 100644 --- a/apps/backend/src/pull-check/pull-piece-stream-tracker.service.spec.ts +++ b/apps/backend/src/pull-check/pull-piece-stream-tracker.service.spec.ts @@ -2,7 +2,7 @@ import { PassThrough } from "node:stream"; import { ServiceUnavailableException } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { IConfig, IPullPieceConfig } from "../config/app.config.js"; +import type { IConfig, IPullPieceConfig } from "../config/index.js"; import { PullPieceStreamTracker } from "./pull-piece-stream-tracker.service.js"; /** Helper to wait for stream cleanup events to process */ @@ -18,13 +18,8 @@ describe("PullPieceStreamTracker", () => { let mockConfigService: ConfigService; const defaultConfig: IPullPieceConfig = { - pullChecksPerSpPerHour: 1, - pullCheckJobTimeoutSeconds: 300, - pullCheckPollIntervalSeconds: 2, - pullCheckPieceSizeBytes: 10 * 1024 * 1024, // 10 MB maxConcurrentStreams: 50, maxStreamsPerCid: 3, - pullPieceCleanupIntervalSeconds: 7 * 24 * 3600, // 7 days }; beforeEach(() => { diff --git a/apps/backend/src/pull-check/pull-piece-stream-tracker.service.ts b/apps/backend/src/pull-check/pull-piece-stream-tracker.service.ts index 16c7f332..17857acf 100644 --- a/apps/backend/src/pull-check/pull-piece-stream-tracker.service.ts +++ b/apps/backend/src/pull-check/pull-piece-stream-tracker.service.ts @@ -1,7 +1,7 @@ import type { Readable } from "node:stream"; import { Injectable, Logger, ServiceUnavailableException } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; -import type { IConfig, IPullPieceConfig } from "../config/app.config.js"; +import type { IConfig, IPullPieceConfig } from "../config/index.js"; /** * Tracks active pull-piece streams to enforce global and per-pieceCid concurrency limits. @@ -154,6 +154,6 @@ export class PullPieceStreamTracker { } private getPullPieceConfig(): IPullPieceConfig { - return this.configService.get("pullPiece", { infer: true }); + return this.configService.get("pullPiece", { infer: true }); } } From e72921ee4ba8cbc5290432b54adb6aeb7661bfec Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 16:57:49 +0530 Subject: [PATCH 14/23] fix: keep clickhouse chain agnostic --- .../src/clickhouse/clickhouse.service.ts | 2 +- apps/backend/src/config/constants.ts | 4 --- apps/backend/src/config/env.schema.ts | 13 +++++---- apps/backend/src/config/legacy-env-compat.ts | 4 --- apps/backend/src/config/loader.ts | 18 ++++++------- apps/backend/src/config/types.ts | 27 +++++++++---------- 6 files changed, 31 insertions(+), 37 deletions(-) diff --git a/apps/backend/src/clickhouse/clickhouse.service.ts b/apps/backend/src/clickhouse/clickhouse.service.ts index 2cdfc34a..c10eef9a 100644 --- a/apps/backend/src/clickhouse/clickhouse.service.ts +++ b/apps/backend/src/clickhouse/clickhouse.service.ts @@ -3,7 +3,7 @@ import { Injectable, Logger, OnApplicationShutdown, OnModuleInit } from "@nestjs import { ConfigService } from "@nestjs/config"; import { InjectMetric } from "@willsoto/nestjs-prometheus"; import { Counter, Gauge, Histogram } from "prom-client"; -import type { IClickhouseConfig, IConfig } from "../config/app.config.js"; +import type { IClickhouseConfig, IConfig } from "../config/index.js"; import { buildMigrations } from "./clickhouse.schema.js"; interface BufferedRow { diff --git a/apps/backend/src/config/constants.ts b/apps/backend/src/config/constants.ts index 0eda35ec..b56163e5 100644 --- a/apps/backend/src/config/constants.ts +++ b/apps/backend/src/config/constants.ts @@ -29,10 +29,6 @@ export const networkDefaults = { pullCheckPollIntervalSeconds: 2, pullCheckPieceSizeBytes: 10 * 1024 * 1024, // 10 MiB pullPieceCleanupIntervalSeconds: 7 * 24 * 3600, // 7 days - - clickhouseBatchSize: 500, - clickhouseFlushIntervalMs: 5000, - clickhouseMaxBufferSize: 5000, } satisfies NetworkDefaults; /** diff --git a/apps/backend/src/config/env.schema.ts b/apps/backend/src/config/env.schema.ts index 846f5a9b..bede922c 100644 --- a/apps/backend/src/config/env.schema.ts +++ b/apps/backend/src/config/env.schema.ts @@ -106,6 +106,13 @@ export const pullPieceEnvSchema = { PULL_PIECE_MAX_STREAMS_PER_CID: Joi.number().integer().min(1).default(3), }; +export const clickhouseEnvSchema = { + CLICKHOUSE_URL: Joi.string().uri().optional(), + CLICKHOUSE_BATCH_SIZE: Joi.number().integer().min(1).default(500), + CLICKHOUSE_FLUSH_INTERVAL_MS: Joi.number().integer().min(100).default(5000), + CLICKHOUSE_MAX_BUFFER_SIZE: Joi.number().integer().min(1).default(5000), +}; + export const datasetEnvSchema = { DEALBOT_LOCAL_DATASETS_PATH: Joi.string().default(DEFAULT_LOCAL_DATASETS_PATH), RANDOM_PIECE_SIZES: Joi.string().default("10485760"), @@ -194,11 +201,6 @@ export const createPerNetworkEnvSchema = (prefix: Uppercase | "") => { .integer() .min(3600) .default(7 * 24 * 3600), - - [k("CLICKHOUSE_URL")]: Joi.string().uri().optional(), - [k("CLICKHOUSE_BATCH_SIZE")]: Joi.number().integer().min(1).default(500), - [k("CLICKHOUSE_FLUSH_INTERVAL_MS")]: Joi.number().integer().min(100).default(5000), - [k("CLICKHOUSE_MAX_BUFFER_SIZE")]: Joi.number().integer().min(1).default(5000), }; }; @@ -218,6 +220,7 @@ export function createConfigValidationSchema(processEnv: NodeJS.ProcessEnv = pro ...databaseEnvSchema, ...globalNetworkEnvSchema, ...jobsEnvSchema, + ...clickhouseEnvSchema, ...pullPieceEnvSchema, ...datasetEnvSchema, ...timeoutEnvSchema, diff --git a/apps/backend/src/config/legacy-env-compat.ts b/apps/backend/src/config/legacy-env-compat.ts index d6ca54e3..bd46ff63 100644 --- a/apps/backend/src/config/legacy-env-compat.ts +++ b/apps/backend/src/config/legacy-env-compat.ts @@ -67,10 +67,6 @@ const LEGACY_PER_NETWORK_VARS = [ "PULL_CHECK_POLL_INTERVAL_SECONDS", "PULL_CHECK_PIECE_SIZE_BYTES", "PULL_PIECE_CLEANUP_INTERVAL_SECONDS", - "CLICKHOUSE_URL", - "CLICKHOUSE_BATCH_SIZE", - "CLICKHOUSE_FLUSH_INTERVAL_MS", - "CLICKHOUSE_MAX_BUFFER_SIZE", ] as const; export interface LegacyEnvCompatResult { diff --git a/apps/backend/src/config/loader.ts b/apps/backend/src/config/loader.ts index f21581d0..a2f10281 100644 --- a/apps/backend/src/config/loader.ts +++ b/apps/backend/src/config/loader.ts @@ -21,6 +21,7 @@ import { import type { BaseNetworkConfig, IAppConfig, + IClickhouseConfig, IConfig, IDatabaseConfig, IDatasetConfig, @@ -74,6 +75,13 @@ const loadJobsConfig = (env: NodeJS.ProcessEnv): IJobsConfig => ({ shutdownFinalScrapeDelaySeconds: getNumberEnv(env, "SHUTDOWN_FINAL_SCRAPE_DELAY_SECONDS", 35), }); +const loadClickhouseConfig = (env: NodeJS.ProcessEnv): IClickhouseConfig => ({ + url: env.CLICKHOUSE_URL || undefined, + batchSize: getNumberEnv(env, "CLICKHOUSE_BATCH_SIZE", 500), + flushIntervalMs: getNumberEnv(env, "CLICKHOUSE_FLUSH_INTERVAL_MS", 5000), + maxBufferSize: getNumberEnv(env, "CLICKHOUSE_MAX_BUFFER_SIZE", 5000), +}); + const loadPullPieceConfig = (env: NodeJS.ProcessEnv): IPullPieceConfig => ({ maxConcurrentStreams: getNumberEnv(env, "PULL_PIECE_MAX_CONCURRENT_STREAMS", 5), maxStreamsPerCid: getNumberEnv(env, "PULL_PIECE_MAX_STREAMS_PER_CID", 3), @@ -210,15 +218,6 @@ function loadNetworkEnvPrefix( "PULL_PIECE_CLEANUP_INTERVAL_SECONDS", networkDefaults.pullPieceCleanupIntervalSeconds, ), - - clickhouseUrl: get("CLICKHOUSE_URL") || undefined, - clickhouseBatchSize: getNumberEnv(env, "CLICKHOUSE_BATCH_SIZE", networkDefaults.clickhouseBatchSize), - clickhouseFlushIntervalMs: getNumberEnv( - env, - "CLICKHOUSE_FLUSH_INTERVAL_MS", - networkDefaults.clickhouseFlushIntervalMs, - ), - clickhouseMaxBufferSize: getNumberEnv(env, "CLICKHOUSE_MAX_BUFFER_SIZE", networkDefaults.clickhouseMaxBufferSize), } satisfies Omit; const walletPrivateKey = (get("WALLET_PRIVATE_KEY") || undefined) as `0x${string}` | undefined; @@ -254,6 +253,7 @@ export function loadConfig(): IConfig { database: loadDatabaseConfig(process.env), ...loadNetworkConfigs(process.env), jobs: loadJobsConfig(process.env), + clickhouse: loadClickhouseConfig(process.env), pullPiece: loadPullPieceConfig(process.env), dataset: loadDatasetConfig(process.env), timeouts: loadTimeoutConfig(process.env), diff --git a/apps/backend/src/config/types.ts b/apps/backend/src/config/types.ts index 6c088ad5..f10311f3 100644 --- a/apps/backend/src/config/types.ts +++ b/apps/backend/src/config/types.ts @@ -125,17 +125,6 @@ export type BaseNetworkConfig = { * Defaults to 7 days (604800 s). Minimum 1 hour enforced by Joi. */ pullPieceCleanupIntervalSeconds: number; - - /** Clickhouse Config */ - /** - * ClickHouse connection URL. Must include the database in the path. - * Example: http://default:password@host:8123/dealbot - * If unset, ClickHouse emission is disabled. - */ - clickhouseUrl: string | undefined; - clickhouseBatchSize: number; - clickhouseFlushIntervalMs: number; - clickhouseMaxBufferSize: number; }; type WalletPrivateKeyNetworkConfig = BaseNetworkConfig & { @@ -204,6 +193,18 @@ export interface IJobsConfig { shutdownFinalScrapeDelaySeconds: number; } +export interface IClickhouseConfig { + /** + * ClickHouse connection URL. Must include the database in the path. + * Example: http://default:password@host:8123/dealbot + * If unset, ClickHouse emission is disabled. + */ + url: string | undefined; + batchSize: number; + flushIntervalMs: number; + maxBufferSize: number; +} + export interface IPullPieceConfig { /** * Maximum number of concurrent piece streams across all pieceCids. @@ -242,6 +243,7 @@ export interface IConfig { networks: INetworksConfig; activeNetworks: Network[]; jobs: IJobsConfig; + clickhouse: IClickhouseConfig; pullPiece: IPullPieceConfig; dataset: IDatasetConfig; timeouts: ITimeoutConfig; @@ -273,7 +275,4 @@ export type NetworkDefaults = Pick< | "pullCheckPollIntervalSeconds" | "pullCheckPieceSizeBytes" | "pullPieceCleanupIntervalSeconds" - | "clickhouseBatchSize" - | "clickhouseFlushIntervalMs" - | "clickhouseMaxBufferSize" >; From 2f84dd4d96821d39b3871a579190df852ddd397f Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 17:00:40 +0530 Subject: [PATCH 15/23] test: fix --- apps/backend/src/jobs/jobs.service.spec.ts | 6 ------ apps/backend/src/jobs/jobs.service.ts | 2 +- apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts | 6 ------ 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/apps/backend/src/jobs/jobs.service.spec.ts b/apps/backend/src/jobs/jobs.service.spec.ts index f46c537c..0d47cb19 100644 --- a/apps/backend/src/jobs/jobs.service.spec.ts +++ b/apps/backend/src/jobs/jobs.service.spec.ts @@ -146,13 +146,7 @@ describe("JobsService schedule rows", () => { pullCheckJobTimeoutSeconds: 300, pullCheckPollIntervalSeconds: 2, pullCheckPieceSizeBytes: 10 * 1024 * 1024, - pullPieceMaxConcurrentStreams: 50, - pullPieceMaxStreamsPerCid: 3, pullPieceCleanupIntervalSeconds: 7 * 24 * 3600, - clickhouseUrl: "http://localhost:8123", - clickhouseBatchSize: 500, - clickhouseFlushIntervalMs: 5000, - clickhouseMaxBufferSize: 5000, } satisfies IConfig["networks"]["calibration"]; baseConfigValues = { diff --git a/apps/backend/src/jobs/jobs.service.ts b/apps/backend/src/jobs/jobs.service.ts index fd33c31c..a4bf1fab 100644 --- a/apps/backend/src/jobs/jobs.service.ts +++ b/apps/backend/src/jobs/jobs.service.ts @@ -766,7 +766,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { return "success"; } try { - await this.pullCheckService.runPullCheck(spAddress, abortController.signal, logContext); + await this.pullCheckService.runPullCheck(spAddress, network, abortController.signal, logContext); return "success"; } catch (error) { if (abortController.signal.aborted) { diff --git a/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts b/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts index bf5a50b5..1a9911c4 100644 --- a/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts +++ b/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts @@ -39,13 +39,7 @@ const baseNetworkConfig = { pullCheckJobTimeoutSeconds: 300, pullCheckPollIntervalSeconds: 2, pullCheckPieceSizeBytes: 10 * 1024 * 1024, // 10 MiB - pullPieceMaxConcurrentStreams: 50, - pullPieceMaxStreamsPerCid: 3, pullPieceCleanupIntervalSeconds: 7 * 24 * 3600, // 7 days - clickhouseUrl: "http://localhost:8123", - clickhouseBatchSize: 500, - clickhouseFlushIntervalMs: 5000, - clickhouseMaxBufferSize: 5000, } satisfies IConfig["networks"]["calibration"]; const makeProvider = (overrides: Partial): PDPProviderEx => From 145519db3b444e7690a3920e661a88f391797332 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 17:48:39 +0530 Subject: [PATCH 16/23] migrate providers service to multi network --- .../dto/provider-list-response.dto.ts | 3 + .../src/providers/providers.controller.ts | 18 +++++- .../src/providers/providers.service.spec.ts | 35 +++++++---- .../src/providers/providers.service.ts | 62 ++++++++++++------- 4 files changed, 84 insertions(+), 34 deletions(-) diff --git a/apps/backend/src/providers/dto/provider-list-response.dto.ts b/apps/backend/src/providers/dto/provider-list-response.dto.ts index 130a7649..f2b71494 100644 --- a/apps/backend/src/providers/dto/provider-list-response.dto.ts +++ b/apps/backend/src/providers/dto/provider-list-response.dto.ts @@ -4,6 +4,9 @@ export class StorageProviderDto { @ApiProperty({ description: "Storage provider address", example: "f01234" }) address!: string; + @ApiProperty({ description: "Network this provider belongs to", enum: ["calibration", "mainnet"] }) + network!: string; + @ApiPropertyOptional({ description: "On-chain provider ID", type: String }) providerId?: string; diff --git a/apps/backend/src/providers/providers.controller.ts b/apps/backend/src/providers/providers.controller.ts index c1e31bc3..bad368fb 100644 --- a/apps/backend/src/providers/providers.controller.ts +++ b/apps/backend/src/providers/providers.controller.ts @@ -1,5 +1,7 @@ -import { Controller, DefaultValuePipe, Get, Logger, ParseIntPipe, Query } from "@nestjs/common"; +import { BadRequestException, Controller, DefaultValuePipe, Get, Logger, ParseIntPipe, Query } from "@nestjs/common"; import { ApiOperation, ApiQuery, ApiResponse, ApiTags } from "@nestjs/swagger"; +import { SUPPORTED_NETWORKS } from "../common/constants.js"; +import type { Network } from "../common/types.js"; import { ProviderListResponseDto } from "./dto/provider-list-response.dto.js"; import { ProvidersService } from "./providers.service.js"; @@ -33,6 +35,12 @@ export class ProvidersController { type: Number, description: "Pagination offset (default: 0)", }) + @ApiQuery({ + name: "network", + required: false, + enum: SUPPORTED_NETWORKS, + description: "Filter by network. When omitted, providers from all active networks are returned.", + }) @ApiResponse({ status: 200, description: "List of providers", @@ -41,12 +49,18 @@ export class ProvidersController { async listProviders( @Query("limit", new DefaultValuePipe(20), ParseIntPipe) limit?: number, @Query("offset", new DefaultValuePipe(0), ParseIntPipe) offset?: number, + @Query("network") network?: string, ): Promise { - this.logger.debug(`Listing providers: limit=${limit}, offset=${offset}`); + if (network !== undefined && !(SUPPORTED_NETWORKS as readonly string[]).includes(network)) { + throw new BadRequestException(`Invalid network "${network}". Must be one of: ${SUPPORTED_NETWORKS.join(", ")}`); + } + + this.logger.debug(`Listing providers: limit=${limit}, offset=${offset}, network=${network ?? "all"}`); const { providers, total } = await this.providersService.getProvidersList({ limit, offset, + network: network as Network | undefined, }); return { diff --git a/apps/backend/src/providers/providers.service.spec.ts b/apps/backend/src/providers/providers.service.spec.ts index 312cb51c..884118f5 100644 --- a/apps/backend/src/providers/providers.service.spec.ts +++ b/apps/backend/src/providers/providers.service.spec.ts @@ -22,7 +22,20 @@ describe("ProvidersService", () => { }; configService = { - get: vi.fn().mockReturnValue({ ids: new Set(["123"]), addresses: new Set(["f0123"]) }), + get: vi.fn().mockImplementation((key: string) => { + if (key === "networks") { + return { + calibration: { + blockedSpIds: new Set(["123"]), + blockedSpAddresses: new Set(["f0123"]), + }, + }; + } + if (key === "activeNetworks") { + return ["calibration"]; + } + return undefined; + }), }; const module = await Test.createTestingModule({ @@ -36,19 +49,20 @@ describe("ProvidersService", () => { service = module.get(ProvidersService); }); - it("getProvidersList applies blocklist filters", async () => { + it("getProvidersList applies per-network blocklist filters", async () => { const queryBuilder = repo.createQueryBuilder(); repo.createQueryBuilder.mockReturnValue(queryBuilder); await service.getProvidersList(); expect(queryBuilder.andWhere).toHaveBeenCalledWith( - '("sp"."providerId" IS NULL OR "sp"."providerId" NOT IN (:...blockedIds))', - { blockedIds: [123n] }, + '("sp"."providerId" IS NULL OR NOT ("sp"."network" = :network_calibration AND "sp"."providerId" IN (:...blockedIds_calibration)))', + { network_calibration: "calibration", blockedIds_calibration: [123n] }, + ); + expect(queryBuilder.andWhere).toHaveBeenCalledWith( + 'NOT ("sp"."network" = :network_calibration AND LOWER("sp"."address") IN (:...blockedAddresses_calibration))', + { network_calibration: "calibration", blockedAddresses_calibration: ["f0123"] }, ); - expect(queryBuilder.andWhere).toHaveBeenCalledWith('LOWER("sp"."address") NOT IN (:...blockedAddresses)', { - blockedAddresses: ["f0123"], - }); }); it("getProvidersList preserves providers with null providerId when applying blocklist filters", async () => { @@ -58,13 +72,12 @@ describe("ProvidersService", () => { await service.getProvidersList(); const providerIdFilterCall = queryBuilder.andWhere.mock.calls.find( - ([clause]: [string, { blockedIds?: bigint[] }]) => - typeof clause === "string" && clause.includes('"sp"."providerId"'), + ([clause]: [string]) => typeof clause === "string" && clause.includes('"sp"."providerId"'), ); expect(providerIdFilterCall).toBeDefined(); expect(providerIdFilterCall?.[0]).toContain('("sp"."providerId" IS NULL'); - expect(providerIdFilterCall?.[0]).toContain('"sp"."providerId" NOT IN'); - expect(providerIdFilterCall?.[1]).toEqual({ blockedIds: [123n] }); + expect(providerIdFilterCall?.[0]).toContain('"sp"."providerId" IN'); + expect(providerIdFilterCall?.[1]).toMatchObject({ blockedIds_calibration: [123n] }); }); }); diff --git a/apps/backend/src/providers/providers.service.ts b/apps/backend/src/providers/providers.service.ts index 4cae9031..252d8cc0 100644 --- a/apps/backend/src/providers/providers.service.ts +++ b/apps/backend/src/providers/providers.service.ts @@ -2,7 +2,8 @@ import { Injectable } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import { InjectRepository } from "@nestjs/typeorm"; import type { Repository } from "typeorm"; -import type { IConfig } from "../config/app.config.js"; +import type { Network } from "../common/types.js"; +import type { IConfig } from "../config/index.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; /** @@ -19,15 +20,27 @@ export class ProvidersService { /** * Get paginated/filtered provider list. + * + * When `network` is provided, results are scoped to that network and only + * that network's blocklist is applied. + * + * When `network` is omitted, results span all active networks and the + * blocklist for each network is applied, preserving the pre-multi-network + * behaviour of a single global blocklist. */ async getProvidersList(options?: { activeOnly?: boolean; approvedOnly?: boolean; + network?: Network; limit?: number; offset?: number; }): Promise<{ providers: StorageProvider[]; total: number }> { const query = this.spRepository.createQueryBuilder("sp"); + if (options?.network) { + query.andWhere("sp.network = :network", { network: options.network }); + } + if (options?.activeOnly) { query.andWhere("sp.is_active = true"); } @@ -37,30 +50,37 @@ export class ProvidersService { } // Filter out blocked providers - const blocklists = this.configService.get("spBlocklists", { infer: true }); - if (blocklists.ids.size > 0) { - // providerId is BigInt in the entity, so we convert strings to BigInts for the query - const blockedIds = Array.from(blocklists.ids) - .map((id) => { - try { - return BigInt(id); - } catch { - return null; - } - }) - .filter((id): id is bigint => id !== null); + const networksConfig = this.configService.get("networks", { infer: true }); + const activeNetworks = this.configService.get("activeNetworks", { infer: true }); + const networksToFilter: Network[] = options?.network ? [options.network] : activeNetworks; + + for (const net of networksToFilter) { + const cfg = networksConfig[net]; + + const blockedIds: bigint[] = []; + for (const id of cfg.blockedSpIds) { + try { + blockedIds.push(BigInt(id)); + } catch { + // skip malformed ID strings + } + } + + const blockedAddresses = Array.from(cfg.blockedSpAddresses).map((a) => a.toLowerCase()); if (blockedIds.length > 0) { - query.andWhere('("sp"."providerId" IS NULL OR "sp"."providerId" NOT IN (:...blockedIds))', { - blockedIds, - }); + query.andWhere( + `("sp"."providerId" IS NULL OR NOT ("sp"."network" = :network_${net} AND "sp"."providerId" IN (:...blockedIds_${net})))`, + { [`network_${net}`]: net, [`blockedIds_${net}`]: blockedIds }, + ); } - } - if (blocklists.addresses.size > 0) { - query.andWhere('LOWER("sp"."address") NOT IN (:...blockedAddresses)', { - blockedAddresses: Array.from(blocklists.addresses), - }); + if (blockedAddresses.length > 0) { + query.andWhere( + `NOT ("sp"."network" = :network_${net} AND LOWER("sp"."address") IN (:...blockedAddresses_${net}))`, + { [`network_${net}`]: net, [`blockedAddresses_${net}`]: blockedAddresses }, + ); + } } const total = await query.getCount(); From 77c9b987aa1b68c9f8075f1654074f0640ac5f02 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 18:02:15 +0530 Subject: [PATCH 17/23] migrate backend .env.example to multi-network configuration --- .env.example | 31 ++++- apps/backend/.env.example | 245 +++++++++++++++++++++++++++----------- 2 files changed, 200 insertions(+), 76 deletions(-) diff --git a/.env.example b/.env.example index 938af27c..8742336b 100644 --- a/.env.example +++ b/.env.example @@ -4,9 +4,32 @@ # Keep this file limited to *secrets only* - non-secret configuration lives in # kustomize ConfigMap patches (e.g., kustomize/overlays/local/backend-configmap-local.yaml). # -# Required -WALLET_ADDRESS= -WALLET_PRIVATE_KEY= +# Multi-network support +# --------------------- +# Dealbot now supports running against multiple Filecoin networks from a single +# instance. The `NETWORKS` env var (set via ConfigMap, not here) selects the +# active networks as a comma-separated list, e.g. `NETWORKS=calibration` or +# `NETWORKS=calibration,mainnet`. +# +# Every network-scoped secret is prefixed with the uppercase network name: +# CALIBRATION_WALLET_ADDRESS, CALIBRATION_WALLET_PRIVATE_KEY +# MAINNET_WALLET_ADDRESS, MAINNET_WALLET_PRIVATE_KEY +# +# Each active network must provide either `_WALLET_PRIVATE_KEY` or +# `_SESSION_KEY_PRIVATE_KEY`. When both are set, the session key +# takes precedence (see docs/runbooks/wallet-and-session-keys.md). + +# --- Required for calibration --- +CALIBRATION_WALLET_ADDRESS= +CALIBRATION_WALLET_PRIVATE_KEY= +# Optional: use a session key instead of the raw wallet key. +# CALIBRATION_SESSION_KEY_PRIVATE_KEY= + +# --- Required for mainnet (only if `mainnet` is in NETWORKS) --- +# MAINNET_WALLET_ADDRESS= +# MAINNET_WALLET_PRIVATE_KEY= +# MAINNET_SESSION_KEY_PRIVATE_KEY= + # # Optional (only if using an external DB or a non-default password) -# DATABASE_PASSWORD= +# DATABASE_PASSWORD= \ No newline at end of file diff --git a/apps/backend/.env.example b/apps/backend/.env.example index e614e6f0..daeb4b7c 100644 --- a/apps/backend/.env.example +++ b/apps/backend/.env.example @@ -1,94 +1,195 @@ -# Node Environment -NODE_ENV=development +# ============================================================================= +# Dealbot backend — environment configuration +# ============================================================================= +# See docs/environment-variables.md for the full reference. +# +# Multi-network primer +# -------------------- +# `NETWORKS` is a comma-separated list of Filecoin networks the instance should +# drive (supported: `calibration`, `mainnet`). Every network-scoped variable is +# namespaced with the UPPERCASE network name: +# +# NETWORKS=calibration,mainnet +# CALIBRATION_WALLET_PRIVATE_KEY=0x... +# MAINNET_WALLET_PRIVATE_KEY=0x... +# +# Globals (database, jobs, timeouts, HTTP ports, etc.) remain unprefixed and +# apply to all networks. Variables for INACTIVE networks are ignored; only +# active networks are validated at startup. +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Application +# ----------------------------------------------------------------------------- +NODE_ENV=production + +# api | worker | both (default: both) +DEALBOT_RUN_MODE=both -# Specify port for dealbot service (optional) DEALBOT_PORT=8080 - -# Specify host for dealbot service (optional) DEALBOT_HOST=localhost # Optional public base URL for DealBot HTTP API (used to construct hosted-piece source URLs for SP pull checks) # DEALBOT_API_PUBLIC_URL=https://dealbot.example.com +# Metrics-only HTTP server (used when DEALBOT_RUN_MODE=worker) +DEALBOT_METRICS_PORT=9090 +DEALBOT_METRICS_HOST=0.0.0.0 + # Comma-separated list of allowed origins for CORS (for web dev) DEALBOT_ALLOWED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 -# Database Configuration +# ----------------------------------------------------------------------------- +# Database +# ----------------------------------------------------------------------------- DATABASE_HOST=localhost DATABASE_PORT=5432 DATABASE_USER=dealbot -DATABASE_PASSWORD=dealbot_password -DATABASE_NAME=filecoin_dealbot - -# Blockchain Configuration -NETWORK=calibration # or mainnet -WALLET_ADDRESS=0x0000000000000000000000000000000000000000 -WALLET_PRIVATE_KEY=your_private_key_here -CHECK_DATASET_CREATION_FEES=true -USE_ONLY_APPROVED_PROVIDERS=true -PDP_SUBGRAPH_ENDPOINT=https://api.thegraph.com/subgraphs/filecoin/pdp - -# Minimum number of datasets per SP (default: 1). When > 1, a separate data_set_creation job provisions extra datasets. -MIN_NUM_DATASETS_FOR_CHECKS=1 - -# Dataset Versioning (optional) -# Uncomment and set to enable dataset versioning (e.g., "dealbot-v1", "dealbot-v2") -# This allows creating new logical datasets without changing wallet addresses -# DEALBOT_DATASET_VERSION=dealbot-v1 - -# Scheduling Configuration -# Intervals: How often jobs run (in seconds) -PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 # Run providers refresh every 4 hours -DATA_RETENTION_POLL_INTERVAL_SECONDS=3600 # Run data retention poll every 60 minutes - -# Prometheus Metrics Configuration -# Cache TTL for wallet balance collection (in seconds) -PROMETHEUS_WALLET_BALANCE_TTL_SECONDS=3600 # Refresh wallet balance every 1 hour -PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS=60 # Wait 1 minute before retry after error - -# Maintenance windows (UTC) -DEALBOT_MAINTENANCE_WINDOWS_UTC=07:00,22:00 -DEALBOT_MAINTENANCE_WINDOW_MINUTES=20 - -# pg-boss (DB-backed jobs) Configuration -# See docs/environment-variables.md for details, limits, and fractional examples. -DEALS_PER_SP_PER_HOUR=2 -DATASET_CREATIONS_PER_SP_PER_HOUR=1 -RETRIEVALS_PER_SP_PER_HOUR=1 +DATABASE_PASSWORD=dealbot +DATABASE_NAME=dealbot +DATABASE_POOL_MAX=1 + +# ----------------------------------------------------------------------------- +# Network selection +# ----------------------------------------------------------------------------- +# Comma-separated. Default: calibration. +NETWORKS=calibration,mainnet + +# ----------------------------------------------------------------------------- +# Per-network configuration — CALIBRATION +# ----------------------------------------------------------------------------- +# At least one of WALLET_PRIVATE_KEY or SESSION_KEY_PRIVATE_KEY is required +# for each ACTIVE network. Session key (if present) takes precedence. +CALIBRATION_WALLET_ADDRESS=0x0000000000000000000000000000000000000000 +CALIBRATION_WALLET_PRIVATE_KEY=0x0000000000000000000000000000000000000000000000000000000000000000 +# CALIBRATION_SESSION_KEY_PRIVATE_KEY= + +# Optional: custom RPC (authenticated endpoints avoid public rate limits) +CALIBRATION_RPC_URL= + +CALIBRATION_CHECK_DATASET_CREATION_FEES=true +CALIBRATION_USE_ONLY_APPROVED_PROVIDERS=true +CALIBRATION_PDP_SUBGRAPH_ENDPOINT= + +# Minimum number of datasets per SP. When > 1, a separate data_set_creation +# job provisions extra datasets. +CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS=1 + +# Optional: dataset versioning (e.g. "dealbot-v1", "dealbot-v2"). +# CALIBRATION_DEALBOT_DATASET_VERSION=dealbot-v1 + +# Per-network scheduling rates (per hour; fractional values supported) +CALIBRATION_DEALS_PER_SP_PER_HOUR=4 +CALIBRATION_DATASET_CREATIONS_PER_SP_PER_HOUR=0.01 +CALIBRATION_RETRIEVALS_PER_SP_PER_HOUR=1 +CALIBRATION_PIECE_CLEANUP_PER_SP_PER_HOUR=0.1 +CALIBRATION_DEAL_JOB_TIMEOUT_SECONDS=360 # 6m: max runtime for deal jobs +CALIBRATION_RETRIEVAL_JOB_TIMEOUT_SECONDS=60 # 1m: max runtime for retrieval jobs +CALIBRATION_DATA_SET_CREATION_JOB_TIMEOUT_SECONDS=300 # 5m: max runtime for dataset-creation jobs +CALIBRATION_MAX_PIECE_CLEANUP_RUNTIME_SECONDS=3000 # 50m: max runtime for piece cleanup + +# Per-network job intervals (seconds) +CALIBRATION_PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 # 4 hours +CALIBRATION_DATA_RETENTION_POLL_INTERVAL_SECONDS=3600 # 1 hour + +# Per-network maintenance windows (UTC) +CALIBRATION_MAINTENANCE_WINDOWS_UTC=07:00,22:00 +CALIBRATION_MAINTENANCE_WINDOW_MINUTES=20 + +# Per-network piece cleanup configuration +CALIBRATION_MAX_DATASET_STORAGE_SIZE_BYTES=1048576000 +CALIBRATION_TARGET_DATASET_STORAGE_SIZE_BYTES=1 + +# Per-network pull check configuration +CALIBRATION_PULL_CHECKS_PER_SP_PER_HOUR=1 +CALIBRATION_PULL_CHECK_JOB_TIMEOUT_SECONDS=300 +CALIBRATION_PULL_CHECK_POLL_INTERVAL_SECONDS=2 +CALIBRATION_PULL_CHECK_PIECE_SIZE_BYTES=10485760 +CALIBRATION_PULL_PIECE_CLEANUP_INTERVAL_SECONDS=604800 + +# Per-network SP blocklists +# CALIBRATION_BLOCKED_SP_IDS=1234,5678 +# CALIBRATION_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234... + +# ----------------------------------------------------------------------------- +# Per-network configuration — MAINNET (uncomment when adding to NETWORKS) +# ----------------------------------------------------------------------------- +MAINNET_WALLET_ADDRESS=0x0000000000000000000000000000000000000000 +MAINNET_WALLET_PRIVATE_KEY=0x0000000000000000000000000000000000000000000000000000000000000000 +# MAINNET_SESSION_KEY_PRIVATE_KEY= +# MAINNET_RPC_URL= +MAINNET_CHECK_DATASET_CREATION_FEES=false +MAINNET_USE_ONLY_APPROVED_PROVIDERS=true +MAINNET_PDP_SUBGRAPH_ENDPOINT= +MAINNET_MIN_NUM_DATASETS_FOR_CHECKS=1 +# MAINNET_DEALBOT_DATASET_VERSION= +MAINNET_DEALS_PER_SP_PER_HOUR=1 +MAINNET_DATASET_CREATIONS_PER_SP_PER_HOUR=0.01 +MAINNET_RETRIEVALS_PER_SP_PER_HOUR=1 +MAINNET_PIECE_CLEANUP_PER_SP_PER_HOUR=0.05 +MAINNET_DEAL_JOB_TIMEOUT_SECONDS=360 +MAINNET_RETRIEVAL_JOB_TIMEOUT_SECONDS=60 +MAINNET_DATA_SET_CREATION_JOB_TIMEOUT_SECONDS=300 +MAINNET_PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 +MAINNET_DATA_RETENTION_POLL_INTERVAL_SECONDS=3600 +MAINNET_MAINTENANCE_WINDOWS_UTC=07:00,22:00 +MAINNET_MAINTENANCE_WINDOW_MINUTES=20 +# MAINNET_BLOCKED_SP_IDS=1234,5678 +# MAINNET_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234... +MAINNET_MAX_DATASET_STORAGE_SIZE_BYTES=25769803776 +MAINNET_TARGET_DATASET_STORAGE_SIZE_BYTES=21474836480 +MAINNET_MAX_PIECE_CLEANUP_RUNTIME_SECONDS=3000 +MAINNET_PULL_CHECKS_PER_SP_PER_HOUR=1 +MAINNET_PULL_CHECK_JOB_TIMEOUT_SECONDS=300 +MAINNET_PULL_CHECK_POLL_INTERVAL_SECONDS=2 +MAINNET_PULL_CHECK_PIECE_SIZE_BYTES=10485760 +MAINNET_PULL_PIECE_CLEANUP_INTERVAL_SECONDS=604800 + +# ----------------------------------------------------------------------------- +# Jobs (pg-boss) +# ----------------------------------------------------------------------------- +# See docs/environment-variables.md for details, limits, and sizing guidance. PG_BOSS_LOCAL_CONCURRENCY=20 -JOB_SCHEDULER_POLL_SECONDS=300 +JOB_SCHEDULER_POLL_SECONDS=120 JOB_WORKER_POLL_SECONDS=60 JOB_CATCHUP_MAX_ENQUEUE=10 JOB_SCHEDULE_PHASE_SECONDS=0 JOB_ENQUEUE_JITTER_SECONDS=0 -DEAL_JOB_TIMEOUT_SECONDS=360 # 6m: Max runtime for deal jobs (TODO: reduce default to 3m) -RETRIEVAL_JOB_TIMEOUT_SECONDS=60 # 1m: Max runtime for retrieval jobs (TODO: reduce default to 30s) -IPFS_BLOCK_FETCH_CONCURRENCY=6 # Parallel block fetches when validating IPFS DAGs - -# Pull Check Configuration -PULL_CHECKS_PER_SP_PER_HOUR=1 # SP pull-pathway checks scheduled per provider per hour -PULL_CHECK_JOB_TIMEOUT_SECONDS=300 # 5m: Max runtime for pull-check jobs -PULL_CHECK_POLL_INTERVAL_SECONDS=2 # SP pull status polling interval -PULL_CHECK_PIECE_SIZE_BYTES=10485760 # 10 MiB synthetic test piece size per pull check -PULL_PIECE_MAX_CONCURRENT_STREAMS=50 # Max concurrent streams across all pieces (DoS protection) -PULL_PIECE_MAX_STREAMS_PER_CID=3 # Max concurrent streams per pieceCid (prevents spam of single piece) +IPFS_BLOCK_FETCH_CONCURRENCY=6 # parallel block fetches during IPFS DAG validation DEALBOT_PGBOSS_POOL_MAX=1 DEALBOT_PGBOSS_SCHEDULER_ENABLED=true -# Dataset Configuration -DEALBOT_LOCAL_DATASETS_PATH=./datasets -KAGGLE_DATASET_TOTAL_PAGES=500 - -# Proxy Configuration -PROXY_LIST=http://username:password@host:port,http://username:password@host:port -PROXY_LOCATIONS=l1,l2 - -# Timeout Configuration (in milliseconds) -CONNECT_TIMEOUT_MS=10000 # 10s: Initial connection timeout -HTTP_REQUEST_TIMEOUT_MS=240000 # 4m: Total transfer timeout for HTTP/1.1 (10MiB @ 170KB/s + overhead) -HTTP2_REQUEST_TIMEOUT_MS=240000 # 4m: Total transfer timeout for HTTP/2 (10MiB @ 170KB/s + overhead) - -# SP Blocklists configuration -# BLOCKED_SP_IDS=1234,5678 -# BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234... +# ----------------------------------------------------------------------------- +# Pull Piece +# ----------------------------------------------------------------------------- +PULL_PIECE_MAX_CONCURRENT_STREAMS=50 # Max concurrent streams across all pieces (DoS protection) +PULL_PIECE_MAX_STREAMS_PER_CID=3 # Max concurrent streams per pieceCid (prevents spam of single piece) +# ----------------------------------------------------------------------------- +# Clickhouse +# ----------------------------------------------------------------------------- +CLICKHOUSE_URL= +CLICKHOUSE_BATCH_SIZE= +CLICKHOUSE_FLUSH_INTERVAL_MS= +CLICKHOUSE_MAX_BUFFER_SIZE= + +# ----------------------------------------------------------------------------- +# Prometheus metrics +# ----------------------------------------------------------------------------- +PROMETHEUS_WALLET_BALANCE_TTL_SECONDS=3600 # refresh cache every 1h +PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS=60 # cooldown after fetch failure + +# ----------------------------------------------------------------------------- +# Dataset generation +# ----------------------------------------------------------------------------- +DEALBOT_LOCAL_DATASETS_PATH=./datasets +RANDOM_PIECE_SIZES=10485760 # 10 MiB (comma-separated list supported) + +# ----------------------------------------------------------------------------- +# HTTP timeouts (milliseconds) +# ----------------------------------------------------------------------------- +CONNECT_TIMEOUT_MS=10000 # 10s: initial connection timeout +HTTP_REQUEST_TIMEOUT_MS=240000 # 4m: total transfer timeout (HTTP/1.1) +HTTP2_REQUEST_TIMEOUT_MS=240000 # 4m: total transfer timeout (HTTP/2) +IPNI_VERIFICATION_TIMEOUT_MS=60000 # 60s: IPNI propagation wait +IPNI_VERIFICATION_POLLING_MS=2000 # 2s: IPNI polling interval \ No newline at end of file From a29872a518ab918635c018e894413167e1785018 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 18:27:52 +0530 Subject: [PATCH 18/23] docs: update environment-variables.md --- apps/backend/src/config/loader.ts | 20 +- docs/environment-variables.md | 917 +++++++++++++----------------- 2 files changed, 407 insertions(+), 530 deletions(-) diff --git a/apps/backend/src/config/loader.ts b/apps/backend/src/config/loader.ts index a2f10281..abd40df0 100644 --- a/apps/backend/src/config/loader.ts +++ b/apps/backend/src/config/loader.ts @@ -146,15 +146,23 @@ function loadNetworkEnvPrefix( networkDefaults.minNumDataSetsForChecks, ), dealsPerSpPerHour: getFloatEnv(env, k("DEALS_PER_SP_PER_HOUR"), networkDefaults.dealsPerSpPerHour), - dealJobTimeoutSeconds: getNumberEnv(env, "DEAL_JOB_TIMEOUT_SECONDS", 360), + dealJobTimeoutSeconds: getNumberEnv(env, k("DEAL_JOB_TIMEOUT_SECONDS"), networkDefaults.dealJobTimeoutSeconds), retrievalsPerSpPerHour: getFloatEnv(env, k("RETRIEVALS_PER_SP_PER_HOUR"), networkDefaults.retrievalsPerSpPerHour), - retrievalJobTimeoutSeconds: getNumberEnv(env, "RETRIEVAL_JOB_TIMEOUT_SECONDS", 60), + retrievalJobTimeoutSeconds: getNumberEnv( + env, + k("RETRIEVAL_JOB_TIMEOUT_SECONDS"), + networkDefaults.retrievalJobTimeoutSeconds, + ), dataSetCreationsPerSpPerHour: getFloatEnv( env, k("DATASET_CREATIONS_PER_SP_PER_HOUR"), networkDefaults.dataSetCreationsPerSpPerHour, ), - dataSetCreationJobTimeoutSeconds: getNumberEnv(env, "DATA_SET_CREATION_JOB_TIMEOUT_SECONDS", 300), + dataSetCreationJobTimeoutSeconds: getNumberEnv( + env, + k("DATA_SET_CREATION_JOB_TIMEOUT_SECONDS"), + networkDefaults.dataSetCreationJobTimeoutSeconds, + ), dataRetentionPollIntervalSeconds: getNumberEnv( env, k("DATA_RETENTION_POLL_INTERVAL_SECONDS"), @@ -204,18 +212,18 @@ function loadNetworkEnvPrefix( pullChecksPerSpPerHour: getFloatEnv(env, k("PULL_CHECKS_PER_SP_PER_HOUR"), networkDefaults.pullChecksPerSpPerHour), pullCheckJobTimeoutSeconds: getNumberEnv( env, - "PULL_CHECK_JOB_TIMEOUT_SECONDS", + k("PULL_CHECK_JOB_TIMEOUT_SECONDS"), networkDefaults.pullCheckJobTimeoutSeconds, ), pullCheckPollIntervalSeconds: getNumberEnv( env, - "PULL_CHECK_POLL_INTERVAL_SECONDS", + k("PULL_CHECK_POLL_INTERVAL_SECONDS"), networkDefaults.pullCheckPollIntervalSeconds, ), pullCheckPieceSizeBytes: getNumberEnv(env, "PULL_CHECK_PIECE_SIZE_BYTES", networkDefaults.pullCheckPieceSizeBytes), pullPieceCleanupIntervalSeconds: getNumberEnv( env, - "PULL_PIECE_CLEANUP_INTERVAL_SECONDS", + k("PULL_PIECE_CLEANUP_INTERVAL_SECONDS"), networkDefaults.pullPieceCleanupIntervalSeconds, ), } satisfies Omit; diff --git a/docs/environment-variables.md b/docs/environment-variables.md index be8689b7..3edf4be6 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -2,25 +2,48 @@ This document provides a comprehensive guide to all environment variables used by the Dealbot. Understanding these variables is essential for proper configuration in development, testing, and production environments. +## Multi-Network Support + +Dealbot drives one or more Filecoin networks from a **single process**. The active set is controlled by the [`NETWORKS`](#networks) variable, and every network-scoped variable is namespaced with the UPPERCASE network name. + +```bash +# Run Dealbot against both networks from the same instance +NETWORKS=calibration,mainnet + +CALIBRATION_WALLET_PRIVATE_KEY=0xabc... +CALIBRATION_RPC_URL=https://api.calibration.node.glif.io/rpc/v1 +CALIBRATION_DEALS_PER_SP_PER_HOUR=2 + +MAINNET_WALLET_PRIVATE_KEY=0xdef... +MAINNET_RPC_URL=https://api.node.glif.io/rpc/v1 +MAINNET_DEALS_PER_SP_PER_HOUR=1 +``` + +**Rules** + +- **Global vs. per-network.** Unprefixed variables (database, HTTP ports, job timeouts, etc.) apply to the whole process. Prefixed variables configure a specific network. +- **Active-network validation.** Only networks listed in `NETWORKS` are validated at startup. Variables for inactive networks are ignored, so you can keep a `MAINNET_*` block commented out until you are ready. +- **Wallet vs. session key.** Each active network must provide either `_WALLET_PRIVATE_KEY` or `_SESSION_KEY_PRIVATE_KEY`. When both are present the session key takes precedence (see [`docs/runbooks/wallet-and-session-keys.md`](./runbooks/wallet-and-session-keys.md)). +- **Supported prefixes.** `CALIBRATION_*`, `MAINNET_*`. Additional networks can be added by extending `SUPPORTED_NETWORKS` in the codebase. + ## Quick Reference | Category | Variables | | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| [Application](#application-configuration) | `NODE_ENV`, `DEALBOT_PORT`, `DEALBOT_HOST`, `DEALBOT_API_PUBLIC_URL`, `DEALBOT_RUN_MODE`, `DEALBOT_METRICS_PORT`, `DEALBOT_METRICS_HOST`, `DEALBOT_ALLOWED_ORIGINS`, `ENABLE_DEV_MODE` | +| [Application](#application-configuration) | `NODE_ENV`, `DEALBOT_PORT`, `DEALBOT_HOST`, `DEALBOT_RUN_MODE`, `DEALBOT_METRICS_PORT`, `DEALBOT_METRICS_HOST`, `DEALBOT_ALLOWED_ORIGINS`, `ENABLE_DEV_MODE`, `DEALBOT_API_PUBLIC_URL`, `DEALBOT_PROBE_LOCATION` | | [Database](#database-configuration) | `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_POOL_MAX`, `DATABASE_USER`, `DATABASE_PASSWORD`, `DATABASE_NAME` | -| [Blockchain](#blockchain-configuration) | `NETWORK`, `RPC_URL`, `WALLET_ADDRESS`, `WALLET_PRIVATE_KEY`, `SESSION_KEY_PRIVATE_KEY`, `CHECK_DATASET_CREATION_FEES`, `USE_ONLY_APPROVED_PROVIDERS`, `PDP_SUBGRAPH_ENDPOINT` | -| [Dataset Versioning](#dataset-versioning) | `DEALBOT_DATASET_VERSION` | -| [Scheduling](#scheduling-configuration) | `PROVIDERS_REFRESH_INTERVAL_SECONDS`, `DATA_RETENTION_POLL_INTERVAL_SECONDS`, `DEALBOT_MAINTENANCE_WINDOWS_UTC`, `DEALBOT_MAINTENANCE_WINDOW_MINUTES` | -| [Jobs (pg-boss)](#jobs-pg-boss) | `DEALBOT_PGBOSS_SCHEDULER_ENABLED`, `DEALBOT_PGBOSS_POOL_MAX`, `DEALS_PER_SP_PER_HOUR`, `DATASET_CREATIONS_PER_SP_PER_HOUR`, `RETRIEVALS_PER_SP_PER_HOUR`, `JOB_SCHEDULER_POLL_SECONDS`, `JOB_WORKER_POLL_SECONDS`, `PG_BOSS_LOCAL_CONCURRENCY`, `JOB_CATCHUP_MAX_ENQUEUE`, `JOB_SCHEDULE_PHASE_SECONDS`, `JOB_ENQUEUE_JITTER_SECONDS`, `DEAL_JOB_TIMEOUT_SECONDS`, `RETRIEVAL_JOB_TIMEOUT_SECONDS`, `SHUTDOWN_FINAL_SCRAPE_DELAY_SECONDS`, `IPFS_BLOCK_FETCH_CONCURRENCY` | +| [Per-Network](#per-network-configuration) | `_WALLET_ADDRESS`, `_WALLET_PRIVATE_KEY`, `_SESSION_KEY_PRIVATE_KEY`, `_RPC_URL`, `_CHECK_DATASET_CREATION_FEES`, `_USE_ONLY_APPROVED_PROVIDERS`, `_PDP_SUBGRAPH_ENDPOINT`, `_DEALBOT_DATASET_VERSION`, `_MIN_NUM_DATASETS_FOR_CHECKS` | +| [Per-Network Scheduling](#per-network-scheduling) | `_DEALS_PER_SP_PER_HOUR`, `_DEAL_JOB_TIMEOUT_SECONDS`, `_RETRIEVALS_PER_SP_PER_HOUR`, `_RETRIEVAL_JOB_TIMEOUT_SECONDS`, `_DATASET_CREATIONS_PER_SP_PER_HOUR`, `_DATA_SET_CREATION_JOB_TIMEOUT_SECONDS`, `_PULL_CHECKS_PER_SP_PER_HOUR`, `_PULL_CHECK_JOB_TIMEOUT_SECONDS`, `_PULL_CHECK_POLL_INTERVAL_SECONDS`, `_PULL_PIECE_CLEANUP_INTERVAL_SECONDS`, `_METRICS_PER_HOUR`, `_PROVIDERS_REFRESH_INTERVAL_SECONDS`, `_DATA_RETENTION_POLL_INTERVAL_SECONDS`, `_MAINTENANCE_WINDOWS_UTC`, `_MAINTENANCE_WINDOW_MINUTES`, `_BLOCKED_SP_IDS`, `_BLOCKED_SP_ADDRESSES`, `_MAX_DATASET_STORAGE_SIZE_BYTES`, `_TARGET_DATASET_STORAGE_SIZE_BYTES`, `_PIECE_CLEANUP_PER_SP_PER_HOUR`, `_MAX_PIECE_CLEANUP_RUNTIME_SECONDS` | +| [Jobs (pg-boss)](#jobs-pg-boss) | `DEALBOT_PGBOSS_SCHEDULER_ENABLED`, `DEALBOT_PGBOSS_POOL_MAX`, `JOB_SCHEDULER_POLL_SECONDS`, `JOB_WORKER_POLL_SECONDS`, `PG_BOSS_LOCAL_CONCURRENCY`, `JOB_CATCHUP_MAX_ENQUEUE`, `JOB_SCHEDULE_PHASE_SECONDS`, `JOB_ENQUEUE_JITTER_SECONDS`, `SHUTDOWN_FINAL_SCRAPE_DELAY_SECONDS`, `IPFS_BLOCK_FETCH_CONCURRENCY` | +| [Pull Check](#pull-check-configuration) | `PULL_CHECK_PIECE_SIZE_BYTES`, `PULL_PIECE_MAX_CONCURRENT_STREAMS`, `PULL_PIECE_MAX_STREAMS_PER_CID` | +| [ClickHouse](#clickhouse-configuration) | `CLICKHOUSE_URL`, `CLICKHOUSE_BATCH_SIZE`, `CLICKHOUSE_FLUSH_INTERVAL_MS`, `CLICKHOUSE_MAX_BUFFER_SIZE` | | [Dataset](#dataset-configuration) | `DEALBOT_LOCAL_DATASETS_PATH`, `RANDOM_PIECE_SIZES` | -| [ClickHouse](#clickhouse-configuration) | `CLICKHOUSE_URL`, `CLICKHOUSE_BATCH_SIZE`, `CLICKHOUSE_FLUSH_INTERVAL_MS`, `DEALBOT_PROBE_LOCATION` | | [Timeouts](#timeout-configuration) | `CONNECT_TIMEOUT_MS`, `HTTP_REQUEST_TIMEOUT_MS`, `HTTP2_REQUEST_TIMEOUT_MS`, `IPNI_VERIFICATION_TIMEOUT_MS`, `IPNI_VERIFICATION_POLLING_MS` | -| [Piece Cleanup](#piece-cleanup) | `MAX_DATASET_STORAGE_SIZE_BYTES`, `TARGET_DATASET_STORAGE_SIZE_BYTES`, `JOB_PIECE_CLEANUP_PER_SP_PER_HOUR`, `MAX_PIECE_CLEANUP_RUNTIME_SECONDS` | -| [Pull Check](#pull-check) | `PULL_CHECKS_PER_SP_PER_HOUR`, `PULL_CHECK_JOB_TIMEOUT_SECONDS`, `PULL_CHECK_POLL_INTERVAL_SECONDS`, `PULL_CHECK_PIECE_SIZE_BYTES`, `PULL_PIECE_MAX_CONCURRENT_STREAMS`, `PULL_PIECE_MAX_STREAMS_PER_CID`, `PULL_PIECE_CLEANUP_INTERVAL_SECONDS` | -| [SP Blocklist](#sp-blocklist-configuration) | `BLOCKED_SP_IDS`, `BLOCKED_SP_ADDRESSES` | | [Prometheus Metrics](#prometheus-metrics-configuration) | `PROMETHEUS_WALLET_BALANCE_TTL_SECONDS`, `PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS` | | [Web Frontend](#web-frontend) | `VITE_API_BASE_URL`, `VITE_PLAUSIBLE_DATA_DOMAIN`, `DEALBOT_API_BASE_URL` | +> **Legend.** `` is the uppercase network name (`CALIBRATION`, `MAINNET`). + --- ## Application Configuration @@ -141,29 +164,6 @@ DEALBOT_HOST=0.0.0.0 --- -### `DEALBOT_API_PUBLIC_URL` - -- **Type**: `string` (URL) -- **Required**: No (but required for [Pull Check](./checks/pull-check.md) when SPs cannot reach `DEALBOT_HOST:DEALBOT_PORT` directly) -- **Default**: Empty (falls back to `http://${DEALBOT_HOST}:${DEALBOT_PORT}`) - -**Role**: Public base URL for the Dealbot HTTP API. Used to construct the hosted-piece source URL handed to a storage provider during a pull check (`{DEALBOT_API_PUBLIC_URL}/api/piece/{pieceCid}`). - -**When to update**: - -- Set to the public URL of your Dealbot deployment whenever pull checks are enabled and SPs cannot reach the bind address directly (the typical production case) -- Leave unset for local development where SPs reach Dealbot on `localhost` - -**Example**: - -```bash -DEALBOT_API_PUBLIC_URL=https://dealbot.filoz.org -``` - -**Notes**: Trailing slashes are stripped at load time. The value is also trimmed and treated as empty when blank. - ---- - ### `DEALBOT_ALLOWED_ORIGINS` - **Type**: `string` (comma-separated URLs) @@ -209,6 +209,43 @@ ENABLE_DEV_MODE=true --- +### `DEALBOT_API_PUBLIC_URL` + +- **Type**: `string` (URL) +- **Required**: No +- **Default**: Empty (falls back to `http://:`) + +**Role**: Publicly reachable base URL for the Dealbot API. Used to construct hosted-piece source URLs that storage providers can fetch during pull checks. When unset, Dealbot constructs the URL from `DEALBOT_HOST` and `DEALBOT_PORT`, which may not be routable from external SP nodes. + +**When to update**: + +- Set in production when the Dealbot API is behind a reverse proxy or load balancer +- Required for pull-check functionality to work correctly when SPs cannot reach the internal host/port directly + +**Example**: + +```bash +DEALBOT_API_PUBLIC_URL=https://dealbot.example.com +``` + +--- + +### `DEALBOT_PROBE_LOCATION` + +- **Type**: `string` +- **Required**: No +- **Default**: `unknown` + +**Role**: A label identifying the geographic or logical location of this Dealbot instance. Attached to metrics and check results to distinguish probes running in different regions or datacenters. + +**Example**: + +```bash +DEALBOT_PROBE_LOCATION=us-east-1 +``` + +--- + ## Database Configuration ### `DATABASE_HOST` @@ -316,779 +353,720 @@ DATABASE_POOL_MAX=1 --- -## Blockchain Configuration +## Network Selection -### `NETWORK` +### `NETWORKS` -- **Type**: `string` +- **Type**: `string` (comma-separated list) - **Required**: No - **Default**: `calibration` -- **Valid values**: `mainnet`, `calibration` - -**Role**: Determines which Filecoin network to interact with. This affects contract addresses, RPC endpoints, and token economics. - -**When to update**: - -- Set to `calibration` for testing with test FIL/USDFC tokens -- Set to `mainnet` for production deployments with real FIL/USDFC - -**⚠️ Warning**: Switching to `mainnet` will use real FIL/USDFC tokens. Ensure your wallet is funded and you understand the costs involved. - ---- +- **Valid values (per entry)**: `mainnet`, `calibration` -### `RPC_URL` +**Role**: Selects which Filecoin networks the instance drives. Every active network is validated independently at startup; inactive networks have their `_*` variables ignored entirely. -- **Type**: `string` (HTTP/HTTPS URL) -- **Required**: No -- **Default**: Uses the default public RPC for the configured network - -**Role**: Custom Filecoin RPC endpoint URL. When set, all on-chain calls (Synapse SDK, viem) use this endpoint instead of the default public RPC. Use an authenticated endpoint to avoid rate limiting on shared public infrastructure. - -Providers like Glif/Chain.Love support passing the API key as a query parameter: +**Examples**: ```bash -RPC_URL=https://filecoin.chain.love/rpc/v1?token=YOUR_API_KEY -``` - -**When to update**: +# Single network (default) +NETWORKS=calibration -- When DealBot is hitting 429 rate limits on the default public RPC -- When switching RPC providers -- When rotating API keys +# Multi-network instance +NETWORKS=calibration,mainnet +``` -**Security**: Treat as a secret if the URL contains an API key. +**⚠️ Warning**: Adding `mainnet` to `NETWORKS` will cause the instance to spend real FIL/USDFC for every scheduled job. Ensure the configured wallet is funded and rate limits are reviewed before enabling. --- -### `WALLET_ADDRESS` +## Per-Network Configuration -- **Type**: `string` (Ethereum-style address) -- **Required**: Yes -- **Security**: Public, but should match `WALLET_PRIVATE_KEY` +Every variable below must be prefixed with the uppercase network name (e.g. `CALIBRATION_`, `MAINNET_`). Values only apply when the corresponding network is listed in [`NETWORKS`](#networks). -**Role**: The Ethereum/FEVM address used for signing transactions and paying for storage deals. +### `_WALLET_ADDRESS` -**When to update**: +- **Type**: `string` (Ethereum-style address) +- **Required**: No (defaults to the zero address) +- **Security**: Public, but should match `_WALLET_PRIVATE_KEY` or be the signer registered against `_SESSION_KEY_PRIVATE_KEY`. -- When switching to a different wallet -- When setting up a new Dealbot instance -- When rotating keys for security +**Role**: The FEVM address used for signing transactions and paying for storage deals on the given network. **Example**: ```bash -WALLET_ADDRESS=0x1234567890abcdef1234567890abcdef12345678 +CALIBRATION_WALLET_ADDRESS=0x1234567890abcdef1234567890abcdef12345678 ``` --- -### `WALLET_PRIVATE_KEY` +### `_WALLET_PRIVATE_KEY` -- **Type**: `string` -- **Required**: Yes -- **Security**: **HIGHLY SENSITIVE** - Never commit to version control, use secrets management +- **Type**: `string` (0x-prefixed hex) +- **Required**: One of `_WALLET_PRIVATE_KEY` or `_SESSION_KEY_PRIVATE_KEY` is required for every active network. Both may be set, in which case the session key takes precedence. +- **Security**: **HIGHLY SENSITIVE** — never commit to version control; use Kubernetes Secrets or an equivalent secrets manager. -**Role**: Private key for signing blockchain transactions. Required in direct key mode. Not needed when `SESSION_KEY_PRIVATE_KEY` is set (session key mode), since the session key handles all signing. If both are set, `SESSION_KEY_PRIVATE_KEY` takes precedence and `WALLET_PRIVATE_KEY` is ignored. +**Role**: Private key for signing blockchain transactions on this network. Required in direct-key mode. Ignored when a session key is provided. -**When to update**: +--- + +### `_SESSION_KEY_PRIVATE_KEY` -- When rotating keys for security -- When setting up a new Dealbot instance -- When switching wallets +- **Type**: `string` (0x-prefixed hex) +- **Required**: See `_WALLET_PRIVATE_KEY` above. +- **Security**: **HIGHLY SENSITIVE**. -**Security best practices**: +**Role**: When set, Dealbot uses session-key authentication on this network. The session key must be registered on the `SessionKeyRegistry` contract from `_WALLET_ADDRESS` (typically a Safe multisig). Storage operations (create dataset, add pieces) are signed with this key instead of `_WALLET_PRIVATE_KEY`. -- Use Kubernetes Secrets or a secrets manager (Vault, AWS Secrets Manager) -- Never log or expose this value +Session keys are scoped (only storage operations, not deposits or withdrawals) and time-limited (expiry set during registration). See [runbooks/wallet-and-session-keys.md](runbooks/wallet-and-session-keys.md) for the full setup process. --- -### `SESSION_KEY_PRIVATE_KEY` +### `_RPC_URL` -- **Type**: `string` (0x-prefixed hex private key) +- **Type**: `string` (HTTP/HTTPS URL) - **Required**: No -- **Security**: **HIGHLY SENSITIVE** - Treat like `WALLET_PRIVATE_KEY` +- **Default**: Empty (SDK falls back to its built-in default public RPC for the network) -**Role**: When set, DealBot uses session key authentication. The session key must be registered on the SessionKeyRegistry contract from the `WALLET_ADDRESS` (typically a Safe multisig). Scoped storage operations are signed with this key instead of `WALLET_PRIVATE_KEY`. +**Role**: Custom Filecoin RPC endpoint. When set, all on-chain calls (Synapse SDK, viem) for this network use the configured endpoint. Use an authenticated endpoint to avoid rate-limiting on shared public infrastructure. -Session keys are scoped (only storage operations, not deposits or withdrawals) and time-limited (expiry set during registration). See [runbooks/wallet-and-session-keys.md](runbooks/wallet-and-session-keys.md) for the full setup process. +Providers like Glif/Chain.Love accept the API key as a query parameter: -**When to update**: +```bash +CALIBRATION_RPC_URL=https://filecoin.chain.love/rpc/v1?token=YOUR_API_KEY +``` -- When rotating session keys -- When switching to session key mode from direct key mode -- When the session key has been compromised +**Security**: Treat as a secret if the URL contains an API key. --- -### `CHECK_DATASET_CREATION_FEES` +### `_CHECK_DATASET_CREATION_FEES` - **Type**: `boolean` - **Required**: No - **Default**: `true` -**Role**: When enabled, validates that the wallet has sufficient balance to cover dataset creation fees + 100 GiB of storage costs. +**Role**: When enabled, validates that the network's wallet has sufficient balance to cover dataset-creation fees plus 100 GiB of storage costs before creating a new dataset. **When to update**: -- Set to `false` to skip addition of dataset creation fees into storage costs. +- Set to `false` to skip the balance check (e.g. for CI/test environments where insufficient balance is expected). --- -### `USE_ONLY_APPROVED_PROVIDERS` +### `_USE_ONLY_APPROVED_PROVIDERS` - **Type**: `boolean` - **Required**: No - **Default**: `true` -**Role**: Restricts deal-making to only Filecoin Warm Storage Service (FWSS) approved storage providers. This ensures deals are made with approved providers. +**Role**: Restricts deal-making to Filecoin Warm Storage Service (FWSS) approved storage providers for the network. **When to update**: -- Set to `false` to test with any storage provider available for testing (providers that support "PDP" product in ServiceProviderRegistry) +- Set to `false` to test against any provider that supports the `PDP` product in `ServiceProviderRegistry`. --- -### `PDP_SUBGRAPH_ENDPOINT` +### `_PDP_SUBGRAPH_ENDPOINT` - **Type**: `string` (URL) - **Required**: No -- **Default**: Empty string (feature disabled) - -**Role**: The Graph API endpoint for querying PDP (Proof of Data Possession) subgraph data. This endpoint is used to retrieve data retention info for provider data. - -**When to update**: +- **Default**: Empty (feature disabled for this network) -- When switching between different Graph API endpoints +**Role**: The Graph API endpoint for querying PDP (Proof of Data Possession) subgraph data for this network. Used to retrieve data-retention information for provider datasets. **Example**: ```bash -PDP_SUBGRAPH_ENDPOINT=https://api.thegraph.com/subgraphs/filecoin/pdp +CALIBRATION_PDP_SUBGRAPH_ENDPOINT=https://api.thegraph.com/subgraphs/filecoin/pdp ``` --- -## Dataset Versioning +### `_MIN_NUM_DATASETS_FOR_CHECKS` + +- **Type**: `number` (integer, ≥ 1) +- **Required**: No +- **Default**: `1` + +**Role**: Minimum number of datasets provisioned per storage provider before running checks on this network. When > 1, the `data_set_creation` job is responsible for provisioning any additional datasets. -### `DEALBOT_DATASET_VERSION` +--- + +### `_DEALBOT_DATASET_VERSION` - **Type**: `string` - **Required**: No - **Default**: Not set (no versioning) -**Role**: Creates versioning for datasets, allowing multiple dataset versions without changing wallet addresses. Useful for separating test data from production data or managing dataset migrations. - -**When to update**: - -- When you want to create a fresh set of datasets -- When separating environments (e.g., `dealbot-v1`, `dealbot-staging`) +**Role**: Tags newly created datasets with a version label, enabling multiple generations of datasets on the same wallet. Useful for separating test data from production data or managing dataset migrations per network. -**Example scenario**: Creating a new dataset version for testing: +**Example**: ```bash -DEALBOT_DATASET_VERSION=dealbot-v2 +CALIBRATION_DEALBOT_DATASET_VERSION=dealbot-v2 ``` --- -## Scheduling Configuration +## Per-Network Scheduling -These variables control when and how often the Dealbot runs its automated jobs. +Scheduling rates and intervals are configured per network, so each network can be tuned independently (e.g. an aggressive calibration cadence with a conservative mainnet cadence). Every variable below must be prefixed with the uppercase network name. -**Note**: Dealbot uses pg-boss for rate-based scheduling (see [Jobs (pg-boss)](#jobs-pg-boss)). +Dealbot uses pg-boss for rate-based scheduling — see [Jobs (pg-boss)](#jobs-pg-boss) for global worker/timeout settings. -### `PROVIDERS_REFRESH_INTERVAL_SECONDS` +### `_DEALS_PER_SP_PER_HOUR` - **Type**: `number` - **Required**: No -- **Default**: `14400` (4 hours, recommended) - -**Role**: How often the providers refresh job runs, in seconds. - -**When to update**: - -- Increase for less frequent providers refresh (reduces costs, slower testing) -- Decrease for more aggressive testing (higher costs, faster feedback) +- **Default**: `4` +- **Limits**: capped at `20` to avoid excessive on-chain activity. -**Example scenario**: Running providers refresh every 4 hours (default): +**Role**: Target deal creation rate per storage provider on this network. -```bash -PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 -``` +**Notes**: Fractional values are supported (e.g. `0.25` ⇒ one deal every 4 hours per SP). --- -### `DATA_RETENTION_POLL_INTERVAL_SECONDS` +### `_DEAL_JOB_TIMEOUT_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `3600` (1 hour) +- **Default**: `360` (6 minutes) +- **Minimum**: `120` (2 minutes) -**Role**: How often the data retention polling job runs, in seconds. This job checks and manages data retention stats of providers for stored datasets. +**Role**: Maximum runtime for data-storage jobs on this network before forced abort via `AbortController`. Covers end-to-end execution including provider lookup, upload, and IPNI verification. **When to update**: -- Increase for less frequent data retention checks -- Decrease for more frequent monitoring of data retention policies - -**Example scenario**: Running data retention checks every 2 hours: - -```bash -DATA_RETENTION_POLL_INTERVAL_SECONDS=7200 -``` +- Increase if deal uploads on this network consistently take longer than the default +- Decrease to fail-fast on stuck jobs --- -### `DEAL_START_OFFSET_SECONDS` +### `_RETRIEVALS_PER_SP_PER_HOUR` - **Type**: `number` - **Required**: No -- **Default**: `0` - -**Role**: Delay before the first deal creation job runs after startup. - -**When to update**: +- **Default**: `2` +- **Limits**: capped at `20`. -- Increase to allow other services to initialize first -- Keep at `0` for immediate deal creation on startup +**Role**: Target retrieval test rate per storage provider on this network. --- -### `RETRIEVAL_START_OFFSET_SECONDS` +### `_RETRIEVAL_JOB_TIMEOUT_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `600` (10 minutes) / `300` (5 minutes in .env.example) +- **Default**: `60` (1 minute) +- **Minimum**: `60` -**Role**: Delay before the first retrieval test runs after startup. This offset prevents retrieval tests from running concurrently with deal creation. +**Role**: Maximum runtime for retrieval jobs on this network before forced abort via `AbortController`. **When to update**: -- Adjust to stagger job execution and prevent resource contention -- Increase if deal creation takes longer than expected +- Increase if retrieval tests on this network consistently exceed the default +- Decrease to detect and fail stuck retrievals faster --- -### `DEALBOT_MAINTENANCE_WINDOWS_UTC` +### `_DATASET_CREATIONS_PER_SP_PER_HOUR` -- **Type**: `string` (comma-separated HH:MM times in UTC) +- **Type**: `number` - **Required**: No -- **Default**: `07:00,22:00` - -**Role**: Daily maintenance windows (UTC) during which deal creation and retrieval checks are skipped. - -**Notes**: - -- Times must be in 24-hour `HH:MM` format. -- Applies to both cron and pg-boss modes. - -**Example**: +- **Default**: `1` +- **Limits**: capped at `20`. -```bash -DEALBOT_MAINTENANCE_WINDOWS_UTC=06:30,21:30 -``` +**Role**: Target dataset-creation rate per storage provider on this network. --- -### `DEALBOT_MAINTENANCE_WINDOW_MINUTES` +### `_DATA_SET_CREATION_JOB_TIMEOUT_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `20` -- **Minimum**: `20` -- **Maximum**: `360` (6 hours). With two daily windows, this keeps maintenance time ≤ runtime. +- **Default**: `300` (5 minutes) +- **Minimum**: `60` -**Role**: Duration (minutes) of each maintenance window in `DEALBOT_MAINTENANCE_WINDOWS_UTC`. +**Role**: Maximum runtime for dataset-creation jobs on this network before forced abort via `AbortController`. -**Example**: +**When to update**: -```bash -DEALBOT_MAINTENANCE_WINDOW_MINUTES=30 -``` +- Increase if dataset creation on this network consistently takes longer (e.g. slow networks or large initial piece uploads) +- Decrease to fail faster on stuck provider interactions --- -## Jobs (pg-boss) +### `_METRICS_PER_HOUR` -In this mode, scheduling is -rate-based (per hour) and persisted in Postgres so restarts do not reset timing. +- **Type**: `number` +- **Required**: No +- **Default**: `0.1` +- **Limits**: capped at `3` to limit database load from materialized-view refreshes. +**Role**: Frequency of metrics aggregation runs per hour on this network. + +--- -### `DEALS_PER_SP_PER_HOUR` +### `_PROVIDERS_REFRESH_INTERVAL_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `4` +- **Default**: `14400` (4 hours) + +**Role**: How often the providers-refresh job runs for this network. + +--- -**Role**: Target deal creation rate per storage provider. +### `_DATA_RETENTION_POLL_INTERVAL_SECONDS` -**Limits**: Config schema caps this at 20 to avoid excessive on-chain activity. +- **Type**: `number` +- **Required**: No +- **Default**: `3600` (1 hour) -**Notes**: Fractional values are supported. For example, `0.25` means one deal every 4 hours per storage provider. +**Role**: How often the data-retention polling job runs for this network. The job checks and updates data-retention stats of providers for stored datasets. --- -### `RETRIEVALS_PER_SP_PER_HOUR` +### `_MAINTENANCE_WINDOWS_UTC` -- **Type**: `number` -- **Required**: No -- **Default**: `2` +- **Type**: `string` (comma-separated `HH:MM` times in UTC) +- **Required**: No +- **Default**: `07:00,22:00` -**Role**: Target retrieval test rate per storage provider. +**Role**: Daily maintenance windows (UTC) during which deal-creation and retrieval checks are skipped for this network. Different networks can have different schedules. -**Limits**: Config schema caps this at 20 to avoid overloading providers. +**Example**: -**Notes**: Fractional values are supported. For example, `0.25` means one retrieval every 4 hours per storage provider. +```bash +CALIBRATION_MAINTENANCE_WINDOWS_UTC=06:30,21:30 +MAINNET_MAINTENANCE_WINDOWS_UTC=05:00,17:00 +``` --- -### `DATASET_CREATIONS_PER_SP_PER_HOUR` +### `_MAINTENANCE_WINDOW_MINUTES` - **Type**: `number` - **Required**: No -- **Default**: `1` - -**Role**: Target dataset creation rate per storage provider. - -**Limits**: Config schema caps this at 20 to avoid excessive dataset generation. +- **Default**: `20` +- **Minimum**: `20` +- **Maximum**: `360` (6 hours) -**Notes**: Fractional values are supported. For example, `0.5` means one dataset creation every 2 hours per storage provider. +**Role**: Duration (minutes) of each maintenance window in `_MAINTENANCE_WINDOWS_UTC`. --- -### `JOB_SCHEDULER_POLL_SECONDS` +### `_BLOCKED_SP_IDS` -- **Type**: `number` +- **Type**: `string` (comma-separated provider IDs) - **Required**: No -- **Default**: `300` +- **Default**: `""` (empty — no providers blocked) -**Role**: How often the scheduler polls Postgres for due jobs. +**Role**: Global blocklist by provider numeric ID. Providers listed here are excluded from **all** scheduled +check types (data-storage, retrieval, and data-retention). -**Notes**: Minimum is 60 seconds to avoid excessive polling; default is 300 seconds. +**Example**: `_BLOCKED_SP_IDS=1234,5678` --- -### `JOB_WORKER_POLL_SECONDS` +### `_BLOCKED_SP_ADDRESSES` -- **Type**: `number` +- **Type**: `string` (comma-separated provider Ethereum addresses) - **Required**: No -- **Default**: `60` +- **Default**: `""` (empty — no providers blocked) -**Role**: How often pg-boss workers check for new jobs. +**Role**: Global blocklist by provider address. Providers listed here are excluded from **all** scheduled +check types (data-storage, retrieval, and data-retention). Matching is case-insensitive. -**Notes**: Minimum is 5 seconds. Lower values reduce job pickup latency but increase DB chatter. +**Example**: `_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...` --- -### `PG_BOSS_LOCAL_CONCURRENCY` +### `_MAX_DATASET_STORAGE_SIZE_BYTES` -- **Type**: `number` +- **Type**: `number` (integer, bytes) - **Required**: No -- **Default**: `20` +- **Default**: `25769803776` (24 GiB) - **Minimum**: `1` -**Role**: Per-instance pg-boss worker concurrency for the `sp.work` queue (`localConcurrency`). This is the total concurrency budget shared by deal and retrieval jobs. +**Role**: **High-water mark.** Maximum total stored data per SP (in bytes) before cleanup kicks in. When live storage for a provider exceeds this value, the cleanup job triggers and deletes the oldest pieces until usage drops below `_TARGET_DATASET_STORAGE_SIZE_BYTES` (the low-water mark). **When to update**: -- Increase for faster throughput (more concurrent jobs; higher load) -- Decrease to reduce load or for more conservative testing +- Increase for longer runway before cleanup kicks in (e.g. months vs weeks) +- Decrease if SP storage is constrained or costs are a concern **Example**: ```bash -PG_BOSS_LOCAL_CONCURRENCY=20 +_MAX_DATASET_STORAGE_SIZE_BYTES=12884901888 # 12 GiB per SP ``` -**Sizing note**: A rough estimate for required concurrency is -`(providers * jobs_per_hour_per_provider * avg_duration_seconds) / 3600`. -Use p95 duration for a more conservative default. - --- -### `DEALBOT_PGBOSS_SCHEDULER_ENABLED` +### `_TARGET_DATASET_STORAGE_SIZE_BYTES` -- **Type**: `boolean` +- **Type**: `number` (integer, bytes) - **Required**: No -- **Default**: `true` +- **Default**: `21474836480` (20 GiB) +- **Minimum**: `1` -**Role**: Enables/disables the pg-boss scheduler loop that enqueues due jobs. Set to `false` for worker-only pods that should only process existing jobs. +**Role**: **Low-water mark.** When cleanup triggers (live usage exceeds `_MAX_DATASET_STORAGE_SIZE_BYTES`), pieces are deleted until usage drops below this target. The gap between MAX and TARGET creates headroom so cleanup doesn't re-trigger immediately. + +**Headroom math**: At 4 deals/SP/hour × 10 MiB = ~960 MiB/day growth. With 4 GiB headroom (24 GiB MAX − 20 GiB TARGET), cleanup provides ~4 days of breathing room per run, which aligns with the daily default cadence. + +**When to update**: + +- Decrease for more aggressive cleanup (larger gap = more headroom) +- Increase toward MAX for minimal cleanup (smaller gap = less headroom) +- Must be less than `_MAX_DATASET_STORAGE_SIZE_BYTES` for cleanup to have effect **Example**: ```bash -DEALBOT_PGBOSS_SCHEDULER_ENABLED=false +_TARGET_DATASET_STORAGE_SIZE_BYTES=16106127360 # 15 GiB per SP (9 GiB headroom) ``` --- -### `DEALBOT_PGBOSS_POOL_MAX` +### `_PIECE_CLEANUP_PER_SP_PER_HOUR` - **Type**: `number` - **Required**: No -- **Default**: `1` +- **Default**: `0.0417` (~1/24, approximately once per day) +- **Minimum**: `0.001` +- **Maximum**: `20` -**Role**: Maximum number of pg-boss connections per instance. Lower this when running through a -session-mode pooler (e.g. Supabase) to avoid exceeding pooler `pool_size`. +**Role**: Target number of piece cleanup runs per storage provider per hour. Controls how frequently the cleanup job runs for each SP. The rate is converted to an interval internally (e.g. 1/hr = every 3600s, 1/24/hr ≈ every 86400s = once per day). + +**When to update**: + +- Increase to run cleanup more frequently when SPs are frequently over quota +- Decrease to reduce scheduling overhead **Example**: ```bash -DEALBOT_PGBOSS_POOL_MAX=2 +# Once per hour (more aggressive) +_PIECE_CLEANUP_PER_SP_PER_HOUR=1 + +# Once per week (very conservative) +_PIECE_CLEANUP_PER_SP_PER_HOUR=0.006 ``` --- -### `JOB_CATCHUP_MAX_ENQUEUE` +### `_MAX_PIECE_CLEANUP_RUNTIME_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `10` +- **Default**: `300` (5 minutes) +- **Minimum**: `60` -**Role**: Maximum number of jobs to enqueue per schedule row per poll. Any remaining backlog -is handled by future polls. +**Role**: Maximum runtime for a cleanup job before forced abort via `AbortController`. Prevents stuck cleanup jobs from blocking the SP work queue. + +**When to update**: + +- Increase if piece deletion calls to the Synapse SDK are known to be slow +- Decrease for faster abort detection on stuck jobs --- -### `JOB_SCHEDULE_PHASE_SECONDS` +### `_PULL_CHECKS_PER_SP_PER_HOUR` - **Type**: `number` - **Required**: No -- **Default**: `0` +- **Default**: `1` +- **Limits**: capped at `20`. -**Role**: Per-instance schedule phase offset (seconds) applied when initializing schedules. -Use this to stagger multiple dealbot deployments that are not sharing a database. +**Role**: Target number of pull-check runs per storage provider per hour on this network. Pull checks validate the SP pull-to-park pathway by serving a temporary piece URL from Dealbot and asking the SP to pull and park it. Independent of deal and retrieval rates. + +**Notes**: Fractional values are supported (e.g. `0.5` ⇒ one pull check every 2 hours per SP). --- -### `JOB_ENQUEUE_JITTER_SECONDS` +### `_PULL_CHECK_JOB_TIMEOUT_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `0` +- **Default**: `300` (5 minutes) -**Role**: Random delay (seconds) applied when enqueuing jobs to avoid synchronized bursts. +**Role**: Maximum runtime for pull-check jobs on this network before forced abort via `AbortController`. Bounds the total polling window for a terminal SP pull status. --- -### `DEAL_JOB_TIMEOUT_SECONDS` +### `_PULL_CHECK_POLL_INTERVAL_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `360` (6 minutes) -- **Minimum**: `120` (2 minutes) -- **Enforced**: Yes (config validation) +- **Default**: `2` -**Role**: Maximum runtime for data storage jobs before forced abort. When a deal job exceeds this timeout, it is actively cancelled using `AbortController`. +**Role**: Polling interval (seconds) used while waiting for a terminal SP pull status during a pull-check job on this network. **When to update**: -- Increase if deal uploads consistently take longer than the default (e.g., slower networks, IPNI delays) -- Decrease if you want to fail-fast on stuck jobs - -**Note**: This is independent of HTTP-level timeouts. The job timeout enforces end-to-end execution time of a Data Storage Check job including all operations (provider lookup, upload, IPNI verification, etc.). +- Increase to reduce polling load against SP endpoints +- Decrease for faster detection of completed or failed pulls --- -### `RETRIEVAL_JOB_TIMEOUT_SECONDS` +### `_PULL_PIECE_CLEANUP_INTERVAL_SECONDS` -- **Type**: `number` +- **Type**: `number` (seconds) - **Required**: No -- **Default**: `60` (1 minute) -- **Minimum**: `60` -- **Enforced**: Yes (config validation) +- **Default**: `604800` (7 days) +- **Minimum**: `3600` (1 hour) -**Role**: Maximum runtime for retrieval test jobs before forced abort. When a retrieval job exceeds this timeout, it is actively cancelled using `AbortController`. +**Role**: How often the `pull_piece_cleanup` job runs for this network to delete expired `pull_pieces` rows (those whose `expires_at` is in the past). **When to update**: -- Increase if retrieval tests consistently take longer than the default -- Decrease to detect and fail stuck retrievals faster - -**Note**: This is independent of HTTP-level timeouts. The job timeout enforces end-to-end execution time of a Retrieval Check job. +- Decrease if pull-piece rows are accumulating faster than expected at this network's pull-check rate +- Increase to reduce background cleanup overhead --- -### `SHUTDOWN_FINAL_SCRAPE_DELAY_SECONDS` +## Jobs (pg-boss) -- **Type**: `number` -- **Required**: No -- **Default**: `35` -- **Minimum**: `0` -- **Enforced**: Yes (config validation) +These variables are **global** (not per-network) and control the shared pg-boss worker runtime. Scheduling is rate-based (per hour, per network) and persisted in Postgres so restarts do not reset timing — see [Per-Network Scheduling](#per-network-scheduling) for the rate/interval knobs. -**Role**: Seconds the worker process holds itself alive after pg-boss drain finishes, so Prometheus captures the terminal counter increments emitted during shutdown. Without this hold, the pod exits before its next scrape and the in-memory counter deltas die with it, leaving `dataStorageStatus{value=pending}` rows without matching `success` / `failure.*` increments. +### `JOB_SCHEDULER_POLL_SECONDS` -**When to update**: +- **Type**: `number` +- **Required**: No +- **Default**: `300` -- Increase if the ServiceMonitor scrape interval is longer than 30 seconds (set this to `scrape_interval + 5` for headroom) -- Decrease to `0` to disable the hold (only safe if metrics are sourced elsewhere, e.g., a DB reconciler) +**Role**: How often the scheduler polls Postgres for due jobs. -**Note**: This value also constrains the deployment's `terminationGracePeriodSeconds`. The pod's total grace must cover the longest job timeout + pg-boss stop buffer + this delay. The default 480-second grace assumes default values across all three. +**Notes**: Minimum is 60 seconds to avoid excessive polling; default is 300 seconds. --- -### `IPFS_BLOCK_FETCH_CONCURRENCY` + +### `JOB_WORKER_POLL_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `6` -- **Minimum**: `1` -- **Enforced**: Yes (config validation) - -**Role**: Maximum number of parallel block fetches when validating IPFS retrievals via DAG traversal. - -**When to update**: +- **Default**: `60` -- Increase to speed up validation on fast networks and responsive gateways -- Decrease to reduce pressure on slower storage providers or constrained environments +**Role**: How often pg-boss workers check for new jobs. -**Note**: This affects the number of concurrent `/ipfs/` requests per retrieval. +**Notes**: Minimum is 5 seconds. Lower values reduce job pickup latency but increase DB chatter. --- -## Piece Cleanup - -These variables control the automatic cleanup of old pieces from storage providers to prevent -unbounded data growth. Cleanup runs as a periodic pg-boss job per SP. - -The cleanup flow checks **live provider data** first (via `filecoin-pin`'s `calculateActualStorage()`) to determine how much data an SP is storing. When usage exceeds the high-water mark (`MAX_DATASET_STORAGE_SIZE_BYTES`), the cleanup job deletes the oldest pieces until usage drops below the low-water mark (`TARGET_DATASET_STORAGE_SIZE_BYTES`). This high-water/low-water approach prevents thrashing near the threshold. - -If the live query fails, cleanup falls back to DB-based `SUM(piece_size)` for the quota decision. Deal creation continues regardless of cleanup state. - -### `MAX_DATASET_STORAGE_SIZE_BYTES` +### `PG_BOSS_LOCAL_CONCURRENCY` -- **Type**: `number` (integer, bytes) +- **Type**: `number` - **Required**: No -- **Default**: `25769803776` (24 GiB) +- **Default**: `20` - **Minimum**: `1` -**Role**: **High-water mark.** Maximum total stored data per SP (in bytes) before cleanup kicks in. When live storage for a provider exceeds this value, the cleanup job triggers and deletes the oldest pieces until usage drops below `TARGET_DATASET_STORAGE_SIZE_BYTES` (the low-water mark). +**Role**: Per-instance pg-boss worker concurrency for the `sp.work` queue (`localConcurrency`). This is the total concurrency budget shared by deal and retrieval jobs. **When to update**: -- Increase for longer runway before cleanup kicks in (e.g. months vs weeks) -- Decrease if SP storage is constrained or costs are a concern +- Increase for faster throughput (more concurrent jobs; higher load) +- Decrease to reduce load or for more conservative testing **Example**: ```bash -MAX_DATASET_STORAGE_SIZE_BYTES=12884901888 # 12 GiB per SP +PG_BOSS_LOCAL_CONCURRENCY=20 ``` +**Sizing note**: A rough estimate for required concurrency is +`(providers * jobs_per_hour_per_provider * avg_duration_seconds) / 3600`. +Use p95 duration for a more conservative default. + --- -### `TARGET_DATASET_STORAGE_SIZE_BYTES` +### `DEALBOT_PGBOSS_SCHEDULER_ENABLED` -- **Type**: `number` (integer, bytes) +- **Type**: `boolean` - **Required**: No -- **Default**: `21474836480` (20 GiB) -- **Minimum**: `1` - -**Role**: **Low-water mark.** When cleanup triggers (live usage exceeds `MAX_DATASET_STORAGE_SIZE_BYTES`), pieces are deleted until usage drops below this target. The gap between MAX and TARGET creates headroom so cleanup doesn't re-trigger immediately. - -**Headroom math**: At 4 deals/SP/hour × 10 MiB = ~960 MiB/day growth. With 4 GiB headroom (24 GiB MAX − 20 GiB TARGET), cleanup provides ~4 days of breathing room per run, which aligns with the daily default cadence. - -**When to update**: +- **Default**: `true` -- Decrease for more aggressive cleanup (larger gap = more headroom) -- Increase toward MAX for minimal cleanup (smaller gap = less headroom) -- Must be less than `MAX_DATASET_STORAGE_SIZE_BYTES` for cleanup to have effect +**Role**: Enables/disables the pg-boss scheduler loop that enqueues due jobs. Set to `false` for worker-only pods that should only process existing jobs. **Example**: ```bash -TARGET_DATASET_STORAGE_SIZE_BYTES=16106127360 # 15 GiB per SP (9 GiB headroom) +DEALBOT_PGBOSS_SCHEDULER_ENABLED=false ``` --- -### `JOB_PIECE_CLEANUP_PER_SP_PER_HOUR` +### `DEALBOT_PGBOSS_POOL_MAX` - **Type**: `number` - **Required**: No -- **Default**: `0.0417` (~1/24, approximately once per day) -- **Minimum**: `0.001` -- **Maximum**: `20` - -**Role**: Target number of piece cleanup runs per storage provider per hour. Controls how frequently the cleanup job runs for each SP. The rate is converted to an interval internally (e.g. 1/hr = every 3600s, 1/24/hr ≈ every 86400s = once per day). - -Only used when `DEALBOT_JOBS_MODE=pgboss`. - -**When to update**: +- **Default**: `1` -- Increase to run cleanup more frequently when SPs are frequently over quota -- Decrease to reduce scheduling overhead +**Role**: Maximum number of pg-boss connections per instance. Lower this when running through a +session-mode pooler (e.g. Supabase) to avoid exceeding pooler `pool_size`. **Example**: ```bash -# Once per hour (more aggressive) -JOB_PIECE_CLEANUP_PER_SP_PER_HOUR=1 - -# Once per week (very conservative) -JOB_PIECE_CLEANUP_PER_SP_PER_HOUR=0.006 +DEALBOT_PGBOSS_POOL_MAX=2 ``` --- -### `MAX_PIECE_CLEANUP_RUNTIME_SECONDS` +### `JOB_CATCHUP_MAX_ENQUEUE` - **Type**: `number` - **Required**: No -- **Default**: `300` (5 minutes) -- **Minimum**: `60` - -**Role**: Maximum runtime for a cleanup job before forced abort via `AbortController`. Prevents stuck cleanup jobs from blocking the SP work queue. - -Only used when `DEALBOT_JOBS_MODE=pgboss`. - -**When to update**: +- **Default**: `10` -- Increase if piece deletion calls to the Synapse SDK are known to be slow -- Decrease for faster abort detection on stuck jobs +**Role**: Maximum number of jobs to enqueue per schedule row per poll. Any remaining backlog +is handled by future polls. --- -## Pull Check - -These variables control the [Pull Check](./checks/pull-check.md), which validates the SP pull-to-park pathway. Pull checks are scheduled per SP and exercised through the `sp.work` queue alongside deal, retrieval, and piece-cleanup jobs. - -### `PULL_CHECKS_PER_SP_PER_HOUR` +### `JOB_SCHEDULE_PHASE_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `1` -- **Minimum**: `0.001` -- **Maximum**: `20` +- **Default**: `0` -**Role**: Target number of pull checks per storage provider per hour. The rate is converted to an interval internally (for example `1` = every 3600s, `0.5` = every 7200s). +**Role**: Per-instance schedule phase offset (seconds) applied when initializing schedules. +Use this to stagger multiple dealbot deployments that are not sharing a database. -**Notes**: Fractional values are supported. Pull checks are independent of `DEALS_PER_SP_PER_HOUR` and `RETRIEVALS_PER_SP_PER_HOUR`. +--- -**Example**: +### `JOB_ENQUEUE_JITTER_SECONDS` -```bash -# Twice per day -PULL_CHECKS_PER_SP_PER_HOUR=0.083 -``` +- **Type**: `number` +- **Required**: No +- **Default**: `0` + +**Role**: Random delay (seconds) applied when enqueuing jobs to avoid synchronized bursts. --- -### `PULL_CHECK_JOB_TIMEOUT_SECONDS` +### `SHUTDOWN_FINAL_SCRAPE_DELAY_SECONDS` -- **Type**: `number` (seconds) +- **Type**: `number` - **Required**: No -- **Default**: `300` (5 minutes) -- **Minimum**: `60` -- **Enforced**: Yes (config validation) +- **Default**: `35` -**Role**: Maximum runtime for a pull-check job before forced abort via `AbortController`. Bounds the polling window for terminal SP pull status and the direct `/piece/{pieceCid}` re-fetch combined. +**Role**: Seconds the process stays alive after pg-boss drains and workers finish, so that Prometheus can scrape the final counter increments emitted during shutdown. Without this delay, terminal metrics (job completion/failure counters) may be missed by the scraper. **When to update**: -- Increase if SPs are slow to reach a terminal pull status (large piece sizes or busy SPs) -- Decrease to fail-fast on stuck jobs +- Increase if your Prometheus scrape interval is longer than 35 seconds +- Decrease to speed up container shutdown when fast restarts are more important than final-scrape completeness --- -### `PULL_CHECK_POLL_INTERVAL_SECONDS` +### `IPFS_BLOCK_FETCH_CONCURRENCY` -- **Type**: `number` (seconds) +- **Type**: `number` - **Required**: No -- **Default**: `2` +- **Default**: `6` - **Minimum**: `1` +- **Enforced**: Yes (config validation) -**Role**: Polling interval used by `waitForPullPieces` while waiting for the SP to report a terminal pull status (`complete` or `failed`). +**Role**: Maximum number of parallel block fetches when validating IPFS retrievals via DAG traversal. **When to update**: -- Decrease for faster terminal-status detection at the cost of more SP-side load -- Increase to be gentler on SPs at the cost of slower pull-check throughput +- Increase to speed up validation on fast networks and responsive gateways +- Decrease to reduce pressure on slower storage providers or constrained environments + +**Note**: This affects the number of concurrent `/ipfs/` requests per retrieval. --- +## Pull Check Configuration + +These variables tune global aspects of the pull-check subsystem — piece size and server-side streaming limits. Scheduling rates and timeouts are configured per network in the [Per-Network Scheduling](#per-network-scheduling) section. + ### `PULL_CHECK_PIECE_SIZE_BYTES` -- **Type**: `number` (integer, bytes) +- **Type**: `number` (bytes) - **Required**: No - **Default**: `10485760` (10 MiB) -- **Minimum**: `1024` -**Role**: Size of the synthetic random piece dealbot generates per pull check. The same byte length is used to compute [`pullRequestThroughputBps`](./checks/events-and-metrics.md#pullRequestThroughputBps). +**Role**: Size (bytes) of the synthetic test piece Dealbot generates per pull-check run. **When to update**: -- Decrease for quicker, lower-bandwidth pull tests -- Increase to stress-test the SP's outbound fetch throughput +- Increase for more realistic large-piece pull tests +- Decrease for faster jobs in low-bandwidth environments --- ### `PULL_PIECE_MAX_CONCURRENT_STREAMS` -- **Type**: `number` (integer) +- **Type**: `number` - **Required**: No -- **Default**: `50` -- **Minimum**: `1` +- **Default**: `5` -**Role**: Maximum number of concurrent HTTP/2 streams allowed across all pieces being served at any given time. This is a process-wide cap shared by all in-flight piece requests. +**Role**: Maximum number of concurrent piece-streaming connections across all piece CIDs. Prevents a spike in pull requests from overloading the Dealbot HTTP server. -**When to update**: +--- -- Decrease to reduce load on the Dealbot HTTP server under heavy SP demand -- Increase if many SPs are simultaneously fetching pieces and stream exhaustion is observed +### `PULL_PIECE_MAX_STREAMS_PER_CID` -**Example**: +- **Type**: `number` +- **Required**: No +- **Default**: `3` -```bash -PULL_PIECE_MAX_CONCURRENT_STREAMS=50 -``` +**Role**: Maximum number of concurrent streaming connections allowed per piece CID. Prevents a single CID from monopolizing the global stream budget. --- -### `PULL_PIECE_MAX_STREAMS_PER_CID` +## ClickHouse Configuration -- **Type**: `number` (integer) -- **Required**: No -- **Default**: `3` -- **Minimum**: `1` +ClickHouse is an **optional** long-term analytics store for check results. When `CLICKHOUSE_URL` is unset, ClickHouse emission is silently disabled and Dealbot runs normally using only Postgres and Prometheus. -**Role**: Maximum number of concurrent HTTP/2 streams allowed per individual `pieceCid`. Prevents a single piece from consuming the entire `PULL_PIECE_MAX_CONCURRENT_STREAMS` budget. +### `CLICKHOUSE_URL` -**When to update**: +- **Type**: `string` (HTTP URL) +- **Required**: No +- **Default**: Empty (ClickHouse disabled) +- **Security**: Treat as a secret if the URL contains credentials. -- Decrease to spread stream capacity more evenly across pieces -- Increase if a single large piece must be fetched concurrently by multiple SPs +**Role**: ClickHouse connection URL. Must include the database in the path. When set, Dealbot buffers check results and flushes them to ClickHouse in batches. Write failures are logged and dropped — ClickHouse is not a source of truth. **Example**: ```bash -PULL_PIECE_MAX_STREAMS_PER_CID=3 +CLICKHOUSE_URL=http://default:password@clickhouse.internal:8123/dealbot ``` --- -### `PULL_PIECE_CLEANUP_INTERVAL_SECONDS` +### `CLICKHOUSE_BATCH_SIZE` -- **Type**: `number` (integer, seconds) +- **Type**: `number` - **Required**: No -- **Default**: `604800` (7 days) -- **Minimum**: `3600` (1 hour) -- **Enforced**: Yes (config validation) +- **Default**: `500` -**Role**: How often the global `pull_piece_cleanup` scheduled job runs to delete `pull_pieces` rows whose `expires_at` timestamp has passed. These rows are temporary registrations created per pull check and are automatically expired after `2 × PULL_CHECK_JOB_TIMEOUT_SECONDS`. +**Role**: Number of rows to accumulate before flushing to ClickHouse. Larger batches reduce HTTP overhead; smaller batches reduce memory usage and flush lag. -**When to update**: +--- -- Decrease if you want expired rows cleaned up more aggressively (e.g. high-volume deployments with many pull checks per hour) -- Increase if the default churn rate is acceptable and you want to reduce scheduler overhead +### `CLICKHOUSE_FLUSH_INTERVAL_MS` -**Example**: +- **Type**: `number` (milliseconds) +- **Required**: No +- **Default**: `5000` (5 seconds) -```bash -# Run every 24 hours instead of the default 7 days -PULL_PIECE_CLEANUP_INTERVAL_SECONDS=86400 -``` +**Role**: Maximum time between ClickHouse flushes. Even if `CLICKHOUSE_BATCH_SIZE` is not reached, a flush is triggered after this interval to bound write latency. + +--- + +### `CLICKHOUSE_MAX_BUFFER_SIZE` + +- **Type**: `number` +- **Required**: No +- **Default**: `5000` + +**Role**: Maximum number of rows to hold in the in-memory buffer before back-pressure is applied. When the buffer exceeds this limit, new rows are dropped and a warning is logged to prevent unbounded memory growth. --- @@ -1132,78 +1110,6 @@ RANDOM_PIECE_SIZES=1024,10240,102400 --- -## ClickHouse Configuration - -Dealbot optionally writes check results to ClickHouse for long-term storage and analysis. All ClickHouse writes are disabled when `CLICKHOUSE_URL` is unset. - -### `CLICKHOUSE_URL` - -- **Type**: `string` (HTTP/HTTPS URL) -- **Required**: No -- **Default**: Not set (ClickHouse writes disabled) - -**Role**: ClickHouse connection URL. Must include the database name in the path. When unset, all ClickHouse inserts are silently dropped and no connection is made. - -**Example**: - -```bash -CLICKHOUSE_URL=http://default:password@clickhouse-host:8123/dealbot -``` - ---- - -### `CLICKHOUSE_BATCH_SIZE` - -- **Type**: `number` -- **Required**: No -- **Default**: `500` -- **Minimum**: `1` - -**Role**: Maximum number of rows to accumulate in the in-memory buffer before triggering a flush to ClickHouse. Rows are also flushed on the interval defined by `CLICKHOUSE_FLUSH_INTERVAL_MS`. - -**When to update**: - -- Decrease for lower-throughput deployments where you want more frequent writes -- Increase to reduce write frequency under high load - ---- - -### `CLICKHOUSE_FLUSH_INTERVAL_MS` - -- **Type**: `number` (milliseconds) -- **Required**: No -- **Default**: `5000` (5 seconds) -- **Minimum**: `100` - -**Role**: How often the ClickHouse buffer is flushed, regardless of batch size. - -**When to update**: - -- Decrease for more real-time data visibility -- Increase to reduce write pressure on the ClickHouse server - ---- - -### `DEALBOT_PROBE_LOCATION` - -- **Type**: `string` -- **Required**: No -- **Default**: `unknown` - -**Role**: A label identifying where this dealbot instance is running (e.g. `aws-us-east-1`, `local`). Written to ClickHouse as `probe_location` on every row, allowing multi-region deployments to be distinguished in queries. - -**When to update**: - -- Set to a meaningful geographic or deployment identifier for each dealbot instance - -**Example**: - -```bash -DEALBOT_PROBE_LOCATION=aws-us-east-1 -``` - ---- - ## Timeout Configuration ### `CONNECT_TIMEOUT_MS` @@ -1285,43 +1191,6 @@ DEALBOT_PROBE_LOCATION=aws-us-east-1 --- -## SP Blocklist Configuration - -Both variables are **optional** and default to an empty list (no providers blocked). Values are -comma-separated lists of provider IDs or addresses. Addresses are matched case-insensitively. - -A blocked provider is excluded from **all** scheduled check types: data-storage, retrieval, and -data-retention. Blocking applies to **scheduled automation only** — manual/dev-triggered checks -(via dev-tools endpoints) are not affected. - ---- - -### `BLOCKED_SP_IDS` - -- **Type**: `string` (comma-separated provider IDs) -- **Required**: No -- **Default**: `""` (empty — no providers blocked) - -**Role**: Global blocklist by provider numeric ID. Providers listed here are excluded from **all** scheduled -check types (data-storage, retrieval, and data-retention). - -**Example**: `BLOCKED_SP_IDS=1234,5678` - ---- - -### `BLOCKED_SP_ADDRESSES` - -- **Type**: `string` (comma-separated provider Ethereum addresses) -- **Required**: No -- **Default**: `""` (empty — no providers blocked) - -**Role**: Global blocklist by provider address. Providers listed here are excluded from **all** scheduled -check types (data-storage, retrieval, and data-retention). Matching is case-insensitive. - -**Example**: `BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...` - ---- - ## Prometheus Metrics Configuration ### `PROMETHEUS_WALLET_BALANCE_TTL_SECONDS` @@ -1461,4 +1330,4 @@ docker run \ | `apps/backend/.env.example` | Full backend configuration template | | `apps/web/.env.example` | Frontend configuration template | -For local Kubernetes development, see [DEVELOPMENT.md](./DEVELOPMENT.md) for setup instructions. +For local Kubernetes development, see [DEVELOPMENT.md](./DEVELOPMENT.md) for setup instructions. \ No newline at end of file From bb97328a6637c84d79fdec9db79306c152c13b85 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Mon, 25 May 2026 18:43:50 +0530 Subject: [PATCH 19/23] fix: job timeouts --- apps/backend/src/jobs/jobs.service.spec.ts | 7 +++---- apps/backend/src/jobs/jobs.service.ts | 13 +++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/backend/src/jobs/jobs.service.spec.ts b/apps/backend/src/jobs/jobs.service.spec.ts index 0d47cb19..38e28f20 100644 --- a/apps/backend/src/jobs/jobs.service.spec.ts +++ b/apps/backend/src/jobs/jobs.service.spec.ts @@ -289,10 +289,9 @@ describe("JobsService schedule rows", () => { baseConfigValues = { ...baseConfigValues, - jobs: { - ...baseConfigValues.jobs, - dealJobTimeoutSeconds: 1, - } as IConfig["jobs"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, dealJobTimeoutSeconds: 1 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), diff --git a/apps/backend/src/jobs/jobs.service.ts b/apps/backend/src/jobs/jobs.service.ts index a4bf1fab..882c76ee 100644 --- a/apps/backend/src/jobs/jobs.service.ts +++ b/apps/backend/src/jobs/jobs.service.ts @@ -532,7 +532,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { // Create AbortController for job timeout enforcement const abortController = new AbortController(); - const timeoutSeconds = this.configService.get("jobs").dealJobTimeoutSeconds; + const timeoutSeconds = this.configService.get("networks", { infer: true })[network].dealJobTimeoutSeconds; const timeoutMs = Math.max(120000, timeoutSeconds * 1000); const effectiveTimeoutSeconds = Math.round(timeoutMs / 1000); const abortReason = new Error(`Deal job timeout (${effectiveTimeoutSeconds}s) for ${spAddress}`); @@ -636,7 +636,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { // Create AbortController for job timeout enforcement const abortController = new AbortController(); - const timeoutSeconds = this.configService.get("jobs").retrievalJobTimeoutSeconds; + const timeoutSeconds = this.configService.get("networks", { infer: true })[network].retrievalJobTimeoutSeconds; const timeoutMs = Math.max(60000, timeoutSeconds * 1000); const effectiveTimeoutSeconds = Math.round(timeoutMs / 1000); const abortReason = new Error(`Retrieval job timeout (${effectiveTimeoutSeconds}s) for ${spAddress}`); @@ -811,8 +811,8 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { } const abortController = new AbortController(); - const jobsConfig = this.configService.get("jobs"); - const timeoutSeconds = jobsConfig.maxPieceCleanupRuntimeSeconds; + const networkCfg = this.configService.get("networks", { infer: true })[network]; + const timeoutSeconds = networkCfg.maxPieceCleanupRuntimeSeconds; const timeoutMs = Math.max(60000, timeoutSeconds * 1000); const effectiveTimeoutSeconds = Math.round(timeoutMs / 1000); const abortReason = new Error(`Piece cleanup job timeout (${effectiveTimeoutSeconds}s) for ${spAddress}`); @@ -904,12 +904,13 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { return; } - const minDataSets = this.configService.get("networks", { infer: true })[network].minNumDataSetsForChecks; + const networkCfg = this.configService.get("networks", { infer: true })[network]; + const minDataSets = networkCfg.minNumDataSetsForChecks; const baseDataSetMetadata = this.dealService.getBaseDataSetMetadata(network); // Create AbortController for job timeout enforcement const abortController = new AbortController(); - const timeoutSeconds = this.configService.get("jobs").dataSetCreationJobTimeoutSeconds; + const timeoutSeconds = networkCfg.dataSetCreationJobTimeoutSeconds; const timeoutMs = Math.max(120000, timeoutSeconds * 1000); const effectiveTimeoutSeconds = Math.round(timeoutMs / 1000); const abortReason = new Error(`Data set creation job timeout (${effectiveTimeoutSeconds}s) for ${spAddress}`); From 6f45bdaf7247d8e313de2265424a7940a57fadd1 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Tue, 26 May 2026 18:29:18 +0530 Subject: [PATCH 20/23] chore: address pr comments --- apps/backend/src/config/constants.ts | 4 ++-- apps/backend/src/config/loader.spec.ts | 14 +++++++------- apps/backend/src/config/loader.ts | 8 ++++++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/apps/backend/src/config/constants.ts b/apps/backend/src/config/constants.ts index b56163e5..84cb2bb9 100644 --- a/apps/backend/src/config/constants.ts +++ b/apps/backend/src/config/constants.ts @@ -22,8 +22,8 @@ export const networkDefaults = { providersRefreshIntervalSeconds: 4 * 3600, maintenanceWindowsUtc: ["07:00", "22:00"], maintenanceWindowMinutes: 20, - maxDatasetStorageSizeBytes: 24 * 1024 * 1024 * 1024, // 10GB - targetDatasetStorageSizeBytes: 20 * 1024 * 1024 * 1024, // 8GB + maxDatasetStorageSizeBytes: 24 * 1024 * 1024 * 1024, // 24 GiB + targetDatasetStorageSizeBytes: 20 * 1024 * 1024 * 1024, // 20 GiB pullChecksPerSpPerHour: 1, pullCheckJobTimeoutSeconds: 300, pullCheckPollIntervalSeconds: 2, diff --git a/apps/backend/src/config/loader.spec.ts b/apps/backend/src/config/loader.spec.ts index bccedd91..95d4baa5 100644 --- a/apps/backend/src/config/loader.spec.ts +++ b/apps/backend/src/config/loader.spec.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { loadConfig } from "./loader.js"; /** @@ -36,13 +36,13 @@ beforeEach(() => { process.env.DATABASE_USER = process.env.DATABASE_USER ?? "test"; process.env.DATABASE_PASSWORD = process.env.DATABASE_PASSWORD ?? "test"; process.env.DATABASE_NAME = process.env.DATABASE_NAME ?? "test"; +}); - return () => { - for (const key of KEYS_TO_RESET) { - if (snapshot[key] === undefined) delete process.env[key]; - else process.env[key] = snapshot[key]; - } - }; +afterEach(() => { + for (const key of KEYS_TO_RESET) { + if (snapshot[key] === undefined) delete process.env[key]; + else process.env[key] = snapshot[key]; + } }); describe("loadConfig", () => { diff --git a/apps/backend/src/config/loader.ts b/apps/backend/src/config/loader.ts index abd40df0..61d911f9 100644 --- a/apps/backend/src/config/loader.ts +++ b/apps/backend/src/config/loader.ts @@ -83,7 +83,7 @@ const loadClickhouseConfig = (env: NodeJS.ProcessEnv): IClickhouseConfig => ({ }); const loadPullPieceConfig = (env: NodeJS.ProcessEnv): IPullPieceConfig => ({ - maxConcurrentStreams: getNumberEnv(env, "PULL_PIECE_MAX_CONCURRENT_STREAMS", 5), + maxConcurrentStreams: getNumberEnv(env, "PULL_PIECE_MAX_CONCURRENT_STREAMS", 50), maxStreamsPerCid: getNumberEnv(env, "PULL_PIECE_MAX_STREAMS_PER_CID", 3), }); @@ -220,7 +220,11 @@ function loadNetworkEnvPrefix( k("PULL_CHECK_POLL_INTERVAL_SECONDS"), networkDefaults.pullCheckPollIntervalSeconds, ), - pullCheckPieceSizeBytes: getNumberEnv(env, "PULL_CHECK_PIECE_SIZE_BYTES", networkDefaults.pullCheckPieceSizeBytes), + pullCheckPieceSizeBytes: getNumberEnv( + env, + k("PULL_CHECK_PIECE_SIZE_BYTES"), + networkDefaults.pullCheckPieceSizeBytes, + ), pullPieceCleanupIntervalSeconds: getNumberEnv( env, k("PULL_PIECE_CLEANUP_INTERVAL_SECONDS"), From 4a9c35ac80d6d105c037d37a413ba13f8fa5d73e Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Wed, 17 Jun 2026 17:19:24 +0530 Subject: [PATCH 21/23] chore: add trailing newline to .env.example --- apps/backend/.env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/backend/.env.example b/apps/backend/.env.example index 81c58cd3..aa953593 100644 --- a/apps/backend/.env.example +++ b/apps/backend/.env.example @@ -204,4 +204,4 @@ CONNECT_TIMEOUT_MS=10000 # 10s: initial connection timeout HTTP_REQUEST_TIMEOUT_MS=240000 # 4m: total transfer timeout (HTTP/1.1) HTTP2_REQUEST_TIMEOUT_MS=240000 # 4m: total transfer timeout (HTTP/2) IPNI_VERIFICATION_TIMEOUT_MS=60000 # 60s: IPNI propagation wait -IPNI_VERIFICATION_POLLING_MS=2000 # 2s: IPNI polling interval \ No newline at end of file +IPNI_VERIFICATION_POLLING_MS=2000 # 2s: IPNI polling interval From 564403e3f6e80bfdc48baca8bb269bc23b933908 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Fri, 26 Jun 2026 19:31:14 +0530 Subject: [PATCH 22/23] feat: add network field to sampled-retrieval logging --- .../src/sampled-retrieval/piece-retrieval.service.ts | 6 ++++++ .../src/sampled-retrieval/piece-validation.service.ts | 1 + .../src/sampled-retrieval/sampled-piece-selector.service.ts | 2 ++ .../src/sampled-retrieval/sampled-retrieval.service.ts | 3 +++ 4 files changed, 12 insertions(+) diff --git a/apps/backend/src/sampled-retrieval/piece-retrieval.service.ts b/apps/backend/src/sampled-retrieval/piece-retrieval.service.ts index 440cccb0..8ffd75d5 100644 --- a/apps/backend/src/sampled-retrieval/piece-retrieval.service.ts +++ b/apps/backend/src/sampled-retrieval/piece-retrieval.service.ts @@ -28,6 +28,7 @@ export class PieceRetrievalService { event: "provider_info_not_found", message: "Cannot fetch piece: provider info not found", spAddress, + network, pieceCid, }); @@ -66,6 +67,7 @@ export class PieceRetrievalService { url, pieceCid, spAddress, + network, bytesReceived: metrics.responseSize, ttfbMs: metrics.ttfb, abortReason: result.abortReason, @@ -95,6 +97,7 @@ export class PieceRetrievalService { statusCode: metrics.statusCode, pieceCid, spAddress, + network, }); return { @@ -127,6 +130,7 @@ export class PieceRetrievalService { url, pieceCid, spAddress, + network, bytesReceived: metrics.responseSize, }); @@ -150,6 +154,7 @@ export class PieceRetrievalService { message: "Piece fetched successfully", pieceCid, spAddress, + network, bytesReceived: metrics.responseSize, latencyMs: metrics.totalTime, ttfbMs: metrics.ttfb, @@ -175,6 +180,7 @@ export class PieceRetrievalService { url, pieceCid, spAddress, + network, aborted, error: toStructuredError(error), }); diff --git a/apps/backend/src/sampled-retrieval/piece-validation.service.ts b/apps/backend/src/sampled-retrieval/piece-validation.service.ts index 8ef65c1a..9e4cd70a 100644 --- a/apps/backend/src/sampled-retrieval/piece-validation.service.ts +++ b/apps/backend/src/sampled-retrieval/piece-validation.service.ts @@ -192,6 +192,7 @@ export class PieceValidationService { event: "block_fetch_unexpected_error", message: "Block fetch loop threw unexpectedly", spAddress, + network, error: toStructuredError(error), }); return { diff --git a/apps/backend/src/sampled-retrieval/sampled-piece-selector.service.ts b/apps/backend/src/sampled-retrieval/sampled-piece-selector.service.ts index a2601ec8..2c4e9679 100644 --- a/apps/backend/src/sampled-retrieval/sampled-piece-selector.service.ts +++ b/apps/backend/src/sampled-retrieval/sampled-piece-selector.service.ts @@ -101,6 +101,7 @@ export class SampledPieceSelectorService { event: "sampled_piece_selected", message: "Selected anonymous piece for retrieval test", spAddress, + network, pieceCid: piece.pieceCid, dataSetId: piece.dataSetId, withIPFSIndexing: piece.withIPFSIndexing, @@ -124,6 +125,7 @@ export class SampledPieceSelectorService { event: "sampled_no_candidates", message: "No anonymous piece found after all fallbacks", spAddress, + network, }); return null; diff --git a/apps/backend/src/sampled-retrieval/sampled-retrieval.service.ts b/apps/backend/src/sampled-retrieval/sampled-retrieval.service.ts index a6d0cc22..133eb314 100644 --- a/apps/backend/src/sampled-retrieval/sampled-retrieval.service.ts +++ b/apps/backend/src/sampled-retrieval/sampled-retrieval.service.ts @@ -63,6 +63,7 @@ export class SampledRetrievalService { pieceId: piece.pieceId, withIPFSIndexing: piece.withIPFSIndexing, spAddress, + network, }); const checkStart = Date.now(); @@ -191,6 +192,7 @@ export class SampledRetrievalService { message: "Failed to enqueue anonymous retrieval row to ClickHouse", pieceCid: piece.pieceCid, spAddress, + network, error: toStructuredError(error), }); } @@ -202,6 +204,7 @@ export class SampledRetrievalService { retrievalId, pieceCid: piece.pieceCid, spAddress, + network, success: finalPieceResult.success, aborted: finalPieceResult.aborted === true, latencyMs: finalPieceResult.latencyMs, From ff9f9c677aada5524349a6e8848f7e8a966d0826 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Fri, 26 Jun 2026 19:57:11 +0530 Subject: [PATCH 23/23] chore: address copilo review comments --- apps/backend/src/config/loader.spec.ts | 17 +++++++++++ apps/backend/src/config/loader.ts | 4 ++- .../providers/providers.controller.spec.ts | 30 +++++++++++++++++-- .../src/providers/providers.controller.ts | 23 +++++++++++--- .../src/providers/providers.service.spec.ts | 15 ++++++++++ .../src/providers/providers.service.ts | 6 ++++ 6 files changed, 87 insertions(+), 8 deletions(-) diff --git a/apps/backend/src/config/loader.spec.ts b/apps/backend/src/config/loader.spec.ts index 95d4baa5..e4f5c62c 100644 --- a/apps/backend/src/config/loader.spec.ts +++ b/apps/backend/src/config/loader.spec.ts @@ -23,6 +23,7 @@ const KEYS_TO_RESET = [ "DEALS_PER_SP_PER_HOUR", "BLOCKED_SP_IDS", "SESSION_KEY_PRIVATE_KEY", + "DEALBOT_API_PUBLIC_URL", ]; const snapshot: Record = {}; @@ -65,6 +66,22 @@ describe("loadConfig", () => { } }); + it("normalizes apiPublicUrl by trimming and stripping trailing slashes", () => { + process.env.NETWORKS = "calibration"; + process.env.CALIBRATION_WALLET_PRIVATE_KEY = "0xkey"; + process.env.DEALBOT_API_PUBLIC_URL = " https://dealbot.example.com/// "; + + expect(loadConfig().app.apiPublicUrl).toBe("https://dealbot.example.com"); + }); + + it("treats a blank apiPublicUrl as undefined", () => { + process.env.NETWORKS = "calibration"; + process.env.CALIBRATION_WALLET_PRIVATE_KEY = "0xkey"; + process.env.DEALBOT_API_PUBLIC_URL = " "; + + expect(loadConfig().app.apiPublicUrl).toBeUndefined(); + }); + it("loads both networks when both are listed in NETWORKS", () => { process.env.NETWORKS = "calibration,mainnet"; process.env.CALIBRATION_WALLET_PRIVATE_KEY = "0xcal"; diff --git a/apps/backend/src/config/loader.ts b/apps/backend/src/config/loader.ts index 05766e1c..735f80aa 100644 --- a/apps/backend/src/config/loader.ts +++ b/apps/backend/src/config/loader.ts @@ -41,7 +41,9 @@ const loadAppConfig = (env: NodeJS.ProcessEnv): IAppConfig => ({ runMode: parseRunMode(env), port: getNumberEnv(env, "DEALBOT_PORT", 3000), host: getStringEnv(env, "DEALBOT_HOST", "127.0.0.1"), - apiPublicUrl: env.DEALBOT_API_PUBLIC_URL || undefined, + // Normalize: trim and strip trailing slashes so hosted-piece source URLs + // (e.g. `${apiPublicUrl}/api/...`) never end up with a `//` join. + apiPublicUrl: env.DEALBOT_API_PUBLIC_URL?.trim().replace(/\/+$/, "") || undefined, metricsPort: getNumberEnv(env, "DEALBOT_METRICS_PORT", 9090), metricsHost: getStringEnv(env, "DEALBOT_METRICS_HOST", "0.0.0.0"), enableDevMode: env.ENABLE_DEV_MODE === "true", diff --git a/apps/backend/src/providers/providers.controller.spec.ts b/apps/backend/src/providers/providers.controller.spec.ts index 967cfae0..9800f80e 100644 --- a/apps/backend/src/providers/providers.controller.spec.ts +++ b/apps/backend/src/providers/providers.controller.spec.ts @@ -1,3 +1,5 @@ +import { BadRequestException } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; import { Test } from "@nestjs/testing"; import { describe, expect, it, vi } from "vitest"; import type { StorageProvider } from "../database/entities/storage-provider.entity.js"; @@ -25,17 +27,24 @@ function makeProvider(overrides: Partial = {}): StorageProvider } describe("ProvidersController", () => { - async function setup(providers: StorageProvider[] = [makeProvider()]) { + async function setup(providers: StorageProvider[] = [makeProvider()], activeNetworks: string[] = ["calibration"]) { const mockService = { getProvidersList: vi.fn().mockResolvedValue({ providers, total: providers.length }), }; + const mockConfig = { + get: vi.fn().mockImplementation((key: string) => (key === "activeNetworks" ? activeNetworks : undefined)), + }; + const module = await Test.createTestingModule({ controllers: [ProvidersController], - providers: [{ provide: ProvidersService, useValue: mockService }], + providers: [ + { provide: ProvidersService, useValue: mockService }, + { provide: ConfigService, useValue: mockConfig }, + ], }).compile(); - return { controller: module.get(ProvidersController) }; + return { controller: module.get(ProvidersController), service: mockService }; } it("listProviders serializes BigInt providerId to string (regression: JSON.stringify crash)", async () => { @@ -56,4 +65,19 @@ describe("ProvidersController", () => { // undefined fields are omitted by JSON.stringify — no crash, no providerId key expect(json).not.toContain('"providerId"'); }); + + it("listProviders rejects a supported-but-inactive network", async () => { + const { controller, service } = await setup(undefined, ["calibration"]); + + // `mainnet` is in SUPPORTED_NETWORKS but not active on this instance. + await expect(controller.listProviders(20, 0, "mainnet")).rejects.toThrow(BadRequestException); + expect(service.getProvidersList).not.toHaveBeenCalled(); + }); + + it("listProviders accepts an active network", async () => { + const { controller, service } = await setup(undefined, ["calibration"]); + + await expect(controller.listProviders(20, 0, "calibration")).resolves.toBeDefined(); + expect(service.getProvidersList).toHaveBeenCalledWith(expect.objectContaining({ network: "calibration" })); + }); }); diff --git a/apps/backend/src/providers/providers.controller.ts b/apps/backend/src/providers/providers.controller.ts index bad368fb..469bfa75 100644 --- a/apps/backend/src/providers/providers.controller.ts +++ b/apps/backend/src/providers/providers.controller.ts @@ -1,7 +1,9 @@ import { BadRequestException, Controller, DefaultValuePipe, Get, Logger, ParseIntPipe, Query } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; import { ApiOperation, ApiQuery, ApiResponse, ApiTags } from "@nestjs/swagger"; import { SUPPORTED_NETWORKS } from "../common/constants.js"; import type { Network } from "../common/types.js"; +import type { IConfig } from "../config/index.js"; import { ProviderListResponseDto } from "./dto/provider-list-response.dto.js"; import { ProvidersService } from "./providers.service.js"; @@ -13,7 +15,10 @@ import { ProvidersService } from "./providers.service.js"; export class ProvidersController { private readonly logger = new Logger(ProvidersController.name); - constructor(private readonly providersService: ProvidersService) {} + constructor( + private readonly providersService: ProvidersService, + private readonly configService: ConfigService, + ) {} /** * List all storage providers with their details. @@ -39,7 +44,8 @@ export class ProvidersController { name: "network", required: false, enum: SUPPORTED_NETWORKS, - description: "Filter by network. When omitted, providers from all active networks are returned.", + description: + "Filter by network. Must be an active network on this instance. When omitted, providers from all active networks are returned.", }) @ApiResponse({ status: 200, @@ -51,8 +57,17 @@ export class ProvidersController { @Query("offset", new DefaultValuePipe(0), ParseIntPipe) offset?: number, @Query("network") network?: string, ): Promise { - if (network !== undefined && !(SUPPORTED_NETWORKS as readonly string[]).includes(network)) { - throw new BadRequestException(`Invalid network "${network}". Must be one of: ${SUPPORTED_NETWORKS.join(", ")}`); + // Validate against active networks, not SUPPORTED_NETWORKS: providers are only + // synced/refreshed for active networks, so a supported-but-inactive network has + // no (or stale) rows and no blocklist config. Rejecting here gives an honest 400 + // instead of a misleading `200 []`. + if (network !== undefined) { + const activeNetworks = this.configService.get("activeNetworks", { infer: true }); + if (!activeNetworks.includes(network as Network)) { + throw new BadRequestException( + `Invalid network "${network}". Must be an active network: ${activeNetworks.join(", ")}`, + ); + } } this.logger.debug(`Listing providers: limit=${limit}, offset=${offset}, network=${network ?? "all"}`); diff --git a/apps/backend/src/providers/providers.service.spec.ts b/apps/backend/src/providers/providers.service.spec.ts index 884118f5..7f769f96 100644 --- a/apps/backend/src/providers/providers.service.spec.ts +++ b/apps/backend/src/providers/providers.service.spec.ts @@ -65,6 +65,21 @@ describe("ProvidersService", () => { ); }); + it("getProvidersList does not throw for a supported-but-inactive network with no config", async () => { + const queryBuilder = repo.createQueryBuilder(); + repo.createQueryBuilder.mockReturnValue(queryBuilder); + + // `mainnet` is a supported network but has no entry in `networks` (inactive). + // The blocklist loop must skip it rather than dereferencing undefined config. + await expect(service.getProvidersList({ network: "mainnet" })).resolves.toEqual({ providers: [], total: 0 }); + + // No blocklist clauses are applied for the unconfigured network. + const blocklistCalls = queryBuilder.andWhere.mock.calls.filter( + ([clause]: [string]) => typeof clause === "string" && clause.includes("blockedIds_"), + ); + expect(blocklistCalls).toHaveLength(0); + }); + it("getProvidersList preserves providers with null providerId when applying blocklist filters", async () => { const queryBuilder = repo.createQueryBuilder(); repo.createQueryBuilder.mockReturnValue(queryBuilder); diff --git a/apps/backend/src/providers/providers.service.ts b/apps/backend/src/providers/providers.service.ts index 252d8cc0..a5c49af0 100644 --- a/apps/backend/src/providers/providers.service.ts +++ b/apps/backend/src/providers/providers.service.ts @@ -56,6 +56,12 @@ export class ProvidersService { for (const net of networksToFilter) { const cfg = networksConfig[net]; + // `networks` is only populated for active networks. The controller already + // rejects inactive networks at the boundary, but this is a public method — + // guard against internal/future callers passing an inactive network so we + // skip (no providers, no blocklist) rather than dereferencing undefined and + // 500-ing. + if (!cfg) continue; const blockedIds: bigint[] = []; for (const id of cfg.blockedSpIds) {