From 8b10d2e0e47429a1aeb2e4ce848fd851457f8bff Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Thu, 23 Apr 2026 11:59:49 +0530 Subject: [PATCH 1/5] refactor: restructure config system to support multi-network architecture --- apps/backend/src/app.controller.ts | 21 +- apps/backend/src/app.module.ts | 5 +- apps/backend/src/common/logging.ts | 4 + apps/backend/src/common/sp-blocklist.spec.ts | 18 +- apps/backend/src/common/sp-blocklist.ts | 12 +- apps/backend/src/common/synapse-factory.ts | 8 +- apps/backend/src/config/app.config.ts | 431 --------------- apps/backend/src/config/constants.ts | 30 + apps/backend/src/config/env.helpers.ts | 24 + apps/backend/src/config/env.parsers.ts | 75 +++ apps/backend/src/config/env.schema.spec.ts | 521 ++++++++++++++++++ apps/backend/src/config/env.schema.ts | 245 ++++++++ .../src/config/legacy-env-compat.spec.ts | 142 +++++ apps/backend/src/config/legacy-env-compat.ts | 139 +++++ apps/backend/src/config/loader.spec.ts | 89 +++ apps/backend/src/config/loader.ts | 222 ++++++++ apps/backend/src/config/types.ts | 199 +++++++ .../data-retention.service.spec.ts | 214 ++++--- .../data-retention/data-retention.service.ts | 90 +-- .../src/dataSource/dataSource.service.spec.ts | 2 +- .../src/dataSource/dataSource.service.ts | 2 +- apps/backend/src/database/database.module.ts | 2 +- .../strategies/ipni.strategy.spec.ts | 9 + .../deal-addons/strategies/ipni.strategy.ts | 2 +- apps/backend/src/deal/deal.service.spec.ts | 174 ++++-- apps/backend/src/deal/deal.service.ts | 87 +-- .../src/dev-tools/dev-tools.controller.ts | 38 +- .../src/dev-tools/dev-tools.service.ts | 40 +- .../src/dev-tools/dto/trigger-deal.dto.ts | 15 +- .../dev-tools/dto/trigger-retrieval.dto.ts | 15 +- .../src/http-client/http-client.service.ts | 2 +- .../src/jobs/data-set-creation.handler.ts | 6 +- apps/backend/src/jobs/jobs.service.spec.ts | 330 +++++++---- apps/backend/src/jobs/jobs.service.ts | 404 ++++++++------ .../repositories/job-schedule.repository.ts | 63 ++- .../metrics-prometheus/check-metric-labels.ts | 6 + .../check-metrics.service.ts | 1 + .../metrics-prometheus.module.ts | 71 +-- .../wallet-balance.collector.spec.ts | 12 +- .../wallet-balance.collector.ts | 13 +- .../pdp-subgraph/pdp-subgraph.service.spec.ts | 107 ++-- .../src/pdp-subgraph/pdp-subgraph.service.ts | 30 +- .../piece-cleanup.service.spec.ts | 86 +-- .../piece-cleanup/piece-cleanup.service.ts | 87 +-- .../providers/providers.controller.spec.ts | 2 +- .../strategies/ipfs-block.strategy.ts | 4 +- .../src/retrieval/retrieval.service.ts | 11 +- .../src/wallet-sdk/wallet-sdk.service.spec.ts | 136 +++-- .../src/wallet-sdk/wallet-sdk.service.ts | 239 ++++---- apps/backend/src/worker.module.ts | 8 +- 50 files changed, 3129 insertions(+), 1364 deletions(-) delete mode 100644 apps/backend/src/config/app.config.ts create mode 100644 apps/backend/src/config/constants.ts create mode 100644 apps/backend/src/config/env.helpers.ts create mode 100644 apps/backend/src/config/env.parsers.ts create mode 100644 apps/backend/src/config/env.schema.spec.ts create mode 100644 apps/backend/src/config/env.schema.ts create mode 100644 apps/backend/src/config/legacy-env-compat.spec.ts create mode 100644 apps/backend/src/config/legacy-env-compat.ts create mode 100644 apps/backend/src/config/loader.spec.ts create mode 100644 apps/backend/src/config/loader.ts create mode 100644 apps/backend/src/config/types.ts diff --git a/apps/backend/src/app.controller.ts b/apps/backend/src/app.controller.ts index ae9f337a..6c965b21 100644 --- a/apps/backend/src/app.controller.ts +++ b/apps/backend/src/app.controller.ts @@ -1,6 +1,7 @@ import { Controller, Get } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; -import { type IBlockchainConfig, type IConfig, IJobsConfig } from "./config/app.config.js"; +import type { Network } from "./common/types.js"; +import type { IConfig, INetworksConfig } from "./config/types.js"; @Controller("api") export class AppController { @@ -21,16 +22,18 @@ export class AppController { */ @Get("config") getConfig() { - const blockchain = this.configService.get("blockchain"); - const jobs = this.configService.get("jobs"); + const activeNetworks = this.configService.get("activeNetworks"); + const networks = this.configService.get("networks"); return { - network: blockchain.network, - jobs: { - dealsPerSpPerHour: jobs.dealsPerSpPerHour, - dataSetCreationsPerSpPerHour: jobs.dataSetCreationsPerSpPerHour, - retrievalsPerSpPerHour: jobs.retrievalsPerSpPerHour, - }, + networks: activeNetworks.map((n) => ({ + network: n, + dealsPerSpPerHour: networks[n].dealsPerSpPerHour, + dataSetCreationsPerSpPerHour: networks[n].dataSetCreationsPerSpPerHour, + retrievalsPerSpPerHour: networks[n].retrievalsPerSpPerHour, + dataRetentionPollIntervalSeconds: networks[n].dataRetentionPollIntervalSeconds, + providersRefreshIntervalSeconds: networks[n].providersRefreshIntervalSeconds, + })), }; } } diff --git a/apps/backend/src/app.module.ts b/apps/backend/src/app.module.ts index 29751324..e8a96281 100644 --- a/apps/backend/src/app.module.ts +++ b/apps/backend/src/app.module.ts @@ -3,7 +3,8 @@ import { ConfigModule } from "@nestjs/config"; import { LoggerModule } from "nestjs-pino"; import { AppController } from "./app.controller.js"; import { buildLoggerModuleParams } from "./common/pino.config.js"; -import { configValidationSchema, loadConfig } from "./config/app.config.js"; +import { validateConfig } from "./config/env.schema.js"; +import { loadConfig } from "./config/loader.js"; import { DatabaseModule } from "./database/database.module.js"; import { DataSourceModule } from "./dataSource/dataSource.module.js"; import { DealModule } from "./deal/deal.module.js"; @@ -18,7 +19,7 @@ import { RetrievalModule } from "./retrieval/retrieval.module.js"; LoggerModule.forRoot(buildLoggerModuleParams()), ConfigModule.forRoot({ load: [loadConfig], - validationSchema: configValidationSchema, + validate: validateConfig, isGlobal: true, }), DatabaseModule, diff --git a/apps/backend/src/common/logging.ts b/apps/backend/src/common/logging.ts index 52e09fec..537c0fd6 100644 --- a/apps/backend/src/common/logging.ts +++ b/apps/backend/src/common/logging.ts @@ -1,3 +1,5 @@ +import { Network } from "./types.js"; + type ErrorWithCode = Error & { code?: unknown }; const MAX_ERROR_STACK_LENGTH = 4 * 1024; const ERROR_STACK_REDACTION_BUFFER_LENGTH = 512; @@ -144,6 +146,7 @@ export type ProviderJobContext = { providerAddress: string; providerId: bigint; providerName: string; + network: Network; }; /** @@ -165,6 +168,7 @@ export type DataSetLogContext = { export type JobLogContext = { jobId?: string; providerAddress: string; + network: Network; providerId?: bigint; providerName?: string; }; diff --git a/apps/backend/src/common/sp-blocklist.spec.ts b/apps/backend/src/common/sp-blocklist.spec.ts index daf6a789..ac020fcf 100644 --- a/apps/backend/src/common/sp-blocklist.spec.ts +++ b/apps/backend/src/common/sp-blocklist.spec.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from "vitest"; -import { type ISpBlocklistConfig } from "../config/app.config.js"; +import type { INetworkConfig } from "../config/types.js"; import { isSpBlocked } from "./sp-blocklist.js"; -const cfg = (overrides: Partial = {}): ISpBlocklistConfig => ({ - ids: new Set(), - addresses: new Set(), +const cfg = (overrides: Partial = {}): Pick => ({ + blockedSpIds: new Set(), + blockedSpAddresses: new Set(), ...overrides, }); @@ -15,25 +15,25 @@ describe("isSpBlocked", () => { }); it("blocks by address", () => { - expect(isSpBlocked(cfg({ addresses: new Set(["0xaaa"]) }), "0xaaa")).toBe(true); + expect(isSpBlocked(cfg({ blockedSpAddresses: new Set(["0xaaa"]) }), "0xaaa")).toBe(true); }); it("address matching is case-insensitive", () => { - const c = cfg({ addresses: new Set(["0xaaa"]) }); + const c = cfg({ blockedSpAddresses: new Set(["0xaaa"]) }); expect(isSpBlocked(c, "0xAAA")).toBe(true); expect(isSpBlocked(c, "0XAAA")).toBe(true); }); it("blocks by numeric ID", () => { - expect(isSpBlocked(cfg({ ids: new Set(["42"]) }), "0xaaa", 42n)).toBe(true); + expect(isSpBlocked(cfg({ blockedSpIds: new Set(["42"]) }), "0xaaa", 42n)).toBe(true); }); it("skips ID check when id is undefined", () => { - expect(isSpBlocked(cfg({ ids: new Set(["42"]) }), "0xaaa", undefined)).toBe(false); + expect(isSpBlocked(cfg({ blockedSpIds: new Set(["42"]) }), "0xaaa", undefined)).toBe(false); }); it("does not block unrelated provider", () => { - const c = cfg({ ids: new Set(["5"]), addresses: new Set(["0xaaa"]) }); + const c = cfg({ blockedSpIds: new Set(["5"]), blockedSpAddresses: new Set(["0xaaa"]) }); expect(isSpBlocked(c, "0xbbb", 6n)).toBe(false); }); }); diff --git a/apps/backend/src/common/sp-blocklist.ts b/apps/backend/src/common/sp-blocklist.ts index 8152d99f..084d6612 100644 --- a/apps/backend/src/common/sp-blocklist.ts +++ b/apps/backend/src/common/sp-blocklist.ts @@ -1,11 +1,15 @@ -import type { ISpBlocklistConfig } from "../config/app.config.js"; +import { INetworkConfig } from "src/config/types.js"; /** * Returns true if the provider is in the SP blocklist. * Checks by address (case-insensitive) first, then by numeric ID if provided. */ -export function isSpBlocked(cfg: ISpBlocklistConfig, address: string, id?: bigint | null): boolean { - if (cfg.addresses.has(address.toLowerCase())) return true; - if (id != null && cfg.ids.has(String(id))) return true; +export function isSpBlocked( + cfg: Pick, + address: string, + id?: bigint | null, +): boolean { + if (cfg.blockedSpAddresses.has(address.toLowerCase())) return true; + if (id != null && cfg.blockedSpIds.has(String(id))) return true; return false; } diff --git a/apps/backend/src/common/synapse-factory.ts b/apps/backend/src/common/synapse-factory.ts index d939c44d..a7051717 100644 --- a/apps/backend/src/common/synapse-factory.ts +++ b/apps/backend/src/common/synapse-factory.ts @@ -28,7 +28,7 @@ import * as SessionKey from "@filoz/synapse-core/session-key"; import { calibration, mainnet, Synapse } from "@filoz/synapse-sdk"; import { createClient, custom, http } from "viem"; import { privateKeyToAccount } from "viem/accounts"; -import type { IBlockchainConfig } from "../config/app.config.js"; +import type { INetworkConfig } from "../config/types.js"; export interface SynapseInstanceResult { synapse: Synapse; @@ -36,13 +36,13 @@ export interface SynapseInstanceResult { } /** Create a Synapse instance from blockchain config. See file-level docs. */ -export async function createSynapseFromConfig(config: IBlockchainConfig): Promise { +export async function createSynapseFromConfig(config: INetworkConfig): Promise { const chain = config.network === "mainnet" ? mainnet : calibration; const rpcUrl = config.rpcUrl; const transport = rpcUrl ? http(rpcUrl) : http(); - const sessionKeyPK = config.sessionKeyPrivateKey; - if (sessionKeyPK) { + if ("sessionKeyPrivateKey" in config) { + const sessionKeyPK = config.sessionKeyPrivateKey; const walletAddress = config.walletAddress as `0x${string}`; const sessionKey = SessionKey.fromSecp256k1({ privateKey: sessionKeyPK, diff --git a/apps/backend/src/config/app.config.ts b/apps/backend/src/config/app.config.ts deleted file mode 100644 index a8010bab..00000000 --- a/apps/backend/src/config/app.config.ts +++ /dev/null @@ -1,431 +0,0 @@ -import Joi from "joi"; -import { DEFAULT_LOCAL_DATASETS_PATH } from "../common/constants.js"; -import { parseMaintenanceWindowTimes } from "../common/maintenance-window.js"; -import type { Network } from "../common/types.js"; - -function parseIdList(value: string | undefined): Set { - if (!value || value.trim().length === 0) return new Set(); - return new Set( - value - .split(",") - .map((s) => s.trim()) - .filter((s) => s.length > 0), - ); -} - -function parseAddressList(value: string | undefined): Set { - if (!value || value.trim().length === 0) return new Set(); - return new Set( - value - .split(",") - .map((s) => s.trim().toLowerCase()) - .filter((s) => s.length > 0), - ); -} - -export const configValidationSchema = Joi.object({ - // Application - NODE_ENV: Joi.string().valid("development", "production", "test").default("development"), - DEALBOT_RUN_MODE: Joi.string().lowercase().valid("api", "worker", "both").default("both"), - DEALBOT_PORT: Joi.number().default(3000), - DEALBOT_HOST: Joi.string().default("127.0.0.1"), - DEALBOT_METRICS_PORT: Joi.number().default(9090), - DEALBOT_METRICS_HOST: Joi.string().default("0.0.0.0"), - ENABLE_DEV_MODE: Joi.boolean().default(false), - PROMETHEUS_WALLET_BALANCE_TTL_SECONDS: Joi.number().min(60).default(3600), - PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS: Joi.number().min(1).default(60), - - // Database - DATABASE_HOST: Joi.string().required(), - DATABASE_PORT: Joi.number().default(5432), - DATABASE_POOL_MAX: Joi.number().integer().min(1).default(1), - DATABASE_USER: Joi.string().required(), - DATABASE_PASSWORD: Joi.string().required(), - DATABASE_NAME: Joi.string().required(), - - // Blockchain - NETWORK: Joi.string().valid("mainnet", "calibration").default("calibration"), - WALLET_ADDRESS: Joi.string().required(), - WALLET_PRIVATE_KEY: Joi.string().optional().empty(""), - RPC_URL: Joi.string() - .uri({ scheme: ["http", "https"] }) - .optional() - .allow(""), - SESSION_KEY_PRIVATE_KEY: Joi.string().optional().empty(""), - CHECK_DATASET_CREATION_FEES: Joi.boolean().default(true), - USE_ONLY_APPROVED_PROVIDERS: Joi.boolean().default(true), - DEALBOT_DATASET_VERSION: Joi.string().optional(), - MIN_NUM_DATASETS_FOR_CHECKS: Joi.number().integer().min(1).default(1), - PDP_SUBGRAPH_ENDPOINT: Joi.string().uri().optional().allow(""), - - // Scheduling - PROVIDERS_REFRESH_INTERVAL_SECONDS: Joi.number().default(4 * 3600), - DATA_RETENTION_POLL_INTERVAL_SECONDS: Joi.number().default(3600), - DEALBOT_MAINTENANCE_WINDOWS_UTC: Joi.string() - .default("07:00,22:00") - .custom((value, helpers) => { - try { - parseMaintenanceWindowTimes(value.split(",")); - } catch (error) { - return helpers.error("any.invalid", { - message: error instanceof Error ? error.message : "Invalid maintenance window format", - }); - } - return value; - }), - DEALBOT_MAINTENANCE_WINDOW_MINUTES: Joi.number().min(20).max(360).default(20), - - // Jobs - // Per-hour limits are guardrails to avoid excessive background load. - DEALS_PER_SP_PER_HOUR: Joi.number().min(0.001).max(20).default(4), - DATASET_CREATIONS_PER_SP_PER_HOUR: Joi.number().min(0.001).max(20).default(1), - RETRIEVALS_PER_SP_PER_HOUR: Joi.number().min(0.001).max(20).default(2), - // Polling interval for pg-boss scheduler (lower = more responsive, higher = less DB chatter). - JOB_SCHEDULER_POLL_SECONDS: Joi.number().min(60).default(300), - JOB_WORKER_POLL_SECONDS: Joi.number().min(5).default(60), - PG_BOSS_LOCAL_CONCURRENCY: Joi.number().integer().min(1).default(20), - DEALBOT_PGBOSS_SCHEDULER_ENABLED: Joi.boolean().default(true), - DEALBOT_PGBOSS_POOL_MAX: Joi.number().integer().min(1).default(1), - JOB_CATCHUP_MAX_ENQUEUE: Joi.number().min(1).default(10), - JOB_SCHEDULE_PHASE_SECONDS: Joi.number().min(0).default(0), - JOB_ENQUEUE_JITTER_SECONDS: Joi.number().min(0).default(0), - DEAL_JOB_TIMEOUT_SECONDS: Joi.number().min(120).default(360), // 6 minutes max runtime for data storage jobs (TODO: reduce default to 3 minutes) - RETRIEVAL_JOB_TIMEOUT_SECONDS: Joi.number().min(60).default(60), // 1 minute max runtime for retrieval jobs (TODO: reduce default to 30 seconds) - DATA_SET_CREATION_JOB_TIMEOUT_SECONDS: Joi.number().min(60).default(300), // 5 minutes max runtime for dataset creation jobs - IPFS_BLOCK_FETCH_CONCURRENCY: Joi.number().integer().min(1).max(32).default(6), - - // Piece Cleanup - MAX_DATASET_STORAGE_SIZE_BYTES: Joi.number() - .integer() - .min(1) - .default(24 * 1024 * 1024 * 1024), // 24 GiB per SP - TARGET_DATASET_STORAGE_SIZE_BYTES: Joi.number() - .integer() - .min(1) - .default(20 * 1024 * 1024 * 1024) // 20 GiB per SP - .custom((value, helpers) => { - const max = helpers.state.ancestors?.[0]?.MAX_DATASET_STORAGE_SIZE_BYTES; - if (max != null && value >= max) { - return helpers.error("any.invalid", { - message: `TARGET_DATASET_STORAGE_SIZE_BYTES (${value}) must be less than MAX_DATASET_STORAGE_SIZE_BYTES (${max})`, - }); - } - return value; - }, "target < max validation"), - JOB_PIECE_CLEANUP_PER_SP_PER_HOUR: Joi.number() - .min(0.001) - .max(20) - .default(1 / 24), // ~once per day - MAX_PIECE_CLEANUP_RUNTIME_SECONDS: Joi.number().min(60).default(300), // 5 minutes max runtime for cleanup jobs - - // Dataset - DEALBOT_LOCAL_DATASETS_PATH: Joi.string().default(DEFAULT_LOCAL_DATASETS_PATH), - RANDOM_PIECE_SIZES: Joi.string().default("10485760"), // 10 MiB - - // Timeouts (in milliseconds) - CONNECT_TIMEOUT_MS: Joi.number().min(1000).default(10000), // 10 seconds to establish connection/receive headers - HTTP_REQUEST_TIMEOUT_MS: Joi.number().min(1000).default(240000), // 4 minutes total for HTTP requests (10MiB @ 170KB/s + overhead) - HTTP2_REQUEST_TIMEOUT_MS: Joi.number().min(1000).default(240000), // 4 minutes total for HTTP/2 requests (10MiB @ 170KB/s + overhead) - IPNI_VERIFICATION_TIMEOUT_MS: Joi.number().min(1000).default(60000), // 60 seconds max time to wait for IPNI verification - IPNI_VERIFICATION_POLLING_MS: Joi.number().min(250).default(2000), // 2 seconds between IPNI verification polls - - // SP Blocklists (comma-separated provider IDs or addresses) - BLOCKED_SP_IDS: Joi.string().optional().allow(""), - BLOCKED_SP_ADDRESSES: Joi.string().optional().allow(""), -}).or("WALLET_PRIVATE_KEY", "SESSION_KEY_PRIVATE_KEY"); - -export interface IAppConfig { - env: string; - runMode: "api" | "worker" | "both"; - port: number; - host: string; - metricsPort: number; - metricsHost: string; - enableDevMode: boolean; - prometheusWalletBalanceTtlSeconds: number; - prometheusWalletBalanceErrorCooldownSeconds: number; -} - -export interface IDatabaseConfig { - host: string; - port: number; - poolMax: number; - username: string; - password: string; - database: string; -} - -export interface IBlockchainConfig { - network: Network; - rpcUrl?: string; - sessionKeyPrivateKey?: `0x${string}`; - walletAddress: string; - walletPrivateKey: `0x${string}`; - checkDatasetCreationFees: boolean; - useOnlyApprovedProviders: boolean; - dealbotDataSetVersion?: string; - minNumDataSetsForChecks: number; - pdpSubgraphEndpoint?: string; -} - -export interface ISchedulingConfig { - providersRefreshIntervalSeconds: number; - dataRetentionPollIntervalSeconds: number; - maintenanceWindowsUtc: string[]; - maintenanceWindowMinutes: number; -} - -export interface IJobsConfig { - /** - * Target number of deal creations per storage provider per hour. - * - * Increasing this increases on-chain activity and dataset uploads. - */ - dealsPerSpPerHour: number; - /** - * Target number of retrieval tests per storage provider per hour. - * - * Increasing this increases retrieval load against providers and DB writes. - */ - retrievalsPerSpPerHour: number; - /** - * Target number of dataset creation runs per storage provider per hour. - */ - dataSetCreationsPerSpPerHour: number; - /** - * How often the scheduler polls Postgres for due jobs (seconds). - * - * Lower values reduce scheduling latency but increase DB chatter. - */ - schedulerPollSeconds: number; - /** - * How often workers check for new jobs (seconds). - * - * Lower values reduce job pickup latency but increase DB chatter. - */ - workerPollSeconds: number; - /** - * Per-instance pg-boss worker concurrency for the `sp.work` queue. - */ - pgbossLocalConcurrency: number; - /** - * Enables the pg-boss scheduler loop (enqueueing due jobs). - * - * Set to false to run "worker-only" pods that only process existing jobs. - */ - pgbossSchedulerEnabled: boolean; - /** - * Maximum number of pg-boss connections per instance. - * - * Helpful when using a session-mode pooler with a low pool_size (e.g. Supabase). - */ - pgbossPoolMax: number; - /** - * Maximum number of jobs to enqueue per schedule row per poll. - * - * Prevents large backlogs from flooding workers after downtime. - */ - catchupMaxEnqueue: number; - /** - * Per-instance phase offset (seconds) applied when initializing schedules. - * - * Use this to stagger multiple dealbot deployments that are not sharing a DB. - */ - schedulePhaseSeconds: number; - /** - * Random delay (seconds) added when enqueuing jobs. - * - * Helps avoid synchronized bursts across instances. Only used with pg-boss. - */ - enqueueJitterSeconds: number; - /** - * Maximum runtime (seconds) for deal jobs before forced abort. - * - * Uses AbortController to actively cancel job execution. - */ - dealJobTimeoutSeconds: number; - /** - * Maximum runtime (seconds) for data-set creation jobs before forced abort. - * - * Uses AbortController to actively cancel job execution. - */ - dataSetCreationJobTimeoutSeconds: number; - /** - * Maximum runtime (seconds) for retrieval jobs before forced abort. - * - * Uses AbortController to actively cancel job execution. - */ - retrievalJobTimeoutSeconds: number; - /** - * Target number of piece cleanup runs per storage provider per hour. - * - * Increasing this makes cleanup more aggressive at the cost of more SP API calls. - * Only used when `DEALBOT_JOBS_MODE=pgboss`. - */ - pieceCleanupPerSpPerHour: number; - /** - * Maximum runtime (seconds) for piece cleanup jobs before forced abort. - * - * Uses AbortController to actively cancel job execution. - * Only used when `DEALBOT_JOBS_MODE=pgboss`. - */ - maxPieceCleanupRuntimeSeconds: number; -} - -export interface IDatasetConfig { - localDatasetsPath: string; - randomDatasetSizes: number[]; -} - -export interface ITimeoutConfig { - connectTimeoutMs: number; - httpRequestTimeoutMs: number; - http2RequestTimeoutMs: number; - ipniVerificationTimeoutMs: number; - ipniVerificationPollingMs: number; -} - -export interface IRetrievalConfig { - ipfsBlockFetchConcurrency: number; -} - -export interface IPieceCleanupConfig { - maxDatasetStorageSizeBytes: number; - targetDatasetStorageSizeBytes: number; -} - -export interface ISpBlocklistConfig { - /** Provider numeric IDs to block from all scheduled checks. */ - ids: Set; - /** Provider addresses to block from all scheduled checks (stored lowercase). */ - addresses: Set; -} - -export interface IConfig { - app: IAppConfig; - database: IDatabaseConfig; - blockchain: IBlockchainConfig; - scheduling: ISchedulingConfig; - jobs: IJobsConfig; - dataset: IDatasetConfig; - timeouts: ITimeoutConfig; - retrieval: IRetrievalConfig; - pieceCleanup: IPieceCleanupConfig; - spBlocklists: ISpBlocklistConfig; -} - -export function loadConfig(): IConfig { - return { - app: { - env: process.env.NODE_ENV || "development", - runMode: (() => { - const mode = (process.env.DEALBOT_RUN_MODE || "both").toLowerCase(); - if (mode === "worker") return "worker"; - if (mode === "api") return "api"; - return "both"; - })(), - port: Number.parseInt(process.env.DEALBOT_PORT || "3000", 10), - host: process.env.DEALBOT_HOST || "127.0.0.1", - metricsPort: Number.parseInt(process.env.DEALBOT_METRICS_PORT || "9090", 10), - metricsHost: process.env.DEALBOT_METRICS_HOST || "0.0.0.0", - enableDevMode: process.env.ENABLE_DEV_MODE === "true", - prometheusWalletBalanceTtlSeconds: Number.parseInt( - process.env.PROMETHEUS_WALLET_BALANCE_TTL_SECONDS || "3600", - 10, - ), - prometheusWalletBalanceErrorCooldownSeconds: Number.parseInt( - process.env.PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS || "60", - 10, - ), - }, - database: { - host: process.env.DATABASE_HOST || "localhost", - port: Number.parseInt(process.env.DATABASE_PORT || "5432", 10), - poolMax: Number.parseInt(process.env.DATABASE_POOL_MAX || "1", 10), - username: process.env.DATABASE_USER || "dealbot", - password: process.env.DATABASE_PASSWORD || "dealbot_password", - database: process.env.DATABASE_NAME || "filecoin_dealbot", - }, - blockchain: { - network: (process.env.NETWORK || "calibration") as Network, - rpcUrl: process.env.RPC_URL || undefined, - sessionKeyPrivateKey: (process.env.SESSION_KEY_PRIVATE_KEY || undefined) as `0x${string}` | undefined, - walletAddress: process.env.WALLET_ADDRESS || "0x0000000000000000000000000000000000000000", - walletPrivateKey: (process.env.WALLET_PRIVATE_KEY || undefined) as `0x${string}`, - checkDatasetCreationFees: process.env.CHECK_DATASET_CREATION_FEES !== "false", - useOnlyApprovedProviders: process.env.USE_ONLY_APPROVED_PROVIDERS !== "false", - dealbotDataSetVersion: process.env.DEALBOT_DATASET_VERSION, - minNumDataSetsForChecks: Number.parseInt(process.env.MIN_NUM_DATASETS_FOR_CHECKS || "1", 10), - pdpSubgraphEndpoint: process.env.PDP_SUBGRAPH_ENDPOINT || "", - }, - scheduling: { - providersRefreshIntervalSeconds: Number.parseInt(process.env.PROVIDERS_REFRESH_INTERVAL_SECONDS || "14400", 10), - dataRetentionPollIntervalSeconds: Number.parseInt(process.env.DATA_RETENTION_POLL_INTERVAL_SECONDS || "3600", 10), - maintenanceWindowsUtc: (process.env.DEALBOT_MAINTENANCE_WINDOWS_UTC || "07:00,22:00") - .split(",") - .map((value) => value.trim()) - .filter((value) => value.length > 0), - maintenanceWindowMinutes: Number.parseInt(process.env.DEALBOT_MAINTENANCE_WINDOW_MINUTES || "20", 10), - }, - jobs: { - dealsPerSpPerHour: Number.parseFloat(process.env.DEALS_PER_SP_PER_HOUR || "4"), - retrievalsPerSpPerHour: Number.parseFloat(process.env.RETRIEVALS_PER_SP_PER_HOUR || "2"), - dataSetCreationsPerSpPerHour: Number.parseFloat(process.env.DATASET_CREATIONS_PER_SP_PER_HOUR || "1"), - schedulerPollSeconds: Number.parseInt(process.env.JOB_SCHEDULER_POLL_SECONDS || "300", 10), - workerPollSeconds: Number.parseInt(process.env.JOB_WORKER_POLL_SECONDS || "60", 10), - pgbossLocalConcurrency: Number.parseInt(process.env.PG_BOSS_LOCAL_CONCURRENCY || "20", 10), - pgbossSchedulerEnabled: process.env.DEALBOT_PGBOSS_SCHEDULER_ENABLED !== "false", - pgbossPoolMax: Number.parseInt(process.env.DEALBOT_PGBOSS_POOL_MAX || "1", 10), - catchupMaxEnqueue: Number.parseInt(process.env.JOB_CATCHUP_MAX_ENQUEUE || "10", 10), - schedulePhaseSeconds: Number.parseInt(process.env.JOB_SCHEDULE_PHASE_SECONDS || "0", 10), - enqueueJitterSeconds: Number.parseInt(process.env.JOB_ENQUEUE_JITTER_SECONDS || "0", 10), - dealJobTimeoutSeconds: Number.parseInt(process.env.DEAL_JOB_TIMEOUT_SECONDS || "360", 10), - retrievalJobTimeoutSeconds: Number.parseInt(process.env.RETRIEVAL_JOB_TIMEOUT_SECONDS || "60", 10), - dataSetCreationJobTimeoutSeconds: Number.parseInt(process.env.DATA_SET_CREATION_JOB_TIMEOUT_SECONDS || "300", 10), - pieceCleanupPerSpPerHour: Number.parseFloat(process.env.JOB_PIECE_CLEANUP_PER_SP_PER_HOUR || String(1 / 24)), - maxPieceCleanupRuntimeSeconds: Number.parseInt(process.env.MAX_PIECE_CLEANUP_RUNTIME_SECONDS || "300", 10), - }, - dataset: { - localDatasetsPath: process.env.DEALBOT_LOCAL_DATASETS_PATH || DEFAULT_LOCAL_DATASETS_PATH, - randomDatasetSizes: (() => { - const envValue = process.env.RANDOM_PIECE_SIZES; - if (envValue && envValue.trim().length > 0) { - const parsed = envValue - .split(",") - .map((s) => Number.parseInt(s.trim(), 10)) - .filter((n) => Number.isFinite(n) && !Number.isNaN(n)); - if (parsed.length > 0) { - return parsed; - } - } - return [ - 10 << 20, // 10 MiB - ]; - })(), - }, - timeouts: { - connectTimeoutMs: Number.parseInt(process.env.CONNECT_TIMEOUT_MS || "10000", 10), - httpRequestTimeoutMs: Number.parseInt(process.env.HTTP_REQUEST_TIMEOUT_MS || "240000", 10), - http2RequestTimeoutMs: Number.parseInt(process.env.HTTP2_REQUEST_TIMEOUT_MS || "240000", 10), - ipniVerificationTimeoutMs: Number.parseInt(process.env.IPNI_VERIFICATION_TIMEOUT_MS || "60000", 10), - ipniVerificationPollingMs: Number.parseInt(process.env.IPNI_VERIFICATION_POLLING_MS || "2000", 10), - }, - retrieval: { - ipfsBlockFetchConcurrency: Number.parseInt(process.env.IPFS_BLOCK_FETCH_CONCURRENCY || "6", 10), - }, - pieceCleanup: { - maxDatasetStorageSizeBytes: Number.parseInt( - process.env.MAX_DATASET_STORAGE_SIZE_BYTES || String(24 * 1024 * 1024 * 1024), - 10, - ), - targetDatasetStorageSizeBytes: Number.parseInt( - process.env.TARGET_DATASET_STORAGE_SIZE_BYTES || String(20 * 1024 * 1024 * 1024), - 10, - ), - }, - spBlocklists: { - ids: parseIdList(process.env.BLOCKED_SP_IDS), - addresses: parseAddressList(process.env.BLOCKED_SP_ADDRESSES), - }, - }; -} diff --git a/apps/backend/src/config/constants.ts b/apps/backend/src/config/constants.ts new file mode 100644 index 00000000..d46c261e --- /dev/null +++ b/apps/backend/src/config/constants.ts @@ -0,0 +1,30 @@ +import { SUPPORTED_NETWORKS } from "../common/constants.js"; +import type { Network } from "../common/types.js"; +import { NetworkDefaults } from "./types.js"; + +/** + * Default values applied to every network config when the corresponding env + * var is absent. Override per-network via `_*` env vars. + */ +export const networkDefaults = { + checkDatasetCreationFees: true, + useOnlyApprovedProviders: true, + minNumDataSetsForChecks: 1, + dealsPerSpPerHour: 4, + retrievalsPerSpPerHour: 2, + dataSetCreationsPerSpPerHour: 1, + pieceCleanupPerSpPerHour: 1 / 24, + maxPieceCleanupRuntimeSeconds: 300, + dataRetentionPollIntervalSeconds: 3600, + providersRefreshIntervalSeconds: 4 * 3600, + maintenanceWindowsUtc: ["07:00", "22:00"], + maintenanceWindowMinutes: 20, + maxDatasetStorageSizeBytes: 24 * 1024 * 1024 * 1024, // 10GB + targetDatasetStorageSizeBytes: 20 * 1024 * 1024 * 1024, // 8GB +} satisfies NetworkDefaults; + +/** + * Uppercase env-var prefixes for every supported network, e.g. + * `["CALIBRATION", "MAINNET"]` + */ +export const NETWORK_ENV_PREFIXES = SUPPORTED_NETWORKS.map((n) => n.toUpperCase()) as Uppercase[]; diff --git a/apps/backend/src/config/env.helpers.ts b/apps/backend/src/config/env.helpers.ts new file mode 100644 index 00000000..6718de13 --- /dev/null +++ b/apps/backend/src/config/env.helpers.ts @@ -0,0 +1,24 @@ +/** + * Raw environment variable accessors. + * + * These are purely mechanical: they read a key from `process.env` (or any + * `NodeJS.ProcessEnv` map) and coerce it to the requested primitive type, + * falling back to a default when the key is absent or empty. + */ + +export const getStringEnv = (env: NodeJS.ProcessEnv, key: string, fallback: string): string => env[key] || fallback; + +export const getNumberEnv = (env: NodeJS.ProcessEnv, key: string, fallback: number): number => { + const value = env[key]; + return value ? Number.parseInt(value, 10) : fallback; +}; + +export const getFloatEnv = (env: NodeJS.ProcessEnv, key: string, fallback: number): number => { + const value = env[key]; + return value ? Number.parseFloat(value) : fallback; +}; + +export const getBooleanEnv = (env: NodeJS.ProcessEnv, key: string, fallback: boolean): boolean => { + const value = env[key]; + return value !== undefined ? value !== "false" : fallback; +}; diff --git a/apps/backend/src/config/env.parsers.ts b/apps/backend/src/config/env.parsers.ts new file mode 100644 index 00000000..6205236e --- /dev/null +++ b/apps/backend/src/config/env.parsers.ts @@ -0,0 +1,75 @@ +/** + * Domain-level environment parsers. + * + * Each function here takes a raw `NodeJS.ProcessEnv` map and produces a + * well-typed domain value. + */ + +import { SUPPORTED_NETWORKS } from "../common/constants.js"; +import type { Network } from "../common/types.js"; +import { getStringEnv } from "./env.helpers.js"; +import type { IAppConfig } from "./types.js"; + +/** + * Returns the list of active networks derived from the `NETWORKS` env var. + * Falls back to the first supported network when the variable is absent. + */ +export function parseActiveNetworks(env: NodeJS.ProcessEnv): Network[] { + const raw = getStringEnv(env, "NETWORKS", SUPPORTED_NETWORKS[0]); + return raw + .split(",") + .map((s) => s.trim()) + .filter((s): s is Network => SUPPORTED_NETWORKS.includes(s as Network)); +} + +/** + * Parses the `DEALBOT_RUN_MODE` env var into a typed run-mode string. + * Defaults to `"both"` when absent or unrecognised. + */ +export function parseRunMode(env: NodeJS.ProcessEnv): IAppConfig["runMode"] { + const mode = getStringEnv(env, "DEALBOT_RUN_MODE", "both").toLowerCase(); + if (mode === "worker") return "worker"; + if (mode === "api") return "api"; + return "both"; +} + +/** + * Parses the comma-separated `RANDOM_PIECE_SIZES` env var into an array of + * byte-lengths. Defaults to 10 MiB when absent or unparseable. + */ +export function parseRandomDatasetSizes(env: NodeJS.ProcessEnv): number[] { + const envValue = env.RANDOM_PIECE_SIZES; + + if (envValue && envValue.trim().length > 0) { + const parsed = envValue + .split(",") + .map((entry) => Number.parseInt(entry.trim(), 10)) + .filter((entry) => Number.isFinite(entry) && !Number.isNaN(entry)); + + if (parsed.length > 0) { + return parsed; + } + } + + return [10 << 20]; +} + +export function parseIdList(value: string | undefined): Set { + if (!value || value.trim().length === 0) return new Set(); + return new Set( + value + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0), + ); +} + +export function parseAddressList(value: string | undefined): Set { + if (!value || value.trim().length === 0) return new Set(); + return new Set( + value + .split(",") + .map((s) => s.trim().toLowerCase()) + .filter((s) => s.length > 0), + ); +} diff --git a/apps/backend/src/config/env.schema.spec.ts b/apps/backend/src/config/env.schema.spec.ts new file mode 100644 index 00000000..4f5e7d0e --- /dev/null +++ b/apps/backend/src/config/env.schema.spec.ts @@ -0,0 +1,521 @@ +import { zeroAddress } from "viem"; +import { describe, expect, it } from "vitest"; +import { createConfigValidationSchema } from "./env.schema.js"; + +/** + * Minimum set of env vars required for validation to pass (database block + * is always required, regardless of which networks are active). + */ +const baseEnv = { + DATABASE_HOST: "localhost", + DATABASE_USER: "test", + DATABASE_PASSWORD: "test", + DATABASE_NAME: "test", + CALIBRATION_WALLET_ADDRESS: zeroAddress, + MAINNET_WALLET_ADDRESS: zeroAddress, +}; + +/** + * Env fragment that satisfies the per-network wallet-key `.or()` constraint + * for a given network prefix. Use with `...withWalletKey("CALIBRATION")`. + */ +const withWalletKey = (prefix: string) => ({ [`${prefix}_WALLET_PRIVATE_KEY`]: "0xkey" }); + +/** + * Builds a schema where only the given networks are active. Keeps tests + * deterministic regardless of the process env that the test runner inherits. + */ +const schemaFor = (networks: string) => + createConfigValidationSchema({ ...baseEnv, NETWORKS: networks } as NodeJS.ProcessEnv); + +const validate = (schema: ReturnType, input: Record) => + schema.validate(input, { allowUnknown: true }); + +describe("createConfigValidationSchema", () => { + describe("wallet-key / session-key constraint (active networks)", () => { + it("accepts a network with WALLET_PRIVATE_KEY only", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_WALLET_PRIVATE_KEY: "0xkey", + }); + expect(error).toBeUndefined(); + }); + + it("accepts a network with SESSION_KEY_PRIVATE_KEY only", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_SESSION_KEY_PRIVATE_KEY: "0xdeadbeef", + }); + expect(error).toBeUndefined(); + }); + + it("accepts both keys being provided (loader decides precedence)", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_WALLET_PRIVATE_KEY: "0xkey", + CALIBRATION_SESSION_KEY_PRIVATE_KEY: "0xsession", + }); + expect(error).toBeUndefined(); + }); + + it("rejects an active network that has neither key", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_WALLET_PRIVATE_KEY|CALIBRATION_SESSION_KEY_PRIVATE_KEY/); + }); + + it("treats empty-string wallet key as absent (must fall back to session key)", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_WALLET_PRIVATE_KEY: "", + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_WALLET_PRIVATE_KEY|CALIBRATION_SESSION_KEY_PRIVATE_KEY/); + }); + + it("treats empty-string session key as absent when wallet key is present", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_WALLET_PRIVATE_KEY: "0xkey", + CALIBRATION_SESSION_KEY_PRIVATE_KEY: "", + }); + expect(error).toBeUndefined(); + }); + + it("rejects when every active network is missing both keys (multi-network)", () => { + const { error } = validate(schemaFor("mainnet,calibration"), { + ...baseEnv, + NETWORKS: "mainnet,calibration", + }); + expect(error).toBeDefined(); + }); + + it("requires each active network independently (multi-network)", () => { + // Only mainnet has a key; calibration is active but has neither → invalid + const { error } = validate(schemaFor("mainnet,calibration"), { + ...baseEnv, + NETWORKS: "mainnet,calibration", + ...withWalletKey("MAINNET"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_WALLET_PRIVATE_KEY|CALIBRATION_SESSION_KEY_PRIVATE_KEY/); + }); + + it("accepts multi-network when every active network provides a key", () => { + const { error } = validate(schemaFor("mainnet,calibration"), { + ...baseEnv, + NETWORKS: "mainnet,calibration", + ...withWalletKey("MAINNET"), + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + }); + }); + + describe("inactive networks", () => { + it("does not require keys for an inactive network", () => { + // Only calibration is active; mainnet has no keys → still valid + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + }); + + it("does not reject invalid-looking values on an inactive network", () => { + // MAINNET is inactive, so its fields should be optional (no strict checks). + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + MAINNET_WALLET_PRIVATE_KEY: "", + MAINNET_SESSION_KEY_PRIVATE_KEY: "", + }); + expect(error).toBeUndefined(); + }); + }); + + describe("NETWORKS env var", () => { + it("rejects an unknown network name", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "unknownnet", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/NETWORKS|Invalid network/); + }); + + it("rejects a mixed list containing an unknown network", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration,bogus", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("accepts a list with whitespace around entries", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: " calibration , mainnet ", + ...withWalletKey("CALIBRATION"), + ...withWalletKey("MAINNET"), + }); + expect(error).toBeUndefined(); + }); + + it("falls back to the default when NETWORKS is absent", () => { + const { error, value } = validate(schemaFor("calibration"), { + ...baseEnv, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + expect(value.NETWORKS).toBeTruthy(); + }); + }); + + describe("database block", () => { + it.each(["DATABASE_HOST", "DATABASE_USER", "DATABASE_PASSWORD", "DATABASE_NAME"] as const)("requires %s", (key) => { + const env: Record = { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + }; + delete env[key]; + const { error } = validate(schemaFor("calibration"), env); + expect(error).toBeDefined(); + expect(error?.message).toMatch(new RegExp(key)); + }); + + it("defaults DATABASE_PORT to 5432 when absent", () => { + const { value } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + }); + expect(value.DATABASE_PORT).toBe(5432); + }); + + it("rejects a non-numeric DATABASE_PORT", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + DATABASE_PORT: "abc", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/DATABASE_PORT/); + }); + + it("rejects DATABASE_POOL_MAX below the minimum", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + DATABASE_POOL_MAX: 0, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/DATABASE_POOL_MAX/); + }); + }); + + describe("app block", () => { + it("rejects an unknown NODE_ENV", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + NODE_ENV: "staging", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/NODE_ENV/); + }); + + it("rejects an unknown DEALBOT_RUN_MODE", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + DEALBOT_RUN_MODE: "invalid-mode", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/DEALBOT_RUN_MODE/); + }); + + it("lowercases DEALBOT_RUN_MODE before validation", () => { + const { error, value } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + DEALBOT_RUN_MODE: "WORKER", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + expect(value.DEALBOT_RUN_MODE).toBe("worker"); + }); + + it("applies sensible defaults for all optional app fields", () => { + const { error, value } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + expect(value.NODE_ENV).toBe("development"); + expect(value.DEALBOT_RUN_MODE).toBe("both"); + expect(value.DEALBOT_PORT).toBe(3000); + expect(value.ENABLE_DEV_MODE).toBe(false); + expect(value.PROMETHEUS_WALLET_BALANCE_TTL_SECONDS).toBe(3600); + }); + + it("enforces PROMETHEUS_WALLET_BALANCE_TTL_SECONDS minimum of 60", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + PROMETHEUS_WALLET_BALANCE_TTL_SECONDS: 59, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/PROMETHEUS_WALLET_BALANCE_TTL_SECONDS/); + }); + + it("coerces ENABLE_DEV_MODE boolean strings", () => { + const { error, value } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ENABLE_DEV_MODE: "true", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + expect(value.ENABLE_DEV_MODE).toBe(true); + }); + }); + + describe("per-network fields (active network)", () => { + it("rejects an RPC URL without an http(s) scheme", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_RPC_URL: "ftp://example.com", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_RPC_URL/); + }); + + it("accepts an empty RPC URL string", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_RPC_URL: "", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + }); + + it("rejects METRICS_PER_HOUR above the maximum", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_METRICS_PER_HOUR: 4, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_METRICS_PER_HOUR/); + }); + + it("rejects DEALS_PER_SP_PER_HOUR at zero (below min)", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_DEALS_PER_SP_PER_HOUR: 0, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("rejects MIN_NUM_DATASETS_FOR_CHECKS when non-integer", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS: 1.5, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS/); + }); + }); + + describe("maintenance windows", () => { + it("accepts a valid comma-separated schedule", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOWS_UTC: "06:30,18:00", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + }); + + it("rejects an invalid hour", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOWS_UTC: "25:00", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_MAINTENANCE_WINDOWS_UTC/); + }); + + it("rejects an invalid minute", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOWS_UTC: "07:70", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("rejects a malformed entry", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOWS_UTC: "7am", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("rejects MAINTENANCE_WINDOW_MINUTES below 20", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOW_MINUTES: 10, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/CALIBRATION_MAINTENANCE_WINDOW_MINUTES/); + }); + + it("rejects MAINTENANCE_WINDOW_MINUTES above 360", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CALIBRATION_MAINTENANCE_WINDOW_MINUTES: 361, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + }); + + describe("jobs block", () => { + it("applies defaults when absent", () => { + const { error, value } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeUndefined(); + expect(value.JOB_SCHEDULER_POLL_SECONDS).toBe(300); + expect(value.JOB_WORKER_POLL_SECONDS).toBe(60); + expect(value.PG_BOSS_LOCAL_CONCURRENCY).toBe(20); + expect(value.DEALBOT_PGBOSS_SCHEDULER_ENABLED).toBe(true); + expect(value.DEAL_JOB_TIMEOUT_SECONDS).toBe(360); + }); + + it("rejects JOB_SCHEDULER_POLL_SECONDS below 60", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + JOB_SCHEDULER_POLL_SECONDS: 30, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("rejects DEAL_JOB_TIMEOUT_SECONDS below 120", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + DEAL_JOB_TIMEOUT_SECONDS: 60, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("rejects non-integer PG_BOSS_LOCAL_CONCURRENCY", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + PG_BOSS_LOCAL_CONCURRENCY: 2.5, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/PG_BOSS_LOCAL_CONCURRENCY/); + }); + }); + + describe("timeout block", () => { + it("enforces CONNECT_TIMEOUT_MS >= 1000", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + CONNECT_TIMEOUT_MS: 500, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + + it("enforces IPNI_VERIFICATION_POLLING_MS >= 250", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + IPNI_VERIFICATION_POLLING_MS: 100, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + }); + + describe("retrieval block", () => { + it("rejects IPFS_BLOCK_FETCH_CONCURRENCY above 32", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + IPFS_BLOCK_FETCH_CONCURRENCY: 33, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + expect(error?.message).toMatch(/IPFS_BLOCK_FETCH_CONCURRENCY/); + }); + + it("rejects IPFS_BLOCK_FETCH_CONCURRENCY when zero", () => { + const { error } = validate(schemaFor("calibration"), { + ...baseEnv, + NETWORKS: "calibration", + IPFS_BLOCK_FETCH_CONCURRENCY: 0, + ...withWalletKey("CALIBRATION"), + }); + expect(error).toBeDefined(); + }); + }); + + describe("stability", () => { + it("returns a fresh schema instance per call", () => { + const a = schemaFor("calibration"); + const b = schemaFor("calibration"); + expect(a).not.toBe(b); + }); + }); +}); diff --git a/apps/backend/src/config/env.schema.ts b/apps/backend/src/config/env.schema.ts new file mode 100644 index 00000000..260f9cd3 --- /dev/null +++ b/apps/backend/src/config/env.schema.ts @@ -0,0 +1,245 @@ +/** + * Joi validation schemas for all environment variables. + * + * Structure + * --------- + * Individual schema objects are exported so tests can validate specific + * slices in isolation. `createConfigValidationSchema()` assembles the final + * Joi object schema that NestJS `ConfigModule` consumes. + * + * Per-network rules + * ----------------- + * `createPerNetworkEnvSchema(prefix)` produces the field rules for one + * network prefix (e.g. "CALIBRATION"). `createConfigValidationSchema` + * iterates all prefixes and: + * - applies rules as-is for ACTIVE networks (and queues the wallet-key + * `.or()` constraint) + * - marks every field `.optional()` for INACTIVE networks so those env + * vars are never required. + */ + +import Joi from "joi"; +import { DEFAULT_LOCAL_DATASETS_PATH, SUPPORTED_NETWORKS } from "../common/constants.js"; +import { parseMaintenanceWindowTimes } from "../common/maintenance-window.js"; +import type { Network } from "../common/types.js"; +import { NETWORK_ENV_PREFIXES } from "./constants.js"; +import { parseActiveNetworks } from "./env.parsers.js"; +import { applyLegacyEnvCompat, logLegacyEnvCompatResult } from "./legacy-env-compat.js"; + +// --------------------------------------------------------------------------- +// Custom Joi validators +// --------------------------------------------------------------------------- + +const validateNetworksEnv = (value: string, helpers: Joi.CustomHelpers) => { + const parts = value + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + + for (const part of parts) { + if (!SUPPORTED_NETWORKS.includes(part as Network)) { + return helpers.error("any.invalid", { + message: `Invalid network "${part}". Supported: ${SUPPORTED_NETWORKS.join(", ")}.`, + }); + } + } + + return parts.length === 0 ? SUPPORTED_NETWORKS[0] : value; +}; + +const validateMaintenanceWindowsEnv = (value: string, helpers: Joi.CustomHelpers) => { + try { + parseMaintenanceWindowTimes(value.split(",")); + } catch (error) { + return helpers.error("any.invalid", { + message: error instanceof Error ? error.message : "Invalid maintenance window format", + }); + } + return value; +}; + +// --------------------------------------------------------------------------- +// Static schema slices (one per config section) +// --------------------------------------------------------------------------- + +export const appEnvSchema = { + NODE_ENV: Joi.string().valid("development", "production", "test").default("development"), + DEALBOT_RUN_MODE: Joi.string().lowercase().valid("api", "worker", "both").default("both"), + DEALBOT_PORT: Joi.number().default(3000), + DEALBOT_HOST: Joi.string().default("127.0.0.1"), + DEALBOT_METRICS_PORT: Joi.number().default(9090), + DEALBOT_METRICS_HOST: Joi.string().default("0.0.0.0"), + ENABLE_DEV_MODE: Joi.boolean().default(false), + PROMETHEUS_WALLET_BALANCE_TTL_SECONDS: Joi.number().min(60).default(3600), + PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS: Joi.number().min(1).default(60), +}; + +export const databaseEnvSchema = { + DATABASE_HOST: Joi.string().required(), + DATABASE_PORT: Joi.number().default(5432), + DATABASE_POOL_MAX: Joi.number().integer().min(1).default(1), + DATABASE_USER: Joi.string().required(), + DATABASE_PASSWORD: Joi.string().required(), + DATABASE_NAME: Joi.string().required(), +}; + +export const globalNetworkEnvSchema = { + NETWORKS: Joi.string().default(SUPPORTED_NETWORKS[0]).custom(validateNetworksEnv), +}; + +export const jobsEnvSchema = { + JOB_SCHEDULER_POLL_SECONDS: Joi.number().min(60).default(300), + JOB_WORKER_POLL_SECONDS: Joi.number().min(5).default(60), + PG_BOSS_LOCAL_CONCURRENCY: Joi.number().integer().min(1).default(20), + DEALBOT_PGBOSS_SCHEDULER_ENABLED: Joi.boolean().default(true), + DEALBOT_PGBOSS_POOL_MAX: Joi.number().integer().min(1).default(1), + JOB_CATCHUP_MAX_ENQUEUE: Joi.number().min(1).default(10), + JOB_SCHEDULE_PHASE_SECONDS: Joi.number().min(0).default(0), + JOB_ENQUEUE_JITTER_SECONDS: Joi.number().min(0).default(0), + DEAL_JOB_TIMEOUT_SECONDS: Joi.number().min(120).default(360), + RETRIEVAL_JOB_TIMEOUT_SECONDS: Joi.number().min(60).default(60), + DATA_SET_CREATION_JOB_TIMEOUT_SECONDS: Joi.number().min(60).default(300), +}; + +export const datasetEnvSchema = { + DEALBOT_LOCAL_DATASETS_PATH: Joi.string().default(DEFAULT_LOCAL_DATASETS_PATH), + RANDOM_PIECE_SIZES: Joi.string().default("10485760"), +}; + +export const timeoutEnvSchema = { + CONNECT_TIMEOUT_MS: Joi.number().min(1000).default(10000), + HTTP_REQUEST_TIMEOUT_MS: Joi.number().min(1000).default(240000), + HTTP2_REQUEST_TIMEOUT_MS: Joi.number().min(1000).default(240000), + IPNI_VERIFICATION_TIMEOUT_MS: Joi.number().min(1000).default(60000), + IPNI_VERIFICATION_POLLING_MS: Joi.number().min(250).default(2000), +}; + +export const retrievalEnvSchema = { + IPFS_BLOCK_FETCH_CONCURRENCY: Joi.number().integer().min(1).max(32).default(6), +}; + +// --------------------------------------------------------------------------- +// Per-network schema factory +// --------------------------------------------------------------------------- + +/** + * Returns the Joi field rules for a single network prefix (e.g. `"CALIBRATION"`). + * Fields are defined as optional here; active-network enforcement is handled + * in `createConfigValidationSchema`. + */ +export const createPerNetworkEnvSchema = (prefix: Uppercase | "") => { + const k = (key: string) => `${prefix}_${key}`; + return { + [k("WALLET_ADDRESS")]: Joi.string().required(), + [k("WALLET_PRIVATE_KEY")]: Joi.string().optional().empty(""), + [k("SESSION_KEY_PRIVATE_KEY")]: Joi.string().optional().empty(""), + [k("RPC_URL")]: Joi.string() + .uri({ scheme: ["http", "https"] }) + .optional() + .allow(""), + [k("PDP_SUBGRAPH_ENDPOINT")]: Joi.string().uri().optional().allow(""), + [k("CHECK_DATASET_CREATION_FEES")]: Joi.boolean().optional(), + [k("USE_ONLY_APPROVED_PROVIDERS")]: Joi.boolean().optional(), + [k("DEALBOT_DATASET_VERSION")]: Joi.string().optional(), + [k("MIN_NUM_DATASETS_FOR_CHECKS")]: Joi.number().integer().min(1).optional(), + [k("METRICS_PER_HOUR")]: Joi.number().min(0.001).max(3).optional(), + [k("DEALS_PER_SP_PER_HOUR")]: Joi.number().min(0.001).max(20).optional(), + [k("RETRIEVALS_PER_SP_PER_HOUR")]: Joi.number().min(0.001).max(20).optional(), + [k("DATASET_CREATIONS_PER_SP_PER_HOUR")]: Joi.number().min(0.001).max(20).optional(), + [k("DATA_RETENTION_POLL_INTERVAL_SECONDS")]: Joi.number().optional(), + [k("PROVIDERS_REFRESH_INTERVAL_SECONDS")]: Joi.number().optional(), + [k("MAINTENANCE_WINDOWS_UTC")]: Joi.string().default("07:00,22:00").custom(validateMaintenanceWindowsEnv), + [k("MAINTENANCE_WINDOW_MINUTES")]: Joi.number().min(20).max(360).default(20), + [k("BLOCKED_SP_IDS")]: Joi.string().optional().allow(""), + [k("BLOCKED_SP_ADDRESSES")]: Joi.string().optional().allow(""), + + [k("PIECE_CLEANUP_PER_SP_PER_HOUR")]: Joi.number() + .min(0.001) + .max(20) + .default(1 / 24), + [k("MAX_PIECE_CLEANUP_RUNTIME_SECONDS")]: Joi.number().min(60).default(300), + [k("MAX_DATASET_STORAGE_SIZE_BYTES")]: Joi.number() + .integer() + .min(1) + .default(24 * 1024 * 1024 * 1024), + [k("TARGET_DATASET_STORAGE_SIZE_BYTES")]: Joi.number() + .integer() + .min(1) + .default(20 * 1024 * 1024 * 1024) // 20 GiB per SP + .custom((value, helpers) => { + const max = helpers.state.ancestors?.[0]?.MAX_DATASET_STORAGE_SIZE_BYTES; + if (max != null && value >= max) { + return helpers.error("any.invalid", { + message: `TARGET_DATASET_STORAGE_SIZE_BYTES (${value}) must be less than MAX_DATASET_STORAGE_SIZE_BYTES (${max})`, + }); + } + return value; + }, "target < max validation"), + }; +}; + +// --------------------------------------------------------------------------- +// Dynamic schema factory +// --------------------------------------------------------------------------- + +/** + * Builds the full Joi validation schema, adapting per-network field + * requirements to which networks are present in `processEnv.NETWORKS`. + */ +export function createConfigValidationSchema(processEnv: NodeJS.ProcessEnv = process.env): Joi.ObjectSchema { + const activeNetworks = parseActiveNetworks(processEnv); + + const schemaFields: Record = { + ...appEnvSchema, + ...databaseEnvSchema, + ...globalNetworkEnvSchema, + ...jobsEnvSchema, + ...datasetEnvSchema, + ...timeoutEnvSchema, + ...retrievalEnvSchema, + }; + + const walletKeyOrConditions: [string, string][] = []; + + for (const prefix of NETWORK_ENV_PREFIXES) { + const networkRules = createPerNetworkEnvSchema(prefix); + + if (activeNetworks.includes(prefix.toLowerCase() as Network)) { + Object.assign(schemaFields, networkRules); + walletKeyOrConditions.push([`${prefix}_WALLET_PRIVATE_KEY`, `${prefix}_SESSION_KEY_PRIVATE_KEY`]); + } else { + const optionalRules = Object.fromEntries( + Object.entries(networkRules).map(([key, rule]) => [key, (rule as Joi.AnySchema).optional()]), + ); + Object.assign(schemaFields, optionalRules); + } + } + + let schema = Joi.object(schemaFields); + + for (const [walletKey, sessionKey] of walletKeyOrConditions) { + schema = schema.or(walletKey, sessionKey); + } + + return schema; +} + +// --------------------------------------------------------------------------- +// NestJS ConfigModule `validate` callback +// --------------------------------------------------------------------------- + +/** + * Entry point wired into `ConfigModule.forRoot({ validate })`. Runs AFTER + * `@nestjs/config` has merged `.env` into the env object, so the scheme + * detector sees the operator's real configuration. + */ +export function validateConfig(rawEnv: Record): Record { + logLegacyEnvCompatResult(applyLegacyEnvCompat(rawEnv as NodeJS.ProcessEnv)); + + const schema = createConfigValidationSchema(rawEnv as NodeJS.ProcessEnv); + const { error, value } = schema.validate(rawEnv, { allowUnknown: true, abortEarly: false }); + if (error) { + throw new Error(`Config validation error: ${error.message}`); + } + return value; +} diff --git a/apps/backend/src/config/legacy-env-compat.spec.ts b/apps/backend/src/config/legacy-env-compat.spec.ts new file mode 100644 index 00000000..58662c8d --- /dev/null +++ b/apps/backend/src/config/legacy-env-compat.spec.ts @@ -0,0 +1,142 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { applyLegacyEnvCompat, logLegacyEnvCompatResult } from "./legacy-env-compat.js"; + +/** Build a fresh env map for every test — never mutate `process.env`. */ +const envOf = (overrides: Record = {}): NodeJS.ProcessEnv => ({ ...overrides }) as NodeJS.ProcessEnv; + +describe("applyLegacyEnvCompat", () => { + describe("skips translation", () => { + it("returns skipReason=networks_already_set when NETWORKS is set", () => { + const env = envOf({ NETWORKS: "calibration", NETWORK: "mainnet", WALLET_PRIVATE_KEY: "0xkey" }); + const before = { ...env }; + const result = applyLegacyEnvCompat(env); + expect(result.applied).toBe(false); + expect(result.skipReason).toBe("networks_already_set"); + expect(result.translatedVars).toEqual([]); + // Env left untouched. + expect(env).toEqual(before); + }); + + it("returns skipReason=no_legacy_network when NETWORK is absent", () => { + const env = envOf(); + const result = applyLegacyEnvCompat(env); + expect(result.applied).toBe(false); + expect(result.skipReason).toBe("no_legacy_network"); + }); + + it("returns skipReason=no_legacy_network when NETWORK is whitespace", () => { + const env = envOf({ NETWORK: " " }); + const result = applyLegacyEnvCompat(env); + expect(result.applied).toBe(false); + expect(result.skipReason).toBe("no_legacy_network"); + }); + + it("returns skipReason=invalid_legacy_network when NETWORK is unsupported", () => { + const env = envOf({ NETWORK: "moonbase" }); + const result = applyLegacyEnvCompat(env); + expect(result.applied).toBe(false); + expect(result.skipReason).toBe("invalid_legacy_network"); + // Must not set NETWORKS to anything bogus. + expect(env.NETWORKS).toBeUndefined(); + }); + }); + + describe("translates when legacy NETWORK is set", () => { + it("sets NETWORKS and copies each unprefixed var to its prefixed slot", () => { + const env = envOf({ + NETWORK: "calibration", + WALLET_PRIVATE_KEY: "0xkey", + WALLET_ADDRESS: "0xabc", + RPC_URL: "https://rpc.example", + DEALS_PER_SP_PER_HOUR: "3", + }); + const result = applyLegacyEnvCompat(env); + + expect(result.applied).toBe(true); + expect(result.network).toBe("calibration"); + expect(result.translatedVars).toEqual( + expect.arrayContaining(["WALLET_PRIVATE_KEY", "WALLET_ADDRESS", "RPC_URL", "DEALS_PER_SP_PER_HOUR"]), + ); + expect(env.NETWORKS).toBe("calibration"); + expect(env.CALIBRATION_WALLET_PRIVATE_KEY).toBe("0xkey"); + expect(env.CALIBRATION_WALLET_ADDRESS).toBe("0xabc"); + expect(env.CALIBRATION_RPC_URL).toBe("https://rpc.example"); + expect(env.CALIBRATION_DEALS_PER_SP_PER_HOUR).toBe("3"); + }); + + it("normalises case-variant NETWORK values", () => { + const env = envOf({ NETWORK: "MAINNET", WALLET_PRIVATE_KEY: "0xkey" }); + const result = applyLegacyEnvCompat(env); + + expect(result.applied).toBe(true); + expect(result.network).toBe("mainnet"); + expect(env.NETWORKS).toBe("mainnet"); + expect(env.MAINNET_WALLET_PRIVATE_KEY).toBe("0xkey"); + }); + + it("does not overwrite an already-set prefixed var (explicit wins)", () => { + const env = envOf({ + NETWORK: "calibration", + WALLET_PRIVATE_KEY: "0xlegacy", + CALIBRATION_WALLET_PRIVATE_KEY: "0xexplicit", + }); + const result = applyLegacyEnvCompat(env); + + expect(result.applied).toBe(true); + expect(env.CALIBRATION_WALLET_PRIVATE_KEY).toBe("0xexplicit"); + // The legacy var is still present but was not re-copied. + expect(result.translatedVars).not.toContain("WALLET_PRIVATE_KEY"); + }); + + it("skips empty legacy values", () => { + const env = envOf({ NETWORK: "calibration", WALLET_PRIVATE_KEY: "0xkey", RPC_URL: "" }); + const result = applyLegacyEnvCompat(env); + + expect(result.applied).toBe(true); + expect(result.translatedVars).not.toContain("RPC_URL"); + expect(env.CALIBRATION_RPC_URL).toBeUndefined(); + }); + + it("does not carry data between distinct env maps (no shared state)", () => { + const a = envOf({ NETWORK: "calibration", WALLET_PRIVATE_KEY: "0xa" }); + const b = envOf({ NETWORK: "mainnet", WALLET_PRIVATE_KEY: "0xb" }); + applyLegacyEnvCompat(a); + applyLegacyEnvCompat(b); + expect(a.CALIBRATION_WALLET_PRIVATE_KEY).toBe("0xa"); + expect(a.MAINNET_WALLET_PRIVATE_KEY).toBeUndefined(); + expect(b.MAINNET_WALLET_PRIVATE_KEY).toBe("0xb"); + expect(b.CALIBRATION_WALLET_PRIVATE_KEY).toBeUndefined(); + }); + }); +}); + +describe("logLegacyEnvCompatResult", () => { + let warnSpy: ReturnType; + + beforeEach(() => { + warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + + afterEach(() => { + warnSpy.mockRestore(); + }); + + it("is a no-op when translation was not applied", () => { + logLegacyEnvCompatResult({ applied: false, translatedVars: [], skipReason: "networks_already_set" }); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("emits a structured JSON warning when translation was applied", () => { + logLegacyEnvCompatResult({ + applied: true, + network: "calibration", + translatedVars: ["WALLET_PRIVATE_KEY", "RPC_URL"], + }); + + expect(warnSpy).toHaveBeenCalledTimes(1); + const payload = JSON.parse(warnSpy.mock.calls[0][0] as string); + expect(payload.event).toBe("config_legacy_env_detected"); + expect(payload.network).toBe("calibration"); + expect(payload.translatedVars).toEqual(["WALLET_PRIVATE_KEY", "RPC_URL"]); + }); +}); diff --git a/apps/backend/src/config/legacy-env-compat.ts b/apps/backend/src/config/legacy-env-compat.ts new file mode 100644 index 00000000..d4ec38f1 --- /dev/null +++ b/apps/backend/src/config/legacy-env-compat.ts @@ -0,0 +1,139 @@ +/** + * Backwards-compatibility shim for the legacy single-network env layout. + * + * Before multi-network support, a dealbot deployment was configured with + * unprefixed variables such as `NETWORK`, `WALLET_PRIVATE_KEY`, `RPC_URL`, + * `DEALS_PER_SP_PER_HOUR`, etc. Multi-network support requires a + * `NETWORKS=...` list plus per-network prefixed variables + * (`CALIBRATION_WALLET_PRIVATE_KEY`, `MAINNET_RPC_URL`, ...). + * + * This shim lets existing deployments roll forward without changing their + * ConfigMaps/Secrets in the same release as the code rollout: if `NETWORKS` + * is absent but legacy `NETWORK` is set to a supported value, it is translated + * into the new shape in place (single-network config only). The translation + * runs *before* the Joi validation schema is evaluated, so the rest of the + * code never has to branch on legacy mode. + * + * Lifecycle + * --------- + * Call `applyLegacyEnvCompat(process.env)` at the top of `validate` in + * `ConfigModule.forRoot`. The function mutates `env` in place so + * subsequent reads observe the translated values. + * + * Removal + * ------- + * Once all environments have been cut over to the new prefixed vars + * and this shim is no longer needed, delete this file and its two call sites. + */ + +import { createPinoExitLogger } from "src/common/pino.config.js"; +import { SUPPORTED_NETWORKS } from "../common/constants.js"; +import type { Network } from "../common/types.js"; + +/** + * Env var names (unprefixed) that were moved into a per-network namespace. + * Each corresponds to a `_` variable in the new layout. + * + * Keep this list in sync with `createPerNetworkEnvSchema` in `env.schema.ts`. + */ +const LEGACY_PER_NETWORK_VARS = [ + "WALLET_ADDRESS", + "WALLET_PRIVATE_KEY", + "SESSION_KEY_PRIVATE_KEY", + "RPC_URL", + "PDP_SUBGRAPH_ENDPOINT", + "CHECK_DATASET_CREATION_FEES", + "USE_ONLY_APPROVED_PROVIDERS", + "DEALBOT_DATASET_VERSION", + "MIN_NUM_DATASETS_FOR_CHECKS", + "DEALS_PER_SP_PER_HOUR", + "RETRIEVALS_PER_SP_PER_HOUR", + "DATASET_CREATIONS_PER_SP_PER_HOUR", + "DATA_RETENTION_POLL_INTERVAL_SECONDS", + "PROVIDERS_REFRESH_INTERVAL_SECONDS", + "MAINTENANCE_WINDOWS_UTC", + "MAINTENANCE_WINDOW_MINUTES", + "BLOCKED_SP_IDS", + "BLOCKED_SP_ADDRESSES", + "PIECE_CLEANUP_PER_SP_PER_HOUR", + "MAX_PIECE_CLEANUP_RUNTIME_SECONDS", + "MAX_DATASET_STORAGE_SIZE_BYTES", + "TARGET_DATASET_STORAGE_SIZE_BYTES", +] as const; + +export interface LegacyEnvCompatResult { + /** True if legacy translation was applied. */ + applied: boolean; + /** The network the legacy env was resolved to (only set when applied). */ + network?: Network; + /** Legacy var names that were copied into the new prefixed slot. */ + translatedVars: string[]; + /** Reason translation was skipped, for diagnostics. */ + skipReason?: "networks_already_set" | "no_legacy_network" | "invalid_legacy_network"; +} + +/** + * Translates legacy single-network env vars into the new prefixed layout. + * Mutates `env` in place and returns a summary describing what happened. + * + * Rules: + * - If `NETWORKS` is already set, return untouched (operator has migrated). + * - Else if legacy `NETWORK` is set to a supported value, copy unprefixed + * vars into `_` slots that are currently unset. Already-set + * prefixed vars are never overwritten (explicit wins). + * - Else return untouched; downstream Joi validation will surface the + * missing-config error with its normal diagnostics. + */ +export function applyLegacyEnvCompat(env: NodeJS.ProcessEnv): LegacyEnvCompatResult { + if (typeof env.NETWORKS === "string" && env.NETWORKS.trim().length > 0) { + return { applied: false, translatedVars: [], skipReason: "networks_already_set" }; + } + + const legacyRaw = env.NETWORK; + if (typeof legacyRaw !== "string" || legacyRaw.trim().length === 0) { + return { applied: false, translatedVars: [], skipReason: "no_legacy_network" }; + } + + const legacyNetwork = legacyRaw.trim().toLowerCase(); + if (!SUPPORTED_NETWORKS.includes(legacyNetwork as Network)) { + return { applied: false, translatedVars: [], skipReason: "invalid_legacy_network" }; + } + + const network = legacyNetwork as Network; + const prefix = network.toUpperCase(); + const translatedVars: string[] = []; + + env.NETWORKS = network; + + for (const key of LEGACY_PER_NETWORK_VARS) { + const legacyValue = env[key]; + if (typeof legacyValue !== "string" || legacyValue.length === 0) continue; + + const prefixedKey = `${prefix}_${key}`; + const existing = env[prefixedKey]; + if (typeof existing === "string" && existing.length > 0) continue; + + env[prefixedKey] = legacyValue; + translatedVars.push(key); + } + + return { applied: true, network, translatedVars }; +} + +/** + * One-time console warning describing a legacy translation. + */ +export function logLegacyEnvCompatResult(result: LegacyEnvCompatResult): void { + if (!result.applied) return; + + const logger = createPinoExitLogger().child({ context: "LegacyEnvCompat" }); + logger.warn({ + level: "warn", + event: "config_legacy_env_detected", + message: + "Legacy single-network env vars detected; translated into per-network prefixed vars. " + + "Update your ConfigMap/Secrets to the prefixed names before the next release.", + network: result.network, + translatedVars: result.translatedVars, + }); +} diff --git a/apps/backend/src/config/loader.spec.ts b/apps/backend/src/config/loader.spec.ts new file mode 100644 index 00000000..bccedd91 --- /dev/null +++ b/apps/backend/src/config/loader.spec.ts @@ -0,0 +1,89 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { loadConfig } from "./loader.js"; + +/** + * `loadConfig` reads from `process.env` directly, so tests snapshot and + * restore the real env around each case. Only keys touched by a test are + * cleared on reset. + */ +const KEYS_TO_RESET = [ + "NETWORKS", + "NETWORK", + "CALIBRATION_WALLET_PRIVATE_KEY", + "CALIBRATION_WALLET_ADDRESS", + "CALIBRATION_RPC_URL", + "CALIBRATION_DEALS_PER_SP_PER_HOUR", + "CALIBRATION_BLOCKED_SP_IDS", + "MAINNET_WALLET_PRIVATE_KEY", + "MAINNET_WALLET_ADDRESS", + "MAINNET_RPC_URL", + "WALLET_PRIVATE_KEY", + "WALLET_ADDRESS", + "RPC_URL", + "DEALS_PER_SP_PER_HOUR", + "BLOCKED_SP_IDS", + "SESSION_KEY_PRIVATE_KEY", +]; + +const snapshot: Record = {}; + +beforeEach(() => { + for (const key of KEYS_TO_RESET) { + snapshot[key] = process.env[key]; + delete process.env[key]; + } + process.env.DATABASE_HOST = process.env.DATABASE_HOST ?? "localhost"; + process.env.DATABASE_USER = process.env.DATABASE_USER ?? "test"; + process.env.DATABASE_PASSWORD = process.env.DATABASE_PASSWORD ?? "test"; + process.env.DATABASE_NAME = process.env.DATABASE_NAME ?? "test"; + + return () => { + for (const key of KEYS_TO_RESET) { + if (snapshot[key] === undefined) delete process.env[key]; + else process.env[key] = snapshot[key]; + } + }; +}); + +describe("loadConfig", () => { + it("loads the active network from prefixed vars", () => { + process.env.NETWORKS = "calibration"; + process.env.CALIBRATION_WALLET_PRIVATE_KEY = "0xkey"; + process.env.CALIBRATION_RPC_URL = "https://rpc.example/calibration"; + process.env.CALIBRATION_DEALS_PER_SP_PER_HOUR = "3"; + + const cfg = loadConfig(); + + expect(cfg.activeNetworks).toEqual(["calibration"]); + expect(cfg.networks.calibration.network).toBe("calibration"); + expect(cfg.networks.calibration.rpcUrl).toBe("https://rpc.example/calibration"); + expect(cfg.networks.calibration.dealsPerSpPerHour).toBe(3); + if ("walletPrivateKey" in cfg.networks.calibration) { + expect(cfg.networks.calibration.walletPrivateKey).toBe("0xkey"); + } else { + throw new Error("calibration should be loaded as walletPrivateKey variant"); + } + }); + + it("loads both networks when both are listed in NETWORKS", () => { + process.env.NETWORKS = "calibration,mainnet"; + process.env.CALIBRATION_WALLET_PRIVATE_KEY = "0xcal"; + process.env.MAINNET_WALLET_PRIVATE_KEY = "0xmain"; + process.env.MAINNET_RPC_URL = "https://rpc.example/mainnet"; + + const cfg = loadConfig(); + + expect(cfg.activeNetworks).toEqual(["calibration", "mainnet"]); + expect(cfg.networks.mainnet.rpcUrl).toBe("https://rpc.example/mainnet"); + }); + + it("does not throw when an inactive network lacks wallet keys", () => { + // Pre-refactor, the loader iterated all SUPPORTED_NETWORKS and threw on + // missing keys for inactive networks — operators had to set keys for both. + process.env.NETWORKS = "calibration"; + process.env.CALIBRATION_WALLET_PRIVATE_KEY = "0xkey"; + // MAINNET_WALLET_PRIVATE_KEY intentionally unset. + + expect(() => loadConfig()).not.toThrow(); + }); +}); diff --git a/apps/backend/src/config/loader.ts b/apps/backend/src/config/loader.ts new file mode 100644 index 00000000..540cf87d --- /dev/null +++ b/apps/backend/src/config/loader.ts @@ -0,0 +1,222 @@ +/** + * Configuration loaders. + * + * Each `load*Config` function reads `process.env` (or an injected map) and + * returns a fully-typed config slice. `loadConfig` assembles all slices into + * the top-level `IConfig` object consumed by NestJS `ConfigModule`. + * + */ + +import { DEFAULT_LOCAL_DATASETS_PATH, ZERO_ADDRESS } from "../common/constants.js"; +import type { Network } from "../common/types.js"; +import { networkDefaults } from "./constants.js"; +import { getBooleanEnv, getFloatEnv, getNumberEnv, getStringEnv } from "./env.helpers.js"; +import { + parseActiveNetworks, + parseAddressList, + parseIdList, + parseRandomDatasetSizes, + parseRunMode, +} from "./env.parsers.js"; +import type { + IAppConfig, + IConfig, + IDatabaseConfig, + IDatasetConfig, + IJobsConfig, + INetworkConfig, + IRetrievalConfig, + ITimeoutConfig, +} from "./types.js"; + +// --------------------------------------------------------------------------- +// Per-section loaders +// --------------------------------------------------------------------------- + +const loadAppConfig = (env: NodeJS.ProcessEnv): IAppConfig => ({ + env: getStringEnv(env, "NODE_ENV", "development"), + runMode: parseRunMode(env), + port: getNumberEnv(env, "DEALBOT_PORT", 3000), + host: getStringEnv(env, "DEALBOT_HOST", "127.0.0.1"), + metricsPort: getNumberEnv(env, "DEALBOT_METRICS_PORT", 9090), + metricsHost: getStringEnv(env, "DEALBOT_METRICS_HOST", "0.0.0.0"), + enableDevMode: env.ENABLE_DEV_MODE === "true", + prometheusWalletBalanceTtlSeconds: getNumberEnv(env, "PROMETHEUS_WALLET_BALANCE_TTL_SECONDS", 3600), + prometheusWalletBalanceErrorCooldownSeconds: getNumberEnv( + env, + "PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS", + 60, + ), +}); + +const loadDatabaseConfig = (env: NodeJS.ProcessEnv): IDatabaseConfig => ({ + host: getStringEnv(env, "DATABASE_HOST", "localhost"), + port: getNumberEnv(env, "DATABASE_PORT", 5432), + poolMax: getNumberEnv(env, "DATABASE_POOL_MAX", 1), + username: getStringEnv(env, "DATABASE_USER", "dealbot"), + password: getStringEnv(env, "DATABASE_PASSWORD", "dealbot_password"), + database: getStringEnv(env, "DATABASE_NAME", "filecoin_dealbot"), +}); + +const loadJobsConfig = (env: NodeJS.ProcessEnv): IJobsConfig => ({ + schedulerPollSeconds: getNumberEnv(env, "JOB_SCHEDULER_POLL_SECONDS", 300), + workerPollSeconds: getNumberEnv(env, "JOB_WORKER_POLL_SECONDS", 60), + pgbossLocalConcurrency: getNumberEnv(env, "PG_BOSS_LOCAL_CONCURRENCY", 20), + pgbossSchedulerEnabled: getBooleanEnv(env, "DEALBOT_PGBOSS_SCHEDULER_ENABLED", true), + pgbossPoolMax: getNumberEnv(env, "DEALBOT_PGBOSS_POOL_MAX", 1), + catchupMaxEnqueue: getNumberEnv(env, "JOB_CATCHUP_MAX_ENQUEUE", 10), + schedulePhaseSeconds: getNumberEnv(env, "JOB_SCHEDULE_PHASE_SECONDS", 0), + enqueueJitterSeconds: getNumberEnv(env, "JOB_ENQUEUE_JITTER_SECONDS", 0), + dealJobTimeoutSeconds: getNumberEnv(env, "DEAL_JOB_TIMEOUT_SECONDS", 360), + retrievalJobTimeoutSeconds: getNumberEnv(env, "RETRIEVAL_JOB_TIMEOUT_SECONDS", 60), + dataSetCreationJobTimeoutSeconds: getNumberEnv(env, "DATA_SET_CREATION_JOB_TIMEOUT_SECONDS", 300), +}); + +const loadDatasetConfig = (env: NodeJS.ProcessEnv): IDatasetConfig => ({ + localDatasetsPath: getStringEnv(env, "DEALBOT_LOCAL_DATASETS_PATH", DEFAULT_LOCAL_DATASETS_PATH), + randomDatasetSizes: parseRandomDatasetSizes(env), +}); + +const loadTimeoutConfig = (env: NodeJS.ProcessEnv): ITimeoutConfig => ({ + connectTimeoutMs: getNumberEnv(env, "CONNECT_TIMEOUT_MS", 10000), + httpRequestTimeoutMs: getNumberEnv(env, "HTTP_REQUEST_TIMEOUT_MS", 240000), + http2RequestTimeoutMs: getNumberEnv(env, "HTTP2_REQUEST_TIMEOUT_MS", 240000), + ipniVerificationTimeoutMs: getNumberEnv(env, "IPNI_VERIFICATION_TIMEOUT_MS", 60000), + ipniVerificationPollingMs: getNumberEnv(env, "IPNI_VERIFICATION_POLLING_MS", 2000), +}); + +const loadRetrievalConfig = (env: NodeJS.ProcessEnv): IRetrievalConfig => ({ + ipfsBlockFetchConcurrency: getNumberEnv(env, "IPFS_BLOCK_FETCH_CONCURRENCY", 6), +}); + +// --------------------------------------------------------------------------- +// Per-network loader +// --------------------------------------------------------------------------- + +type DistributiveOmit = T extends any ? Omit : never; + +/** + * Reads all per-network env vars for one network's prefix + * (e.g. `"CALIBRATION"` → reads `CALIBRATION_RPC_URL`, `CALIBRATION_WALLET_ADDRESS`, ...). + * + * Legacy (unprefixed) envs are translated to this prefixed form by + * `applyLegacyEnvCompat` before `loadConfig` runs, so this function only + * needs to handle the prefixed scheme. + */ +function loadNetworkEnvPrefix( + prefix: Uppercase, + env: NodeJS.ProcessEnv, +): DistributiveOmit { + const k = (key: string) => `${prefix}_${key}`; + const get = (key: string) => env[k(key)]; + + const base = { + walletAddress: get("WALLET_ADDRESS") || ZERO_ADDRESS, + rpcUrl: get("RPC_URL") || undefined, + pdpSubgraphEndpoint: get("PDP_SUBGRAPH_ENDPOINT") || "", + checkDatasetCreationFees: getBooleanEnv( + env, + k("CHECK_DATASET_CREATION_FEES"), + networkDefaults.checkDatasetCreationFees, + ), + useOnlyApprovedProviders: getBooleanEnv( + env, + k("USE_ONLY_APPROVED_PROVIDERS"), + networkDefaults.useOnlyApprovedProviders, + ), + dealbotDataSetVersion: get("DEALBOT_DATASET_VERSION") || undefined, + minNumDataSetsForChecks: getNumberEnv( + env, + k("MIN_NUM_DATASETS_FOR_CHECKS"), + networkDefaults.minNumDataSetsForChecks, + ), + dealsPerSpPerHour: getFloatEnv(env, k("DEALS_PER_SP_PER_HOUR"), networkDefaults.dealsPerSpPerHour), + retrievalsPerSpPerHour: getFloatEnv(env, k("RETRIEVALS_PER_SP_PER_HOUR"), networkDefaults.retrievalsPerSpPerHour), + dataSetCreationsPerSpPerHour: getFloatEnv( + env, + k("DATASET_CREATIONS_PER_SP_PER_HOUR"), + networkDefaults.dataSetCreationsPerSpPerHour, + ), + dataRetentionPollIntervalSeconds: getNumberEnv( + env, + k("DATA_RETENTION_POLL_INTERVAL_SECONDS"), + networkDefaults.dataRetentionPollIntervalSeconds, + ), + providersRefreshIntervalSeconds: getNumberEnv( + env, + k("PROVIDERS_REFRESH_INTERVAL_SECONDS"), + networkDefaults.providersRefreshIntervalSeconds, + ), + maintenanceWindowsUtc: get("MAINTENANCE_WINDOWS_UTC") + ? get("MAINTENANCE_WINDOWS_UTC")! + .split(",") + .map((v) => v.trim()) + .filter((v) => v.length > 0) + : networkDefaults.maintenanceWindowsUtc, + maintenanceWindowMinutes: getNumberEnv( + env, + k("MAINTENANCE_WINDOW_MINUTES"), + networkDefaults.maintenanceWindowMinutes, + ), + blockedSpIds: parseIdList(get("BLOCKED_SP_IDS")), + blockedSpAddresses: parseAddressList(get("BLOCKED_SP_ADDRESSES")), + pieceCleanupPerSpPerHour: getFloatEnv( + env, + k("PIECE_CLEANUP_PER_SP_PER_HOUR"), + networkDefaults.pieceCleanupPerSpPerHour, + ), + maxPieceCleanupRuntimeSeconds: getNumberEnv( + env, + k("MAX_PIECE_CLEANUP_RUNTIME_SECONDS"), + networkDefaults.maxPieceCleanupRuntimeSeconds, + ), + maxDatasetStorageSizeBytes: getNumberEnv( + env, + k("MAX_DATASET_STORAGE_SIZE_BYTES"), + networkDefaults.maxDatasetStorageSizeBytes, + ), + targetDatasetStorageSizeBytes: getNumberEnv( + env, + k("TARGET_DATASET_STORAGE_SIZE_BYTES"), + networkDefaults.targetDatasetStorageSizeBytes, + ), + }; + + const walletPrivateKey = (get("WALLET_PRIVATE_KEY") || undefined) as `0x${string}` | undefined; + const sessionKeyPrivateKey = (get("SESSION_KEY_PRIVATE_KEY") || undefined) as `0x${string}` | undefined; + + if (sessionKeyPrivateKey) return { ...base, sessionKeyPrivateKey }; + if (walletPrivateKey) return { ...base, walletPrivateKey }; + + // Joi's .or() constraint on `${prefix}_WALLET_PRIVATE_KEY` / + // `${prefix}_SESSION_KEY_PRIVATE_KEY` ensures this branch is unreachable + throw new Error(`[config] Neither WALLET_PRIVATE_KEY nor SESSION_KEY_PRIVATE_KEY is set for ${prefix}`); +} + +const loadNetworkConfigs = (env: NodeJS.ProcessEnv): Pick => { + const activeNetworks = parseActiveNetworks(env); + const networks = {} as Record; + + for (const network of activeNetworks) { + const prefix = network.toUpperCase() as Uppercase; + networks[network] = { network, ...loadNetworkEnvPrefix(prefix, env) }; + } + + return { networks, activeNetworks }; +}; + +// --------------------------------------------------------------------------- +// Top-level loader (NestJS ConfigModule entry-point) +// --------------------------------------------------------------------------- + +export function loadConfig(): IConfig { + return { + app: loadAppConfig(process.env), + database: loadDatabaseConfig(process.env), + ...loadNetworkConfigs(process.env), + jobs: loadJobsConfig(process.env), + dataset: loadDatasetConfig(process.env), + timeouts: loadTimeoutConfig(process.env), + retrieval: loadRetrievalConfig(process.env), + }; +} diff --git a/apps/backend/src/config/types.ts b/apps/backend/src/config/types.ts new file mode 100644 index 00000000..c3ea2fad --- /dev/null +++ b/apps/backend/src/config/types.ts @@ -0,0 +1,199 @@ +import { Network } from "../common/types.js"; + +export interface IAppConfig { + env: string; + runMode: "api" | "worker" | "both"; + port: number; + host: string; + metricsPort: number; + metricsHost: string; + enableDevMode: boolean; + prometheusWalletBalanceTtlSeconds: number; + prometheusWalletBalanceErrorCooldownSeconds: number; +} + +export interface IDatabaseConfig { + host: string; + port: number; + poolMax: number; + username: string; + password: string; + database: string; +} + +type BaseNetworkConfig = { + network: Network; + + /** Blockchain Config */ + rpcUrl?: string; + walletAddress: string; + checkDatasetCreationFees: boolean; + useOnlyApprovedProviders: boolean; + dealbotDataSetVersion?: string; + pdpSubgraphEndpoint?: string; + minNumDataSetsForChecks: number; + + /** + * Target number of deal creations per storage provider per hour. + * + * Increasing this increases on-chain activity and dataset uploads. + */ + dealsPerSpPerHour: number; + /** + * Target number of retrieval tests per storage provider per hour. + * + * Increasing this increases retrieval load against providers and DB writes. + */ + retrievalsPerSpPerHour: number; + /** + * Target number of dataset creation runs per storage provider per hour. + */ + dataSetCreationsPerSpPerHour: number; + /** + * Target number of piece cleanup runs per storage provider per hour. + * + * Increasing this makes cleanup more aggressive at the cost of more SP API calls. + */ + pieceCleanupPerSpPerHour: number; + maxPieceCleanupRuntimeSeconds: number; + dataRetentionPollIntervalSeconds: number; + providersRefreshIntervalSeconds: number; + + /** Maintenance Config */ + maintenanceWindowsUtc: string[]; + maintenanceWindowMinutes: number; + + /** Blocked Providers Config */ + blockedSpIds: Set; + blockedSpAddresses: Set; + + /** Piece Cleanup Config */ + maxDatasetStorageSizeBytes: number; + targetDatasetStorageSizeBytes: number; +}; + +type WalletPrivateKeyNetworkConfig = BaseNetworkConfig & { + walletPrivateKey: `0x${string}`; +}; + +type SessionKeyNetworkConfig = BaseNetworkConfig & { + sessionKeyPrivateKey: `0x${string}`; +}; + +export type INetworkConfig = WalletPrivateKeyNetworkConfig | SessionKeyNetworkConfig; + +export type INetworksConfig = Record; + +export interface IJobsConfig { + /** + * How often the scheduler polls Postgres for due jobs (seconds). + * + * Lower values reduce scheduling latency but increase DB chatter. + */ + schedulerPollSeconds: number; + /** + * How often workers check for new jobs (seconds). + * + * Lower values reduce job pickup latency but increase DB chatter. + */ + workerPollSeconds: number; + /** + * Per-instance pg-boss worker concurrency for the `sp.work` queue. + */ + pgbossLocalConcurrency: number; + /** + * Enables the pg-boss scheduler loop (enqueueing due jobs). + * + * Set to false to run "worker-only" pods that only process existing jobs. + */ + pgbossSchedulerEnabled: boolean; + /** + * Maximum number of pg-boss connections per instance. + * + * Helpful when using a session-mode pooler with a low pool_size (e.g. Supabase). + */ + pgbossPoolMax: number; + /** + * Maximum number of jobs to enqueue per schedule row per poll. + * + * Prevents large backlogs from flooding workers after downtime. + */ + catchupMaxEnqueue: number; + /** + * Per-instance phase offset (seconds) applied when initializing schedules. + * + * Use this to stagger multiple dealbot deployments that are not sharing a DB. + */ + schedulePhaseSeconds: number; + /** + * Random delay (seconds) added when enqueuing jobs. + * + * Helps avoid synchronized bursts across instances. Only used with pg-boss. + */ + enqueueJitterSeconds: number; + /** + * Maximum runtime (seconds) for deal jobs before forced abort. + * + * Uses AbortController to actively cancel job execution. + */ + dealJobTimeoutSeconds: number; + /** + * Maximum runtime (seconds) for data-set creation jobs before forced abort. + * + * Uses AbortController to actively cancel job execution. + */ + dataSetCreationJobTimeoutSeconds: number; + /** + * Maximum runtime (seconds) for retrieval jobs before forced abort. + * + * Uses AbortController to actively cancel job execution. + */ + retrievalJobTimeoutSeconds: number; +} + +export interface IDatasetConfig { + localDatasetsPath: string; + randomDatasetSizes: number[]; +} + +export interface ITimeoutConfig { + connectTimeoutMs: number; + httpRequestTimeoutMs: number; + http2RequestTimeoutMs: number; + ipniVerificationTimeoutMs: number; + ipniVerificationPollingMs: number; +} + +export interface IRetrievalConfig { + ipfsBlockFetchConcurrency: number; +} + +export interface IConfig { + app: IAppConfig; + database: IDatabaseConfig; + networks: INetworksConfig; + activeNetworks: Network[]; + jobs: IJobsConfig; + dataset: IDatasetConfig; + timeouts: ITimeoutConfig; + retrieval: IRetrievalConfig; +} + +export type NetworkDefaults = Pick< + INetworkConfig, + | "dealbotDataSetVersion" + | "checkDatasetCreationFees" + | "useOnlyApprovedProviders" + | "minNumDataSetsForChecks" + | "dealsPerSpPerHour" + | "retrievalsPerSpPerHour" + | "dataSetCreationsPerSpPerHour" + | "pieceCleanupPerSpPerHour" + | "maxPieceCleanupRuntimeSeconds" + | "dataRetentionPollIntervalSeconds" + | "providersRefreshIntervalSeconds" + | "maintenanceWindowsUtc" + | "maintenanceWindowMinutes" + | "maxDatasetStorageSizeBytes" + | "targetDatasetStorageSizeBytes" +>; diff --git a/apps/backend/src/data-retention/data-retention.service.spec.ts b/apps/backend/src/data-retention/data-retention.service.spec.ts index b30ed0aa..aa2da60a 100644 --- a/apps/backend/src/data-retention/data-retention.service.spec.ts +++ b/apps/backend/src/data-retention/data-retention.service.spec.ts @@ -2,7 +2,7 @@ import type { ConfigService } from "@nestjs/config"; import type { Counter, Gauge } from "prom-client"; import { Repository } from "typeorm"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { IConfig } from "../config/app.config.js"; +import type { IConfig } from "../config/types.js"; import type { DataRetentionBaseline } from "../database/entities/data-retention-baseline.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; import { buildCheckMetricLabels } from "../metrics-prometheus/check-metric-labels.js"; @@ -61,11 +61,18 @@ describe("DataRetentionService", () => { beforeEach(() => { configServiceMock = { get: vi.fn((key: keyof IConfig) => { - if (key === "blockchain") { - return { pdpSubgraphEndpoint: "https://example.com/subgraph" }; + if (key === "activeNetworks") { + return ["calibration"]; } - if (key === "spBlocklists") { - return { ids: new Set(), addresses: new Set() }; + if (key === "networks") { + return { + calibration: { + pdpSubgraphEndpoint: "https://example.com/subgraph", + network: "calibration", + blockedSpIds: new Set(), + blockedSpAddresses: new Set(), + }, + }; } return undefined; }), @@ -133,11 +140,13 @@ describe("DataRetentionService", () => { }); it("returns early when pdpSubgraphEndpoint is empty", async () => { - (configServiceMock.get as ReturnType).mockReturnValue({ - pdpSubgraphEndpoint: "", + (configServiceMock.get as ReturnType).mockImplementation((key: keyof IConfig) => { + if (key === "activeNetworks") return ["calibration"]; + if (key === "networks") return { calibration: { pdpSubgraphEndpoint: "", network: "calibration" } }; + return undefined; }); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(pdpSubgraphServiceMock.fetchSubgraphMeta).not.toHaveBeenCalled(); expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).not.toHaveBeenCalled(); @@ -146,34 +155,48 @@ describe("DataRetentionService", () => { it("returns early when no testing providers configured", async () => { walletSdkServiceMock.getTestingProviders.mockReturnValueOnce(null); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).not.toHaveBeenCalled(); }); it("returns early when all providers are blocked for data-retention", async () => { (configServiceMock.get as ReturnType).mockImplementation((key: string) => { - if (key === "blockchain") return { pdpSubgraphEndpoint: "https://example.com/subgraph" }; - if (key === "spBlocklists") return { ids: new Set(), addresses: new Set([PROVIDER_A, PROVIDER_B]) }; + if (key === "networks") + return { + calibration: { + pdpSubgraphEndpoint: "https://example.com/subgraph", + network: "calibration", + blockedSpIds: new Set(), + blockedSpAddresses: new Set([PROVIDER_A, PROVIDER_B]), + }, + }; }); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).not.toHaveBeenCalled(); }); it("excludes blocked providers from data-retention polling while retaining unblocked ones", async () => { (configServiceMock.get as ReturnType).mockImplementation((key: string) => { - if (key === "blockchain") return { pdpSubgraphEndpoint: "https://example.com/subgraph" }; - if (key === "spBlocklists") return { ids: new Set(), addresses: new Set([PROVIDER_A]) }; + if (key === "networks") + return { + calibration: { + pdpSubgraphEndpoint: "https://example.com/subgraph", + network: "calibration", + blockedSpIds: new Set(), + blockedSpAddresses: new Set([PROVIDER_A]), + }, + }; }); pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); const allAddressesPolled: string[] = ( - pdpSubgraphServiceMock.fetchProvidersWithDatasets.mock.calls as [{ addresses: string[] }][] - ).flatMap(([{ addresses }]) => addresses); + pdpSubgraphServiceMock.fetchProvidersWithDatasets.mock.calls as [string, { addresses: string[] }][] + ).flatMap(([, { addresses }]) => addresses); expect(allAddressesPolled).toContain(PROVIDER_B.toLowerCase()); expect(allAddressesPolled).not.toContain(PROVIDER_A.toLowerCase()); }); @@ -181,7 +204,7 @@ describe("DataRetentionService", () => { it("returns early when testing providers array is empty", async () => { walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).not.toHaveBeenCalled(); }); @@ -189,10 +212,10 @@ describe("DataRetentionService", () => { it("sets baseline on first poll without emitting counters (fresh deploy / new provider)", async () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(pdpSubgraphServiceMock.fetchSubgraphMeta).toHaveBeenCalled(); - expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).toHaveBeenCalledWith({ + expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).toHaveBeenCalledWith("https://example.com/subgraph", { blockNumber: 1200, addresses: [PROVIDER_A, PROVIDER_B], }); @@ -206,6 +229,7 @@ describe("DataRetentionService", () => { expect(mockBaselineRepository.upsert).toHaveBeenCalledWith( { providerAddress: PROVIDER_A, + network: "calibration", faultedPeriods: "10", successPeriods: "90", lastBlockNumber: "1200", @@ -217,7 +241,7 @@ describe("DataRetentionService", () => { it("computes deltas correctly on consecutive polls", async () => { // First poll: blockNumber=1200 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); const firstCallCount = counterMock.labels.mock.calls.length; @@ -236,7 +260,7 @@ describe("DataRetentionService", () => { }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll should have incremented counters with the delta expect(counterMock.labels.mock.calls.length).toBeGreaterThan(firstCallCount); @@ -246,11 +270,11 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValue([makeProvider()]); // First poll - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); counterMock.labels.mockClear(); // Second poll with same data and same block number - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // No new increments since deltas are zero expect(counterMock.labels).not.toHaveBeenCalled(); @@ -267,7 +291,7 @@ describe("DataRetentionService", () => { const providerB = makeProvider({ address: PROVIDER_B, totalFaultedPeriods: 20n }); pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([providerA, providerB]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); const labelCalls = counterMock.labels.mock.calls; const providerAFaulted = labelCalls.some( @@ -289,12 +313,13 @@ describe("DataRetentionService", () => { const provider = makeProvider(); pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([provider]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // totalFaultedPeriods = 10, totalProvingPeriods = 100 // confirmedTotalSuccess = 100 - 10 = 90 expect(counterMock.labels).toHaveBeenCalledWith({ checkType: "dataRetention", + network: "calibration", providerId: "1", providerName: "Provider A", providerStatus: "approved", @@ -302,6 +327,7 @@ describe("DataRetentionService", () => { }); expect(counterMock.labels).toHaveBeenCalledWith({ checkType: "dataRetention", + network: "calibration", providerId: "1", providerName: "Provider A", providerStatus: "approved", @@ -312,7 +338,7 @@ describe("DataRetentionService", () => { it("handles empty providers array without errors", async () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(counterMock.labels).not.toHaveBeenCalled(); }); @@ -326,12 +352,13 @@ describe("DataRetentionService", () => { const provider = makeProvider(); pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([provider]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // totalFaultedPeriods = 10, totalProvingPeriods = 100 // confirmedTotalSuccess = 100 - 10 = 90 expect(counterMock.labels).toHaveBeenCalledWith({ checkType: "dataRetention", + network: "calibration", providerId: "1", providerName: "Provider A", providerStatus: "approved", @@ -339,6 +366,7 @@ describe("DataRetentionService", () => { }); expect(counterMock.labels).toHaveBeenCalledWith({ checkType: "dataRetention", + network: "calibration", providerId: "1", providerName: "Provider A", providerStatus: "approved", @@ -350,7 +378,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockRejectedValueOnce(new Error("subgraph down")); // Should not throw - await expect(service.pollDataRetention()).resolves.toBeUndefined(); + await expect(service.pollDataRetention("calibration")).resolves.toBeUndefined(); }); it("resets baseline on negative deltas without incrementing counters", async () => { @@ -358,14 +386,14 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 100n, totalProvingPeriods: 200n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); counterMock.labels.mockClear(); // Second poll: lower values (e.g., chain reorg or subgraph correction) pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 50n, totalProvingPeriods: 100n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Both deltas are negative, so counters should not be incremented expect(counterMock.labels).not.toHaveBeenCalled(); @@ -374,7 +402,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 52n, totalProvingPeriods: 105n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should now increment based on new baseline (52-50=2 faulted, 55-50=5 success) expect(counterMock.labels).toHaveBeenCalled(); @@ -393,7 +421,7 @@ describe("DataRetentionService", () => { makeProvider({ totalFaultedPeriods: largeValue, totalProvingPeriods: largeValue * 2n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should have been called multiple times (chunked increments) expect(counterMock.inc).toHaveBeenCalled(); @@ -417,7 +445,7 @@ describe("DataRetentionService", () => { makeProvider({ totalFaultedPeriods: maxSafeInt, totalProvingPeriods: maxSafeInt * 2n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should increment without chunking since it's exactly at the boundary expect(counterMock.inc).toHaveBeenCalled(); @@ -435,7 +463,7 @@ describe("DataRetentionService", () => { }); pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([provider]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Uses subgraph totals directly: faulted=5*5=25, success=45*5=225 const incCalls = counterMock.inc.mock.calls; @@ -454,16 +482,17 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValue([]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should be called twice: once for first 50, once for remaining 25 expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).toHaveBeenCalledTimes(2); - expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).toHaveBeenNthCalledWith(1, { - addresses: expect.arrayContaining([expect.any(String)]), - blockNumber: 1200, - }); - expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets.mock.calls[0][0].addresses).toHaveLength(50); - expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets.mock.calls[1][0].addresses).toHaveLength(25); + expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).toHaveBeenNthCalledWith( + 1, + "https://example.com/subgraph", + { addresses: expect.arrayContaining([expect.any(String)]), blockNumber: 1200 }, + ); + expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets.mock.calls[0][1].addresses).toHaveLength(50); + expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets.mock.calls[1][1].addresses).toHaveLength(25); }); it("continues processing next batch if one batch fails", async () => { @@ -480,7 +509,7 @@ describe("DataRetentionService", () => { .mockRejectedValueOnce(new Error("Subgraph timeout")) .mockResolvedValueOnce([]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Both batches should be attempted expect(pdpSubgraphServiceMock.fetchProvidersWithDatasets).toHaveBeenCalledTimes(2); @@ -491,7 +520,7 @@ describe("DataRetentionService", () => { const PROVIDER_C = "0x1234567890123456789012345678901234567890"; pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_C })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should not increment counters for missing provider expect(counterMock.labels).not.toHaveBeenCalled(); @@ -505,7 +534,7 @@ describe("DataRetentionService", () => { makeProvider({ address: PROVIDER_B }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Repository should not be queried since no stale providers expect(mockSPRepository.find).not.toHaveBeenCalled(); @@ -514,7 +543,7 @@ describe("DataRetentionService", () => { it("successfully cleans up stale provider with valid database entry", async () => { // First poll: establish baseline for PROVIDER_A pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: PROVIDER_A removed from active list, only PROVIDER_B active walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -537,23 +566,28 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should fetch stale provider info from database expect(mockSPRepository.find).toHaveBeenCalledWith({ - where: { address: expect.anything() }, + where: { + network: "calibration", + address: expect.anything(), + }, select: ["address", "providerId", "name", "isApproved"], }); // Should remove all counter label combinations const approvedLabels = buildCheckMetricLabels({ checkType: "dataRetention", + network: "calibration", providerId: 1n, providerName: "Provider A", providerIsApproved: true, }); const unapprovedLabels = buildCheckMetricLabels({ checkType: "dataRetention", + network: "calibration", providerId: 1n, providerName: "Provider A", providerIsApproved: false, @@ -567,7 +601,7 @@ describe("DataRetentionService", () => { it("skips cleanup entirely when database fetch fails", async () => { // First poll: establish baseline pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed, but DB fails walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -583,7 +617,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should attempt to fetch from database expect(mockSPRepository.find).toHaveBeenCalled(); @@ -606,7 +640,7 @@ describe("DataRetentionService", () => { ]); counterMock.labels.mockClear(); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should compute delta from original baseline, not from zero expect(counterMock.labels).toHaveBeenCalled(); @@ -615,7 +649,7 @@ describe("DataRetentionService", () => { it("retains baseline when provider not found in database", async () => { // First poll: establish baseline pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed from active list walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -632,7 +666,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should NOT remove counters (provider not in DB) expect(counterMock.remove).not.toHaveBeenCalled(); @@ -652,7 +686,7 @@ describe("DataRetentionService", () => { ]); counterMock.labels.mockClear(); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should use old baseline (delta from 10 to 12 = 2) expect(counterMock.labels).toHaveBeenCalled(); @@ -661,7 +695,7 @@ describe("DataRetentionService", () => { it("retains baseline when provider has null providerId", async () => { // First poll: establish baseline pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -685,7 +719,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should NOT remove counters (missing providerId) expect(counterMock.remove).not.toHaveBeenCalled(); @@ -694,7 +728,7 @@ describe("DataRetentionService", () => { it("retains baseline when counter removal throws error", async () => { // First poll: establish baseline pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -722,7 +756,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should attempt removal expect(counterMock.remove).toHaveBeenCalled(); @@ -742,7 +776,7 @@ describe("DataRetentionService", () => { ]); counterMock.labels.mockClear(); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should compute delta from original baseline expect(counterMock.labels).toHaveBeenCalled(); @@ -764,7 +798,7 @@ describe("DataRetentionService", () => { makeProvider({ address: PROVIDER_C }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: only PROVIDER_A remains active walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -778,11 +812,14 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should fetch both stale providers in one query expect(mockSPRepository.find).toHaveBeenCalledWith({ - where: { address: expect.anything() }, + where: { + network: "calibration", + address: expect.anything(), + }, select: ["address", "providerId", "name", "isApproved"], }); @@ -793,7 +830,7 @@ describe("DataRetentionService", () => { it("skips cleanup when processing errors occurred", async () => { // First poll: establish baseline pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed, but processing has errors walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -803,7 +840,7 @@ describe("DataRetentionService", () => { // Simulate processing error pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockRejectedValueOnce(new Error("Processing failed")); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should NOT attempt cleanup due to processing errors expect(mockSPRepository.find).not.toHaveBeenCalled(); @@ -822,7 +859,7 @@ describe("DataRetentionService", () => { makeProvider({ address: PROVIDER_MIXED_CASE.toLowerCase() as `0x${string}` }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: provider removed walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -840,7 +877,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should successfully find and clean up provider despite case difference expect(counterMock.remove).toHaveBeenCalled(); @@ -864,7 +901,7 @@ describe("DataRetentionService", () => { // With DB baseline: faultedDelta = 10 - 10 = 0, successDelta = 90 - 90 = 0 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Key assertion: counters should NOT be incremented because deltas are zero expect(counterMock.labels).not.toHaveBeenCalled(); @@ -886,7 +923,7 @@ describe("DataRetentionService", () => { // faultedDelta = 10 - 8 = 2, successDelta = 90 - 85 = 5 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should increment by only the delta, not the full cumulative values expect(counterMock.labels).toHaveBeenCalledWith(expect.objectContaining({ value: "failure" })); @@ -901,9 +938,9 @@ describe("DataRetentionService", () => { it("only loads baselines from DB once across multiple polls", async () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValue([makeProvider()]); - await service.pollDataRetention(); - await service.pollDataRetention(); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); + await service.pollDataRetention("calibration"); + await service.pollDataRetention("calibration"); // DB find should only be called once (lazy init) expect(mockBaselineRepository.find).toHaveBeenCalledTimes(1); @@ -922,13 +959,13 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValue([makeProvider()]); // First poll: DB load fails, poll bails out to avoid emitting bloated values - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(mockBaselineRepository.find).toHaveBeenCalledTimes(1); expect(pdpSubgraphServiceMock.fetchSubgraphMeta).not.toHaveBeenCalled(); expect(counterMock.labels).not.toHaveBeenCalled(); // Second poll: DB load succeeds, baselines restored, normal delta computation - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(mockBaselineRepository.find).toHaveBeenCalledTimes(2); // Deltas from DB baseline: faultedDelta = 10 - 10 = 0, successDelta = 90 - 90 = 0 expect(counterMock.labels).not.toHaveBeenCalled(); @@ -938,7 +975,7 @@ describe("DataRetentionService", () => { // First poll: fresh deploy, no baselines in DB // Baseline set to: faultedPeriods=10, successPeriods=90 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); counterMock.labels.mockClear(); counterMock.inc.mockClear(); @@ -950,7 +987,7 @@ describe("DataRetentionService", () => { makeProvider({ totalFaultedPeriods: 12n, totalProvingPeriods: 105n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // faultedDelta = (12 - 10) * 5 = 10, successDelta = ((105 - 12) - 90) * 5 = 15 expect(counterMock.labels).toHaveBeenCalled(); @@ -961,7 +998,7 @@ describe("DataRetentionService", () => { it("deletes baseline from DB when stale provider is cleaned up", async () => { // First poll: establish baseline for PROVIDER_A pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: PROVIDER_A removed from active list walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -974,10 +1011,13 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should delete the baseline from DB - expect(mockBaselineRepository.delete).toHaveBeenCalledWith({ providerAddress: PROVIDER_A }); + expect(mockBaselineRepository.delete).toHaveBeenCalledWith({ + providerAddress: PROVIDER_A, + network: "calibration", + }); }); }); @@ -987,7 +1027,7 @@ describe("DataRetentionService", () => { // estimatedOverduePeriods = (1200 - 901) / 100 = 2.99 -> 2 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(gaugeMock.labels).toHaveBeenCalledWith( expect.objectContaining({ @@ -1004,7 +1044,7 @@ describe("DataRetentionService", () => { // nextDeadline=2000 > currentBlock=1200 pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ proofSets: [] })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(gaugeMock.set).toHaveBeenCalledWith(0); }); @@ -1014,7 +1054,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 100n, totalProvingPeriods: 200n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); gaugeMock.labels.mockClear(); gaugeMock.set.mockClear(); @@ -1022,7 +1062,7 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([ makeProvider({ totalFaultedPeriods: 50n, totalProvingPeriods: 100n }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Gauge should still be emitted despite negative deltas on counters expect(gaugeMock.labels).toHaveBeenCalled(); @@ -1032,7 +1072,7 @@ describe("DataRetentionService", () => { it("naturally resets gauge to 0 when subgraph catches up", async () => { // First poll: provider is overdue (currentBlock=1200, nextDeadline=1000) pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider()]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); expect(gaugeMock.set).toHaveBeenCalledWith(2); @@ -1048,7 +1088,7 @@ describe("DataRetentionService", () => { }), ]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Gauge should reset to 0 because nextDeadline (1300) > currentBlock (1200) expect(gaugeMock.set).toHaveBeenCalledWith(0); @@ -1057,7 +1097,7 @@ describe("DataRetentionService", () => { it("removes overdue gauge when stale provider is cleaned up", async () => { // First poll: establish baseline for PROVIDER_A pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_A })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Second poll: PROVIDER_A removed from active list walletSdkServiceMock.getTestingProviders.mockReturnValueOnce([ @@ -1070,17 +1110,19 @@ describe("DataRetentionService", () => { pdpSubgraphServiceMock.fetchProvidersWithDatasets.mockResolvedValueOnce([makeProvider({ address: PROVIDER_B })]); - await service.pollDataRetention(); + await service.pollDataRetention("calibration"); // Should remove overdue gauge for stale provider (both approved and unapproved labels) const approvedLabels = buildCheckMetricLabels({ checkType: "dataRetention", + network: "calibration", providerId: 1n, providerName: "Provider A", providerIsApproved: true, }); const unapprovedLabels = buildCheckMetricLabels({ checkType: "dataRetention", + network: "calibration", providerId: 1n, providerName: "Provider A", providerIsApproved: false, diff --git a/apps/backend/src/data-retention/data-retention.service.ts b/apps/backend/src/data-retention/data-retention.service.ts index 1a824f0f..899fc714 100644 --- a/apps/backend/src/data-retention/data-retention.service.ts +++ b/apps/backend/src/data-retention/data-retention.service.ts @@ -6,7 +6,8 @@ import { Counter, Gauge } from "prom-client"; import { Raw, Repository } from "typeorm"; import { toStructuredError } from "../common/logging.js"; import { isSpBlocked } from "../common/sp-blocklist.js"; -import { IConfig } from "../config/app.config.js"; +import type { Network } from "../common/types.js"; +import type { IConfig, INetworkConfig, INetworksConfig } from "../config/types.js"; import { DataRetentionBaseline } from "../database/entities/data-retention-baseline.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; import { buildCheckMetricLabels, CheckMetricLabels } from "../metrics-prometheus/check-metric-labels.js"; @@ -24,7 +25,7 @@ export class DataRetentionService { private static readonly CHALLENGES_PER_PROVING_PERIOD = 5n; /** - * Tracks cumulative faulted/success period totals per provider address. + * Tracks cumulative faulted/success period totals keyed by "network:providerAddress". * Used to compute deltas between consecutive polls for Prometheus counter increments. * Populated from the database on first poll, then kept in sync. * Note: Baselines are stored in periods, but emitted metrics are converted to challenges @@ -38,8 +39,8 @@ export class DataRetentionService { } >; - /** Whether baselines have been loaded from the database */ - private baselinesLoaded = false; + /** Per-network baseline load flags */ + private readonly baselinesLoadedByNetwork: Map = new Map(); constructor( private readonly configService: ConfigService, @@ -57,32 +58,41 @@ export class DataRetentionService { this.providerCumulativeTotals = new Map(); } + private cumulativeTotalsKey(network: Network, address: string): string { + return `${network}:${address}`; + } + /** * Polls the PDP subgraph for provider proof-set data, computes proving period deltas, * converts them to challenge counts, and increments Prometheus counters with the * challenge delta since the last poll. */ - async pollDataRetention(): Promise { - const pdpSubgraphEndpoint = this.configService.get("blockchain").pdpSubgraphEndpoint; + async pollDataRetention(network: Network): Promise { + const networkConfig = this.configService.get("networks")[network]; + const pdpSubgraphEndpoint = networkConfig.pdpSubgraphEndpoint; if (!pdpSubgraphEndpoint) { this.logger.warn({ event: "pdp_subgraph_endpoint_not_configured", message: "No PDP subgraph endpoint configured", + network, }); return; } - await this.loadBaselinesFromDb(); + await this.loadBaselinesFromDb(network); - if (!this.baselinesLoaded) { + if (!this.baselinesLoadedByNetwork.get(network)) { // Cannot safely compute deltas without baselines — would emit full cumulative history + this.logger.log({ + event: "failed_to_load_baselines_by_network", + }); return; } try { - const subgraphMeta = await this.pdpSubgraphService.fetchSubgraphMeta(); - const allProviderInfos = this.walletSdkService.getTestingProviders(); - const spBlocklists = this.configService.get("spBlocklists"); + const subgraphMeta = await this.pdpSubgraphService.fetchSubgraphMeta(pdpSubgraphEndpoint); + const allProviderInfos = this.walletSdkService.getTestingProviders(network); + const spBlocklists = this.configService.get("networks")[network]; const providerInfos = allProviderInfos?.filter((p) => !isSpBlocked(spBlocklists, p.serviceProvider, p.id)); if (!providerInfos || providerInfos.length === 0) { @@ -109,7 +119,7 @@ export class DataRetentionService { ); try { - const providersFromSubgraph = await this.pdpSubgraphService.fetchProvidersWithDatasets({ + const providersFromSubgraph = await this.pdpSubgraphService.fetchProvidersWithDatasets(pdpSubgraphEndpoint, { blockNumber, addresses: batchAddresses, }); @@ -125,7 +135,7 @@ export class DataRetentionService { ), ); } - return this.processProvider(provider, providerInfo, blockNumberBigInt); + return this.processProvider(provider, providerInfo, blockNumberBigInt, network); }), ); @@ -142,11 +152,12 @@ export class DataRetentionService { providerAddress: addr, providerId: providerInfo?.id, providerName: providerInfo?.name, + network, error: toStructuredError(result.reason), }); } else { const addr = providersFromSubgraph[index].address.toLowerCase(); - upsertPromises.push(this.persistBaseline(addr, result.value, blockNumberBigInt)); + upsertPromises.push(this.persistBaseline(addr, result.value, blockNumberBigInt, network)); } }); @@ -175,17 +186,19 @@ export class DataRetentionService { // Only cleanup stale providers after successful poll to preserve baselines during transient failures if (!hasProcessingErrors) { - await this.cleanupStaleProviders(providerAddresses); + await this.cleanupStaleProviders(providerAddresses, network); } else { this.logger.warn({ event: "stale_provider_cleanup_skipped", message: "Skipping stale provider cleanup due to processing errors", + network, }); } } catch (error) { this.logger.error({ event: "data_retention_poll_failed", message: "Failed to poll data retention", + network, error: toStructuredError(error), }); } @@ -201,12 +214,13 @@ export class DataRetentionService { * * @param activeProviderAddresses - Array of currently active provider addresses (normalized to lowercase) */ - private async cleanupStaleProviders(activeProviderAddresses: string[]): Promise { + private async cleanupStaleProviders(activeProviderAddresses: string[], network: Network): Promise { const activeAddressSet = new Set(activeProviderAddresses); const staleAddresses: string[] = []; - for (const [address] of this.providerCumulativeTotals) { - if (!activeAddressSet.has(address)) { + for (const [key] of this.providerCumulativeTotals) { + const [keyNetwork, address] = key.split(":", 2); + if (keyNetwork === network && address && !activeAddressSet.has(address)) { staleAddresses.push(address); } } @@ -224,7 +238,10 @@ export class DataRetentionService { let staleProviders: StorageProvider[] = []; try { staleProviders = await this.storageProviderRepository.find({ - where: { address: Raw((alias) => `LOWER(${alias}) IN (:...addresses)`, { addresses: staleAddresses }) }, + where: { + network, + address: Raw((alias) => `LOWER(${alias}) IN (:...addresses)`, { addresses: staleAddresses }), + }, select: ["address", "providerId", "name", "isApproved"], }); } catch (error) { @@ -240,17 +257,20 @@ export class DataRetentionService { const providerLookup = new Map(staleProviders.map((p) => [p.address.toLowerCase(), p])); for (const address of staleAddresses) { + const totalsKey = this.cumulativeTotalsKey(network, address); try { const provider = providerLookup.get(address); if (provider && provider.providerId != null) { const approvedLabels = buildCheckMetricLabels({ + network, checkType: "dataRetention", providerId: provider.providerId, providerName: provider.name, providerIsApproved: true, }); const unapprovedLabels = buildCheckMetricLabels({ + network, checkType: "dataRetention", providerId: provider.providerId, providerName: provider.name, @@ -266,14 +286,15 @@ export class DataRetentionService { this.estimatedOverduePeriodsGauge.remove(unapprovedLabels); // Only delete local memory if Prometheus removal succeeded without throwing - this.providerCumulativeTotals.delete(address); + this.providerCumulativeTotals.delete(totalsKey); // Also remove persisted baseline from DB - this.baselineRepository.delete({ providerAddress: address }).catch((err) => { + this.baselineRepository.delete({ providerAddress: address, network }).catch((err) => { this.logger.warn({ event: "baseline_db_delete_failed", message: "Failed to delete persisted baseline for stale provider", providerAddress: address, + network, error: toStructuredError(err), }); }); @@ -320,6 +341,7 @@ export class DataRetentionService { provider: ProviderDataSetResponse["providers"][number], pdpProvider: PDPProviderEx, currentBlock: bigint, + network: Network, ): Promise<{ faultedPeriods: bigint; successPeriods: bigint }> { const { address, totalFaultedPeriods, totalProvingPeriods, proofSets } = provider; // Note: Query filters proofSets with nextDeadline_lt: $blockNumber, so all deadlines are in the past @@ -333,7 +355,8 @@ export class DataRetentionService { const confirmedTotalSuccess = totalProvingPeriods - totalFaultedPeriods; const normalizedAddress = address.toLowerCase(); - const previous = this.providerCumulativeTotals.get(normalizedAddress); + const totalsKey = this.cumulativeTotalsKey(network, normalizedAddress); + const previous = this.providerCumulativeTotals.get(totalsKey); const newBaseline = { faultedPeriods: totalFaultedPeriods, @@ -341,6 +364,7 @@ export class DataRetentionService { }; const providerLabels = buildCheckMetricLabels({ + network, checkType: "dataRetention", providerId: pdpProvider.id, providerName: pdpProvider.name, @@ -371,7 +395,7 @@ export class DataRetentionService { faultedPeriods: totalFaultedPeriods.toString(), successPeriods: confirmedTotalSuccess.toString(), }); - this.providerCumulativeTotals.set(normalizedAddress, newBaseline); + this.providerCumulativeTotals.set(totalsKey, newBaseline); return newBaseline; } @@ -393,7 +417,7 @@ export class DataRetentionService { successChallengesDelta: successChallengesDelta.toString(), }); // Reset baseline without incrementing counters - this.providerCumulativeTotals.set(normalizedAddress, newBaseline); + this.providerCumulativeTotals.set(totalsKey, newBaseline); return newBaseline; } @@ -405,38 +429,40 @@ export class DataRetentionService { this.safeIncrementCounter(this.dataSetChallengeStatusCounter, providerLabels, "success", successChallengesDelta); } - this.providerCumulativeTotals.set(normalizedAddress, newBaseline); + this.providerCumulativeTotals.set(totalsKey, newBaseline); return newBaseline; } /** * Loads persisted baselines from the database into the in-memory map. - * Only runs once; if the DB read fails, retries on the next poll. + * Only runs once per network; if the DB read fails, retries on the next poll. */ - private async loadBaselinesFromDb(): Promise { - if (this.baselinesLoaded) { + private async loadBaselinesFromDb(network: Network): Promise { + if (this.baselinesLoadedByNetwork.get(network)) { return; } try { - const rows = await this.baselineRepository.find(); + const rows = await this.baselineRepository.find({ where: { network } }); for (const row of rows) { - this.providerCumulativeTotals.set(row.providerAddress, { + this.providerCumulativeTotals.set(this.cumulativeTotalsKey(network, row.providerAddress), { faultedPeriods: BigInt(row.faultedPeriods), successPeriods: BigInt(row.successPeriods), }); } - this.baselinesLoaded = true; + this.baselinesLoadedByNetwork.set(network, true); this.logger.log({ event: "baselines_loaded_from_db", message: "Loaded baseline(s) from database", + network, baselineCount: rows.length, }); } catch (error) { this.logger.error({ event: "baseline_load_failed", message: "Failed to load baselines from database. Will retry on next poll.", + network, error: toStructuredError(error), }); } @@ -449,10 +475,12 @@ export class DataRetentionService { providerAddress: string, baseline: { faultedPeriods: bigint; successPeriods: bigint }, blockNumber: bigint, + network: Network, ): Promise { await this.baselineRepository.upsert( { providerAddress, + network, faultedPeriods: baseline.faultedPeriods.toString(), successPeriods: baseline.successPeriods.toString(), lastBlockNumber: blockNumber.toString(), diff --git a/apps/backend/src/dataSource/dataSource.service.spec.ts b/apps/backend/src/dataSource/dataSource.service.spec.ts index 8e8848c1..bcc2e3c9 100644 --- a/apps/backend/src/dataSource/dataSource.service.spec.ts +++ b/apps/backend/src/dataSource/dataSource.service.spec.ts @@ -1,7 +1,7 @@ import { ConfigService } from "@nestjs/config"; import * as fs from "fs"; import * as path from "path"; -import { IConfig } from "src/config/app.config.js"; +import type { IConfig } from "src/config/types.js"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DataSourceService } from "./dataSource.service.js"; diff --git a/apps/backend/src/dataSource/dataSource.service.ts b/apps/backend/src/dataSource/dataSource.service.ts index b03c7400..28d173a1 100644 --- a/apps/backend/src/dataSource/dataSource.service.ts +++ b/apps/backend/src/dataSource/dataSource.service.ts @@ -6,7 +6,7 @@ import * as path from "path"; import { toStructuredError } from "../common/logging.js"; import { writeWithBackpressure } from "../common/stream-utils.js"; import type { DataFile } from "../common/types.js"; -import type { IConfig, IDatasetConfig } from "../config/app.config.js"; +import type { IConfig, IDatasetConfig } from "../config/types.js"; @Injectable() export class DataSourceService { diff --git a/apps/backend/src/database/database.module.ts b/apps/backend/src/database/database.module.ts index 9249c3a9..2f9bdaab 100644 --- a/apps/backend/src/database/database.module.ts +++ b/apps/backend/src/database/database.module.ts @@ -6,7 +6,7 @@ import { DataSource, type DataSourceOptions } from "typeorm"; import { fileURLToPath } from "url"; import { toStructuredError } from "../common/logging.js"; import { createPinoExitLogger } from "../common/pino.config.js"; -import type { IAppConfig, IConfig, IDatabaseConfig } from "../config/app.config.js"; +import type { IAppConfig, IConfig, IDatabaseConfig } from "../config/types.js"; import { DataRetentionBaseline } from "./entities/data-retention-baseline.entity.js"; import { Deal } from "./entities/deal.entity.js"; import { JobScheduleState } from "./entities/job-schedule-state.entity.js"; diff --git a/apps/backend/src/deal-addons/strategies/ipni.strategy.spec.ts b/apps/backend/src/deal-addons/strategies/ipni.strategy.spec.ts index fdd9c304..0621410c 100644 --- a/apps/backend/src/deal-addons/strategies/ipni.strategy.spec.ts +++ b/apps/backend/src/deal-addons/strategies/ipni.strategy.spec.ts @@ -1,6 +1,7 @@ import { CID } from "multiformats/cid"; import type { Mock } from "vitest"; import { describe, expect, it, vi } from "vitest"; +import { Network } from "../../common/types.js"; import { Deal } from "../../database/entities/deal.entity.js"; import { StorageProvider } from "../../database/entities/storage-provider.entity.js"; import { IpniStatus, ServiceType } from "../../database/types.js"; @@ -12,6 +13,7 @@ import { IpniAddonStrategy } from "./ipni.strategy.js"; describe("IpniAddonStrategy getPieceStatus", () => { type DealForMetrics = { spAddress?: string; + network: Network; storageProvider?: { providerId?: bigint; name?: string; @@ -44,6 +46,7 @@ describe("IpniAddonStrategy getPieceStatus", () => { Object.assign(new Deal(), { id: "deal-1", spAddress: "0xsp", + network: "calibration", fileName: "file", fileSize: 1, walletAddress: "0xwallet", @@ -67,6 +70,7 @@ describe("IpniAddonStrategy getPieceStatus", () => { buildLabelsForDeal: vi.fn().mockImplementation((deal: DealForMetrics) => { if (!deal?.spAddress) return null; return buildCheckMetricLabels({ + network: deal.network, checkType: "dataStorage", providerId: deal.storageProvider?.providerId, providerName: deal.storageProvider?.name, @@ -254,6 +258,7 @@ describe("IpniAddonStrategy getPieceStatus", () => { const labels = { checkType: "dataStorage", + network: "calibration", providerId: "9", providerName: "SP", providerStatus: "approved", @@ -343,6 +348,7 @@ describe("IpniAddonStrategy getPieceStatus", () => { const labels = { checkType: "dataStorage", + network: "calibration", providerId: "9", providerName: "SP", providerStatus: "approved", @@ -385,6 +391,7 @@ describe("IpniAddonStrategy getPieceStatus", () => { const labels = { checkType: "dataStorage", + network: "calibration", providerId: "9", providerName: "SP", providerStatus: "approved", @@ -424,6 +431,7 @@ describe("IpniAddonStrategy getPieceStatus", () => { const labels = { checkType: "dataStorage", + network: "calibration", providerId: "9", providerName: "SP", providerStatus: "approved", @@ -464,6 +472,7 @@ describe("IpniAddonStrategy getPieceStatus", () => { const labels = { checkType: "dataStorage", + network: "calibration", providerId: "9", providerName: "SP", providerStatus: "approved", diff --git a/apps/backend/src/deal-addons/strategies/ipni.strategy.ts b/apps/backend/src/deal-addons/strategies/ipni.strategy.ts index ab61d922..c2a6b656 100644 --- a/apps/backend/src/deal-addons/strategies/ipni.strategy.ts +++ b/apps/backend/src/deal-addons/strategies/ipni.strategy.ts @@ -8,7 +8,7 @@ import type { Repository } from "typeorm"; import { delay } from "../../common/abort-utils.js"; import { buildUnixfsCar } from "../../common/car-utils.js"; import { type DealLogContext, toStructuredError } from "../../common/logging.js"; -import type { IConfig } from "../../config/app.config.js"; +import type { IConfig } from "../../config/types.js"; import { Deal } from "../../database/entities/deal.entity.js"; import type { DealMetadata, IpniMetadata } from "../../database/types.js"; import { IpniStatus, ServiceType } from "../../database/types.js"; diff --git a/apps/backend/src/deal/deal.service.spec.ts b/apps/backend/src/deal/deal.service.spec.ts index 77ad2eae..31a8fd49 100644 --- a/apps/backend/src/deal/deal.service.spec.ts +++ b/apps/backend/src/deal/deal.service.spec.ts @@ -101,11 +101,24 @@ describe("DealService", () => { metricsStartOffsetSeconds: 900, }; } - if (key === "blockchain") { + if (key === "activeNetworks") { + return ["calibration"]; + } + if (key === "networks") { return { - walletPrivateKey: generatePrivateKey(), - network: "calibration", - walletAddress: "0x123", + calibration: { + walletPrivateKey: generatePrivateKey(), + network: "calibration", + walletAddress: "0x123", + checkDatasetCreationFees: false, + useOnlyApprovedProviders: false, + minNumDataSetsForChecks: 1, + dealsPerSpPerHour: 4, + retrievalsPerSpPerHour: 2, + dataSetCreationsPerSpPerHour: 1, + dataRetentionPollIntervalSeconds: 3600, + providersRefreshIntervalSeconds: 14400, + }, }; } return undefined; @@ -263,7 +276,13 @@ describe("DealService", () => { testedAt: new Date(), }); - const deal = await service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload); + const deal = await service.createDeal( + mockSynapseInstance, + mockProviderInfo, + mockDealInput, + uploadPayload, + "calibration", + ); expect(createContextMock).toHaveBeenCalledWith( expect.objectContaining({ @@ -353,10 +372,17 @@ describe("DealService", () => { }; }); - const deal = await service.createDeal(mockSynapseInstance, providerInfo, mockDealInput, uploadPayload); + const deal = await service.createDeal( + mockSynapseInstance, + providerInfo, + mockDealInput, + uploadPayload, + "calibration", + ); const labels = { checkType: "dataStorage", + network: "calibration", providerId: "42", providerName: "Test Provider", providerStatus: "approved", @@ -418,7 +444,7 @@ describe("DealService", () => { }); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, "calibration"), ).rejects.toThrow("Dealbot did not receive onProgress events during upload"); expect(mockDataStorageMetrics.observeIngestMs).not.toHaveBeenCalled(); @@ -458,7 +484,13 @@ describe("DealService", () => { testedAt: new Date(), }); - const deal = await service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload); + const deal = await service.createDeal( + mockSynapseInstance, + mockProviderInfo, + mockDealInput, + uploadPayload, + "calibration", + ); expect(deal.ingestLatencyMs).toBeNull(); expect(deal.ingestThroughputBps).toBeNull(); @@ -510,7 +542,13 @@ describe("DealService", () => { testedAt: new Date(), }); - const deal = await service.createDeal(mockSynapseInstance, mockProviderInfo, zeroSizeDealInput, uploadPayload); + const deal = await service.createDeal( + mockSynapseInstance, + mockProviderInfo, + zeroSizeDealInput, + uploadPayload, + "calibration", + ); expect(deal.ingestLatencyMs).toBe(1000); expect(deal.ingestThroughputBps).toBeNull(); @@ -537,12 +575,13 @@ describe("DealService", () => { (executeUpload as Mock).mockRejectedValue(new Error("timed out waiting for upload")); - await expect(service.createDeal(mockSynapseInstance, providerInfo, mockDealInput, uploadPayload)).rejects.toThrow( - "timed out", - ); + await expect( + service.createDeal(mockSynapseInstance, providerInfo, mockDealInput, uploadPayload, "calibration"), + ).rejects.toThrow("timed out"); const labels = { checkType: "dataStorage", + network: "calibration", providerId: "7", providerName: "Test Provider", providerStatus: "unapproved", @@ -571,12 +610,13 @@ describe("DealService", () => { (executeUpload as Mock).mockRejectedValue(new Error("connection refused")); - await expect(service.createDeal(mockSynapseInstance, providerInfo, mockDealInput, uploadPayload)).rejects.toThrow( - "connection refused", - ); + await expect( + service.createDeal(mockSynapseInstance, providerInfo, mockDealInput, uploadPayload, "calibration"), + ).rejects.toThrow("connection refused"); const labels = { checkType: "dataStorage", + network: "calibration", providerId: "7", providerName: "Test Provider", providerStatus: "unapproved", @@ -627,6 +667,7 @@ describe("DealService", () => { mockProviderInfo, mockDealInput, uploadPayload, + "calibration", undefined, abortController.signal, ), @@ -659,7 +700,7 @@ describe("DealService", () => { (executeUpload as Mock).mockRejectedValue(error); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, "calibration"), ).rejects.toThrow("Upload failed"); expect(mockDeal.status).toBe(DealStatus.FAILED); @@ -677,7 +718,7 @@ describe("DealService", () => { createContextMock.mockRejectedValue(error); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, "calibration"), ).rejects.toThrow("Storage creation failed"); expect(mockDeal.status).toBe(DealStatus.FAILED); @@ -695,7 +736,7 @@ describe("DealService", () => { createContextMock.mockRejectedValue(error); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, "calibration"), ).rejects.toThrow("secret-token"); expect(mockDeal.status).toBe(DealStatus.FAILED); @@ -719,6 +760,7 @@ describe("DealService", () => { mockProviderInfo, mockDealInput, uploadPayload, + "calibration", undefined, abortController.signal, ), @@ -753,7 +795,7 @@ describe("DealService", () => { dealAddonsMock.handleStored.mockRejectedValueOnce(ipniError); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, "calibration"), ).rejects.toThrow("IPNI verification failed"); expect(mockDeal.status).toBe(DealStatus.FAILED); @@ -789,7 +831,7 @@ describe("DealService", () => { }); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, "calibration"), ).rejects.toThrow("Retrieval gate failed"); expect(mockDeal.status).toBe(DealStatus.FAILED); @@ -821,11 +863,12 @@ describe("DealService", () => { ); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, "calibration"), ).rejects.toThrow("timed out"); const labels = { checkType: "dataStorage", + network: "calibration", providerId: "42", providerName: "Test Provider", providerStatus: "approved", @@ -870,11 +913,12 @@ describe("DealService", () => { retrievalAddonsMock.testAllRetrievalMethods.mockRejectedValue(new Error("retrieval timed out")); await expect( - service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload), + service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload, "calibration"), ).rejects.toThrow("retrieval timed out"); const labels = { checkType: "dataStorage", + network: "calibration", providerId: "42", providerName: "Test Provider", providerStatus: "approved", @@ -918,7 +962,13 @@ describe("DealService", () => { testedAt: new Date(), }); - const deal = await service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload); + const deal = await service.createDeal( + mockSynapseInstance, + mockProviderInfo, + mockDealInput, + uploadPayload, + "calibration", + ); expect(deal.dealLatencyMs).toBeGreaterThanOrEqual(0); expect(deal.dealLatencyWithIpniMs).toBeUndefined(); @@ -958,11 +1008,24 @@ describe("DealService", () => { }); const createServiceWithVersion = async (dealbotDataSetVersion: string | undefined) => { - mockConfigService.get.mockReturnValue({ + const versionNetworkConfig = { walletPrivateKey: generatePrivateKey(), network: "calibration", walletAddress: "0x123", + checkDatasetCreationFees: false, + useOnlyApprovedProviders: false, + minNumDataSetsForChecks: 1, + dealsPerSpPerHour: 4, + retrievalsPerSpPerHour: 2, + dataSetCreationsPerSpPerHour: 1, + dataRetentionPollIntervalSeconds: 3600, + providersRefreshIntervalSeconds: 14400, dealbotDataSetVersion, + }; + mockConfigService.get.mockImplementation((key: string) => { + if (key === "activeNetworks") return ["calibration"]; + if (key === "networks") return { calibration: versionNetworkConfig }; + return undefined; }); const module: TestingModule = await Test.createTestingModule({ @@ -992,7 +1055,13 @@ describe("DealService", () => { carData: Uint8Array.from([1, 2, 3]), rootCid: CID.parse(mockRootCid), }; - await testService.createDeal(mockSynapseInstance, mockProviderInfo, dealInputWithMetadata, uploadPayload); + await testService.createDeal( + mockSynapseInstance, + mockProviderInfo, + dealInputWithMetadata, + uploadPayload, + "calibration", + ); expect(createContextMock).toHaveBeenCalledWith({ providerId: 101n, @@ -1009,7 +1078,13 @@ describe("DealService", () => { carData: Uint8Array.from([1, 2, 3]), rootCid: CID.parse(mockRootCid), }; - await testService.createDeal(mockSynapseInstance, mockProviderInfo, dealInputWithMetadata, uploadPayload); + await testService.createDeal( + mockSynapseInstance, + mockProviderInfo, + dealInputWithMetadata, + uploadPayload, + "calibration", + ); expect(createContextMock).toHaveBeenCalledWith({ providerId: 101n, @@ -1025,7 +1100,13 @@ describe("DealService", () => { carData: Uint8Array.from([1, 2, 3]), rootCid: CID.parse(mockRootCid), }; - await testService.createDeal(mockSynapseInstance, mockProviderInfo, dealInputWithMetadata, uploadPayload); + await testService.createDeal( + mockSynapseInstance, + mockProviderInfo, + dealInputWithMetadata, + uploadPayload, + "calibration", + ); expect(createContextMock).toHaveBeenCalledWith({ providerId: 101n, @@ -1054,7 +1135,13 @@ describe("DealService", () => { }, }; - await testService.createDeal(mockSynapseInstance, mockProviderInfo, dealInputWithConflict, uploadPayload); + await testService.createDeal( + mockSynapseInstance, + mockProviderInfo, + dealInputWithConflict, + uploadPayload, + "calibration", + ); // Verify config value overwrites dealInput value expect(createContextMock).toHaveBeenCalledWith({ @@ -1094,7 +1181,13 @@ describe("DealService", () => { testedAt: new Date(), }); - const deal = await service.createDeal(mockSynapseInstance, mockProviderInfo, mockDealInput, uploadPayload); + const deal = await service.createDeal( + mockSynapseInstance, + mockProviderInfo, + mockDealInput, + uploadPayload, + "calibration", + ); // Verify that pieceId from onPiecesConfirmed (123) is preserved and not overwritten by undefined expect(deal.pieceId).toBe(123); @@ -1134,7 +1227,7 @@ describe("DealService", () => { vi.spyOn(service as any, "createSynapseInstance").mockImplementation(() => synapseMock as unknown as Synapse); - const result = await service.checkDataSetExists("0xprovider", { dealbotDS: "1" }); + const result = await service.checkDataSetExists("0xprovider", { dealbotDS: "1" }, "calibration"); expect(result).toBe(true); expect(synapseMock.storage.createContext).toHaveBeenCalledWith({ @@ -1152,7 +1245,7 @@ describe("DealService", () => { }; vi.spyOn(service as any, "createSynapseInstance").mockImplementation(() => synapseMock as unknown as Synapse); - const result = await service.checkDataSetExists("0xprovider", { dealbotDS: "1" }); + const result = await service.checkDataSetExists("0xprovider", { dealbotDS: "1" }, "calibration"); expect(result).toBe(false); }); @@ -1160,14 +1253,15 @@ describe("DealService", () => { describe("getBaseDataSetMetadata", () => { it("always includes IPNI metadata key", () => { - const metadata = service.getBaseDataSetMetadata(); + const metadata = service.getBaseDataSetMetadata("calibration"); expect(metadata).toEqual(expect.objectContaining({ [METADATA_KEYS.WITH_IPFS_INDEXING]: "" })); }); it("includes dataset version when configured along with IPNI metadata", () => { - (service as any).blockchainConfig.dealbotDataSetVersion = "v1"; + const networks = mockConfigService.get("networks") as any; + networks.calibration.dealbotDataSetVersion = "v1"; - const metadata = service.getBaseDataSetMetadata(); + const metadata = service.getBaseDataSetMetadata("calibration"); expect(metadata).toEqual({ [METADATA_KEYS.WITH_IPFS_INDEXING]: "", @@ -1201,7 +1295,7 @@ describe("DealService", () => { it("throws when provider is not found in registry", async () => { vi.spyOn(mockWalletSdkService, "getProviderInfo").mockReturnValue(undefined); - await expect(service.createDataSetWithPiece("0xunknown", { dealbotDS: "1" })).rejects.toThrow( + await expect(service.createDataSetWithPiece("0xunknown", { dealbotDS: "1" }, "calibration")).rejects.toThrow( "Provider 0xunknown not found in registry", ); }); @@ -1220,7 +1314,7 @@ describe("DealService", () => { return { pieceCid: "bafk-seed", pieceId: 1, transactionHash: "0xhash" }; }); - await service.createDataSetWithPiece("0xprovider", { dealbotDS: "1" }); + await service.createDataSetWithPiece("0xprovider", { dealbotDS: "1" }, "calibration"); expect(createContextMock).toHaveBeenCalledWith({ providerId: 101n, @@ -1263,7 +1357,7 @@ describe("DealService", () => { return { pieceCid: "bafk-seed" }; }); - await service.createDataSetWithPiece("0xprovider", {}); + await service.createDataSetWithPiece("0xprovider", {}, "calibration"); expect(mockDataStorageMetrics.observeIngestMs).not.toHaveBeenCalled(); expect(mockDataStorageMetrics.recordUploadStatus).not.toHaveBeenCalled(); @@ -1283,7 +1377,7 @@ describe("DealService", () => { (executeUpload as Mock).mockResolvedValue({}); - await expect(service.createDataSetWithPiece("0xprovider", {})).rejects.toThrow( + await expect(service.createDataSetWithPiece("0xprovider", {}, "calibration")).rejects.toThrow( "Data-set creation upload completed without producing a pieceCid", ); expect(mockDataSetCreationMetrics.recordStatus).not.toHaveBeenCalledWith( @@ -1309,7 +1403,7 @@ describe("DealService", () => { await opts?.onProgress?.({ type: "onStored", data: { pieceCid: "bafk" } }); }); - await expect(service.createDataSetWithPiece("0xprovider", {})).resolves.toBeUndefined(); + await expect(service.createDataSetWithPiece("0xprovider", {}, "calibration")).resolves.toBeUndefined(); expect(mockDataSetCreationMetrics.recordStatus).toHaveBeenCalledWith( expect.objectContaining({ checkType: "dataSetCreation" }), "success", @@ -1330,7 +1424,7 @@ describe("DealService", () => { }); const controller = new AbortController(); - const resultPromise = service.createDataSetWithPiece("0xprovider", {}, controller.signal); + const resultPromise = service.createDataSetWithPiece("0xprovider", {}, "calibration", controller.signal); controller.abort(new Error("Job aborted")); await expect(resultPromise).rejects.toThrow("Job aborted"); diff --git a/apps/backend/src/deal/deal.service.ts b/apps/backend/src/deal/deal.service.ts index ee1738d9..4fdfd63f 100644 --- a/apps/backend/src/deal/deal.service.ts +++ b/apps/backend/src/deal/deal.service.ts @@ -16,8 +16,8 @@ import { toStructuredError, } from "../common/logging.js"; import { createSynapseFromConfig } from "../common/synapse-factory.js"; -import type { DataFile, Hex } from "../common/types.js"; -import type { IBlockchainConfig, IConfig } from "../config/app.config.js"; +import type { DataFile, Hex, Network } from "../common/types.js"; +import type { IConfig, INetworkConfig } from "../config/types.js"; import { Deal } from "../database/entities/deal.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; import { DealStatus, ServiceType } from "../database/types.js"; @@ -49,8 +49,7 @@ type UploadResultSummary = { @Injectable() export class DealService implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(DealService.name); - private readonly blockchainConfig: IBlockchainConfig; - private sharedSynapse?: Synapse; + private readonly sharedSynapseByNetwork: Map = new Map(); constructor( private readonly dataSourceService: DataSourceService, @@ -65,27 +64,33 @@ export class DealService implements OnModuleInit, OnModuleDestroy { private readonly dataStorageMetrics: DataStorageCheckMetrics, private readonly retrievalMetrics: RetrievalCheckMetrics, private readonly dataSetCreationMetrics: DataSetCreationCheckMetrics, - ) { - this.blockchainConfig = this.configService.get("blockchain"); - } + ) {} async onModuleInit() { - this.logger.log({ - event: "synapse_initialization", - message: "Creating shared Synapse instance", - }); - this.sharedSynapse = await this.createSynapseInstance(); + const activeNetworks = this.configService.get("activeNetworks"); + for (const network of activeNetworks) { + this.logger.log({ + event: "synapse_initialization", + message: "Creating shared Synapse instance", + network, + }); + const synapse = await this.createSynapseInstance(network); + this.sharedSynapseByNetwork.set(network, synapse); + } } async onModuleDestroy(): Promise { - if (this.sharedSynapse) { - this.sharedSynapse = undefined; - } + this.sharedSynapseByNetwork.clear(); + } + + private getNetworkConfig(network: Network): INetworkConfig { + return this.configService.get("networks")[network]; } async createDealForProvider( pdpProvider: PDPProviderEx, options: { + network: Network; existingDealId?: string; signal?: AbortSignal; extraDataSetMetadata?: Record; @@ -96,13 +101,15 @@ export class DealService implements OnModuleInit, OnModuleDestroy { const { preprocessed, cleanup } = await this.prepareDealInput(options.signal, options.logContext); try { - const synapse = this.sharedSynapse ?? (await this.createSynapseInstance()); + const synapse = + this.sharedSynapseByNetwork.get(options.network) ?? (await this.createSynapseInstance(options.network)); const uploadPayload = await this.prepareUploadPayload(preprocessed, options.signal); return await this.createDeal( synapse, pdpProvider, preprocessed, uploadPayload, + options.network, options.existingDealId, options.signal, options.extraDataSetMetadata, @@ -137,19 +144,20 @@ export class DealService implements OnModuleInit, OnModuleDestroy { return { preprocessed, cleanup }; } - getBaseDataSetMetadata(): Record { + getBaseDataSetMetadata(network: Network): Record { // IPNI is always enabled for all deals const metadata: Record = { [METADATA_KEYS.WITH_IPFS_INDEXING]: "", }; - if (this.blockchainConfig.dealbotDataSetVersion) { - metadata.dealbotDataSetVersion = this.blockchainConfig.dealbotDataSetVersion; + const networkConfig = this.getNetworkConfig(network); + if (networkConfig.dealbotDataSetVersion) { + metadata.dealbotDataSetVersion = networkConfig.dealbotDataSetVersion; } return metadata; } - getWalletAddress(): string { - return this.blockchainConfig.walletAddress; + getWalletAddress(network: Network): string { + return this.getNetworkConfig(network).walletAddress; } async createDeal( @@ -157,6 +165,7 @@ export class DealService implements OnModuleInit, OnModuleDestroy { pdpProvider: PDPProviderEx, dealInput: DealPreprocessingResult, uploadPayload: UploadPayload, + network: Network, existingDealId?: string, signal?: AbortSignal, extraDataSetMetadata?: Record, @@ -165,6 +174,7 @@ export class DealService implements OnModuleInit, OnModuleDestroy { const providerAddress = pdpProvider.serviceProvider; const checkType = "dataStorage" as const; let providerLabels = buildCheckMetricLabels({ + network, checkType, providerId: pdpProvider.id, providerName: pdpProvider.name, @@ -202,11 +212,13 @@ export class DealService implements OnModuleInit, OnModuleDestroy { deal.id = randomUUID(); } + const networkCfg = this.getNetworkConfig(network); deal.fileName = dealInput.processedData.name; deal.fileSize = dealInput.processedData.size; deal.spAddress = providerAddress; + deal.network = network; deal.status = DealStatus.PENDING; - deal.walletAddress = this.blockchainConfig.walletAddress; + deal.walletAddress = networkCfg.walletAddress; deal.metadata = dealInput.metadata; deal.serviceTypes = dealInput.appliedAddons; @@ -222,10 +234,11 @@ export class DealService implements OnModuleInit, OnModuleDestroy { try { // Load storageProvider relation deal.storageProvider = await this.storageProviderRepository.findOne({ - where: { address: deal.spAddress }, + where: { address: deal.spAddress, network }, }); dealLogContext.providerId = deal.storageProvider?.providerId ?? dealLogContext.providerId; providerLabels = buildCheckMetricLabels({ + network, checkType, providerId: deal.storageProvider?.providerId, providerName: pdpProvider.name ?? deal.storageProvider?.name, @@ -236,8 +249,8 @@ export class DealService implements OnModuleInit, OnModuleDestroy { const dataSetMetadata = { ...dealInput.synapseConfig.dataSetMetadata, ...extraDataSetMetadata }; - if (this.blockchainConfig.dealbotDataSetVersion) { - dataSetMetadata.dealbotDataSetVersion = this.blockchainConfig.dealbotDataSetVersion; + if (networkCfg.dealbotDataSetVersion) { + dataSetMetadata.dealbotDataSetVersion = networkCfg.dealbotDataSetVersion; } const filecoinPinLogger = createFilecoinPinLogger(this.logger, dealLogContext); @@ -537,13 +550,14 @@ export class DealService implements OnModuleInit, OnModuleDestroy { async checkDataSetExists( providerAddress: string, metadata: Record, + network: Network, signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); - const synapse = this.sharedSynapse ?? (await this.createSynapseInstance()); - const providerInfo = this.walletSdkService.getProviderInfo(providerAddress); + const synapse = this.sharedSynapseByNetwork.get(network) ?? (await this.createSynapseInstance(network)); + const providerInfo = this.walletSdkService.getProviderInfo(providerAddress, network); if (!providerInfo) { - throw new Error(`Provider ${providerAddress} not found in registry`); + throw new Error(`Provider ${providerAddress} not found in registry for network ${network}`); } const context = await awaitWithAbort( synapse.storage.createContext({ @@ -567,14 +581,16 @@ export class DealService implements OnModuleInit, OnModuleDestroy { async createDataSetWithPiece( providerAddress: string, metadata: Record, + network: Network, signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); - const providerInfo = this.walletSdkService.getProviderInfo(providerAddress); + const providerInfo = this.walletSdkService.getProviderInfo(providerAddress, network); if (!providerInfo) { - throw new Error(`Provider ${providerAddress} not found in registry`); + throw new Error(`Provider ${providerAddress} not found in registry for network ${network}`); } const labels = buildCheckMetricLabels({ + network, checkType: "dataSetCreation", providerId: providerInfo.id, providerName: providerInfo.name, @@ -599,7 +615,7 @@ export class DealService implements OnModuleInit, OnModuleDestroy { let transactionHash: string | undefined; try { - const synapse = this.sharedSynapse ?? (await this.createSynapseInstance()); + const synapse = this.sharedSynapseByNetwork.get(network) ?? (await this.createSynapseInstance(network)); signal?.throwIfAborted(); const DATA_SET_CREATION_PIECE_SIZE = 200 * 1024; // 200 KiB @@ -738,14 +754,16 @@ export class DealService implements OnModuleInit, OnModuleDestroy { // Deal Creation Helpers // ============================================================================ - private async createSynapseInstance(): Promise { + private async createSynapseInstance(network: Network): Promise { + const networkConfig = this.getNetworkConfig(network); try { - const { synapse, isSessionKeyMode } = await createSynapseFromConfig(this.blockchainConfig); + const { synapse, isSessionKeyMode } = await createSynapseFromConfig(networkConfig); if (isSessionKeyMode) { this.logger.log({ event: "synapse_session_key_init", message: "Initializing Synapse with session key", - walletAddress: this.blockchainConfig.walletAddress, + network, + walletAddress: networkConfig.walletAddress, }); } return synapse; @@ -753,6 +771,7 @@ export class DealService implements OnModuleInit, OnModuleDestroy { this.logger.error({ event: "synapse_init_failed", message: "Failed to initialize Synapse for deal job", + network, error: toStructuredError(error), }); throw error; diff --git a/apps/backend/src/dev-tools/dev-tools.controller.ts b/apps/backend/src/dev-tools/dev-tools.controller.ts index 7ae09d0e..55ccd2a3 100644 --- a/apps/backend/src/dev-tools/dev-tools.controller.ts +++ b/apps/backend/src/dev-tools/dev-tools.controller.ts @@ -1,6 +1,6 @@ import { Controller, Get, Logger, Param, Query, UsePipes, ValidationPipe } from "@nestjs/common"; import { ApiOperation, ApiQuery, ApiResponse, ApiTags } from "@nestjs/swagger"; -import { DevToolsService } from "./dev-tools.service.js"; +import { DEFAULT_NETWORK, DevToolsService } from "./dev-tools.service.js"; import { TriggerDealQueryDto, TriggerDealResponseDto } from "./dto/trigger-deal.dto.js"; import { TriggerRetrievalQueryDto, TriggerRetrievalResponseDto } from "./dto/trigger-retrieval.dto.js"; @@ -13,18 +13,28 @@ export class DevToolsController { @Get("providers") @ApiOperation({ summary: "List available storage providers" }) + @ApiQuery({ + name: "network", + default: DEFAULT_NETWORK, + required: false, + description: "Network to query (mainnet or calibration, default: calibration)", + example: "calibration", + enum: ["mainnet", "calibration"], + }) @ApiResponse({ status: 200, description: "List of available storage providers for testing", }) - listProviders() { + @UsePipes(new ValidationPipe({ transform: true })) + listProviders(@Query("network") network?: "mainnet" | "calibration") { this.logger.log({ event: "api_request", message: "GET /api/dev/providers", endpoint: "/api/dev/providers", method: "GET", + network, }); - return this.devToolsService.listProviders(); + return this.devToolsService.listProviders(network); } @Get("deal") @@ -35,6 +45,14 @@ export class DevToolsController { description: "Storage provider address", example: "0x1234567890abcdef1234567890abcdef12345678", }) + @ApiQuery({ + name: "network", + default: DEFAULT_NETWORK, + required: false, + description: "Network to use (mainnet or calibration, default: calibration)", + example: "calibration", + enum: ["mainnet", "calibration"], + }) @ApiResponse({ status: 200, description: "Deal accepted - use /api/dev/deals/:dealId to check progress", @@ -56,8 +74,9 @@ export class DevToolsController { endpoint: "/api/dev/deal", method: "GET", spAddress: query.spAddress, + network: query.network, }); - return this.devToolsService.triggerDeal(query.spAddress); + return this.devToolsService.triggerDeal(query.spAddress, query.network); } @Get("deals/:dealId") @@ -96,6 +115,14 @@ export class DevToolsController { description: "Storage provider address (uses most recent deal for this SP)", example: "0x1234567890abcdef1234567890abcdef12345678", }) + @ApiQuery({ + name: "network", + default: DEFAULT_NETWORK, + required: false, + description: "Network to query (mainnet or calibration, default: calibration)", + example: "calibration", + enum: ["mainnet", "calibration"], + }) @ApiResponse({ status: 200, description: "Test results", @@ -118,7 +145,8 @@ export class DevToolsController { method: "GET", dealId: query.dealId, spAddress: query.spAddress, + network: query.network, }); - return this.devToolsService.triggerRetrieval(query.dealId, query.spAddress); + return this.devToolsService.triggerRetrieval(query.dealId, query.spAddress, query.network); } } diff --git a/apps/backend/src/dev-tools/dev-tools.service.ts b/apps/backend/src/dev-tools/dev-tools.service.ts index 8b08e046..26952302 100644 --- a/apps/backend/src/dev-tools/dev-tools.service.ts +++ b/apps/backend/src/dev-tools/dev-tools.service.ts @@ -1,7 +1,9 @@ import { BadRequestException, Injectable, Logger, NotFoundException } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; import type { Repository } from "typeorm"; +import { SUPPORTED_NETWORKS } from "../common/constants.js"; import { type DealLogContext, toStructuredError } from "../common/logging.js"; +import type { Network } from "../common/types.js"; import { Deal } from "../database/entities/deal.entity.js"; import { DealStatus, RetrievalStatus } from "../database/types.js"; import { DealService } from "../deal/deal.service.js"; @@ -10,6 +12,8 @@ import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; import type { TriggerDealResponseDto } from "./dto/trigger-deal.dto.js"; import type { RetrievalMethodResultDto, TriggerRetrievalResponseDto } from "./dto/trigger-retrieval.dto.js"; +export const DEFAULT_NETWORK: Network = SUPPORTED_NETWORKS[0]; + @Injectable() export class DevToolsService { private readonly logger = new Logger(DevToolsService.name); @@ -25,11 +29,12 @@ export class DevToolsService { /** * List all available storage providers for testing */ - listProviders(): unknown[] { - const providers = this.walletSdkService.getTestingProviders(); + listProviders(network: Network = DEFAULT_NETWORK): unknown[] { + const providers = this.walletSdkService.getTestingProviders(network); this.logger.log({ event: "providers_listed", message: "Listing available providers", + network, count: providers.length, }); // Serialize BigInt values to strings for JSON response @@ -69,15 +74,16 @@ export class DevToolsService { * Trigger a deal for a specific storage provider. * Returns immediately with deal ID - processing happens in background. */ - async triggerDeal(spAddress: string): Promise { + async triggerDeal(spAddress: string, network: Network = DEFAULT_NETWORK): Promise { this.logger.log({ event: "deal_trigger_requested", message: "Triggering deal for storage provider", spAddress, + network, }); // Validate SP exists - const providerInfo = this.walletSdkService.getProviderInfo(spAddress); + const providerInfo = this.walletSdkService.getProviderInfo(spAddress, network); if (!providerInfo) { throw new NotFoundException(`Storage provider not found: ${spAddress}`); } @@ -92,7 +98,8 @@ export class DevToolsService { // Create a pending deal record first so we can return the ID immediately const pendingDeal = this.dealRepository.create({ spAddress, - walletAddress: this.dealService.getWalletAddress(), + network, + walletAddress: this.dealService.getWalletAddress(network), fileName: "pending", fileSize: 0, status: DealStatus.PENDING, @@ -117,7 +124,7 @@ export class DevToolsService { }); // Fire off the deal creation in the background (don't await) - this.processDealInBackground(dealId, providerInfo, dealLogContext).catch((err) => { + this.processDealInBackground(dealId, providerInfo, network, dealLogContext).catch((err) => { this.logger.error({ ...dealLogContext, event: "background_deal_processing_failed", @@ -144,6 +151,7 @@ export class DevToolsService { private async processDealInBackground( dealId: string, providerInfo: ReturnType, + network: Network, dealLogContext: DealLogContext, ): Promise { if (!providerInfo || providerInfo.id == null) { @@ -151,8 +159,10 @@ export class DevToolsService { } try { const deal = await this.dealService.createDealForProvider(providerInfo, { + network, existingDealId: dealId, logContext: { + network, jobId: "dev_tools_manual_deal", providerAddress: providerInfo.serviceProvider, providerId: providerInfo.id, @@ -220,13 +230,17 @@ export class DevToolsService { /** * Trigger data fetch for a deal by ID or most recent deal for an SP */ - async triggerRetrieval(dealId?: string, spAddress?: string): Promise { + async triggerRetrieval( + dealId?: string, + spAddress?: string, + network: Network = DEFAULT_NETWORK, + ): Promise { if (!dealId && !spAddress) { throw new BadRequestException("Either dealId or spAddress must be provided"); } // Find the deal - const deal = await this.findDeal(dealId, spAddress); + const deal = await this.findDeal(dealId, spAddress, network); const pieceCid = deal.pieceCid; if (!pieceCid) { throw new BadRequestException(`Deal ${deal.id} has no piece CID - cannot perform data fetch`); @@ -296,7 +310,7 @@ export class DevToolsService { /** * Find a deal by ID or most recent deal for an SP */ - private async findDeal(dealId?: string, spAddress?: string): Promise { + private async findDeal(dealId?: string, spAddress?: string, network: Network = DEFAULT_NETWORK): Promise { let deal: Deal | null = null; if (dealId) { @@ -308,17 +322,17 @@ export class DevToolsService { throw new NotFoundException(`Deal not found: ${dealId}`); } } else if (spAddress) { - // Find most recent successful deal for this SP + // Find most recent successful deal for this SP on the given network deal = await this.dealRepository.findOne({ where: [ - { spAddress, status: DealStatus.DEAL_CREATED }, - { spAddress, status: DealStatus.PIECE_ADDED }, + { spAddress, network, status: DealStatus.DEAL_CREATED }, + { spAddress, network, status: DealStatus.PIECE_ADDED }, ], order: { createdAt: "DESC" }, }); if (!deal) { - throw new NotFoundException(`No successful deals found for SP: ${spAddress}`); + throw new NotFoundException(`No successful deals found for SP: ${spAddress} on network: ${network}`); } } diff --git a/apps/backend/src/dev-tools/dto/trigger-deal.dto.ts b/apps/backend/src/dev-tools/dto/trigger-deal.dto.ts index 297af800..59c6f9cb 100644 --- a/apps/backend/src/dev-tools/dto/trigger-deal.dto.ts +++ b/apps/backend/src/dev-tools/dto/trigger-deal.dto.ts @@ -1,5 +1,6 @@ import { ApiProperty } from "@nestjs/swagger"; -import { IsNotEmpty, IsString } from "class-validator"; +import { IsIn, IsNotEmpty, IsOptional, IsString } from "class-validator"; +import type { Network } from "../../common/types.js"; export class TriggerDealQueryDto { @ApiProperty({ @@ -9,6 +10,18 @@ export class TriggerDealQueryDto { @IsString() @IsNotEmpty() spAddress: string; + + @ApiProperty({ + description: "Network to use", + example: "calibration", + required: false, + enum: ["mainnet", "calibration"], + default: "calibration", + }) + @IsString() + @IsIn(["mainnet", "calibration"]) + @IsOptional() + network?: Network; } export class TriggerDealResponseDto { diff --git a/apps/backend/src/dev-tools/dto/trigger-retrieval.dto.ts b/apps/backend/src/dev-tools/dto/trigger-retrieval.dto.ts index 6911ff58..76387f32 100644 --- a/apps/backend/src/dev-tools/dto/trigger-retrieval.dto.ts +++ b/apps/backend/src/dev-tools/dto/trigger-retrieval.dto.ts @@ -1,5 +1,6 @@ import { ApiProperty } from "@nestjs/swagger"; -import { IsOptional, IsString, IsUUID, ValidateIf } from "class-validator"; +import { IsIn, IsOptional, IsString, IsUUID, ValidateIf } from "class-validator"; +import type { Network } from "../../common/types.js"; export class TriggerRetrievalQueryDto { @ApiProperty({ @@ -21,6 +22,18 @@ export class TriggerRetrievalQueryDto { @IsOptional() @ValidateIf((o) => !o.dealId) spAddress?: string; + + @ApiProperty({ + description: "Network to use", + example: "calibration", + required: false, + enum: ["mainnet", "calibration"], + default: "calibration", + }) + @IsString() + @IsIn(["mainnet", "calibration"]) + @IsOptional() + network?: Network; } export class RetrievalMethodResultDto { diff --git a/apps/backend/src/http-client/http-client.service.ts b/apps/backend/src/http-client/http-client.service.ts index 48e10e5c..94be13a9 100644 --- a/apps/backend/src/http-client/http-client.service.ts +++ b/apps/backend/src/http-client/http-client.service.ts @@ -6,7 +6,7 @@ import type { AxiosRequestConfig } from "axios"; import { firstValueFrom } from "rxjs"; import { request as undiciRequest } from "undici"; import { toStructuredError } from "../common/logging.js"; -import type { IConfig } from "../config/app.config.js"; +import type { IConfig } from "../config/types.js"; import type { HttpVersion, RequestMetrics, RequestWithMetrics } from "./types.js"; @Injectable() diff --git a/apps/backend/src/jobs/data-set-creation.handler.ts b/apps/backend/src/jobs/data-set-creation.handler.ts index 1fb3faac..a85f97f5 100644 --- a/apps/backend/src/jobs/data-set-creation.handler.ts +++ b/apps/backend/src/jobs/data-set-creation.handler.ts @@ -1,5 +1,6 @@ import type { Logger } from "@nestjs/common"; import type { DataSetLogContext, ProviderJobContext } from "../common/logging.js"; +import type { Network } from "../common/types.js"; import type { DealService } from "../deal/deal.service.js"; export interface DataSetCreationDeps { @@ -23,6 +24,7 @@ export interface DataSetCreationDeps { export async function provisionNextMissingDataSet( deps: DataSetCreationDeps, spAddress: string, + network: Network, minDataSets: number, baseDataSetMetadata: Record, dataSetLogContext: ProviderJobContext, @@ -46,7 +48,7 @@ export async function provisionNextMissingDataSet( }; // Check if data-set already exists by attempting to resolve its context - const exists = await dealService.checkDataSetExists(spAddress, metadata, signal); + const exists = await dealService.checkDataSetExists(spAddress, metadata, network, signal); if (exists) { existingCount++; @@ -58,7 +60,7 @@ export async function provisionNextMissingDataSet( event: "creating_provisioned_data_set", message: "Creating provisioned data-set", }); - await dealService.createDataSetWithPiece(spAddress, metadata, signal); + await dealService.createDataSetWithPiece(spAddress, metadata, network, signal); logger.log({ ...logContext, event: "data_set_provisioning_progress", diff --git a/apps/backend/src/jobs/jobs.service.spec.ts b/apps/backend/src/jobs/jobs.service.spec.ts index 5b8c58bc..1ed1bafc 100644 --- a/apps/backend/src/jobs/jobs.service.spec.ts +++ b/apps/backend/src/jobs/jobs.service.spec.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { IConfig, ISpBlocklistConfig } from "../config/app.config.js"; +import { Network } from "../common/types.js"; +import type { IConfig, INetworkConfig, INetworksConfig } from "../config/types.js"; import { DATA_RETENTION_POLL_QUEUE, PROVIDERS_REFRESH_QUEUE, SP_WORK_QUEUE } from "./job-queues.js"; import { JobsService } from "./jobs.service.js"; @@ -109,20 +110,32 @@ describe("JobsService schedule rows", () => { storageProvidersTested: { set: vi.fn() } as unknown as JobsServiceDeps[19], }; - const emptySpBlocklists: ISpBlocklistConfig = { - ids: new Set(), - addresses: new Set(), - }; + const baseNetworkConfig = { + walletPrivateKey: "0x", + network: "calibration" as const, + useOnlyApprovedProviders: false, + minNumDataSetsForChecks: 1, + dealsPerSpPerHour: 4, + retrievalsPerSpPerHour: 2, + dataSetCreationsPerSpPerHour: 1, + dataRetentionPollIntervalSeconds: 3600, + providersRefreshIntervalSeconds: 14400, + walletAddress: "0x0000000000000000000000000000000000000000", + checkDatasetCreationFees: true, + maintenanceWindowsUtc: ["07:00", "22:00"], + maintenanceWindowMinutes: 20, + blockedSpIds: new Set(), + blockedSpAddresses: new Set(), + pieceCleanupPerSpPerHour: 1, + maxPieceCleanupRuntimeSeconds: 300, + maxDatasetStorageSizeBytes: 24 * 1024 * 1024 * 1024, + targetDatasetStorageSizeBytes: 20 * 1024 * 1024 * 1024, + } satisfies IConfig["networks"]["calibration"]; baseConfigValues = { app: { runMode: "both" } as IConfig["app"], - blockchain: { useOnlyApprovedProviders: false, minNumDataSetsForChecks: 1 } as IConfig["blockchain"], - scheduling: { - providersRefreshIntervalSeconds: 4 * 3600, - dataRetentionPollIntervalSeconds: 3600, - maintenanceWindowsUtc: ["07:00", "22:00"], - maintenanceWindowMinutes: 20, - } as IConfig["scheduling"], + activeNetworks: ["calibration"] as IConfig["activeNetworks"], + networks: { calibration: baseNetworkConfig } as unknown as IConfig["networks"], jobs: { schedulePhaseSeconds: 0, catchupMaxEnqueue: 10, @@ -130,8 +143,6 @@ describe("JobsService schedule rows", () => { pgbossSchedulerEnabled: true, workerPollSeconds: 60, dataSetCreationJobTimeoutSeconds: 300, - pieceCleanupPerSpPerHour: 1, - maxPieceCleanupRuntimeSeconds: 300, } as IConfig["jobs"], database: { host: "localhost", @@ -140,10 +151,6 @@ describe("JobsService schedule rows", () => { password: "pass", database: "dealbot", } as IConfig["database"], - pieceCleanup: { - maxDatasetStorageSizeBytes: 24 * 1024 * 1024 * 1024, - } as IConfig["pieceCleanup"], - spBlocklists: emptySpBlocklists, }; configService = { @@ -197,12 +204,16 @@ describe("JobsService schedule rows", () => { return "success"; }); - await callPrivate(service, "recordJobExecution", "deal", run); + await callPrivate(service, "recordJobExecution", "deal", "calibration", run); expect(run).toHaveBeenCalled(); - expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal" }); - expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", handler_result: "success" }); - expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal" }, 5); + expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }); + expect(completedCounter.inc).toHaveBeenCalledWith({ + job_type: "deal", + network: "calibration", + handler_result: "success", + }); + expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }, 5); }); it("records metrics for failed job execution", async () => { @@ -219,11 +230,15 @@ describe("JobsService schedule rows", () => { throw new Error("boom"); }); - await expect(callPrivate(service, "recordJobExecution", "deal", run)).rejects.toThrow("boom"); + await expect(callPrivate(service, "recordJobExecution", "deal", "calibration", run)).rejects.toThrow("boom"); - expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal" }); - expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", handler_result: "error" }); - expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal" }, 2); + expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }); + expect(completedCounter.inc).toHaveBeenCalledWith({ + job_type: "deal", + network: "calibration", + handler_result: "error", + }); + expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }, 2); }); it("records metrics for aborted job execution", async () => { @@ -240,11 +255,15 @@ describe("JobsService schedule rows", () => { return "aborted" as const; }); - await callPrivate(service, "recordJobExecution", "deal", run); + await callPrivate(service, "recordJobExecution", "deal", "calibration", run); - expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal" }); - expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", handler_result: "aborted" }); - expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal" }, 3); + expect(startedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }); + expect(completedCounter.inc).toHaveBeenCalledWith({ + job_type: "deal", + network: "calibration", + handler_result: "aborted", + }); + expect(durationHistogram.observe).toHaveBeenCalledWith({ job_type: "deal", network: "calibration" }, 3); }); it("deal job records aborted when abort signal fires", async () => { @@ -297,6 +316,7 @@ describe("JobsService schedule rows", () => { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60, + network: "calibration", }, }); @@ -304,7 +324,11 @@ describe("JobsService schedule rows", () => { await vi.advanceTimersByTimeAsync(120_000); await jobPromise; - expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", handler_result: "aborted" }); + expect(completedCounter.inc).toHaveBeenCalledWith({ + job_type: "deal", + network: "calibration", + handler_result: "aborted", + }); }); it("retrieval job records aborted when abort signal fires", async () => { @@ -326,7 +350,7 @@ describe("JobsService schedule rows", () => { } as unknown as JobsServiceDeps[0]; const retrievalService = { - performRandomRetrievalForProvider: vi.fn(async (_sp: string, signal: AbortSignal) => { + performRandomRetrievalForProvider: vi.fn(async (_sp: string, _network: Network, signal: AbortSignal) => { await new Promise((resolve) => { if (signal.aborted) return resolve(); signal.addEventListener("abort", () => resolve(), { once: true }); @@ -355,13 +379,18 @@ describe("JobsService schedule rows", () => { jobType: "retrieval", spAddress: "0xaaa", intervalSeconds: 60, + network: "calibration", }, }); await vi.advanceTimersByTimeAsync(60_000); await jobPromise; - expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "retrieval", handler_result: "aborted" }); + expect(completedCounter.inc).toHaveBeenCalledWith({ + job_type: "retrieval", + network: "calibration", + handler_result: "aborted", + }); }); it("retrieval job resolves providerId from storage_providers when wallet cache misses", async () => { @@ -387,6 +416,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleRetrievalJob", { id: "job-retrieval-provider-fallback", data: { + network: "calibration", jobType: "retrieval", spAddress: "0xaaa", intervalSeconds: 60, @@ -395,6 +425,7 @@ describe("JobsService schedule rows", () => { expect(retrievalService.performRandomRetrievalForProvider).toHaveBeenCalledWith( "0xaaa", + "calibration", expect.any(AbortSignal), expect.objectContaining({ jobId: "job-retrieval-provider-fallback", @@ -431,12 +462,17 @@ describe("JobsService schedule rows", () => { jobType: "retrieval", spAddress: "0xaaa", intervalSeconds: 60, + network: "calibration", }, }), ).rejects.toThrow("providerId is required for job execution"); expect(retrievalService.performRandomRetrievalForProvider).not.toHaveBeenCalled(); - expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "retrieval", handler_result: "error" }); + expect(completedCounter.inc).toHaveBeenCalledWith({ + job_type: "retrieval", + network: "calibration", + handler_result: "error", + }); }); it("updates queue metrics from pg-boss state and age queries", async () => { @@ -595,7 +631,9 @@ describe("JobsService schedule rows", () => { it("adds schedule rows for newly seen providers", async () => { baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -608,8 +646,8 @@ describe("JobsService schedule rows", () => { storageProviderRepositoryMock.find.mockResolvedValueOnce([providerA]).mockResolvedValueOnce([providerA, providerB]); - await callPrivate(service, "ensureScheduleRows"); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", "calibration"); + await callPrivate(service, "ensureScheduleRows", "calibration"); // Check upserts for providerB const upsertCalls = jobScheduleRepositoryMock.upsertSchedule.mock.calls; @@ -627,15 +665,18 @@ describe("JobsService schedule rows", () => { const providerA = { address: "0xaaa" }; storageProviderRepositoryMock.find.mockResolvedValueOnce([providerA]); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", "calibration"); - expect(jobScheduleRepositoryMock.deleteSchedulesForInactiveProviders).toHaveBeenCalledWith([providerA.address]); + expect(jobScheduleRepositoryMock.deleteSchedulesForInactiveProviders).toHaveBeenCalledWith( + [providerA.address], + "calibration", + ); }); it("does not delete schedule rows when no active providers exist", async () => { storageProviderRepositoryMock.find.mockResolvedValueOnce([]); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", "calibration"); expect(jobScheduleRepositoryMock.deleteSchedulesForInactiveProviders).not.toHaveBeenCalled(); }); @@ -643,7 +684,9 @@ describe("JobsService schedule rows", () => { it("uses approved-only filter when configured", async () => { baseConfigValues = { ...baseConfigValues, - blockchain: { useOnlyApprovedProviders: true } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, useOnlyApprovedProviders: true }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -652,30 +695,32 @@ describe("JobsService schedule rows", () => { service = buildService({ configService }); storageProviderRepositoryMock.find.mockResolvedValueOnce([]); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", "calibration"); expect(storageProviderRepositoryMock.find).toHaveBeenCalledWith({ select: { address: true, providerId: true }, - where: { isActive: true, isApproved: true }, + where: { network: "calibration", isActive: true, isApproved: true }, }); }); it("always inserts global data_retention_poll and providers_refresh schedules", async () => { storageProviderRepositoryMock.find.mockResolvedValueOnce([]); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", "calibration"); expect(jobScheduleRepositoryMock.upsertSchedule).toHaveBeenCalledWith( "providers_refresh", "", expect.any(Number), expect.any(Date), + "calibration", ); expect(jobScheduleRepositoryMock.upsertSchedule).toHaveBeenCalledWith( "data_retention_poll", "", expect.any(Number), expect.any(Date), + "calibration", ); }); @@ -705,18 +750,19 @@ describe("JobsService schedule rows", () => { id: 1, job_type: "deal", sp_address: "0xaaa", + network: "calibration", interval_seconds: 1, next_run_at: "2024-01-01T00:00:00Z", }, ]); - await callPrivate(service, "enqueueDueJobs"); + await callPrivate(service, "enqueueDueJobs", "calibration"); expect(send).toHaveBeenCalledTimes(3); for (const call of send.mock.calls) { expect(call[0]).toBe("sp.work"); - expect(call[1]).toMatchObject({ jobType: "deal", spAddress: "0xaaa" }); - expect(call[2]).toMatchObject({ singletonKey: "0xaaa", retryLimit: 0 }); + expect(call[1]).toMatchObject({ jobType: "deal", spAddress: "0xaaa", network: "calibration" }); + expect(call[2]).toMatchObject({ singletonKey: "calibration:0xaaa", retryLimit: 0 }); expect(call[2]?.startAfter).toBeUndefined(); } @@ -740,12 +786,13 @@ describe("JobsService schedule rows", () => { id: 10, job_type: "providers_refresh", sp_address: "", + network: "calibration", interval_seconds: 1800, next_run_at: "2024-01-01T00:00:00Z", }, ]); - await callPrivate(service, "enqueueDueJobs"); + await callPrivate(service, "enqueueDueJobs", "calibration"); // Should only enqueue once despite being 8 intervals overdue expect(send).toHaveBeenCalledTimes(1); @@ -772,16 +819,17 @@ describe("JobsService schedule rows", () => { id: 11, job_type: "providers_refresh", sp_address: "", + network: "calibration", interval_seconds: 14400, next_run_at: "2024-01-01T00:00:00Z", }, ]); - await callPrivate(service, "enqueueDueJobs"); + await callPrivate(service, "enqueueDueJobs", "calibration"); expect(send).toHaveBeenCalledTimes(1); expect(send.mock.calls[0][2]).toMatchObject({ - singletonKey: "providers_refresh", + singletonKey: "calibration:providers_refresh", retryLimit: 0, }); }); @@ -789,11 +837,13 @@ describe("JobsService schedule rows", () => { it("global jobs are skipped during maintenance windows", async () => { baseConfigValues = { ...baseConfigValues, - scheduling: { - ...baseConfigValues.scheduling, - maintenanceWindowsUtc: ["03:00"], - maintenanceWindowMinutes: 60, - } as IConfig["scheduling"], + networks: { + calibration: { + ...baseConfigValues.networks?.calibration, + maintenanceWindowsUtc: ["03:00"], + maintenanceWindowMinutes: 60, + } as INetworkConfig, + } as INetworksConfig, }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -818,7 +868,7 @@ describe("JobsService schedule rows", () => { }, ]); - await callPrivate(service, "enqueueDueJobs"); + await callPrivate(service, "enqueueDueJobs", "calibration"); // Global job should not be enqueued during maintenance expect(send).not.toHaveBeenCalled(); @@ -832,11 +882,13 @@ describe("JobsService schedule rows", () => { it("defers jobs until maintenance window ends (same-day)", async () => { baseConfigValues = { ...baseConfigValues, - scheduling: { - ...baseConfigValues.scheduling, - maintenanceWindowsUtc: ["07:00"], - maintenanceWindowMinutes: 20, - } as IConfig["scheduling"], + networks: { + calibration: { + ...baseConfigValues.networks?.calibration, + maintenanceWindowsUtc: ["07:00"], + maintenanceWindowMinutes: 20, + } as INetworkConfig, + } as INetworksConfig, }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -848,13 +900,13 @@ describe("JobsService schedule rows", () => { (service as unknown as { safeSend: typeof safeSend }).safeSend = safeSend; const now = new Date("2024-01-01T07:05:00Z"); - const maintenance = callPrivate(service, "getMaintenanceWindowStatus", now) as any; + const maintenance = callPrivate(service, "getMaintenanceWindowStatus", now, "calibration") as any; await callPrivate( service, "deferJobForMaintenance", "deal", - { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60 }, + { jobType: "deal", spAddress: "0xaaa", network: "calibration", intervalSeconds: 60 }, maintenance, now, ); @@ -863,7 +915,7 @@ describe("JobsService schedule rows", () => { expect(safeSend).toHaveBeenCalledWith( "deal", "sp.work", - { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60 }, + { jobType: "deal", spAddress: "0xaaa", network: "calibration", intervalSeconds: 60 }, { startAfter: expectedResumeAt }, ); }); @@ -871,11 +923,13 @@ describe("JobsService schedule rows", () => { it("defers jobs until maintenance window ends (wraps midnight)", async () => { baseConfigValues = { ...baseConfigValues, - scheduling: { - ...baseConfigValues.scheduling, - maintenanceWindowsUtc: ["23:50"], - maintenanceWindowMinutes: 20, - } as IConfig["scheduling"], + networks: { + calibration: { + ...baseConfigValues.networks?.calibration, + maintenanceWindowsUtc: ["23:50"], + maintenanceWindowMinutes: 20, + } as INetworkConfig, + } as INetworksConfig, }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -887,13 +941,13 @@ describe("JobsService schedule rows", () => { (service as unknown as { safeSend: typeof safeSend }).safeSend = safeSend; const now = new Date("2024-01-01T23:55:00Z"); - const maintenance = callPrivate(service, "getMaintenanceWindowStatus", now) as any; + const maintenance = callPrivate(service, "getMaintenanceWindowStatus", now, "calibration") as any; await callPrivate( service, "deferJobForMaintenance", "retrieval", - { jobType: "retrieval", spAddress: "0xbbb", intervalSeconds: 60 }, + { jobType: "retrieval", spAddress: "0xbbb", network: "calibration", intervalSeconds: 60 }, maintenance, now, ); @@ -902,7 +956,7 @@ describe("JobsService schedule rows", () => { expect(safeSend).toHaveBeenCalledWith( "retrieval", "sp.work", - { jobType: "retrieval", spAddress: "0xbbb", intervalSeconds: 60 }, + { jobType: "retrieval", spAddress: "0xbbb", network: "calibration", intervalSeconds: 60 }, { startAfter: expectedResumeAt }, ); }); @@ -930,7 +984,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDealJob", { id: "job-deal-1", - data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60 }, + data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60, network: "calibration" }, }); expect(dealService.createDealForProvider).toHaveBeenCalledTimes(1); @@ -970,7 +1024,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDealJob", { id: "job-deal-no-quota-gate", - data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60 }, + data: { jobType: "deal", network: "calibration", spAddress: "0xaaa", intervalSeconds: 60 }, }); expect(pieceCleanupService.cleanupPiecesForProvider).not.toHaveBeenCalled(); @@ -982,7 +1036,9 @@ describe("JobsService schedule rows", () => { vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -1011,12 +1067,13 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDealJob", { id: "job-deal-2", - data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60 }, + data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60, network: "calibration" }, }); expect(dealService.checkDataSetExists).toHaveBeenCalledWith( "0xaaa", { dealbotDataSetVersion: "v1", withIpniIndexing: "", dealbotDS: "1" }, + "calibration", expect.any(AbortSignal), ); expect(dealService.createDealForProvider).toHaveBeenCalledTimes(1); @@ -1024,6 +1081,7 @@ describe("JobsService schedule rows", () => { expect.objectContaining({ serviceProvider: "0xaaa" }), expect.objectContaining({ extraDataSetMetadata: { dealbotDS: "1" }, + network: "calibration", }), ); @@ -1035,7 +1093,9 @@ describe("JobsService schedule rows", () => { vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -1064,18 +1124,19 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDealJob", { id: "job-deal-3", - data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60 }, + data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60, network: "calibration" }, }); expect(dealService.checkDataSetExists).toHaveBeenCalledWith( "0xaaa", { dealbotDataSetVersion: "v1", dealbotDS: "2" }, + "calibration", expect.any(AbortSignal), ); expect(dealService.createDealForProvider).toHaveBeenCalledTimes(1); expect(dealService.createDealForProvider).toHaveBeenCalledWith( expect.objectContaining({ serviceProvider: "0xaaa" }), - expect.objectContaining({ extraDataSetMetadata: undefined }), + expect.objectContaining({ extraDataSetMetadata: undefined, network: "calibration" }), ); vi.spyOn(Math, "random").mockRestore(); @@ -1086,7 +1147,9 @@ describe("JobsService schedule rows", () => { vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -1117,7 +1180,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDealJob", { id: "job-deal-4", - data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60 }, + data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60, network: "calibration" }, }); expect(dealService.createDealForProvider).toHaveBeenCalledWith( @@ -1133,7 +1196,9 @@ describe("JobsService schedule rows", () => { baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], jobs: { ...baseConfigValues.jobs, dealJobTimeoutSeconds: 1, @@ -1172,11 +1237,15 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDealJob", { id: "job-deal-selection-abort", - data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60 }, + data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60, network: "calibration" }, }); expect(dealService.createDealForProvider).not.toHaveBeenCalled(); - expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: "deal", handler_result: "aborted" }); + expect(completedCounter.inc).toHaveBeenCalledWith({ + job_type: "deal", + network: "calibration", + handler_result: "aborted", + }); vi.spyOn(Math, "random").mockRestore(); }); @@ -1202,13 +1271,14 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDataSetCreationJob", { id: "job-ds-1", - data: { jobType: "data_set_creation", spAddress: "0xaaa", intervalSeconds: 3600 }, + data: { jobType: "data_set_creation", spAddress: "0xaaa", intervalSeconds: 3600, network: "calibration" }, }); expect(dealService.createDataSetWithPiece).toHaveBeenCalledTimes(1); expect(dealService.createDataSetWithPiece).toHaveBeenCalledWith( "0xaaa", { withIpniIndexing: "" }, + "calibration", expect.any(AbortSignal), ); }); @@ -1219,7 +1289,9 @@ describe("JobsService schedule rows", () => { baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -1243,13 +1315,14 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDataSetCreationJob", { id: "job-ds-2", - data: { jobType: "data_set_creation", spAddress: "0xaaa", intervalSeconds: 3600 }, + data: { jobType: "data_set_creation", spAddress: "0xaaa", intervalSeconds: 3600, network: "calibration" }, }); expect(dealService.createDataSetWithPiece).not.toHaveBeenCalled(); expect(dealService.checkDataSetExists).toHaveBeenCalledWith( "0xaaa", { dealbotDataSetVersion: "v1" }, + "calibration", expect.any(AbortSignal), ); }); @@ -1259,7 +1332,9 @@ describe("JobsService schedule rows", () => { vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -1283,7 +1358,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDataSetCreationJob", { id: "job-ds-3", - data: { jobType: "data_set_creation", spAddress: "0xaaa", intervalSeconds: 3600 }, + data: { jobType: "data_set_creation", spAddress: "0xaaa", intervalSeconds: 3600, network: "calibration" }, }); // Only the first missing data set (index 0) should be created @@ -1291,6 +1366,7 @@ describe("JobsService schedule rows", () => { expect(dealService.createDataSetWithPiece).toHaveBeenCalledWith( "0xaaa", { dealbotDataSetVersion: "v1" }, + "calibration", expect.any(AbortSignal), ); }); @@ -1300,7 +1376,9 @@ describe("JobsService schedule rows", () => { vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); baseConfigValues = { ...baseConfigValues, - blockchain: { ...baseConfigValues.blockchain, minNumDataSetsForChecks: 3 } as IConfig["blockchain"], + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, minNumDataSetsForChecks: 3 }, + } as unknown as IConfig["networks"], }; configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), @@ -1325,7 +1403,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDataSetCreationJob", { id: "job-ds-3b", - data: { jobType: "data_set_creation", spAddress: "0xaaa", intervalSeconds: 3600 }, + data: { jobType: "data_set_creation", spAddress: "0xaaa", intervalSeconds: 3600, network: "calibration" }, }); // Should skip index 0 (exists) and create only index 1 @@ -1333,6 +1411,7 @@ describe("JobsService schedule rows", () => { expect(dealService.createDataSetWithPiece).toHaveBeenCalledWith( "0xaaa", { dealbotDataSetVersion: "v1", dealbotDS: "1" }, + "calibration", expect.any(AbortSignal), ); }); @@ -1355,6 +1434,7 @@ describe("JobsService schedule rows", () => { provisionNextMissingDataSet( { dealService, logger }, "0xaaa", + "calibration", 5, {}, { providerAddress: "0xaaa", jobId: "job-ds-4", providerId: 1n, providerName: "test-provider" }, @@ -1375,34 +1455,36 @@ describe("JobsService schedule rows", () => { const activeGauge = metricsMocks.storageProvidersActive as unknown as { set: ReturnType }; const testedGauge = metricsMocks.storageProvidersTested as unknown as { set: ReturnType }; - await callPrivate(service, "updateStorageProviderGauges"); + await callPrivate(service, "updateStorageProviderGauges", "calibration"); - expect(activeGauge.set).toHaveBeenCalledWith({ status: "active" }, 7); - expect(activeGauge.set).toHaveBeenCalledWith({ status: "inactive" }, 3); - expect(testedGauge.set).toHaveBeenCalledWith(7); + expect(activeGauge.set).toHaveBeenCalledWith({ network: "calibration", status: "active" }, 7); + expect(activeGauge.set).toHaveBeenCalledWith({ network: "calibration", status: "inactive" }, 3); + expect(testedGauge.set).toHaveBeenCalledWith({ network: "calibration" }, 7); }); it("filters tested providers by isApproved when useOnlyApprovedProviders is enabled", async () => { - baseConfigValues.blockchain = { - useOnlyApprovedProviders: true, - minNumDataSetsForChecks: 1, - } as IConfig["blockchain"]; - service = buildService(); + baseConfigValues = { + ...baseConfigValues, + networks: { + calibration: { ...(baseConfigValues.networks as any).calibration, useOnlyApprovedProviders: true }, + } as unknown as IConfig["networks"], + }; + service = buildService({ + configService: { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]) } as unknown as JobsServiceDeps[0], + }); storageProviderRepositoryMock.count.mockResolvedValueOnce(10).mockResolvedValueOnce(7).mockResolvedValueOnce(5); // testedCount (only approved) await callPrivate(service, "updateStorageProviderGauges"); expect(storageProviderRepositoryMock.count).toHaveBeenNthCalledWith(3, { - where: { isActive: true, isApproved: true }, + where: { network: "calibration", isActive: true, isApproved: true }, }); }); it("subtracts globally blocked providers from tested gauge when global blocklist is non-empty", async () => { - baseConfigValues.spBlocklists = { - ids: new Set(), - addresses: new Set(["0xblocked"]), - }; + if (baseConfigValues.networks?.calibration) + baseConfigValues.networks.calibration.blockedSpAddresses = new Set(["0xblocked"]); configService = { get: vi.fn((key: keyof IConfig) => baseConfigValues[key]), } as unknown as JobsServiceDeps[0]; @@ -1421,9 +1503,9 @@ describe("JobsService schedule rows", () => { const testedGauge = metricsMocks.storageProvidersTested as unknown as { set: ReturnType }; - await callPrivate(service, "updateStorageProviderGauges"); + await callPrivate(service, "updateStorageProviderGauges", "calibration"); - expect(testedGauge.set).toHaveBeenCalledWith(2); // 3 providers minus 1 globally blocked + expect(testedGauge.set).toHaveBeenCalledWith({ network: "calibration" }, 2); // 3 providers minus 1 globally blocked }); it("catches storage provider gauge errors without rethrowing", async () => { @@ -1435,10 +1517,10 @@ describe("JobsService schedule rows", () => { const providerA = { address: "0xaaa", providerId: 1n }; storageProviderRepositoryMock.find.mockResolvedValueOnce([providerA]); - baseConfigValues.spBlocklists = { ids: new Set(["1"]), addresses: new Set() }; + if (baseConfigValues.networks?.calibration) baseConfigValues.networks.calibration.blockedSpIds = new Set(["1"]); service = buildService(); - await callPrivate(service, "ensureScheduleRows"); + await callPrivate(service, "ensureScheduleRows", "calibration"); const upsertCalls = jobScheduleRepositoryMock.upsertSchedule.mock.calls; const jobTypes = upsertCalls.filter((c) => c[1] === providerA.address).map((c) => c[0]); @@ -1447,14 +1529,14 @@ describe("JobsService schedule rows", () => { expect(jobTypes).not.toContain("retrieval"); // Blocked provider is excluded from the active-address list passed to cleanup, // so its existing schedule rows will be deleted. - expect(jobScheduleRepositoryMock.deleteSchedulesForInactiveProviders).toHaveBeenCalledWith([]); + expect(jobScheduleRepositoryMock.deleteSchedulesForInactiveProviders).toHaveBeenCalledWith([], "calibration"); }); it("deal job is skipped at runtime when provider is blocked", async () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); - baseConfigValues.spBlocklists = { ids: new Set(["1"]), addresses: new Set() }; + if (baseConfigValues.networks?.calibration) baseConfigValues.networks.calibration.blockedSpIds = new Set(["1"]); const dealService = { createDealForProvider: vi.fn(), @@ -1473,7 +1555,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDealJob", { id: "job-blocked-deal", - data: { jobType: "deal", spAddress: "0xaaa", intervalSeconds: 60 }, + data: { jobType: "deal", network: "calibration", spAddress: "0xaaa", intervalSeconds: 60 }, }); expect(dealService.createDealForProvider).not.toHaveBeenCalled(); @@ -1483,7 +1565,7 @@ describe("JobsService schedule rows", () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); - baseConfigValues.spBlocklists = { ids: new Set(["2"]), addresses: new Set() }; + if (baseConfigValues.networks?.calibration) baseConfigValues.networks.calibration.blockedSpIds = new Set(["2"]); const retrievalService = { performRandomRetrievalForProvider: vi.fn() }; const walletSdkService = { @@ -1497,7 +1579,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleRetrievalJob", { id: "job-blocked-retrieval", - data: { jobType: "retrieval", spAddress: "0xaaa", intervalSeconds: 60 }, + data: { jobType: "retrieval", network: "calibration", spAddress: "0xaaa", intervalSeconds: 60 }, }); expect(retrievalService.performRandomRetrievalForProvider).not.toHaveBeenCalled(); @@ -1507,7 +1589,7 @@ describe("JobsService schedule rows", () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); - baseConfigValues.spBlocklists = { ids: new Set(["3"]), addresses: new Set() }; + if (baseConfigValues.networks?.calibration) baseConfigValues.networks.calibration.blockedSpIds = new Set(["3"]); const dealService = { getBaseDataSetMetadata: vi.fn(() => ({})), @@ -1525,7 +1607,7 @@ describe("JobsService schedule rows", () => { await callPrivate(service, "handleDataSetCreationJob", { id: "job-blocked-ds", - data: { jobType: "data_set_creation", spAddress: "0xaaa", intervalSeconds: 3600 }, + data: { jobType: "data_set_creation", network: "calibration", spAddress: "0xaaa", intervalSeconds: 3600 }, }); expect(dealService.createDataSetWithPiece).not.toHaveBeenCalled(); @@ -1535,7 +1617,8 @@ describe("JobsService schedule rows", () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2024-01-01T12:00:00Z")); - baseConfigValues.spBlocklists = { ids: new Set(), addresses: new Set(["0xaaa"]) }; + if (baseConfigValues.networks?.calibration) + baseConfigValues.networks.calibration.blockedSpAddresses = new Set(["0xaaa"]); const completedCounter = metricsMocks.jobsCompletedCounter as unknown as { inc: ReturnType }; const dealService = { @@ -1590,11 +1673,20 @@ describe("JobsService schedule rows", () => { for (const testCase of cases) { await callPrivate(testCase.service, testCase.handler, { id: `job-address-blocked-${testCase.jobType}`, - data: { jobType: testCase.jobType, spAddress: "0xaaa", intervalSeconds: testCase.intervalSeconds }, + data: { + jobType: testCase.jobType, + network: "calibration", + spAddress: "0xaaa", + intervalSeconds: testCase.intervalSeconds, + }, }); testCase.expectCheckNotRun(); - expect(completedCounter.inc).toHaveBeenCalledWith({ job_type: testCase.jobType, handler_result: "success" }); + expect(completedCounter.inc).toHaveBeenCalledWith({ + job_type: testCase.jobType, + network: "calibration", + handler_result: "success", + }); } expect(storageProviderRepositoryMock.findOne).not.toHaveBeenCalled(); diff --git a/apps/backend/src/jobs/jobs.service.ts b/apps/backend/src/jobs/jobs.service.ts index 01357225..a1bd3c80 100644 --- a/apps/backend/src/jobs/jobs.service.ts +++ b/apps/backend/src/jobs/jobs.service.ts @@ -5,10 +5,12 @@ import { InjectMetric } from "@willsoto/nestjs-prometheus"; import { type Job, PgBoss, type SendOptions } from "pg-boss"; import type { Counter, Gauge, Histogram } from "prom-client"; import type { Repository } from "typeorm"; +import { SUPPORTED_NETWORKS } from "../common/constants.js"; import { type JobLogContext, type ProviderJobContext, toStructuredError } from "../common/logging.js"; import { getMaintenanceWindowStatus } from "../common/maintenance-window.js"; import { isSpBlocked } from "../common/sp-blocklist.js"; -import type { IConfig, ISpBlocklistConfig } from "../config/app.config.js"; +import type { Network } from "../common/types.js"; +import type { IConfig, INetworksConfig } from "../config/types.js"; import { DataRetentionService } from "../data-retention/data-retention.service.js"; import type { JobType } from "../database/entities/job-schedule-state.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; @@ -18,7 +20,7 @@ import { RetrievalService } from "../retrieval/retrieval.service.js"; import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; import { provisionNextMissingDataSet } from "./data-set-creation.handler.js"; import { DATA_RETENTION_POLL_QUEUE, PROVIDERS_REFRESH_QUEUE, SP_WORK_QUEUE } from "./job-queues.js"; -import { JobScheduleRepository } from "./repositories/job-schedule.repository.js"; +import { JobScheduleRepository, ScheduleRow } from "./repositories/job-schedule.repository.js"; type SpJobType = "deal" | "retrieval" | "data_set_creation" | "piece_cleanup"; const SP_JOB_TYPES: ReadonlySet = new Set(["deal", "retrieval", "data_set_creation", "piece_cleanup"]); @@ -26,18 +28,10 @@ function isSpJobType(jobType: string): jobType is SpJobType { return SP_JOB_TYPES.has(jobType); } -type SpJobData = { jobType: SpJobType; spAddress: string; intervalSeconds: number }; -type ProvidersRefreshJobData = { intervalSeconds: number }; +type SpJobData = { jobType: SpJobType; spAddress: string; intervalSeconds: number; network: Network }; +type ProvidersRefreshJobData = { intervalSeconds: number; network: Network }; type SpJob = Job; -type DataRetentionJobData = { intervalSeconds: number }; - -type ScheduleRow = { - id: number; - job_type: JobType; - sp_address: string; - interval_seconds: number; - next_run_at: string; -}; +type DataRetentionJobData = { intervalSeconds: number; network: Network }; type JobRunStatus = "success" | "error" | "aborted"; @@ -105,8 +99,11 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { const workersEnabled = runMode !== "api"; if (process.env.DEALBOT_DISABLE_CHAIN !== "true") { - await this.walletSdkService.ensureWalletAllowances(); - await this.walletSdkService.ensureProvidersLoaded(); + const activeNetworks = this.configService.get("activeNetworks"); + for (const network of activeNetworks) { + await this.walletSdkService.ensureWalletAllowances(network); + await this.walletSdkService.ensureProvidersLoaded(network); + } } await this.startBoss(); if (!this.boss) { @@ -295,8 +292,8 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { message: "Skipping unknown SP job type", jobType: job.data.jobType, providerAddress: job.data.spAddress, - providerId: this.walletSdkService.getProviderInfo(job.data.spAddress)?.id, - providerName: this.walletSdkService.getProviderInfo(job.data.spAddress)?.name, + providerId: this.walletSdkService.getProviderInfo(job.data.spAddress, job.data.network)?.id, + providerName: this.walletSdkService.getProviderInfo(job.data.spAddress, job.data.network)?.name, }); }, ) @@ -336,17 +333,21 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { ); } - private getMaintenanceWindowStatus(now: Date = new Date()) { - const scheduling = this.configService.get("scheduling"); - return getMaintenanceWindowStatus(now, scheduling.maintenanceWindowsUtc, scheduling.maintenanceWindowMinutes); + private getMaintenanceWindowStatus(now: Date = new Date(), network: Network) { + const networkConfig = this.configService.get("networks")[network]; + return getMaintenanceWindowStatus(now, networkConfig.maintenanceWindowsUtc, networkConfig.maintenanceWindowMinutes); } - private async resolveProviderJobContext(spAddress: string, jobId: string): Promise { - let providerInfo = this.walletSdkService.getProviderInfo(spAddress); + private async resolveProviderJobContext( + spAddress: string, + jobId: string, + network: Network, + ): Promise { + let providerInfo = this.walletSdkService.getProviderInfo(spAddress, network); if (providerInfo == null && process.env.DEALBOT_DISABLE_CHAIN !== "true") { - await this.walletSdkService.loadProviders(); - providerInfo = this.walletSdkService.getProviderInfo(spAddress); + await this.walletSdkService.loadProviders(network); + providerInfo = this.walletSdkService.getProviderInfo(spAddress, network); } let providerId = providerInfo?.id; @@ -355,7 +356,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { // Fall back to DB if either providerId or providerName is missing if (providerId == null || !providerName) { const provider = await this.storageProviderRepository.findOne({ - where: { address: spAddress }, + where: { address: spAddress, network }, select: { providerId: true, name: true }, }); providerId = providerId ?? provider?.providerId ?? undefined; @@ -375,6 +376,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { providerAddress: spAddress, providerId, providerName, + network, }; } @@ -383,9 +385,10 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { spAddress: string, jobId: string, message: string, + network: Network, ): Promise { - const spBlocklists = this.configService.get("spBlocklists"); - if (isSpBlocked(spBlocklists, spAddress)) { + const networkCfg = this.configService.get("networks")[network]; + if (isSpBlocked(networkCfg, spAddress)) { this.logger.log({ jobId, providerAddress: spAddress, @@ -395,8 +398,8 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { return null; } - const logContext = await this.resolveProviderJobContext(spAddress, jobId); - if (isSpBlocked(spBlocklists, spAddress, logContext.providerId)) { + const logContext = await this.resolveProviderJobContext(spAddress, jobId, network); + if (isSpBlocked(networkCfg, spAddress, logContext.providerId)) { this.logger.log({ ...logContext, event: `${jobType}_job_blocked`, @@ -408,27 +411,33 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { return logContext; } - private logMaintenanceSkip(taskLabel: string, windowLabel?: string, logContext?: Partial) { - const scheduling = this.configService.get("scheduling"); + private logMaintenanceSkip( + taskLabel: string, + network: Network, + windowLabel?: string, + logContext?: Partial, + ) { + const networkConfig = this.configService.get("networks")[network]; const label = windowLabel ?? "unknown"; this.logger.log({ ...logContext, event: "maintenance_window_active", - message: `Maintenance window active (${label} UTC, ${scheduling.maintenanceWindowMinutes}m); deferring ${taskLabel}`, + message: `Maintenance window active (${label} UTC, ${networkConfig.maintenanceWindowMinutes}m); deferring ${taskLabel}`, }); } private async handleDealJob(job: SpJob): Promise { const data = job.data; const spAddress = data.spAddress; + const network = data.network; const now = new Date(); - const maintenance = this.getMaintenanceWindowStatus(now); + const maintenance = this.getMaintenanceWindowStatus(now, network); if (maintenance.active) { - this.logMaintenanceSkip(`deal job for ${spAddress}`, maintenance.window?.label, { + this.logMaintenanceSkip(`deal job for ${spAddress}`, network, maintenance.window?.label, { jobId: job.id, providerAddress: spAddress, - providerId: this.walletSdkService.getProviderInfo(spAddress)?.id, - providerName: this.walletSdkService.getProviderInfo(spAddress)?.name, + providerId: this.walletSdkService.getProviderInfo(spAddress, network)?.id, + providerName: this.walletSdkService.getProviderInfo(spAddress, network)?.name, }); await this.deferJobForMaintenance("deal", data, maintenance, now); return; @@ -444,24 +453,25 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { abortController.abort(abortReason); }, timeoutMs); - await this.recordJobExecution("deal", async () => { + await this.recordJobExecution("deal", network, async () => { const logContext = await this.resolveRunnableProviderJobContext( "deal", spAddress, job.id, "Deal job skipped: provider is blocked for scheduled data-storage checks", + network, ); if (logContext == null) { clearTimeout(timeoutId); return "success"; } try { - let provider = this.walletSdkService.getTestingProviders().find((p) => p.serviceProvider === spAddress); + let provider = this.walletSdkService.getTestingProviders(network).find((p) => p.serviceProvider === spAddress); if (!provider) { if (process.env.DEALBOT_DISABLE_CHAIN !== "true") { - await this.walletSdkService.loadProviders(); + await this.walletSdkService.loadProviders(network); } - provider = this.walletSdkService.getTestingProviders().find((p) => p.serviceProvider === spAddress); + provider = this.walletSdkService.getTestingProviders(network).find((p) => p.serviceProvider === spAddress); if (!provider) { this.logger.warn({ ...logContext, @@ -473,8 +483,9 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { } // Data-set-aware deal creation - const minDataSets = this.configService.get("blockchain").minNumDataSetsForChecks; - const baseDataSetMetadata = this.dealService.getBaseDataSetMetadata(); + const networkCfg = this.configService.get("networks")[network]; + const minDataSets = networkCfg.minNumDataSetsForChecks; + const baseDataSetMetadata = this.dealService.getBaseDataSetMetadata(network); let extraDataSetMetadata: Record | undefined; if (minDataSets > 1) { @@ -486,6 +497,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { const exists = await this.dealService.checkDataSetExists( spAddress, expectedMetadata, + network, abortController.signal, ); @@ -519,7 +531,9 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { await this.dealService.createDealForProvider(provider, { signal: abortController.signal, extraDataSetMetadata, + network, logContext: { + network, jobId: logContext.jobId, providerAddress: logContext.providerAddress, providerId: provider.id ?? logContext.providerId, @@ -557,14 +571,15 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { private async handleRetrievalJob(job: SpJob): Promise { const data = job.data; const spAddress = data.spAddress; + const network = data.network; const now = new Date(); - const maintenance = this.getMaintenanceWindowStatus(now); + const maintenance = this.getMaintenanceWindowStatus(now, network); if (maintenance.active) { - this.logMaintenanceSkip(`retrieval job for ${spAddress}`, maintenance.window?.label, { + this.logMaintenanceSkip(`retrieval job for ${spAddress}`, network, maintenance.window?.label, { jobId: job.id, providerAddress: spAddress, - providerId: this.walletSdkService.getProviderInfo(spAddress)?.id, - providerName: this.walletSdkService.getProviderInfo(spAddress)?.name, + providerId: this.walletSdkService.getProviderInfo(spAddress, network)?.id, + providerName: this.walletSdkService.getProviderInfo(spAddress, network)?.name, }); await this.deferJobForMaintenance("retrieval", data, maintenance, now); return; @@ -580,19 +595,25 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { abortController.abort(abortReason); }, timeoutMs); - await this.recordJobExecution("retrieval", async () => { + await this.recordJobExecution("retrieval", network, async () => { const logContext = await this.resolveRunnableProviderJobContext( "retrieval", spAddress, job.id, "Retrieval job skipped: provider is blocked for scheduled retrieval checks", + network, ); if (logContext == null) { clearTimeout(timeoutId); return "success"; } try { - await this.retrievalService.performRandomRetrievalForProvider(spAddress, abortController.signal, logContext); + await this.retrievalService.performRandomRetrievalForProvider( + spAddress, + network, + abortController.signal, + logContext, + ); return "success"; } catch (error) { if (abortController.signal.aborted) { @@ -622,25 +643,26 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { } private async handleDataRetentionJob(data: DataRetentionJobData): Promise { - void data; - await this.recordJobExecution("data_retention_poll", async () => { - await this.dataRetentionService.pollDataRetention(); + const network = data.network; + await this.recordJobExecution("data_retention_poll", network, async () => { + await this.dataRetentionService.pollDataRetention(network); return "success"; }); } private async handleProvidersRefreshJob(data: ProvidersRefreshJobData): Promise { - void data; - await this.recordJobExecution("providers_refresh", async () => { + const network = data.network; + await this.recordJobExecution("providers_refresh", network, async () => { if (process.env.DEALBOT_DISABLE_CHAIN === "true") { this.logger.warn({ event: "chain_integration_disabled", message: "Chain integration disabled; skipping provider refresh job.", + network, }); } else { - await this.walletSdkService.loadProviders(); + await this.walletSdkService.loadProviders(network); } - await this.updateStorageProviderGauges(); + await this.updateStorageProviderGauges(network); return "success"; }); } @@ -648,13 +670,14 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { private async handlePieceCleanupJob(job: SpJob): Promise { const data = job.data; const spAddress = data.spAddress; + const network = data.network; const now = new Date(); - const maintenance = this.getMaintenanceWindowStatus(now); + const maintenance = this.getMaintenanceWindowStatus(now, network); if (maintenance.active) { - this.logMaintenanceSkip(`piece_cleanup job for ${spAddress}`, maintenance.window?.label, { + this.logMaintenanceSkip(`piece_cleanup job for ${spAddress}`, network, maintenance.window?.label, { jobId: job.id, providerAddress: spAddress, - providerId: this.walletSdkService.getProviderInfo(spAddress)?.id, + providerId: this.walletSdkService.getProviderInfo(spAddress, network)?.id, }); await this.deferJobForMaintenance("piece_cleanup", data, maintenance, now); return; @@ -670,10 +693,10 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { abortController.abort(abortReason); }, timeoutMs); - await this.recordJobExecution("piece_cleanup", async () => { - const logContext = await this.resolveProviderJobContext(spAddress, job.id); + await this.recordJobExecution("piece_cleanup", network, async () => { + const logContext = await this.resolveProviderJobContext(spAddress, job.id, network); try { - await this.pieceCleanupService.cleanupPiecesForProvider(spAddress, abortController.signal, logContext); + await this.pieceCleanupService.cleanupPiecesForProvider(spAddress, network, abortController.signal, logContext); return "success"; } catch (error) { if (abortController.signal.aborted) { @@ -701,30 +724,38 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { }); } - private async updateStorageProviderGauges(): Promise { + private async updateStorageProviderGauges(network: Network): Promise { try { - const totalProviders = await this.storageProviderRepository.count(); - const activeCount = await this.storageProviderRepository.count({ where: { isActive: true } }); + const networkFilter = { network }; + const totalProviders = await this.storageProviderRepository.count({ where: networkFilter }); + const activeCount = await this.storageProviderRepository.count({ where: { ...networkFilter, isActive: true } }); const inactiveCount = Math.max(0, totalProviders - activeCount); - this.storageProvidersActive.set({ status: "active" }, activeCount); - this.storageProvidersActive.set({ status: "inactive" }, inactiveCount); - - const useOnlyApprovedProviders = this.configService.get("blockchain").useOnlyApprovedProviders; - const testedWhere = useOnlyApprovedProviders ? { isActive: true, isApproved: true } : { isActive: true }; - const spBlocklists = this.configService.get("spBlocklists"); - const hasGlobalBlocklist = spBlocklists.addresses.size > 0 || spBlocklists.ids.size > 0; - let testedCount: number; - if (hasGlobalBlocklist) { - const testedProviders = await this.storageProviderRepository.find({ - select: { address: true, providerId: true }, - where: testedWhere, - }); - testedCount = testedProviders.filter((p) => !isSpBlocked(spBlocklists, p.address, p.providerId)).length; - } else { - testedCount = await this.storageProviderRepository.count({ where: testedWhere }); + this.storageProvidersActive.set({ network, status: "active" }, activeCount); + this.storageProvidersActive.set({ network, status: "inactive" }, inactiveCount); + + const activeNetworks = this.configService.get("activeNetworks"); + const networks = network ? [network] : activeNetworks; + let testedCount = 0; + for (const net of networks) { + const networkCfg = this.configService.get("networks")[net]; + const testedWhere = networkCfg.useOnlyApprovedProviders + ? { network: net, isActive: true, isApproved: true } + : { network: net, isActive: true }; + const hasGlobalBlocklist = networkCfg.blockedSpIds.size > 0 || networkCfg.blockedSpAddresses.size > 0; + if (hasGlobalBlocklist) { + const testedProviders = await this.storageProviderRepository.find({ + select: { address: true, providerId: true }, + where: testedWhere, + }); + testedCount += testedProviders.filter((p) => !isSpBlocked(networkCfg, p.address, p.providerId)).length; + } else { + testedCount += await this.storageProviderRepository.count({ + where: testedWhere, + }); + } } - this.storageProvidersTested.set(testedCount); + this.storageProvidersTested.set({ network }, testedCount); } catch (error) { this.logger.warn({ event: "update_storage_provider_metrics_failed", @@ -737,21 +768,23 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { private async handleDataSetCreationJob(job: SpJob): Promise { const data = job.data; const spAddress = data.spAddress; + const network = data.network; const now = new Date(); - const maintenance = this.getMaintenanceWindowStatus(now); + const maintenance = this.getMaintenanceWindowStatus(now, network); if (maintenance.active) { - this.logMaintenanceSkip(`data_set_creation job for ${spAddress}`, maintenance.window?.label, { + this.logMaintenanceSkip(`data_set_creation job for ${spAddress}`, network, maintenance.window?.label, { jobId: job.id, providerAddress: spAddress, - providerId: this.walletSdkService.getProviderInfo(spAddress)?.id, - providerName: this.walletSdkService.getProviderInfo(spAddress)?.name, + providerId: this.walletSdkService.getProviderInfo(spAddress, network)?.id, + providerName: this.walletSdkService.getProviderInfo(spAddress, network)?.name, }); await this.deferJobForMaintenance("data_set_creation", data, maintenance, now); return; } - const minDataSets = this.configService.get("blockchain").minNumDataSetsForChecks; - const baseDataSetMetadata = this.dealService.getBaseDataSetMetadata(); + const networkCfg = this.configService.get("networks")[network]; + const minDataSets = networkCfg.minNumDataSetsForChecks; + const baseDataSetMetadata = this.dealService.getBaseDataSetMetadata(network); // Create AbortController for job timeout enforcement const abortController = new AbortController(); @@ -763,12 +796,13 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { abortController.abort(abortReason); }, timeoutMs); - await this.recordJobExecution("data_set_creation", async () => { + await this.recordJobExecution("data_set_creation", network, async () => { const dataSetLogContext = await this.resolveRunnableProviderJobContext( "data_set_creation", spAddress, job.id, "Data set creation job skipped: provider is blocked for scheduled data-storage checks", + network, ); if (dataSetLogContext == null) { clearTimeout(timeoutId); @@ -778,6 +812,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { await provisionNextMissingDataSet( { dealService: this.dealService, logger: this.logger }, spAddress, + network, minDataSets, baseDataSetMetadata, dataSetLogContext, @@ -810,12 +845,16 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { }); } - private maintenanceResumeAt(now: Date, maintenance: ReturnType): Date | null { + private maintenanceResumeAt( + now: Date, + network: Network, + maintenance: ReturnType, + ): Date | null { if (!maintenance.active || !maintenance.window) { return null; } - const scheduling = this.configService.get("scheduling"); - const durationMinutes = scheduling.maintenanceWindowMinutes; + const networkConfig = this.configService.get("networks")[network]; + const durationMinutes = networkConfig.maintenanceWindowMinutes; if (durationMinutes <= 0) { return null; } @@ -842,7 +881,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { maintenance: ReturnType, now: Date, ): Promise { - const resumeAt = this.maintenanceResumeAt(now, maintenance); + const resumeAt = this.maintenanceResumeAt(now, data.network, maintenance); if (resumeAt == null) { return; } @@ -874,29 +913,32 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { * Runs one scheduler tick and updates queue metrics. */ private async runTick(): Promise { - try { - await this.ensureScheduleRows(); - await this.enqueueDueJobs(); - } catch (error) { - this.logger.error({ - event: "pgboss_scheduler_tick_failed", - message: "pg-boss scheduler core tick failed", - error: toStructuredError(error), - }); - } + const activeNetworks = this.configService.get("activeNetworks"); + for (const network of activeNetworks) { + try { + await this.ensureScheduleRows(network); + await this.enqueueDueJobs(network); + } catch (error) { + this.logger.error({ + event: "pgboss_scheduler_tick_failed", + message: "pg-boss scheduler core tick failed", + error: toStructuredError(error), + }); + } - try { - await this.updateQueueMetrics(); - } catch (error) { - this.logger.error({ - event: "pgboss_scheduler_metrics_update_failed", - message: "pg-boss scheduler metrics update failed", - error: toStructuredError(error), - }); + try { + await this.updateQueueMetrics(network); + } catch (error) { + this.logger.error({ + event: "pgboss_scheduler_metrics_update_failed", + message: "pg-boss scheduler metrics update failed", + error: toStructuredError(error), + }); + } } } - private getIntervalSecondsForRates(): { + private getIntervalSecondsForNetwork(network: Network): { dealIntervalSeconds: number; retrievalIntervalSeconds: number; dataSetCreationIntervalSeconds: number; @@ -904,20 +946,21 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { providersRefreshIntervalSeconds: number; pieceCleanupIntervalSeconds: number; } { - const jobsConfig = this.configService.get("jobs", { infer: true }); - const scheduling = this.configService.get("scheduling", { infer: true }); + const networkCfg = this.configService.get("networks")[network]; - const dealsPerHour = jobsConfig.dealsPerSpPerHour; - const retrievalsPerHour = jobsConfig.retrievalsPerSpPerHour; - const dataSetCreationsPerHour = jobsConfig.dataSetCreationsPerSpPerHour; - const pieceCleanupPerHour = jobsConfig.pieceCleanupPerSpPerHour; + const { + dealsPerSpPerHour, + retrievalsPerSpPerHour, + dataSetCreationsPerSpPerHour, + pieceCleanupPerSpPerHour, + dataRetentionPollIntervalSeconds, + providersRefreshIntervalSeconds, + } = networkCfg; - const dealIntervalSeconds = Math.max(1, Math.round(3600 / dealsPerHour)); - const retrievalIntervalSeconds = Math.max(1, Math.round(3600 / retrievalsPerHour)); - const dataSetCreationIntervalSeconds = Math.max(1, Math.round(3600 / dataSetCreationsPerHour)); - const pieceCleanupIntervalSeconds = Math.max(1, Math.round(3600 / pieceCleanupPerHour)); - const dataRetentionPollIntervalSeconds = scheduling.dataRetentionPollIntervalSeconds; - const providersRefreshIntervalSeconds = scheduling.providersRefreshIntervalSeconds; + const dealIntervalSeconds = Math.max(1, Math.round(3600 / dealsPerSpPerHour)); + const retrievalIntervalSeconds = Math.max(1, Math.round(3600 / retrievalsPerSpPerHour)); + const dataSetCreationIntervalSeconds = Math.max(1, Math.round(3600 / dataSetCreationsPerSpPerHour)); + const pieceCleanupIntervalSeconds = Math.max(1, Math.round(3600 / pieceCleanupPerSpPerHour)); return { dealIntervalSeconds, @@ -936,7 +979,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { * - Pauses rows for providers that are no longer active. * - Ensures global data_retention_poll and providers_refresh jobs exist. */ - private async ensureScheduleRows(): Promise { + private async ensureScheduleRows(network: Network): Promise { const now = new Date(); const { dealIntervalSeconds, @@ -945,14 +988,15 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { dataRetentionPollIntervalSeconds, providersRefreshIntervalSeconds, pieceCleanupIntervalSeconds, - } = this.getIntervalSecondsForRates(); + } = this.getIntervalSecondsForNetwork(network); - const useOnlyApprovedProviders = this.configService.get("blockchain").useOnlyApprovedProviders; + const networkCfg = this.configService.get("networks")[network]; + const useOnlyApprovedProviders = networkCfg.useOnlyApprovedProviders; // Active providers are guaranteed to support ipniIpfs // as validated by WalletSdkService.loadProvidersInternal() const providers = await this.storageProviderRepository.find({ select: { address: true, providerId: true }, - where: useOnlyApprovedProviders ? { isActive: true, isApproved: true } : { isActive: true }, + where: { network, isActive: true, ...(useOnlyApprovedProviders && { isApproved: true }) }, }); const phaseMs = this.schedulePhaseSeconds() * 1000; @@ -962,12 +1006,11 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { const dataRetentionPollStartAt = new Date(now.getTime() + phaseMs); const providersRefreshStartAt = new Date(now.getTime() + phaseMs); - const minDataSets = this.configService.get("blockchain").minNumDataSetsForChecks; + const minDataSets = networkCfg.minNumDataSetsForChecks; const cleanupStartAt = new Date(now.getTime() + phaseMs); - const spBlocklistsCfg = this.configService.get("spBlocklists"); const unblockedAddresses = providers - .filter(({ address, providerId }) => !isSpBlocked(spBlocklistsCfg, address, providerId)) + .filter(({ address, providerId }) => !isSpBlocked(networkCfg, address, providerId)) .map(({ address }) => address); const blockedCount = providers.length - unblockedAddresses.length; if (blockedCount > 0) { @@ -979,14 +1022,21 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { } for (const address of unblockedAddresses) { - await this.jobScheduleRepository.upsertSchedule("deal", address, dealIntervalSeconds, dealStartAt); - await this.jobScheduleRepository.upsertSchedule("retrieval", address, retrievalIntervalSeconds, retrievalStartAt); + await this.jobScheduleRepository.upsertSchedule("deal", address, dealIntervalSeconds, dealStartAt, network); + await this.jobScheduleRepository.upsertSchedule( + "retrieval", + address, + retrievalIntervalSeconds, + retrievalStartAt, + network, + ); if (minDataSets >= 1) { await this.jobScheduleRepository.upsertSchedule( "data_set_creation", address, dataSetCreationIntervalSeconds, dataSetCreationStartAt, + network, ); } await this.jobScheduleRepository.upsertSchedule( @@ -994,15 +1044,20 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { address, pieceCleanupIntervalSeconds, cleanupStartAt, + network, ); } if (providers.length > 0) { - const deletedAddresses = await this.jobScheduleRepository.deleteSchedulesForInactiveProviders(unblockedAddresses); + const deletedAddresses = await this.jobScheduleRepository.deleteSchedulesForInactiveProviders( + unblockedAddresses, + network, + ); if (deletedAddresses.length > 0) { this.logger.warn({ event: "job_schedules_deleted", message: "Deleted job schedules for providers no longer in active list", + network, deletedCount: deletedAddresses.length, deletedAddresses, }); @@ -1011,21 +1066,24 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { this.logger.warn({ event: "job_schedule_deletion_skipped", message: "No active providers found; skipping job schedule deletion to prevent accidental mass-deletion", + network, }); } - // Global job schedules (sp_address = '') + // Per-network global job schedules (sp_address = '') await this.jobScheduleRepository.upsertSchedule( "data_retention_poll", "", dataRetentionPollIntervalSeconds, dataRetentionPollStartAt, + network, ); await this.jobScheduleRepository.upsertSchedule( "providers_refresh", "", providersRefreshIntervalSeconds, providersRefreshStartAt, + network, ); } @@ -1055,19 +1113,19 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { return { intervalMs, nextRunAt, runsDue }; } - private async enqueueDueJobs(): Promise { + private async enqueueDueJobs(network: Network): Promise { if (!this.boss) return; const now = new Date(); - const maintenance = this.getMaintenanceWindowStatus(now); + const maintenance = this.getMaintenanceWindowStatus(now, network); const catchupMax = this.catchupMaxEnqueue(); if (maintenance.active) { - this.logMaintenanceSkip("Global job enqueues", maintenance.window?.label); + this.logMaintenanceSkip("Global job enqueues", network, maintenance.window?.label); } await this.jobScheduleRepository.runTransaction(async (manager) => { - const rows = await this.jobScheduleRepository.findDueSchedulesWithManager(manager, now); + const rows = await this.jobScheduleRepository.findDueSchedulesWithManager(manager, now, network); for (const row of rows) { // Skip legacy job types that are no longer supported. @@ -1154,15 +1212,19 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { } private mapJobPayload(row: ScheduleRow): SpJobData | ProvidersRefreshJobData | DataRetentionJobData { + const network = (row.network ?? SUPPORTED_NETWORKS[0]) as Network; if ( row.job_type === "deal" || row.job_type === "retrieval" || row.job_type === "data_set_creation" || row.job_type === "piece_cleanup" ) { - return { jobType: row.job_type, spAddress: row.sp_address, intervalSeconds: row.interval_seconds }; + return { jobType: row.job_type, spAddress: row.sp_address, intervalSeconds: row.interval_seconds, network }; } - return { intervalSeconds: row.interval_seconds }; + if (row.job_type === "providers_refresh" || row.job_type === "data_retention_poll") { + return { intervalSeconds: row.interval_seconds, network }; + } + return { intervalSeconds: row.interval_seconds, network }; } /** @@ -1181,14 +1243,16 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { if (isSpJobType(jobType)) { const spData = data as SpJobData; if (!finalOptions.singletonKey) { - finalOptions.singletonKey = spData.spAddress; + // Include network in singleton key to prevent cross-network deduplication collisions + finalOptions.singletonKey = `${spData.network}:${spData.spAddress}`; } } else { - // Global jobs: use job type as singleton key. - finalOptions.singletonKey = jobType; + // Global jobs: include network in singleton key for per-network isolation + const networkKey = (data as any).network as string | undefined; + finalOptions.singletonKey = networkKey ? `${networkKey}:${jobType}` : jobType; } await this.boss.send(name, data, finalOptions); - this.jobsEnqueueAttemptsCounter.inc({ job_type: jobType, outcome: "success" }); + this.jobsEnqueueAttemptsCounter.inc({ network: data.network, job_type: jobType, outcome: "success" }); return true; } catch (error) { this.logger.warn({ @@ -1198,7 +1262,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { jobType, error: toStructuredError(error), }); - this.jobsEnqueueAttemptsCounter.inc({ job_type: jobType, outcome: "error" }); + this.jobsEnqueueAttemptsCounter.inc({ network: data.network, job_type: jobType, outcome: "error" }); return false; } } @@ -1206,18 +1270,22 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { /** * Records handler start/end metrics around a job execution. */ - private async recordJobExecution(jobType: JobType, run: () => Promise): Promise { + private async recordJobExecution( + jobType: JobType, + network: Network | "global", + run: () => Promise, + ): Promise { const startedAt = Date.now(); - this.jobsStartedCounter.inc({ job_type: jobType }); + this.jobsStartedCounter.inc({ network, job_type: jobType }); try { const status = await run(); const finishedAt = Date.now(); - this.jobDuration.observe({ job_type: jobType }, (finishedAt - startedAt) / 1000); - this.jobsCompletedCounter.inc({ job_type: jobType, handler_result: status }); + this.jobDuration.observe({ network, job_type: jobType }, (finishedAt - startedAt) / 1000); + this.jobsCompletedCounter.inc({ network, job_type: jobType, handler_result: status }); } catch (error) { const finishedAt = Date.now(); - this.jobDuration.observe({ job_type: jobType }, (finishedAt - startedAt) / 1000); - this.jobsCompletedCounter.inc({ job_type: jobType, handler_result: "error" }); + this.jobDuration.observe({ network, job_type: jobType }, (finishedAt - startedAt) / 1000); + this.jobsCompletedCounter.inc({ network, job_type: jobType, handler_result: "error" }); throw error; } } @@ -1225,7 +1293,7 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { /** * Refreshes queue depth and age gauges from pg-boss tables. */ - private async updateQueueMetrics(): Promise { + private async updateQueueMetrics(network: Network): Promise { const jobTypes: JobType[] = [ "deal", "retrieval", @@ -1235,26 +1303,26 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { "providers_refresh", ]; for (const jobType of jobTypes) { - this.jobsQueuedGauge.set({ job_type: jobType }, 0); - this.jobsRetryScheduledGauge.set({ job_type: jobType }, 0); - this.jobsInFlightGauge.set({ job_type: jobType }, 0); - this.jobsPausedGauge.set({ job_type: jobType }, 0); - this.oldestQueuedAgeGauge.set({ job_type: jobType }, 0); - this.oldestInFlightAgeGauge.set({ job_type: jobType }, 0); + this.jobsQueuedGauge.set({ network, job_type: jobType }, 0); + this.jobsRetryScheduledGauge.set({ network, job_type: jobType }, 0); + this.jobsInFlightGauge.set({ network, job_type: jobType }, 0); + this.jobsPausedGauge.set({ network, job_type: jobType }, 0); + this.oldestQueuedAgeGauge.set({ network, job_type: jobType }, 0); + this.oldestInFlightAgeGauge.set({ network, job_type: jobType }, 0); } - const rows = await this.jobScheduleRepository.countBossJobStates(["created", "retry", "active"]); + const rows = await this.jobScheduleRepository.countBossJobStates(["created", "retry", "active"], network); if (rows.length > 0) { for (const row of rows) { const jobType = row.job_type as JobType; if (!jobTypes.includes(jobType)) continue; const state = String(row.state).toLowerCase(); if (state === "active") { - this.jobsInFlightGauge.set({ job_type: jobType }, row.count); + this.jobsInFlightGauge.set({ network, job_type: jobType }, row.count); } else if (state === "retry") { - this.jobsRetryScheduledGauge.set({ job_type: jobType }, row.count); + this.jobsRetryScheduledGauge.set({ network, job_type: jobType }, row.count); } else { - this.jobsQueuedGauge.set({ job_type: jobType }, row.count); + this.jobsQueuedGauge.set({ network, job_type: jobType }, row.count); } } } else { @@ -1265,24 +1333,24 @@ export class JobsService implements OnModuleInit, OnApplicationShutdown { }); } - const pausedSchedules = await this.jobScheduleRepository.countPausedSchedules(); + const pausedSchedules = await this.jobScheduleRepository.countPausedSchedules(network); for (const row of pausedSchedules) { - this.jobsPausedGauge.set({ job_type: row.job_type }, row.count); + this.jobsPausedGauge.set({ network, job_type: row.job_type }, row.count); } const now = new Date(); - const queuedAges = await this.jobScheduleRepository.minBossJobAgeSecondsByState("created", now); + const queuedAges = await this.jobScheduleRepository.minBossJobAgeSecondsByState("created", now, network); for (const row of queuedAges) { const jobType = row.job_type as JobType; if (!jobTypes.includes(jobType)) continue; - this.oldestQueuedAgeGauge.set({ job_type: jobType }, Math.max(0, row.min_age_seconds ?? 0)); + this.oldestQueuedAgeGauge.set({ network, job_type: jobType }, Math.max(0, row.min_age_seconds ?? 0)); } - const activeAges = await this.jobScheduleRepository.minBossJobAgeSecondsByState("active", now); + const activeAges = await this.jobScheduleRepository.minBossJobAgeSecondsByState("active", now, network); for (const row of activeAges) { const jobType = row.job_type as JobType; if (!jobTypes.includes(jobType)) continue; - this.oldestInFlightAgeGauge.set({ job_type: jobType }, Math.max(0, row.min_age_seconds ?? 0)); + this.oldestInFlightAgeGauge.set({ network, job_type: jobType }, Math.max(0, row.min_age_seconds ?? 0)); } } } diff --git a/apps/backend/src/jobs/repositories/job-schedule.repository.ts b/apps/backend/src/jobs/repositories/job-schedule.repository.ts index e698c2d2..3bce2a0e 100644 --- a/apps/backend/src/jobs/repositories/job-schedule.repository.ts +++ b/apps/backend/src/jobs/repositories/job-schedule.repository.ts @@ -2,6 +2,7 @@ import { Injectable, Logger } from "@nestjs/common"; import { InjectDataSource } from "@nestjs/typeorm"; import type { DataSource } from "typeorm"; import { toStructuredError } from "../../common/logging.js"; +import { Network } from "../../common/types.js"; import type { JobType } from "../../database/entities/job-schedule-state.entity.js"; import { DATA_RETENTION_POLL_QUEUE, @@ -17,6 +18,7 @@ export type ScheduleRow = { id: number; job_type: JobType; sp_address: string; + network: string; interval_seconds: number; next_run_at: string; }; @@ -39,17 +41,23 @@ export class JobScheduleRepository { * @param intervalSeconds - The frequency of the job in seconds * @param nextRunAt - The scheduled time for the next run */ - async upsertSchedule(jobType: JobType, spAddress: string, intervalSeconds: number, nextRunAt: Date): Promise { + async upsertSchedule( + jobType: JobType, + spAddress: string, + intervalSeconds: number, + nextRunAt: Date, + network: Network, + ): Promise { await this.dataSource.query( ` - INSERT INTO job_schedule_state (job_type, sp_address, interval_seconds, next_run_at) - VALUES ($1, $2, $3, $4) + INSERT INTO job_schedule_state (job_type, sp_address, network, interval_seconds, next_run_at) + VALUES ($1, $2, $3, $4, $5) ON CONFLICT (job_type, sp_address, network) DO UPDATE SET interval_seconds = EXCLUDED.interval_seconds, paused = job_schedule_state.paused, updated_at = NOW() `, - [jobType, spAddress, intervalSeconds, nextRunAt], + [jobType, spAddress, network, intervalSeconds, nextRunAt], ); } @@ -63,21 +71,24 @@ export class JobScheduleRepository { * @param activeAddresses - List of currently active provider addresses to keep. * @returns Array of storage provider addresses whose schedules were deleted. */ - async deleteSchedulesForInactiveProviders(activeAddresses: string[]): Promise { + async deleteSchedulesForInactiveProviders(activeAddresses: string[], network: Network): Promise { try { if (activeAddresses.length === 0) { this.logger.warn({ event: "delete_all_provider_schedules_warning", message: "Deleting all provider schedules because activeAddresses is empty. Ensure this is intended to avoid mass deletion.", + network, }); const [rows] = (await this.dataSource.query( ` DELETE FROM job_schedule_state WHERE job_type IN ('deal', 'retrieval', 'data_set_creation', 'piece_cleanup') AND sp_address <> '' + AND network = $1 RETURNING sp_address `, + [network], )) || [[]]; return rows.map((row: { sp_address: string }) => row.sp_address); } @@ -87,16 +98,18 @@ export class JobScheduleRepository { DELETE FROM job_schedule_state WHERE job_type IN ('deal', 'retrieval', 'data_set_creation', 'piece_cleanup') AND sp_address <> '' - AND sp_address <> ALL($1::text[]) + AND network = $1 + AND sp_address <> ALL($2::text[]) RETURNING sp_address `, - [activeAddresses], + [network, activeAddresses], )) || [[]]; return rows.map((row: { sp_address: string }) => row.sp_address); } catch (error) { this.logger.error({ event: "delete_inactive_provider_schedules_failed", message: "Failed to delete schedules for inactive providers", + network, error: toStructuredError(error), }); throw error; @@ -106,14 +119,16 @@ export class JobScheduleRepository { /** * Counts manually paused jobs by type. */ - async countPausedSchedules(): Promise<{ job_type: string; count: number }[]> { + async countPausedSchedules(network?: Network): Promise<{ job_type: string; count: number }[]> { return this.dataSource.query( ` SELECT job_type, COUNT(*)::int AS count FROM job_schedule_state - WHERE paused = true + WHERE paused = true + AND ($1::text IS NULL OR network = $1) GROUP BY job_type `, + [network ?? null], ); } @@ -127,17 +142,19 @@ export class JobScheduleRepository { async findDueSchedulesWithManager( manager: { query: (sql: string, params?: any[]) => Promise }, now: Date, + network?: string, ): Promise { return manager.query( ` - SELECT id, job_type, sp_address, interval_seconds, next_run_at - FROM job_schedule_state - WHERE paused = false - AND next_run_at <= $1 - ORDER BY next_run_at ASC - FOR UPDATE SKIP LOCKED - `, - [now], + SELECT id, job_type, sp_address, network, interval_seconds, next_run_at + FROM job_schedule_state + WHERE paused = false + AND next_run_at <= $1 + AND ($2::text IS NULL OR network = $2) + ORDER BY next_run_at ASC + FOR UPDATE SKIP LOCKED + `, + [now, network ?? null], ); } @@ -202,10 +219,14 @@ export class JobScheduleRepository { /** * Counts pg-boss jobs by dealbot job type and state for the requested states. + * Filters by network if provided. * Uses `data->>'jobType'` for the shared sp.work queue and maps legacy queue names. * Casts state to text so drivers always return a string (pg-boss uses job_state enum). */ - async countBossJobStates(states: string[]): Promise<{ job_type: string; state: string; count: number }[]> { + async countBossJobStates( + states: string[], + network?: Network, + ): Promise<{ job_type: string; state: string; count: number }[]> { return this.dataSource.query( ` SELECT @@ -223,6 +244,7 @@ export class JobScheduleRepository { COUNT(*)::int AS count FROM pgboss.job WHERE state::text = ANY($1::text[]) + AND ($9::text IS NULL OR data->>'network' = $9) GROUP BY 1, 2 `, [ @@ -234,17 +256,20 @@ export class JobScheduleRepository { LEGACY_RETRIEVAL_QUEUE, DATA_RETENTION_POLL_QUEUE, PROVIDERS_REFRESH_QUEUE, + network, // Added as $9 ], ); } /** * Returns the minimum age (seconds) for jobs in a given pg-boss state, grouped by job type. + * Filters by network if provided. * Uses created_on for queued jobs and started_on for active jobs (pg-boss schema column names). */ async minBossJobAgeSecondsByState( state: "created" | "active", now: Date, + network?: Network, ): Promise<{ job_type: string; min_age_seconds: number | null }[]> { return this.dataSource.query( ` @@ -271,6 +296,7 @@ export class JobScheduleRepository { ) AS min_age_seconds FROM pgboss.job WHERE state::text = $2 + AND ($10::text IS NULL OR data->>'network' = $10) GROUP BY 1 `, [ @@ -283,6 +309,7 @@ export class JobScheduleRepository { LEGACY_RETRIEVAL_QUEUE, DATA_RETENTION_POLL_QUEUE, PROVIDERS_REFRESH_QUEUE, + network, // Added as $10 ], ); } diff --git a/apps/backend/src/metrics-prometheus/check-metric-labels.ts b/apps/backend/src/metrics-prometheus/check-metric-labels.ts index d8447160..364c9c5c 100644 --- a/apps/backend/src/metrics-prometheus/check-metric-labels.ts +++ b/apps/backend/src/metrics-prometheus/check-metric-labels.ts @@ -1,3 +1,5 @@ +import { Network } from "../common/types.js"; + export type CheckType = "dataStorage" | "retrieval" | "dataRetention" | "dataSetCreation"; export type ProviderStatus = "approved" | "unapproved"; @@ -6,10 +8,12 @@ export type CheckMetricLabels = { providerId: string; providerName: string; providerStatus: ProviderStatus; + network: Network; }; export type CheckMetricLabelInput = { checkType: CheckType; + network: Network; providerId?: bigint | null; providerName?: string | null; providerIsApproved?: boolean | null; @@ -17,6 +21,7 @@ export type CheckMetricLabelInput = { export const buildCheckMetricLabels = ({ checkType, + network, providerId, providerName, providerIsApproved, @@ -26,6 +31,7 @@ export const buildCheckMetricLabels = ({ return { checkType, + network, providerId: normalizedProviderId, providerName: providerName ?? "unknown", providerStatus, diff --git a/apps/backend/src/metrics-prometheus/check-metrics.service.ts b/apps/backend/src/metrics-prometheus/check-metrics.service.ts index 55975cad..a3d3ae71 100644 --- a/apps/backend/src/metrics-prometheus/check-metrics.service.ts +++ b/apps/backend/src/metrics-prometheus/check-metrics.service.ts @@ -223,6 +223,7 @@ export class DiscoverabilityCheckMetrics { buildLabelsForDeal(deal: Deal): CheckMetricLabels | null { if (!deal.spAddress) return null; return buildCheckMetricLabels({ + network: deal.network, checkType: "dataStorage", providerId: deal.storageProvider?.providerId, providerName: deal.storageProvider?.name, diff --git a/apps/backend/src/metrics-prometheus/metrics-prometheus.module.ts b/apps/backend/src/metrics-prometheus/metrics-prometheus.module.ts index 18bda30d..6120c4d1 100644 --- a/apps/backend/src/metrics-prometheus/metrics-prometheus.module.ts +++ b/apps/backend/src/metrics-prometheus/metrics-prometheus.module.ts @@ -59,98 +59,98 @@ const metricProviders = [ // docs/checks/events-and-metrics.md#ingestMs name: "ingestMs", help: "Time to upload a piece to a storage provider (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [10, 50, 100, 500, 1000, 2000, 5000, 10000, 30000, 60000, 120000, 300000], }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#ingestThroughputBps name: "ingestThroughputBps", help: "Ingest throughput in bytes per second", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: throughputBuckets, }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#pieceAddedOnChainMs name: "pieceAddedOnChainMs", help: "Time from upload end to piece added on-chain (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [10, 50, 100, 500, 1000, 2000, 5000, 10000, 30000, 60000, 120000, 300000], }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#pieceConfirmedOnChainMs name: "pieceConfirmedOnChainMs", help: "Time from piece added to piece confirmed on-chain (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [10, 50, 100, 500, 1000, 2000, 5000, 10000, 30000, 60000, 120000, 300000], }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#spIndexLocallyMs name: "spIndexLocallyMs", help: "Time from upload end to SP indexing locally (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [10, 50, 100, 500, 1000, 2000, 5000, 10000, 30000, 60000, 120000, 300000], }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#spAnnounceAdvertisementMs name: "spAnnounceAdvertisementMs", help: "Time from upload end to SP advertisement to IPNI (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [10, 50, 100, 500, 1000, 2000, 5000, 10000, 30000, 60000, 120000, 300000], }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#ipniVerifyMs name: "ipniVerifyMs", help: "IPNI verification duration (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [10, 50, 100, 500, 1000, 2000, 5000, 10000, 30000, 60000, 120000, 300000], }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#ipfsRetrievalFirstByteMs name: "ipfsRetrievalFirstByteMs", help: "Time to first byte for IPFS retrievals (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [1, 5, 10, 50, 100, 250, 500, 1000, 2000, 5000, 10000, 30000], }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#ipfsRetrievalBlockFirstByteMs name: "ipfsRetrievalBlockFirstByteMs", help: "Time to first byte for individual IPFS block fetches (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [1, 5, 10, 50, 100, 250, 500, 1000, 2000, 5000, 10000, 30000], }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#ipfsRetrievalLastByteMs name: "ipfsRetrievalLastByteMs", help: "Time to last byte for IPFS retrievals (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [1, 5, 10, 50, 100, 250, 500, 1000, 2000, 5000, 10000, 30000], }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#ipfsRetrievalThroughputBps name: "ipfsRetrievalThroughputBps", help: "IPFS retrieval throughput in bytes per second", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: throughputBuckets, }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#dataStorageCheckMs name: "dataStorageCheckMs", help: "End-to-end data storage check duration (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [100, 500, 1000, 2000, 5000, 10000, 30000, 60000, 120000, 300000, 600000], }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#retrievalCheckMs name: "retrievalCheckMs", help: "End-to-end retrieval check duration (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [100, 500, 1000, 2000, 5000, 10000, 30000, 60000, 120000, 300000, 600000], }), makeHistogramProvider({ // docs/checks/events-and-metrics.md#dataSetCreationMs name: "dataSetCreationMs", help: "End-to-end data-set creation upload duration (ms)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, buckets: [100, 500, 1000, 2000, 5000, 10000, 30000, 60000, 120000, 300000, 600000], }), // Sub-status metrics (docs/checks/data-storage.md) @@ -158,70 +158,71 @@ const metricProviders = [ // docs/checks/data-storage.md#sub-status-meanings (Upload Status) name: "dataStorageUploadStatus", help: "Data storage upload sub-status counts", - labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value", "network"] as const, }), makeCounterProvider({ // docs/checks/data-storage.md#sub-status-meanings (Onchain Status) name: "dataStorageOnchainStatus", help: "Data storage onchain sub-status counts", - labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value", "network"] as const, }), makeCounterProvider({ // docs/checks/data-storage.md#deal-status-progression (Overall Status) name: "dataStorageStatus", help: "Data storage check overall status counts (success when all sub-statuses succeed, failure.timedout/failure.other otherwise)", - labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value", "network"] as const, }), makeCounterProvider({ // docs/checks/data-storage.md#sub-status-meanings (Discoverability Status) name: "discoverabilityStatus", help: "Discoverability sub-status counts", - labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value", "network"] as const, }), makeCounterProvider({ // docs/checks/data-storage.md#sub-status-meanings (Retrieval Status) name: "retrievalStatus", help: "Retrieval sub-status counts", - labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value", "network"] as const, }), makeCounterProvider({ // docs/checks/events-and-metrics.md#ipfsRetrievalHttpResponseCode name: "ipfsRetrievalHttpResponseCode", help: "HTTP response codes for IPFS retrievals", - labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value", "network"] as const, }), makeCounterProvider({ // docs/checks/events-and-metrics.md#dataSetCreationStatus name: "dataSetCreationStatus", help: "Data-set creation status counts", - labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value", "network"] as const, }), // Data Retention Metrics makeCounterProvider({ name: "dataSetChallengeStatus", help: "Provider dataset challenge status", - labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "value", "network"] as const, }), makeGaugeProvider({ name: "pdp_provider_estimated_overdue_periods", help: "Estimated number of unrecorded overdue proving periods per provider. Resets to 0 when the subgraph catches up.", - labelNames: ["checkType", "providerId", "providerName", "providerStatus"] as const, + labelNames: ["checkType", "providerId", "providerName", "providerStatus", "network"] as const, }), // Storage provider metrics: absolute counts, independent of query filters. makeGaugeProvider({ name: "storage_providers_active", help: "Number of active storage providers", - labelNames: ["status"] as const, + labelNames: ["status", "network"] as const, }), makeGaugeProvider({ name: "storage_providers_tested", help: "Number of storage providers being tested", + labelNames: ["network"] as const, }), // Wallet metrics: balances in base units as returned by chain services. makeGaugeProvider({ name: "wallet_balance", help: "Wallet balance in base units (per currency)", - labelNames: ["currency", "wallet"] as const, + labelNames: ["currency", "wallet", "network"] as const, }), // Job scheduler metrics (pg-boss) /** @@ -230,7 +231,7 @@ const metricProviders = [ makeGaugeProvider({ name: "jobs_queued", help: "Number of queued jobs (pg-boss state: created)", - labelNames: ["job_type"] as const, + labelNames: ["job_type", "network"] as const, }), /** * Jobs scheduled for retry per type (pg-boss state: retry). @@ -238,7 +239,7 @@ const metricProviders = [ makeGaugeProvider({ name: "jobs_retry_scheduled", help: "Number of jobs in retry state (pg-boss state: retry)", - labelNames: ["job_type"] as const, + labelNames: ["job_type", "network"] as const, }), /** * Oldest queued job age per type (seconds). @@ -246,7 +247,7 @@ const metricProviders = [ makeGaugeProvider({ name: "oldest_queued_age_seconds", help: "Age in seconds of the oldest queued job (pg-boss state: created)", - labelNames: ["job_type"] as const, + labelNames: ["job_type", "network"] as const, }), /** * Oldest in-flight job age per type (seconds). @@ -254,7 +255,7 @@ const metricProviders = [ makeGaugeProvider({ name: "oldest_in_flight_age_seconds", help: "Age in seconds of the oldest active job (pg-boss state: active)", - labelNames: ["job_type"] as const, + labelNames: ["job_type", "network"] as const, }), /** * Currently executing jobs per type (pg-boss state: active). @@ -262,7 +263,7 @@ const metricProviders = [ makeGaugeProvider({ name: "jobs_in_flight", help: "Number of active jobs currently executing", - labelNames: ["job_type"] as const, + labelNames: ["job_type", "network"] as const, }), /** * Manually paused jobs per type (paused = true in job_schedule_state). @@ -270,7 +271,7 @@ const metricProviders = [ makeGaugeProvider({ name: "jobs_paused", help: "Number of manually paused jobs in job_schedule_state", - labelNames: ["job_type"] as const, + labelNames: ["job_type", "network"] as const, }), /** * Enqueue attempts per type (success/error). @@ -278,7 +279,7 @@ const metricProviders = [ makeCounterProvider({ name: "jobs_enqueue_attempts_total", help: "Total number of enqueue attempts", - labelNames: ["job_type", "outcome"] as const, + labelNames: ["job_type", "outcome", "network"] as const, }), /** * Jobs started by handlers per type. @@ -286,7 +287,7 @@ const metricProviders = [ makeCounterProvider({ name: "jobs_started_total", help: "Total number of jobs started", - labelNames: ["job_type"] as const, + labelNames: ["job_type", "network"] as const, }), /** * Handler completion results per type. @@ -299,7 +300,7 @@ const metricProviders = [ makeCounterProvider({ name: "jobs_completed_total", help: "Total number of jobs completed", - labelNames: ["job_type", "handler_result"] as const, + labelNames: ["job_type", "handler_result", "network"] as const, }), /** * Handler execution duration per type (seconds). @@ -307,7 +308,7 @@ const metricProviders = [ makeHistogramProvider({ name: "job_duration_seconds", help: "Job execution duration in seconds", - labelNames: ["job_type"] as const, + labelNames: ["job_type", "network"] as const, buckets: [0.1, 0.5, 1, 2, 3, 4, 5, 10, 15, 20, 30, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330, 360, 420, 600], }), ]; diff --git a/apps/backend/src/metrics-prometheus/wallet-balance.collector.spec.ts b/apps/backend/src/metrics-prometheus/wallet-balance.collector.spec.ts index e61d42a2..08d9f58b 100644 --- a/apps/backend/src/metrics-prometheus/wallet-balance.collector.spec.ts +++ b/apps/backend/src/metrics-prometheus/wallet-balance.collector.spec.ts @@ -1,14 +1,14 @@ import type { ConfigService } from "@nestjs/config"; import type { Gauge } from "prom-client"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { IConfig } from "../config/app.config.js"; +import type { IConfig } from "../config/types.js"; import type { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; import { WalletBalanceCollector } from "./wallet-balance.collector.js"; describe("WalletBalanceCollector", () => { let collector: WalletBalanceCollector; let gaugeMock: { set: ReturnType; collect?: () => Promise }; - let walletSdkMock: { getWalletBalances: ReturnType }; + let walletSdkMock: { getWalletBalances: ReturnType; getNetworkConfig: ReturnType }; let configMock: { get: ReturnType }; let collectFn: () => Promise; @@ -16,6 +16,7 @@ describe("WalletBalanceCollector", () => { gaugeMock = { set: vi.fn() }; walletSdkMock = { getWalletBalances: vi.fn(async () => ({ usdfc: 50_000_000n, fil: 1_000_000_000n })), + getNetworkConfig: vi.fn(() => ({ walletAddress: "0xABCDEF1234567890" })), }; configMock = { get: vi.fn((key: string) => { @@ -25,8 +26,11 @@ describe("WalletBalanceCollector", () => { prometheusWalletBalanceErrorCooldownSeconds: 60, }; } - if (key === "blockchain") { - return { walletAddress: "0xABCDEF1234567890" }; + if (key === "activeNetworks") { + return ["calibration"]; + } + if (key === "networks") { + return { calibration: { walletAddress: "0xABCDEF1234567890", network: "calibration" } }; } return {}; }), diff --git a/apps/backend/src/metrics-prometheus/wallet-balance.collector.ts b/apps/backend/src/metrics-prometheus/wallet-balance.collector.ts index c804ac15..188e0c25 100644 --- a/apps/backend/src/metrics-prometheus/wallet-balance.collector.ts +++ b/apps/backend/src/metrics-prometheus/wallet-balance.collector.ts @@ -3,7 +3,7 @@ import { ConfigService } from "@nestjs/config"; import { InjectMetric } from "@willsoto/nestjs-prometheus"; import type { Gauge } from "prom-client"; import { toStructuredError } from "../common/logging.js"; -import type { IConfig } from "../config/app.config.js"; +import type { IConfig, INetworksConfig } from "../config/types.js"; import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; @Injectable() @@ -42,10 +42,13 @@ export class WalletBalanceCollector implements OnModuleInit { this.refreshPromise = (async () => { try { - const { usdfc, fil } = await this.walletSdkService.getWalletBalances(); - const walletShort = this.configService.get("blockchain").walletAddress.slice(0, 8); - this.walletBalanceGauge.set({ currency: "USDFC", wallet: walletShort }, Number(usdfc)); - this.walletBalanceGauge.set({ currency: "FIL", wallet: walletShort }, Number(fil)); + const activeNetworks = this.configService.get("activeNetworks"); + for (const network of activeNetworks) { + const { usdfc, fil } = await this.walletSdkService.getWalletBalances(network); + const walletShort = this.configService.get("networks")[network].walletAddress.slice(0, 8); + this.walletBalanceGauge.set({ currency: "USDFC", wallet: walletShort }, Number(usdfc)); + this.walletBalanceGauge.set({ currency: "FIL", wallet: walletShort }, Number(fil)); + } this.cachedAt = Date.now(); } catch (error) { this.logger.warn({ diff --git a/apps/backend/src/pdp-subgraph/pdp-subgraph.service.spec.ts b/apps/backend/src/pdp-subgraph/pdp-subgraph.service.spec.ts index cd3a1ea8..7337b3d7 100644 --- a/apps/backend/src/pdp-subgraph/pdp-subgraph.service.spec.ts +++ b/apps/backend/src/pdp-subgraph/pdp-subgraph.service.spec.ts @@ -1,6 +1,4 @@ -import type { ConfigService } from "@nestjs/config"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { IConfig } from "../config/app.config.js"; import { PDPSubgraphService } from "./pdp-subgraph.service.js"; const VALID_ADDRESS = "0xd8da6bf26964af9d7eed9e03e53415d37aa96045" as const; @@ -40,16 +38,7 @@ describe("PDPSubgraphService", () => { let fetchMock: ReturnType; beforeEach(() => { - const configService = { - get: vi.fn((key: keyof IConfig) => { - if (key === "blockchain") { - return { pdpSubgraphEndpoint: SUBGRAPH_ENDPOINT }; - } - return undefined; - }), - } as unknown as ConfigService; - - service = new PDPSubgraphService(configService); + service = new PDPSubgraphService(); fetchMock = vi.fn(); vi.stubGlobal("fetch", fetchMock); @@ -69,7 +58,7 @@ describe("PDPSubgraphService", () => { json: async () => makeSubgraphResponse([makeValidProvider()]), }); - const providers = await service.fetchProvidersWithDatasets({ + const providers = await service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses: [VALID_ADDRESS], }); @@ -93,7 +82,7 @@ describe("PDPSubgraphService", () => { json: async () => makeSubgraphResponse([]), }); - const providers = await service.fetchProvidersWithDatasets({ + const providers = await service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses: [VALID_ADDRESS], }); @@ -101,7 +90,7 @@ describe("PDPSubgraphService", () => { }); it("returns empty array when addresses array is empty", async () => { - const providers = await service.fetchProvidersWithDatasets({ + const providers = await service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses: [], }); @@ -116,7 +105,7 @@ describe("PDPSubgraphService", () => { status: 500, }); - const promise = service.fetchProvidersWithDatasets({ + const promise = service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses: [VALID_ADDRESS], }); @@ -139,7 +128,7 @@ describe("PDPSubgraphService", () => { }), }); - const promise = service.fetchProvidersWithDatasets({ + const promise = service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses: [VALID_ADDRESS], }); @@ -155,7 +144,7 @@ describe("PDPSubgraphService", () => { it("throws on network failure", async () => { fetchMock.mockRejectedValueOnce(new Error("Network error")); - const promise = service.fetchProvidersWithDatasets({ + const promise = service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses: [VALID_ADDRESS], }); @@ -177,10 +166,7 @@ describe("PDPSubgraphService", () => { }); await expect( - service.fetchProvidersWithDatasets({ - blockNumber: 5000, - addresses: [VALID_ADDRESS], - }), + service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses: [VALID_ADDRESS] }), ).rejects.toThrow("Data validation failed"); // Should only be called once - no retries for validation errors @@ -196,10 +182,7 @@ describe("PDPSubgraphService", () => { }); await expect( - service.fetchProvidersWithDatasets({ - blockNumber: 5000, - addresses: [VALID_ADDRESS], - }), + service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses: [VALID_ADDRESS] }), ).rejects.toThrow("Data validation failed"); // Should only be called once - no retries for validation errors @@ -212,10 +195,7 @@ describe("PDPSubgraphService", () => { json: async () => makeSubgraphResponse([makeValidProvider()]), }); - await service.fetchProvidersWithDatasets({ - blockNumber: 12345, - addresses: [VALID_ADDRESS], - }); + await service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 12345, addresses: [VALID_ADDRESS] }); const body = JSON.parse(fetchMock.mock.calls[0][1].body); expect(body.variables.blockNumber).toBe("12345"); @@ -233,7 +213,7 @@ describe("PDPSubgraphService", () => { }), }); - const promise = service.fetchProvidersWithDatasets({ + const promise = service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses: [VALID_ADDRESS], }); @@ -255,10 +235,7 @@ describe("PDPSubgraphService", () => { }); const addresses = [VALID_ADDRESS, "0xAb5801a7D398351b8bE11C439e05C5B3259aeC9B"]; - await service.fetchProvidersWithDatasets({ - blockNumber: 5000, - addresses, - }); + await service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses }); const body = JSON.parse(fetchMock.mock.calls[0][1].body); expect(body.variables.addresses).toEqual(addresses); @@ -273,10 +250,7 @@ describe("PDPSubgraphService", () => { json: async () => makeSubgraphResponse([]), }); - await service.fetchProvidersWithDatasets({ - blockNumber: 5000, - addresses, - }); + await service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses }); // Should make 2 requests expect(fetchMock).toHaveBeenCalledTimes(2); @@ -289,7 +263,7 @@ describe("PDPSubgraphService", () => { json: async () => makeSubgraphResponse([makeValidProvider()]), }); - const promise = service.fetchProvidersWithDatasets({ + const promise = service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses: [VALID_ADDRESS], }); @@ -321,10 +295,7 @@ describe("PDPSubgraphService", () => { }; }); - const fetchPromise = service.fetchProvidersWithDatasets({ - blockNumber: 5000, - addresses, - }); + const fetchPromise = service.fetchProvidersWithDatasets(SUBGRAPH_ENDPOINT, { blockNumber: 5000, addresses }); await vi.runAllTimersAsync(); @@ -343,7 +314,7 @@ describe("PDPSubgraphService", () => { json: async () => makeSubgraphMetaResponse(12345), }); - const meta = await service.fetchSubgraphMeta(); + const meta = await service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT); expect(fetchMock).toHaveBeenCalledWith(SUBGRAPH_ENDPOINT, { method: "POST", @@ -361,13 +332,9 @@ describe("PDPSubgraphService", () => { }); it("throws when PDP subgraph endpoint is not configured", async () => { - const configService = { - get: vi.fn(() => ({ pdpSubgraphEndpoint: "" })), - } as unknown as ConfigService; - - const serviceWithoutEndpoint = new PDPSubgraphService(configService); + const serviceWithoutEndpoint = new PDPSubgraphService(); - await expect(serviceWithoutEndpoint.fetchSubgraphMeta()).rejects.toThrow("No PDP subgraph endpoint configured"); + await expect(serviceWithoutEndpoint.fetchSubgraphMeta("")).rejects.toThrow("No PDP subgraph endpoint configured"); }); it("throws on HTTP error response", async () => { @@ -377,7 +344,7 @@ describe("PDPSubgraphService", () => { statusText: "Internal Server Error", }); - const promise = service.fetchSubgraphMeta(); + const promise = service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT); promise.catch(() => {}); await vi.runAllTimersAsync(); @@ -394,7 +361,7 @@ describe("PDPSubgraphService", () => { }), }); - const promise = service.fetchSubgraphMeta(); + const promise = service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT); promise.catch(() => {}); await vi.runAllTimersAsync(); @@ -417,7 +384,7 @@ describe("PDPSubgraphService", () => { }), }); - await expect(service.fetchSubgraphMeta()).rejects.toThrow("Data validation failed"); + await expect(service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)).rejects.toThrow("Data validation failed"); expect(fetchMock).toHaveBeenCalledTimes(1); // Should not retry validation errors }); @@ -435,7 +402,7 @@ describe("PDPSubgraphService", () => { }), }); - await expect(service.fetchSubgraphMeta()).rejects.toThrow("Data validation failed"); + await expect(service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)).rejects.toThrow("Data validation failed"); expect(fetchMock).toHaveBeenCalledTimes(1); }); @@ -445,7 +412,7 @@ describe("PDPSubgraphService", () => { json: async () => makeSubgraphMetaResponse(12345), }); - const promise = service.fetchSubgraphMeta(); + const promise = service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT); await vi.runAllTimersAsync(); @@ -459,7 +426,7 @@ describe("PDPSubgraphService", () => { it("throws after MAX_RETRIES attempts on persistent network errors", async () => { fetchMock.mockRejectedValue(new Error("Network timeout")); - const promise = service.fetchSubgraphMeta(); + const promise = service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT); promise.catch(() => {}); await vi.runAllTimersAsync(); @@ -480,7 +447,7 @@ describe("PDPSubgraphService", () => { const startTime = Date.now(); // Make 5 requests - should all go through immediately - const promises = Array.from({ length: 5 }, () => service.fetchSubgraphMeta()); + const promises = Array.from({ length: 5 }, () => service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)); await Promise.all(promises); @@ -499,13 +466,13 @@ describe("PDPSubgraphService", () => { }); // Fill up the rate limit window with 50 requests - const initialPromises = Array.from({ length: 50 }, () => service.fetchSubgraphMeta()); + const initialPromises = Array.from({ length: 50 }, () => service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)); await Promise.all(initialPromises); fetchMock.mockClear(); // Try to make one more request - should wait for oldest to expire - const promise = service.fetchSubgraphMeta(); + const promise = service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT); // Advance past the 10 second window + buffer await vi.advanceTimersByTimeAsync(10010); @@ -528,7 +495,7 @@ describe("PDPSubgraphService", () => { }); // Fill 48 slots - const initialPromises = Array.from({ length: 48 }, () => service.fetchSubgraphMeta()); + const initialPromises = Array.from({ length: 48 }, () => service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)); await vi.runAllTimersAsync(); await Promise.all(initialPromises); @@ -556,7 +523,7 @@ describe("PDPSubgraphService", () => { }); // Make 30 requests at t=0 - const batch1 = Array.from({ length: 30 }, () => service.fetchSubgraphMeta()); + const batch1 = Array.from({ length: 30 }, () => service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)); await vi.runAllTimersAsync(); await Promise.all(batch1); @@ -564,7 +531,7 @@ describe("PDPSubgraphService", () => { await vi.advanceTimersByTimeAsync(5000); // Make 20 more requests at t=5000 - const batch2 = Array.from({ length: 20 }, () => service.fetchSubgraphMeta()); + const batch2 = Array.from({ length: 20 }, () => service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)); await vi.runAllTimersAsync(); await Promise.all(batch2); @@ -575,7 +542,7 @@ describe("PDPSubgraphService", () => { fetchMock.mockClear(); // Should be able to make 30 more requests immediately - const batch3 = Array.from({ length: 30 }, () => service.fetchSubgraphMeta()); + const batch3 = Array.from({ length: 30 }, () => service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)); await vi.runAllTimersAsync(); await Promise.all(batch3); @@ -589,13 +556,13 @@ describe("PDPSubgraphService", () => { }); // Fill the window - const initialPromises = Array.from({ length: 50 }, () => service.fetchSubgraphMeta()); + const initialPromises = Array.from({ length: 50 }, () => service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)); await vi.runAllTimersAsync(); await Promise.all(initialPromises); fetchMock.mockClear(); - const promise = service.fetchSubgraphMeta(); + const promise = service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT); // Advance past the window + buffer await vi.advanceTimersByTimeAsync(10010); @@ -611,7 +578,7 @@ describe("PDPSubgraphService", () => { }); // Fill window with 50 requests - const batch1 = Array.from({ length: 50 }, () => service.fetchSubgraphMeta()); + const batch1 = Array.from({ length: 50 }, () => service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)); await vi.runAllTimersAsync(); await Promise.all(batch1); @@ -642,7 +609,7 @@ describe("PDPSubgraphService", () => { }); // Fill 47 slots - const initial = Array.from({ length: 47 }, () => service.fetchSubgraphMeta()); + const initial = Array.from({ length: 47 }, () => service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)); await vi.runAllTimersAsync(); await Promise.all(initial); @@ -674,7 +641,7 @@ describe("PDPSubgraphService", () => { }); // Make 20 requests - const batch1 = Array.from({ length: 20 }, () => service.fetchSubgraphMeta()); + const batch1 = Array.from({ length: 20 }, () => service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT)); await vi.runAllTimersAsync(); await Promise.all(batch1); @@ -684,7 +651,7 @@ describe("PDPSubgraphService", () => { fetchMock.mockClear(); // Make another request - should have full window available - await service.fetchSubgraphMeta(); + await service.fetchSubgraphMeta(SUBGRAPH_ENDPOINT); const timestamps = (service as any).requestTimestamps; // Should only have 1 timestamp (the new one), old ones filtered out diff --git a/apps/backend/src/pdp-subgraph/pdp-subgraph.service.ts b/apps/backend/src/pdp-subgraph/pdp-subgraph.service.ts index aedd8bce..88407e24 100644 --- a/apps/backend/src/pdp-subgraph/pdp-subgraph.service.ts +++ b/apps/backend/src/pdp-subgraph/pdp-subgraph.service.ts @@ -1,7 +1,5 @@ import { Injectable, Logger } from "@nestjs/common"; -import { ConfigService } from "@nestjs/config"; import { toStructuredError } from "../common/logging.js"; -import type { IBlockchainConfig, IConfig } from "../config/app.config.js"; import { Queries } from "./queries.js"; import type { GraphQLResponse, ProviderDataSetResponse, ProvidersWithDataSetsOptions, SubgraphMeta } from "./types.js"; import { validateProviderDataSetResponse, validateSubgraphMetaResponse } from "./types.js"; @@ -23,7 +21,6 @@ class ValidationError extends Error { @Injectable() export class PDPSubgraphService { private readonly logger: Logger = new Logger(PDPSubgraphService.name); - private readonly blockchainConfig: IBlockchainConfig; private static readonly MAX_PROVIDERS_PER_QUERY = 100; private static readonly MAX_CONCURRENT_REQUESTS = 50; @@ -33,10 +30,6 @@ export class PDPSubgraphService { private requestTimestamps: number[] = []; - constructor(private readonly configService: ConfigService) { - this.blockchainConfig = this.configService.get("blockchain"); - } - /** * Fetch subgraph metadata including the latest indexed block number * @@ -44,15 +37,15 @@ export class PDPSubgraphService { * @returns Subgraph metadata with block number * @throws Error if endpoint is not configured or after MAX_RETRIES attempts */ - async fetchSubgraphMeta(attempt: number = 1): Promise { - if (!this.blockchainConfig.pdpSubgraphEndpoint) { + async fetchSubgraphMeta(endpoint: string, attempt: number = 1): Promise { + if (!endpoint) { throw new Error("No PDP subgraph endpoint configured"); } try { await this.enforceRateLimit(); - const response = await fetch(this.blockchainConfig.pdpSubgraphEndpoint, { + const response = await fetch(endpoint, { method: "POST", headers: { "Content-Type": "application/json", @@ -106,7 +99,7 @@ export class PDPSubgraphService { error: toStructuredError(error), }); await new Promise((resolve) => setTimeout(resolve, delay)); - return this.fetchSubgraphMeta(attempt + 1); + return this.fetchSubgraphMeta(endpoint, attempt + 1); } this.logger.error({ @@ -128,6 +121,7 @@ export class PDPSubgraphService { * @returns Array of providers with their data sets currently proving */ async fetchProvidersWithDatasets( + endpoint: string, options: ProvidersWithDataSetsOptions, ): Promise { const { blockNumber, addresses } = options; @@ -137,16 +131,17 @@ export class PDPSubgraphService { } if (addresses.length <= PDPSubgraphService.MAX_PROVIDERS_PER_QUERY) { - return this.fetchWithRetry(blockNumber, addresses); + return this.fetchWithRetry(endpoint, blockNumber, addresses); } - return this.fetchMultipleBatchesWithRateLimit(blockNumber, addresses); + return this.fetchMultipleBatchesWithRateLimit(endpoint, blockNumber, addresses); } /** * Fetch multiple batches with rate limiting and concurrency control */ private async fetchMultipleBatchesWithRateLimit( + endpoint: string, blockNumber: number, addresses: string[], ): Promise { @@ -161,7 +156,7 @@ export class PDPSubgraphService { for (let i = 0; i < batches.length; i += PDPSubgraphService.MAX_CONCURRENT_REQUESTS) { const batchGroup = batches.slice(i, i + PDPSubgraphService.MAX_CONCURRENT_REQUESTS); - const results = await Promise.all(batchGroup.map((batch) => this.fetchWithRetry(blockNumber, batch))); + const results = await Promise.all(batchGroup.map((batch) => this.fetchWithRetry(endpoint, blockNumber, batch))); allProviders.push(...results.flat()); } @@ -174,11 +169,12 @@ export class PDPSubgraphService { * Assuming initial request to be first attempt */ private async fetchWithRetry( + endpoint: string, blockNumber: number, addresses: string[], attempt: number = 1, ): Promise { - if (!this.blockchainConfig.pdpSubgraphEndpoint) { + if (!endpoint) { throw new Error("No PDP subgraph endpoint configured"); } @@ -190,7 +186,7 @@ export class PDPSubgraphService { try { await this.enforceRateLimit(); - const response = await fetch(this.blockchainConfig.pdpSubgraphEndpoint, { + const response = await fetch(endpoint, { method: "POST", headers: { "Content-Type": "application/json", @@ -247,7 +243,7 @@ export class PDPSubgraphService { error: toStructuredError(error), }); await new Promise((resolve) => setTimeout(resolve, delay)); - return this.fetchWithRetry(blockNumber, addresses, attempt + 1); + return this.fetchWithRetry(endpoint, blockNumber, addresses, attempt + 1); } this.logger.error({ diff --git a/apps/backend/src/piece-cleanup/piece-cleanup.service.spec.ts b/apps/backend/src/piece-cleanup/piece-cleanup.service.spec.ts index b2ef7c48..d54e4591 100644 --- a/apps/backend/src/piece-cleanup/piece-cleanup.service.spec.ts +++ b/apps/backend/src/piece-cleanup/piece-cleanup.service.spec.ts @@ -3,7 +3,7 @@ import { Test, TestingModule } from "@nestjs/testing"; import { getRepositoryToken } from "@nestjs/typeorm"; import { calculateActualStorage, listDataSets } from "filecoin-pin/core/data-set"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { IConfig } from "../config/app.config.js"; +import type { IConfig } from "../config/types.js"; import { Deal } from "../database/entities/deal.entity.js"; import { DealStatus } from "../database/types.js"; import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; @@ -79,17 +79,15 @@ describe("PieceCleanupService", () => { function createConfigMock() { return { get: vi.fn((key: keyof IConfig) => { - if (key === "pieceCleanup") { + if (key === "networks") { return { - maxDatasetStorageSizeBytes: THRESHOLD_BYTES, - targetDatasetStorageSizeBytes: TARGET_BYTES, - }; - } - if (key === "blockchain") { - return { - walletPrivateKey: "0x1234567890123456789012345678901234567890123456789012345678901234", - network: "calibration", - walletAddress: "0x123", + calibration: { + walletPrivateKey: "0x1234567890123456789012345678901234567890123456789012345678901234", + network: "calibration", + walletAddress: "0x123", + maxDatasetStorageSizeBytes: THRESHOLD_BYTES, + targetDatasetStorageSizeBytes: TARGET_BYTES, + }, }; } return undefined; @@ -149,7 +147,7 @@ describe("PieceCleanupService", () => { it("returns total bytes from the query builder", async () => { mockQueryBuilder(50 * MiB); - const result = await service.getStoredBytesForProvider("0xProvider"); + const result = await service.getStoredBytesForProvider("0xProvider", "calibration"); expect(result).toBe(50 * MiB); expect(dealRepoMock.createQueryBuilder).toHaveBeenCalledWith("deal"); @@ -158,7 +156,7 @@ describe("PieceCleanupService", () => { it("returns 0 when no deals exist", async () => { mockQueryBuilder(0); - const result = await service.getStoredBytesForProvider("0xProvider"); + const result = await service.getStoredBytesForProvider("0xProvider", "calibration"); expect(result).toBe(0); }); @@ -172,7 +170,7 @@ describe("PieceCleanupService", () => { }; dealRepoMock.createQueryBuilder.mockReturnValue(qb); - const result = await service.getStoredBytesForProvider("0xProvider"); + const result = await service.getStoredBytesForProvider("0xProvider", "calibration"); expect(result).toBe(0); }); @@ -183,7 +181,7 @@ describe("PieceCleanupService", () => { const abortController = new AbortController(); vi.mocked(listDataSets).mockReturnValueOnce(new Promise(() => {}) as any); - const result = service.getLiveStoredBytesForProvider("0xProvider", abortController.signal); + const result = service.getLiveStoredBytesForProvider("0xProvider", "calibration", abortController.signal); abortController.abort(new Error("listing timed out")); await expect(result).rejects.toThrow("listing timed out"); @@ -210,7 +208,7 @@ describe("PieceCleanupService", () => { }, ] as any); - await service.getLiveStoredBytesForProvider("0xProvider", signal); + await service.getLiveStoredBytesForProvider("0xProvider", "calibration", signal); expect(calculateActualStorage).toHaveBeenCalledWith( expect.anything(), @@ -246,7 +244,9 @@ describe("PieceCleanupService", () => { timedOut: true, }); - await expect(service.getLiveStoredBytesForProvider("0xProvider")).rejects.toThrow("Live storage query timed out"); + await expect(service.getLiveStoredBytesForProvider("0xProvider", "calibration")).rejects.toThrow( + "Live storage query timed out", + ); }); }); @@ -255,7 +255,7 @@ describe("PieceCleanupService", () => { const deals = [makeDeal({ createdAt: new Date("2024-01-01") }), makeDeal({ createdAt: new Date("2024-01-02") })]; dealRepoMock.find.mockResolvedValue(deals); - const result = await service.getCleanupCandidates("0xProvider", 10); + const result = await service.getCleanupCandidates("0xProvider", "calibration", 10); expect(result).toEqual(deals); expect(dealRepoMock.find).toHaveBeenCalledWith( @@ -274,7 +274,7 @@ describe("PieceCleanupService", () => { it("respects the limit parameter", async () => { dealRepoMock.find.mockResolvedValue([]); - await service.getCleanupCandidates("0xProvider", 5); + await service.getCleanupCandidates("0xProvider", "calibration", 5); expect(dealRepoMock.find).toHaveBeenCalledWith( expect.objectContaining({ @@ -288,7 +288,7 @@ describe("PieceCleanupService", () => { it("skips cleanup when stored bytes are below threshold", async () => { vi.spyOn(service, "getLiveStoredBytesForProvider").mockResolvedValue(50 * MiB); // 50 MiB < 100 MiB threshold - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.skipped).toBe(true); expect(result.deleted).toBe(0); @@ -300,7 +300,7 @@ describe("PieceCleanupService", () => { it("skips cleanup when stored bytes equal threshold", async () => { vi.spyOn(service, "getLiveStoredBytesForProvider").mockResolvedValue(THRESHOLD_BYTES); // exactly at threshold - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.skipped).toBe(true); expect(result.deleted).toBe(0); @@ -310,7 +310,7 @@ describe("PieceCleanupService", () => { vi.spyOn(service, "getLiveStoredBytesForProvider").mockRejectedValue(new Error("network error")); mockQueryBuilder(50 * MiB); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.skipped).toBe(true); expect(result.storedBytes).toBe(50 * MiB); @@ -329,7 +329,7 @@ describe("PieceCleanupService", () => { dealRepoMock.find.mockResolvedValue([deal1, deal2, deal3, deal4, deal5]); const deletePieceSpy = vi.spyOn(service, "deletePiece").mockResolvedValue(undefined); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.skipped).toBe(false); expect(result.deleted).toBe(5); @@ -343,9 +343,9 @@ describe("PieceCleanupService", () => { vi.spyOn(service, "getLiveStoredBytesForProvider").mockRejectedValue(new Error("cleanup timeout")); mockQueryBuilder(THRESHOLD_BYTES + 1); - await expect(service.cleanupPiecesForProvider("0xProvider", abortController.signal)).rejects.toThrow( - "cleanup timeout", - ); + await expect( + service.cleanupPiecesForProvider("0xProvider", "calibration", abortController.signal), + ).rejects.toThrow("cleanup timeout"); expect(dealRepoMock.find).not.toHaveBeenCalled(); expect(dealRepoMock.createQueryBuilder).not.toHaveBeenCalled(); @@ -379,7 +379,9 @@ describe("PieceCleanupService", () => { }); mockQueryBuilder(THRESHOLD_BYTES + 1); - await expect(service.cleanupPiecesForProvider("0xProvider")).rejects.toThrow("Live storage query timed out"); + await expect(service.cleanupPiecesForProvider("0xProvider", "calibration")).rejects.toThrow( + "Live storage query timed out", + ); expect(dealRepoMock.find).not.toHaveBeenCalled(); expect(dealRepoMock.createQueryBuilder).not.toHaveBeenCalled(); @@ -389,7 +391,7 @@ describe("PieceCleanupService", () => { vi.spyOn(service, "getLiveStoredBytesForProvider").mockResolvedValue(200 * MiB); // above threshold dealRepoMock.find.mockResolvedValue([]); // no candidates - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.skipped).toBe(false); expect(result.deleted).toBe(0); @@ -410,7 +412,7 @@ describe("PieceCleanupService", () => { const deletePieceSpy = vi.spyOn(service, "deletePiece").mockResolvedValue(undefined); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.deleted).toBe(5); // 50 MiB = 5 × 10 MiB expect(result.failed).toBe(0); @@ -428,7 +430,7 @@ describe("PieceCleanupService", () => { vi.spyOn(service, "deletePiece").mockRejectedValueOnce(new Error("SDK error")).mockResolvedValueOnce(undefined); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.deleted).toBe(1); expect(result.failed).toBe(1); @@ -443,7 +445,9 @@ describe("PieceCleanupService", () => { const abortController = new AbortController(); abortController.abort(new Error("aborted")); - await expect(service.cleanupPiecesForProvider("0xProvider", abortController.signal)).rejects.toThrow("aborted"); + await expect( + service.cleanupPiecesForProvider("0xProvider", "calibration", abortController.signal), + ).rejects.toThrow("aborted"); }); it("bails out when all deletions in a batch fail", async () => { @@ -455,7 +459,7 @@ describe("PieceCleanupService", () => { vi.spyOn(service, "deletePiece").mockRejectedValue(new Error("persistent failure")); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.deleted).toBe(0); expect(result.failed).toBe(2); @@ -472,7 +476,7 @@ describe("PieceCleanupService", () => { const deletePieceSpy = vi.spyOn(service, "deletePiece").mockResolvedValue(undefined); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); // Piece is still deleted expect(result.deleted).toBe(1); @@ -495,7 +499,7 @@ describe("PieceCleanupService", () => { const deletePieceSpy = vi.spyOn(service, "deletePiece").mockResolvedValue(undefined); - const result = await service.cleanupPiecesForProvider("0xProvider"); + const result = await service.cleanupPiecesForProvider("0xProvider", "calibration"); expect(result.deleted).toBe(4); expect(deletePieceSpy).toHaveBeenCalledTimes(4); @@ -508,13 +512,13 @@ describe("PieceCleanupService", () => { it("throws when deal is missing pieceId", async () => { const deal = makeDeal({ pieceId: undefined }); - await expect(service.deletePiece(deal)).rejects.toThrow("missing pieceId"); + await expect(service.deletePiece(deal, "calibration")).rejects.toThrow("missing pieceId"); }); it("throws when deal is missing dataSetId", async () => { const deal = makeDeal({ dataSetId: undefined }); - await expect(service.deletePiece(deal)).rejects.toThrow("missing dataSetId"); + await expect(service.deletePiece(deal, "calibration")).rejects.toThrow("missing dataSetId"); }); it("calls Synapse SDK to delete piece and marks deal as cleaned up", async () => { @@ -535,7 +539,7 @@ describe("PieceCleanupService", () => { const deal = makeDeal({ pieceId: 42, dataSetId: 1n, spAddress: "0xProvider" }); dealRepoMock.save.mockResolvedValue(deal); - await service.deletePiece(deal); + await service.deletePiece(deal, "calibration"); expect(createContextMock).toHaveBeenCalledWith({ providerId: 9, @@ -565,7 +569,7 @@ describe("PieceCleanupService", () => { const deal = makeDeal({ pieceId: 42, dataSetId: 1n, spAddress: "0xProvider" }); dealRepoMock.save.mockResolvedValue(deal); - await service.deletePiece(deal); + await service.deletePiece(deal, "calibration"); expect(deal.cleanedUp).toBe(true); expect(deal.cleanedUpAt).toBeInstanceOf(Date); @@ -590,7 +594,7 @@ describe("PieceCleanupService", () => { const deal = makeDeal({ pieceId: 42, dataSetId: 1n, spAddress: "0xProvider" }); dealRepoMock.save.mockResolvedValue(deal); - await service.deletePiece(deal); + await service.deletePiece(deal, "calibration"); expect(deal.cleanedUp).toBe(true); expect(dealRepoMock.save).toHaveBeenCalledWith(deal); @@ -613,7 +617,7 @@ describe("PieceCleanupService", () => { const deal = makeDeal({ pieceId: 42, dataSetId: 1n, spAddress: "0xProvider" }); - await expect(service.deletePiece(deal)).rejects.toThrow("network timeout"); + await expect(service.deletePiece(deal, "calibration")).rejects.toThrow("network timeout"); }); it("respects abort signal before SDK call", async () => { @@ -621,7 +625,7 @@ describe("PieceCleanupService", () => { const abortController = new AbortController(); abortController.abort(new Error("cancelled")); - await expect(service.deletePiece(deal, abortController.signal)).rejects.toThrow("cancelled"); + await expect(service.deletePiece(deal, "calibration", abortController.signal)).rejects.toThrow("cancelled"); }); }); }); diff --git a/apps/backend/src/piece-cleanup/piece-cleanup.service.ts b/apps/backend/src/piece-cleanup/piece-cleanup.service.ts index 45454347..3840a30a 100644 --- a/apps/backend/src/piece-cleanup/piece-cleanup.service.ts +++ b/apps/backend/src/piece-cleanup/piece-cleanup.service.ts @@ -6,7 +6,8 @@ import { calculateActualStorage, listDataSets } from "filecoin-pin/core/data-set import { IsNull, Not, Repository } from "typeorm"; import { type PieceCleanupLogContext, type ProviderJobContext, toStructuredError } from "../common/logging.js"; import { createSynapseFromConfig } from "../common/synapse-factory.js"; -import type { IBlockchainConfig, IConfig } from "../config/app.config.js"; +import { Network } from "../common/types.js"; +import type { IConfig, INetworkConfig } from "../config/types.js"; import { Deal } from "../database/entities/deal.entity.js"; import { DealStatus } from "../database/types.js"; import { WalletSdkService } from "../wallet-sdk/wallet-sdk.service.js"; @@ -29,17 +30,14 @@ class LiveStorageQueryTimedOutError extends Error {} @Injectable() export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(PieceCleanupService.name); - private readonly blockchainConfig: IBlockchainConfig; - private sharedSynapse?: Synapse; + private readonly sharedSynapseByNetwork: Map = new Map(); constructor( private readonly configService: ConfigService, @InjectRepository(Deal) private readonly dealRepository: Repository, private readonly walletSdkService: WalletSdkService, - ) { - this.blockchainConfig = this.configService.get("blockchain"); - } + ) {} async onModuleInit(): Promise { if (process.env.DEALBOT_DISABLE_CHAIN === "true") { @@ -47,9 +45,17 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { return; } try { - this.logger.log("Initializing shared Synapse instance for piece cleanup."); - const { synapse } = await createSynapseFromConfig(this.blockchainConfig); - this.sharedSynapse = synapse; + const activeNetworks = this.configService.get("activeNetworks"); + for (const network of activeNetworks) { + this.logger.log({ + event: "synapse_initialization", + message: "Creating shared Synapse instance", + network, + }); + const networkCfg = this.getNetworkConfig(network); + const { synapse } = await createSynapseFromConfig(networkCfg); + this.sharedSynapseByNetwork.set(network, synapse); + } } catch (error) { this.logger.error({ event: "piece_cleanup_synapse_init_failed", @@ -60,19 +66,31 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { } async onModuleDestroy(): Promise { - if (this.sharedSynapse) { - this.sharedSynapse = undefined; - } + this.sharedSynapseByNetwork.clear(); } - private async createSynapseInstance(): Promise { + private getNetworkConfig(network: Network): INetworkConfig { + return this.configService.get("networks")[network]; + } + + private async createSynapseInstance(network: Network): Promise { + const networkConfig = this.getNetworkConfig(network); try { - const { synapse } = await createSynapseFromConfig(this.blockchainConfig); + const { synapse, isSessionKeyMode } = await createSynapseFromConfig(networkConfig); + if (isSessionKeyMode) { + this.logger.log({ + event: "synapse_session_key_init", + message: "Initializing Synapse with session key", + network, + walletAddress: networkConfig.walletAddress, + }); + } return synapse; } catch (error) { this.logger.error({ event: "synapse_init_failed", - message: "Failed to initialize Synapse for piece cleanup", + message: "Failed to initialize Synapse for deal job", + network, error: toStructuredError(error), }); throw error; @@ -113,13 +131,15 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { */ private async checkProviderQuota( spAddress: string, + network: Network, signal?: AbortSignal, logContext?: ProviderJobContext, ): Promise<{ isOverQuota: boolean; storedBytes: number; thresholdBytes: number }> { - const thresholdBytes = this.configService.get("pieceCleanup").maxDatasetStorageSizeBytes; + const networkCfg = this.getNetworkConfig(network); + const thresholdBytes = networkCfg.maxDatasetStorageSizeBytes; let storedBytes: number; try { - storedBytes = await this.getLiveStoredBytesForProvider(spAddress, signal); + storedBytes = await this.getLiveStoredBytesForProvider(spAddress, network, signal); } catch (error) { if (signal?.aborted || error instanceof LiveStorageQueryTimedOutError) { throw error; @@ -131,7 +151,7 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { providerAddress: spAddress, error: toStructuredError(error), }); - storedBytes = await this.getStoredBytesForProvider(spAddress); + storedBytes = await this.getStoredBytesForProvider(spAddress, network); } return { isOverQuota: storedBytes > thresholdBytes, storedBytes, thresholdBytes }; } @@ -147,13 +167,14 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { */ async cleanupPiecesForProvider( spAddress: string, + network: Network, signal?: AbortSignal, logContext?: ProviderJobContext, ): Promise { const { maxDatasetStorageSizeBytes: thresholdBytes, targetDatasetStorageSizeBytes: targetBytes } = - this.configService.get("pieceCleanup"); + this.getNetworkConfig(network); - const { storedBytes, isOverQuota } = await this.checkProviderQuota(spAddress, signal, logContext); + const { storedBytes, isOverQuota } = await this.checkProviderQuota(spAddress, network, signal, logContext); const cleanupLogContext: PieceCleanupLogContext = { ...logContext, @@ -184,13 +205,13 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { let failed = 0; let bytesRemoved = 0; - const synapse = this.sharedSynapse ?? (await this.createSynapseInstance()); + const synapse = this.sharedSynapseByNetwork.get(network) ?? (await this.createSynapseInstance(network)); // Fetch candidates in batches. Keep deleting until back under quota or runtime cap. while (bytesRemoved < excessBytes) { signal?.throwIfAborted(); - const candidates = await this.getCleanupCandidates(spAddress, 50); + const candidates = await this.getCleanupCandidates(spAddress, network, 50); if (candidates.length === 0) { this.logger.warn({ @@ -218,7 +239,7 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { } try { - await this.deletePiece(deal, signal, synapse, cleanupLogContext); + await this.deletePiece(deal, network, signal, synapse, cleanupLogContext); deleted++; batchDeletedCount++; bytesRemoved += Number(deal.pieceSize || 0); @@ -274,8 +295,8 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { /** * Query the provider's actual storage via filecoin-pin. */ - async getLiveStoredBytesForProvider(spAddress: string, signal?: AbortSignal): Promise { - const synapse = this.sharedSynapse ?? (await this.createSynapseInstance()); + async getLiveStoredBytesForProvider(spAddress: string, network: Network, signal?: AbortSignal): Promise { + const synapse = this.sharedSynapseByNetwork.get(network) ?? (await this.createSynapseInstance(network)); const datasets = await this.awaitWithAbort( listDataSets(synapse, { @@ -319,12 +340,13 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { * Calculate total stored bytes for a provider from the deals table. * Only counts completed deals that have not already been cleaned up. */ - async getStoredBytesForProvider(spAddress: string): Promise { - const walletAddress = this.blockchainConfig.walletAddress; + async getStoredBytesForProvider(spAddress: string, network: Network): Promise { + const walletAddress = this.getNetworkConfig(network).walletAddress; const result = await this.dealRepository .createQueryBuilder("deal") .select("COALESCE(SUM(deal.piece_size), 0)", "totalBytes") .where("deal.sp_address = :spAddress", { spAddress }) + .andWhere("deal.network = :network", { network }) .andWhere("deal.wallet_address = :walletAddress", { walletAddress }) .andWhere("deal.status = :status", { status: DealStatus.DEAL_CREATED }) .andWhere("deal.piece_id IS NOT NULL") @@ -338,12 +360,13 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { /** * Get the oldest completed deals (candidates for cleanup). */ - async getCleanupCandidates(spAddress: string, limit: number): Promise { - const walletAddress = this.blockchainConfig.walletAddress; + async getCleanupCandidates(spAddress: string, network: Network, limit: number): Promise { + const walletAddress = this.getNetworkConfig(network).walletAddress; return this.dealRepository.find({ where: { spAddress, walletAddress, + network, status: DealStatus.DEAL_CREATED, pieceId: Not(IsNull()), dataSetId: Not(IsNull()), @@ -359,6 +382,7 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { */ async deletePiece( deal: Deal, + network: Network, signal?: AbortSignal, existingSynapse?: Synapse, logContext?: PieceCleanupLogContext, @@ -372,11 +396,12 @@ export class PieceCleanupService implements OnModuleInit, OnModuleDestroy { signal?.throwIfAborted(); - const providerId = this.walletSdkService.getProviderInfo(deal.spAddress)?.id; + const providerId = this.walletSdkService.getProviderInfo(deal.spAddress, network)?.id; if (providerId === undefined) { throw new Error(`Provider ID not found for SP address ${deal.spAddress}`); } - const synapse = existingSynapse ?? this.sharedSynapse ?? (await this.createSynapseInstance()); + const synapse = + existingSynapse ?? this.sharedSynapseByNetwork.get(network) ?? (await this.createSynapseInstance(network)); const storage = await synapse.storage.createContext({ providerId, dataSetId: deal.dataSetId, diff --git a/apps/backend/src/providers/providers.controller.spec.ts b/apps/backend/src/providers/providers.controller.spec.ts index 967cfae0..3ad01572 100644 --- a/apps/backend/src/providers/providers.controller.spec.ts +++ b/apps/backend/src/providers/providers.controller.spec.ts @@ -6,9 +6,9 @@ import { ProvidersService } from "./providers.service.js"; function makeProvider(overrides: Partial = {}): StorageProvider { return { - network: "calibration", address: "f01234", providerId: 99n, + network: "calibration", name: "Test SP", description: "desc", payee: "0xabc", diff --git a/apps/backend/src/retrieval-addons/strategies/ipfs-block.strategy.ts b/apps/backend/src/retrieval-addons/strategies/ipfs-block.strategy.ts index d2676897..c4f0f495 100644 --- a/apps/backend/src/retrieval-addons/strategies/ipfs-block.strategy.ts +++ b/apps/backend/src/retrieval-addons/strategies/ipfs-block.strategy.ts @@ -6,7 +6,7 @@ import { CID } from "multiformats/cid"; import * as raw from "multiformats/codecs/raw"; import { sha256 } from "multiformats/hashes/sha2"; import { toStructuredError } from "../../common/logging.js"; -import type { IConfig } from "../../config/app.config.js"; +import type { IConfig } from "../../config/types.js"; import { ServiceType } from "../../database/types.js"; import { HttpClientService } from "../../http-client/http-client.service.js"; import { WalletSdkService } from "../../wallet-sdk/wallet-sdk.service.js"; @@ -89,7 +89,7 @@ export class IpfsBlockRetrievalStrategy implements IRetrievalAddon { } private getSpEndpoint(config: RetrievalConfiguration): string { - const providerInfo = this.walletSdkService.getProviderInfo(config.storageProvider); + const providerInfo = this.walletSdkService.getProviderInfo(config.storageProvider, config.deal.network); if (!providerInfo) { throw new Error(`Provider ${config.storageProvider} not found in approved providers`); diff --git a/apps/backend/src/retrieval/retrieval.service.ts b/apps/backend/src/retrieval/retrieval.service.ts index 1bf2d517..d61e646d 100644 --- a/apps/backend/src/retrieval/retrieval.service.ts +++ b/apps/backend/src/retrieval/retrieval.service.ts @@ -4,8 +4,8 @@ import { InjectRepository } from "@nestjs/typeorm"; import { CID } from "multiformats/cid"; import type { Repository } from "typeorm"; import { type ProviderJobContext, type RetrievalLogContext, toStructuredError } from "../common/logging.js"; -import type { Hex } from "../common/types.js"; -import type { IConfig } from "../config/app.config.js"; +import type { Hex, Network } from "../common/types.js"; +import type { IConfig } from "../config/types.js"; import { Deal } from "../database/entities/deal.entity.js"; import { Retrieval } from "../database/entities/retrieval.entity.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; @@ -44,10 +44,11 @@ export class RetrievalService { async performRandomRetrievalForProvider( spAddress: string, + network: Network, signal?: AbortSignal, logContext?: ProviderJobContext, ): Promise { - const deal = await this.selectRandomSuccessfulDealForProvider(spAddress); + const deal = await this.selectRandomSuccessfulDealForProvider(spAddress, network); if (!deal) { this.logger.warn({ ...logContext, @@ -86,6 +87,7 @@ export class RetrievalService { throw new Error(`Storage provider ${deal.spAddress} not found`); } const providerLabels = buildCheckMetricLabels({ + network: deal.network, checkType: "retrieval", providerId: provider.providerId, providerName: provider.name, @@ -313,12 +315,13 @@ export class RetrievalService { * We select a random successful deal (DEAL_CREATED only) for a given provider. * Uses Postgres ORDER BY RANDOM() since Dealbot is Postgres-only. */ - private async selectRandomSuccessfulDealForProvider(spAddress: string): Promise { + private async selectRandomSuccessfulDealForProvider(spAddress: string, network: Network): Promise { const randomDatasetSizes = this.getRandomDatasetSizes(); const query = this.dealRepository .createQueryBuilder("deal") .innerJoin("deal.storageProvider", "sp", "sp.isActive = :isActive", { isActive: true }) .where("deal.sp_address = :spAddress", { spAddress }) + .andWhere("deal.network = :network", { network }) .andWhere("deal.status IN (:...statuses)", { statuses: [DealStatus.DEAL_CREATED], }) diff --git a/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts b/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts index 9888e25a..2e9d3e14 100644 --- a/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts +++ b/apps/backend/src/wallet-sdk/wallet-sdk.service.spec.ts @@ -1,6 +1,6 @@ import type { ConfigService } from "@nestjs/config"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { configValidationSchema, type IBlockchainConfig, type IConfig } from "../config/app.config.js"; +import type { IConfig } from "../config/types.js"; import { WalletSdkService } from "./wallet-sdk.service.js"; import type { PDPProviderEx } from "./wallet-sdk.types.js"; @@ -10,15 +10,28 @@ type LoggerLike = { log: (message: string) => void; }; -const baseConfig: IBlockchainConfig = { - network: "calibration", +const baseNetworkConfig = { + network: "calibration" as const, walletAddress: "0x0000000000000000000000000000000000000000", - walletPrivateKey: "0xtest", + walletPrivateKey: "0xtest" as `0x${string}`, checkDatasetCreationFees: false, useOnlyApprovedProviders: false, minNumDataSetsForChecks: 1, pdpSubgraphEndpoint: "https://api.thegraph.com/subgraphs/filecoin/pdp", -}; + dealsPerSpPerHour: 4, + retrievalsPerSpPerHour: 2, + dataSetCreationsPerSpPerHour: 1, + dataRetentionPollIntervalSeconds: 3600, + providersRefreshIntervalSeconds: 14400, + maintenanceWindowsUtc: ["07:00", "22:00"], + maintenanceWindowMinutes: 20, + pieceCleanupPerSpPerHour: 1, + maxPieceCleanupRuntimeSeconds: 300, + maxDatasetStorageSizeBytes: 24 * 1024 * 1024 * 1024, + targetDatasetStorageSizeBytes: 20 * 1024 * 1024 * 1024, + blockedSpIds: new Set(), + blockedSpAddresses: new Set(), +} satisfies IConfig["networks"]["calibration"]; const makeProvider = (overrides: Partial): PDPProviderEx => ({ @@ -36,56 +49,6 @@ const makeProvider = (overrides: Partial): PDPProviderEx => ...overrides, }) as PDPProviderEx; -describe("config validation", () => { - const requiredEnv = { - DATABASE_HOST: "localhost", - DATABASE_USER: "test", - DATABASE_PASSWORD: "test", - DATABASE_NAME: "test", - WALLET_ADDRESS: "0x1234567890123456789012345678901234567890", - }; - - it("requires WALLET_PRIVATE_KEY when SESSION_KEY_PRIVATE_KEY is absent", () => { - const { error } = configValidationSchema.validate(requiredEnv, { allowUnknown: true }); - expect(error).toBeDefined(); - expect(error?.message).toMatch(/WALLET_PRIVATE_KEY/); - }); - - it("accepts missing WALLET_PRIVATE_KEY when SESSION_KEY_PRIVATE_KEY is set", () => { - const { error } = configValidationSchema.validate( - { ...requiredEnv, SESSION_KEY_PRIVATE_KEY: "0xdeadbeef" }, - { allowUnknown: true }, - ); - expect(error).toBeUndefined(); - }); - - it("accepts both WALLET_PRIVATE_KEY and SESSION_KEY_PRIVATE_KEY (session key takes precedence)", () => { - const { error } = configValidationSchema.validate( - { ...requiredEnv, WALLET_PRIVATE_KEY: "0xkey", SESSION_KEY_PRIVATE_KEY: "0xsession" }, - { allowUnknown: true }, - ); - expect(error).toBeUndefined(); - }); - - it("treats empty string WALLET_PRIVATE_KEY as absent", () => { - const { error } = configValidationSchema.validate( - { ...requiredEnv, WALLET_PRIVATE_KEY: "" }, - { allowUnknown: true }, - ); - // Empty string is normalized to undefined by .empty(""), so .or() treats it as absent - expect(error).toBeDefined(); - expect(error?.message).toMatch(/WALLET_PRIVATE_KEY|SESSION_KEY_PRIVATE_KEY/); - }); - - it("treats empty string SESSION_KEY_PRIVATE_KEY as absent", () => { - const { error } = configValidationSchema.validate( - { ...requiredEnv, WALLET_PRIVATE_KEY: "0xkey", SESSION_KEY_PRIVATE_KEY: "" }, - { allowUnknown: true }, - ); - expect(error).toBeUndefined(); - }); -}); - describe("WalletSdkService", () => { let service: WalletSdkService; let repoMock: { create: ReturnType; upsert: ReturnType }; @@ -98,7 +61,11 @@ describe("WalletSdkService", () => { }; const configService = { - get: vi.fn((key: keyof IConfig) => (key === "blockchain" ? baseConfig : undefined)), + get: vi.fn((key: keyof IConfig) => { + if (key === "activeNetworks") return ["calibration"]; + if (key === "networks") return { calibration: baseNetworkConfig }; + return undefined; + }), } as unknown as ConfigService; service = new WalletSdkService(configService, repoMock as any); @@ -149,8 +116,8 @@ describe("WalletSdkService", () => { expect(options).toEqual(expect.objectContaining({ conflictPaths: ["address", "network"] })); expect(entities).toEqual( expect.arrayContaining([ - expect.objectContaining({ network: "calibration", address: "0xdup", providerId: 21n, name: "new" }), - expect.objectContaining({ network: "calibration", address: "0xother", providerId: 22n }), + expect.objectContaining({ address: "0xdup", network: "calibration", providerId: 21n, name: "new" }), + expect.objectContaining({ address: "0xother", network: "calibration", providerId: 22n }), ]), ); }); @@ -184,7 +151,9 @@ describe("WalletSdkService", () => { const [entities] = repoMock.upsert.mock.calls[0]; expect(entities).toEqual( - expect.arrayContaining([expect.objectContaining({ address: "0xdup2", providerId: 30n, name: "active" })]), + expect.arrayContaining([ + expect.objectContaining({ address: "0xdup2", network: "calibration", providerId: 30n, name: "active" }), + ]), ); }); @@ -214,7 +183,9 @@ describe("WalletSdkService", () => { const [entities] = repoMock.upsert.mock.calls[0]; expect(entities).toEqual( - expect.arrayContaining([expect.objectContaining({ address: "0xdup3", providerId: 41n, name: "second" })]), + expect.arrayContaining([ + expect.objectContaining({ address: "0xdup3", network: "calibration", providerId: 41n, name: "second" }), + ]), ); }); @@ -224,10 +195,16 @@ describe("WalletSdkService", () => { resolveLoad = resolve; }); const loadProvidersInternal = vi.fn(() => loadPromise); + // Inject a mock network state for calibration + const mockState = { + providersLoadedOnce: false, + providersLoadPromise: null, + }; + (service as any).networkStates.set("calibration", mockState); (service as any).loadProvidersInternal = loadProvidersInternal; - const first = service.ensureProvidersLoaded(); - const second = service.ensureProvidersLoaded(); + const first = service.ensureProvidersLoaded("calibration"); + const second = service.ensureProvidersLoaded("calibration"); expect(loadProvidersInternal).toHaveBeenCalledTimes(1); @@ -235,34 +212,47 @@ describe("WalletSdkService", () => { await Promise.all([first, second]); expect(loadProvidersInternal).toHaveBeenCalledTimes(1); - expect((service as any).providersLoadedOnce).toBe(true); + expect(mockState.providersLoadedOnce).toBe(true); }); it("retries ensureProvidersLoaded after a failed load", async () => { const loadProvidersInternal = vi.fn().mockResolvedValueOnce(false).mockResolvedValueOnce(true); + const mockState = { + providersLoadedOnce: false, + providersLoadPromise: null, + }; + (service as any).networkStates.set("calibration", mockState); (service as any).loadProvidersInternal = loadProvidersInternal; - await service.ensureProvidersLoaded(); - await service.ensureProvidersLoaded(); + await service.ensureProvidersLoaded("calibration"); + await service.ensureProvidersLoaded("calibration"); expect(loadProvidersInternal).toHaveBeenCalledTimes(2); - expect((service as any).providersLoadedOnce).toBe(true); + expect(mockState.providersLoadedOnce).toBe(true); }); describe("ensureWalletAllowances", () => { it("performs read-only check in session key mode", async () => { - (service as any)._isSessionKeyMode = true; - // getUploadCosts needs _synapseClient but will fail without a real RPC + const mockState = { + isSessionKeyMode: true, + synapseClient: null, + config: baseNetworkConfig, + }; + (service as any).networkStates.set("calibration", mockState); + // getUploadCosts needs synapseClient but will fail without a real RPC // Verify it doesn't fall through to the storageManager.prepare path - (service as any)._synapseClient = null; - await expect(service.ensureWalletAllowances()).rejects.toThrow(); - // storageManager.prepare was never called (it would also throw, but differently) + await expect(service.ensureWalletAllowances("calibration")).rejects.toThrow(); }); it("attempts allowances in direct key mode", async () => { - (service as any)._isSessionKeyMode = false; + const mockState = { + isSessionKeyMode: false, + storageManager: undefined, + config: baseNetworkConfig, + }; + (service as any).networkStates.set("calibration", mockState); // storageManager is not initialized so prepare() will throw - await expect(service.ensureWalletAllowances()).rejects.toThrow(); + await expect(service.ensureWalletAllowances("calibration")).rejects.toThrow(); }); }); }); diff --git a/apps/backend/src/wallet-sdk/wallet-sdk.service.ts b/apps/backend/src/wallet-sdk/wallet-sdk.service.ts index a46d58b4..e7ba4c98 100644 --- a/apps/backend/src/wallet-sdk/wallet-sdk.service.ts +++ b/apps/backend/src/wallet-sdk/wallet-sdk.service.ts @@ -7,37 +7,39 @@ import { Injectable, Logger, type OnModuleInit } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import { InjectRepository } from "@nestjs/typeorm"; import type { Repository } from "typeorm"; -import type { Hex } from "viem"; +import type { Chain, Client, Hex, Transport } from "viem"; import { toStructuredError } from "../common/logging.js"; import { createSynapseFromConfig } from "../common/synapse-factory.js"; -import { Network } from "../common/types.js"; -import type { IBlockchainConfig, IConfig } from "../config/app.config.js"; +import type { Network } from "../common/types.js"; +import type { IConfig, INetworkConfig } from "../config/types.js"; import { StorageProvider } from "../database/entities/storage-provider.entity.js"; import type { PDPProviderEx, WalletServices } from "./wallet-sdk.types.js"; +interface NetworkState { + config: INetworkConfig; + paymentsService: PaymentsService; + warmStorageService: WarmStorageService; + spRegistry: SPRegistryService; + storageManager: StorageManager; + synapseClient: Client; + isSessionKeyMode: boolean; + providerCache: Map; + activeProviderAddresses: Set; + approvedProviderAddresses: Set; + providersLoadPromise: Promise | null; + providersLoadedOnce: boolean; +} + @Injectable() export class WalletSdkService implements OnModuleInit { private readonly logger = new Logger(WalletSdkService.name); - private readonly blockchainConfig: IBlockchainConfig; - private paymentsService: PaymentsService; - private warmStorageService: WarmStorageService; - private spRegistry: SPRegistryService; - private storageManager: StorageManager; - private providerCache: Map = new Map(); - private activeProviderAddresses: Set = new Set(); - private approvedProviderAddresses: Set = new Set(); - private providersLoadPromise: Promise | null = null; - private providersLoadedOnce = false; - private _isSessionKeyMode = false; - private _synapseClient: any; + private readonly networkStates: Map = new Map(); constructor( private readonly configService: ConfigService, @InjectRepository(StorageProvider) private readonly spRepository: Repository, - ) { - this.blockchainConfig = this.configService.get("blockchain"); - } + ) {} async onModuleInit() { if (process.env.DEALBOT_DISABLE_CHAIN === "true") { @@ -48,35 +50,57 @@ export class WalletSdkService implements OnModuleInit { }); return; } - await this.initializeServices(); - await this.ensureProvidersLoaded(); + this.logger.log({ + event: "env_log", + message: "Environment variables", + networks: this.configService.get("networks"), + }); + const activeNetworks = this.configService.get("activeNetworks"); + for (const network of activeNetworks) { + await this.initializeServicesForNetwork(network); + await this.ensureProvidersLoaded(network); + } } /** - * Initialize wallet services with provider and signer + * Initialize wallet services for a specific network. */ - private async initializeServices(): Promise { - const { synapse, isSessionKeyMode } = await createSynapseFromConfig(this.blockchainConfig); + private async initializeServicesForNetwork(network: Network): Promise { + const networkConfig = this.configService.get("networks")[network]; + const { synapse, isSessionKeyMode } = await createSynapseFromConfig(networkConfig); this.logger.log({ event: "wallet_sdk_initialized", message: isSessionKeyMode ? "Initialized wallet SDK services (session key mode)" : "Initialized wallet SDK services", - network: this.blockchainConfig.network, - walletAddress: this.blockchainConfig.walletAddress, + network, + walletAddress: networkConfig.walletAddress, }); - this.warmStorageService = new WarmStorageService({ - client: synapse.client, + this.networkStates.set(network, { + config: networkConfig, + paymentsService: synapse.payments, + warmStorageService: new WarmStorageService({ client: synapse.client }), + spRegistry: new SPRegistryService({ client: synapse.client }), + storageManager: synapse.storage, + synapseClient: synapse.client, + isSessionKeyMode, + providerCache: new Map(), + activeProviderAddresses: new Set(), + approvedProviderAddresses: new Set(), + providersLoadPromise: null, + providersLoadedOnce: false, }); - this.spRegistry = new SPRegistryService({ - client: synapse.client, - }); - this.paymentsService = synapse.payments; - this.storageManager = synapse.storage; - this._synapseClient = synapse.client; - this._isSessionKeyMode = isSessionKeyMode; + } + + private getNetworkState(network: Network): NetworkState { + const target = network; + const state = this.networkStates.get(target); + if (!state) { + throw new Error(`No initialized state for network "${target}". Ensure NETWORKS includes this network.`); + } + return state; } /** @@ -84,42 +108,46 @@ export class WalletSdkService implements OnModuleInit { * This allows dealbot to test all FWSS SPs, even those not yet approved * Only loads active, approved providers that support the PDP product */ - async loadProviders(): Promise { - if (this.providersLoadPromise) { - await this.providersLoadPromise; + async loadProviders(network: Network): Promise { + const state = this.getNetworkState(network); + if (state.providersLoadPromise) { + await state.providersLoadPromise; return; } - this.providersLoadPromise = this.loadProvidersInternal(); + state.providersLoadPromise = this.loadProvidersInternal(network); try { - const success = await this.providersLoadPromise; + const success = await state.providersLoadPromise; if (success) { - this.providersLoadedOnce = true; + state.providersLoadedOnce = true; } } finally { - this.providersLoadPromise = null; + state.providersLoadPromise = null; } } - async ensureProvidersLoaded(): Promise { - if (this.providersLoadedOnce) { + async ensureProvidersLoaded(network: Network): Promise { + const state = this.getNetworkState(network); + if (state.providersLoadedOnce) { return; } - await this.loadProviders(); + await this.loadProviders(network); } - private async loadProvidersInternal(): Promise { + private async loadProvidersInternal(network: Network): Promise { + const state = this.getNetworkState(network); try { this.logger.log({ event: "providers_load_started", message: "Loading all service providers from sp-registry", + network, }); - const approvedIds = await this.warmStorageService.getApprovedProviderIds(); + const approvedIds = await state.warmStorageService.getApprovedProviderIds(); - const totalProviders = await this.spRegistry.getProviderCount(); + const totalProviders = await state.spRegistry.getProviderCount(); - const activeProviders = await this.spRegistry.getAllActiveProviders(); + const activeProviders = await state.spRegistry.getAllActiveProviders(); const activeProviderIds = new Set(activeProviders.map((info) => info.id)); const allProviderIds = Array.from({ length: Number(totalProviders) }, (_, i) => BigInt(i + 1)); const inactiveProviderIds = allProviderIds.filter((id) => !activeProviderIds.has(id)); @@ -130,7 +158,7 @@ export class WalletSdkService implements OnModuleInit { // (empty capabilities), which causes getPDPProvidersByIds to throw. for (const id of inactiveProviderIds) { try { - const provider = await this.spRegistry.getProvider({ providerId: id }); + const provider = await state.spRegistry.getProvider({ providerId: id }); if (provider) { providerInfos.push(provider); } @@ -139,6 +167,7 @@ export class WalletSdkService implements OnModuleInit { event: "inactive_provider_skip", message: `Skipping inactive provider ${id} — no PDP product or invalid data`, providerId: id, + network, }); } } @@ -146,9 +175,9 @@ export class WalletSdkService implements OnModuleInit { const validProviders = providerInfos.filter((info) => !!info); - this.providerCache.clear(); - this.activeProviderAddresses.clear(); - this.approvedProviderAddresses.clear(); + state.providerCache.clear(); + state.activeProviderAddresses.clear(); + state.approvedProviderAddresses.clear(); const extendedProviders = validProviders.map((info) => { const supportsIpniIpfs = !!info.pdp.ipniIpfs; const isApproved = approvedIds.includes(info.id); @@ -161,13 +190,14 @@ export class WalletSdkService implements OnModuleInit { providerId: info.id, providerName: info.name, providerAddress: info.serviceProvider, + network, }); } // select approved, active providers - if (info.isActive) this.activeProviderAddresses.add(info.serviceProvider); - if (isApproved && info.isActive) this.approvedProviderAddresses.add(info.serviceProvider); - this.providerCache.set(info.serviceProvider, { + if (info.isActive) state.activeProviderAddresses.add(info.serviceProvider); + if (isApproved && info.isActive) state.approvedProviderAddresses.add(info.serviceProvider); + state.providerCache.set(info.serviceProvider, { ...info, isApproved, }); @@ -178,10 +208,11 @@ export class WalletSdkService implements OnModuleInit { }; }); - this.syncProvidersToDatabase(extendedProviders, this.blockchainConfig.network).catch((err) => + this.syncProvidersToDatabase(extendedProviders, network).catch((err) => this.logger.error({ event: "providers_sync_to_db_failed", message: "Failed to sync providers to DB", + network, error: toStructuredError(err), }), ); @@ -189,21 +220,23 @@ export class WalletSdkService implements OnModuleInit { this.logger.log({ event: "providers_load_completed", message: "Loaded providers from on-chain", - totalProviders: this.providerCache.size, - testingProviders: this.activeProviderAddresses.size, - approvedProviders: this.approvedProviderAddresses.size, + network, + totalProviders: state.providerCache.size, + testingProviders: state.activeProviderAddresses.size, + approvedProviders: state.approvedProviderAddresses.size, }); return true; } catch (error) { this.logger.error({ event: "providers_load_failed", message: "Failed to load registered providers from on-chain", + network, error: toStructuredError(error), }); // Fallback to empty array, let the application handle this gracefully - this.providerCache.clear(); - this.activeProviderAddresses.clear(); - this.approvedProviderAddresses.clear(); + state.providerCache.clear(); + state.activeProviderAddresses.clear(); + state.approvedProviderAddresses.clear(); return false; } } @@ -211,34 +244,36 @@ export class WalletSdkService implements OnModuleInit { /** * Get count of approved providers */ - getApprovedProvidersCount(): number { - return this.approvedProviderAddresses.size; + getApprovedProvidersCount(network: Network): number { + return this.getNetworkState(network).approvedProviderAddresses.size; } /** * Get count of all active providers supporting ipniIpfs */ - getAllActiveProvidersCount(): number { - return this.activeProviderAddresses.size; + getAllActiveProvidersCount(network: Network): number { + return this.getNetworkState(network).activeProviderAddresses.size; } /** * Get count of testing providers */ - getTestingProvidersCount(): number { - return this.blockchainConfig.useOnlyApprovedProviders - ? this.getApprovedProvidersCount() - : this.getAllActiveProvidersCount(); + getTestingProvidersCount(network: Network): number { + const state = this.getNetworkState(network); + return state.config.useOnlyApprovedProviders + ? state.approvedProviderAddresses.size + : state.activeProviderAddresses.size; } /** * Get approved providers */ - getApprovedProviders(): PDPProviderEx[] { + getApprovedProviders(network: Network): PDPProviderEx[] { + const state = this.getNetworkState(network); const approvedProviders: PDPProviderEx[] = []; - for (const address of this.approvedProviderAddresses) { - const provider = this.providerCache.get(address); + for (const address of state.approvedProviderAddresses) { + const provider = state.providerCache.get(address); if (provider) approvedProviders.push(provider); } @@ -248,11 +283,12 @@ export class WalletSdkService implements OnModuleInit { /** * Get all active providers */ - getAllActiveProviders(): PDPProviderEx[] { + getAllActiveProviders(network: Network): PDPProviderEx[] { + const state = this.getNetworkState(network); const activeProviders: PDPProviderEx[] = []; - for (const address of this.activeProviderAddresses) { - const provider = this.providerCache.get(address); + for (const address of state.activeProviderAddresses) { + const provider = state.providerCache.get(address); if (provider) activeProviders.push(provider); } @@ -262,17 +298,21 @@ export class WalletSdkService implements OnModuleInit { /** * Get testing providers */ - getTestingProviders(): PDPProviderEx[] { - return this.blockchainConfig.useOnlyApprovedProviders ? this.getApprovedProviders() : this.getAllActiveProviders(); + getTestingProviders(network: Network): PDPProviderEx[] { + const state = this.getNetworkState(network); + return state.config.useOnlyApprovedProviders + ? this.getApprovedProviders(network) + : this.getAllActiveProviders(network); } /** * Get wallet services (now returns instance variables) */ - getWalletServices(): WalletServices { + getWalletServices(network: Network): WalletServices { + const state = this.getNetworkState(network); return { - paymentsService: this.paymentsService, - warmStorageService: this.warmStorageService, + paymentsService: state.paymentsService, + warmStorageService: state.warmStorageService, }; } @@ -280,9 +320,10 @@ export class WalletSdkService implements OnModuleInit { * Get wallet balances in base units. * USDFC is the available balance in the Filecoin Pay contract (funds minus lockups). */ - async getWalletBalances(): Promise<{ usdfc: bigint; fil: bigint }> { - const accountInfo = await this.paymentsService.accountInfo(); - const filBalance = await this.paymentsService.walletBalance(); + async getWalletBalances(network: Network): Promise<{ usdfc: bigint; fil: bigint }> { + const state = this.getNetworkState(network); + const accountInfo = await state.paymentsService.accountInfo(); + const filBalance = await state.paymentsService.walletBalance(); return { usdfc: accountInfo.availableFunds, fil: filBalance, @@ -290,10 +331,10 @@ export class WalletSdkService implements OnModuleInit { } /** - * Get approved provider info by address + * Get provider info by address for a specific network. */ - getProviderInfo(address: string): PDPProviderEx | undefined { - return this.providerCache.get(address); + getProviderInfo(address: string, network: Network): PDPProviderEx | undefined { + return this.getNetworkState(network).providerCache.get(address); } /** @@ -301,11 +342,12 @@ export class WalletSdkService implements OnModuleInit { * Skipped in session key mode, deposits and operator approvals must be * done separately via the Safe multisig UI. */ - async ensureWalletAllowances(): Promise { - if (this._isSessionKeyMode) { + async ensureWalletAllowances(network: Network): Promise { + const state = this.getNetworkState(network); + if (state.isSessionKeyMode) { const { getUploadCosts } = await import("@filoz/synapse-core/warm-storage"); - const costs = await getUploadCosts(this._synapseClient, { - clientAddress: this.blockchainConfig.walletAddress as `0x${string}`, + const costs = await getUploadCosts(state.synapseClient, { + clientAddress: state.config.walletAddress as Hex, dataSize: 100n * 1024n * 1024n * 1024n, }); @@ -313,6 +355,7 @@ export class WalletSdkService implements OnModuleInit { this.logger.log({ event: "wallet_status_check_completed", message: "Session key mode: account is funded and approved", + network, costs: this.serializeBigInt(costs), }); } else { @@ -320,6 +363,7 @@ export class WalletSdkService implements OnModuleInit { event: "wallet_not_ready", message: "Session key mode: account is NOT ready. Deposit USDFC and/or approve FWSS operator via the Safe multisig.", + network, depositNeeded: costs.depositNeeded.toString(), needsApproval: costs.needsFwssMaxApproval, costs: this.serializeBigInt(costs), @@ -331,12 +375,13 @@ export class WalletSdkService implements OnModuleInit { return; } const STORAGE_SIZE_GB = 100n; - const { costs, transaction } = await this.storageManager.prepare({ + const { costs, transaction } = await state.storageManager.prepare({ dataSize: STORAGE_SIZE_GB * 1024n * 1024n * 1024n, }); this.logger.log({ event: "wallet_status_check_completed", + network, depositAmount: transaction?.depositAmount, includesApproval: transaction?.includesApproval, costs, @@ -345,6 +390,7 @@ export class WalletSdkService implements OnModuleInit { if (transaction) { this.logger.log({ event: "wallet_deposit_started", + network, depositAmount: transaction.depositAmount.toString(), includesApproval: transaction?.includesApproval, costs, @@ -354,6 +400,7 @@ export class WalletSdkService implements OnModuleInit { this.logger.log({ event: "wallet_deposit_succeeded", + network, transactionHash: hash, depositAmount: transaction.depositAmount.toString(), includesApproval: transaction.includesApproval, @@ -397,7 +444,7 @@ export class WalletSdkService implements OnModuleInit { } /** - * Create or update provider in database + * Create or update provider in database with network scoping. */ async syncProvidersToDatabase(providerInfos: PDPProviderEx[], network: Network): Promise { try { @@ -414,6 +461,7 @@ export class WalletSdkService implements OnModuleInit { event: "duplicate_provider_address", message: "Duplicate provider address detected", address, + network, existingProviderId: existing.id, newProviderId: info.id, }); @@ -459,6 +507,7 @@ export class WalletSdkService implements OnModuleInit { event: "duplicate_provider_addresses_unresolved", message: "Duplicate provider addresses without active/inactive resolution; keeping highest providerId entries", + network, details: formatDetails(conflictAddresses), }); } @@ -468,6 +517,7 @@ export class WalletSdkService implements OnModuleInit { this.logger.warn({ event: "duplicate_provider_addresses_resolved", message: "Duplicate provider addresses detected; replaced inactive entries with active ones", + network, details: formatDetails(resolvedOnly), }); } @@ -475,8 +525,8 @@ export class WalletSdkService implements OnModuleInit { const entities = Array.from(dedupedProviders.values()).map((info) => this.spRepository.create({ - network, address: info.serviceProvider as Hex, + network, providerId: info.id, name: info.name, description: info.description, @@ -497,6 +547,7 @@ export class WalletSdkService implements OnModuleInit { this.logger.warn({ event: "track_providers_failed", message: "Failed to track providers", + network, error: toStructuredError(error), }); throw error; diff --git a/apps/backend/src/worker.module.ts b/apps/backend/src/worker.module.ts index 886e1db3..ffaff1f5 100644 --- a/apps/backend/src/worker.module.ts +++ b/apps/backend/src/worker.module.ts @@ -1,18 +1,16 @@ import { Module } from "@nestjs/common"; import { ConfigModule } from "@nestjs/config"; -import { LoggerModule } from "nestjs-pino"; -import { buildLoggerModuleParams } from "./common/pino.config.js"; -import { configValidationSchema, loadConfig } from "./config/app.config.js"; +import { validateConfig } from "./config/env.schema.js"; +import { loadConfig } from "./config/loader.js"; import { DatabaseModule } from "./database/database.module.js"; import { JobsModule } from "./jobs/jobs.module.js"; import { MetricsPrometheusModule } from "./metrics-prometheus/metrics-prometheus.module.js"; @Module({ imports: [ - LoggerModule.forRoot(buildLoggerModuleParams()), ConfigModule.forRoot({ load: [loadConfig], - validationSchema: configValidationSchema, + validate: validateConfig, isGlobal: true, }), DatabaseModule, From 88e8056b9c3d004285486cdf70bd7c29ee8c060b Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Thu, 23 Apr 2026 12:03:43 +0530 Subject: [PATCH 2/5] chore: fix import order --- .../src/config/legacy-env-compat.spec.ts | 35 ++----------------- apps/backend/src/config/legacy-env-compat.ts | 2 +- 2 files changed, 3 insertions(+), 34 deletions(-) diff --git a/apps/backend/src/config/legacy-env-compat.spec.ts b/apps/backend/src/config/legacy-env-compat.spec.ts index 58662c8d..9cb759e6 100644 --- a/apps/backend/src/config/legacy-env-compat.spec.ts +++ b/apps/backend/src/config/legacy-env-compat.spec.ts @@ -1,5 +1,5 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { applyLegacyEnvCompat, logLegacyEnvCompatResult } from "./legacy-env-compat.js"; +import { describe, expect, it } from "vitest"; +import { applyLegacyEnvCompat } from "./legacy-env-compat.js"; /** Build a fresh env map for every test — never mutate `process.env`. */ const envOf = (overrides: Record = {}): NodeJS.ProcessEnv => ({ ...overrides }) as NodeJS.ProcessEnv; @@ -109,34 +109,3 @@ describe("applyLegacyEnvCompat", () => { }); }); }); - -describe("logLegacyEnvCompatResult", () => { - let warnSpy: ReturnType; - - beforeEach(() => { - warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - }); - - afterEach(() => { - warnSpy.mockRestore(); - }); - - it("is a no-op when translation was not applied", () => { - logLegacyEnvCompatResult({ applied: false, translatedVars: [], skipReason: "networks_already_set" }); - expect(warnSpy).not.toHaveBeenCalled(); - }); - - it("emits a structured JSON warning when translation was applied", () => { - logLegacyEnvCompatResult({ - applied: true, - network: "calibration", - translatedVars: ["WALLET_PRIVATE_KEY", "RPC_URL"], - }); - - expect(warnSpy).toHaveBeenCalledTimes(1); - const payload = JSON.parse(warnSpy.mock.calls[0][0] as string); - expect(payload.event).toBe("config_legacy_env_detected"); - expect(payload.network).toBe("calibration"); - expect(payload.translatedVars).toEqual(["WALLET_PRIVATE_KEY", "RPC_URL"]); - }); -}); diff --git a/apps/backend/src/config/legacy-env-compat.ts b/apps/backend/src/config/legacy-env-compat.ts index d4ec38f1..a68d25ed 100644 --- a/apps/backend/src/config/legacy-env-compat.ts +++ b/apps/backend/src/config/legacy-env-compat.ts @@ -26,8 +26,8 @@ * and this shim is no longer needed, delete this file and its two call sites. */ -import { createPinoExitLogger } from "src/common/pino.config.js"; import { SUPPORTED_NETWORKS } from "../common/constants.js"; +import { createPinoExitLogger } from "../common/pino.config.js"; import type { Network } from "../common/types.js"; /** From dc09c9c08b6937ab7c3ac849ad13f9fc76eeff7e Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Thu, 23 Apr 2026 12:30:52 +0530 Subject: [PATCH 3/5] docs: update configuration for multi-network --- .env.example | 29 +- apps/backend/.env.example | 195 ++++++++---- docs/environment-variables.md | 540 +++++++++++++++------------------- 3 files changed, 399 insertions(+), 365 deletions(-) diff --git a/.env.example b/.env.example index 938af27c..78b41714 100644 --- a/.env.example +++ b/.env.example @@ -4,9 +4,32 @@ # Keep this file limited to *secrets only* - non-secret configuration lives in # kustomize ConfigMap patches (e.g., kustomize/overlays/local/backend-configmap-local.yaml). # -# Required -WALLET_ADDRESS= -WALLET_PRIVATE_KEY= +# Multi-network support +# --------------------- +# Dealbot now supports running against multiple Filecoin networks from a single +# instance. The `NETWORKS` env var (set via ConfigMap, not here) selects the +# active networks as a comma-separated list, e.g. `NETWORKS=calibration` or +# `NETWORKS=calibration,mainnet`. +# +# Every network-scoped secret is prefixed with the uppercase network name: +# CALIBRATION_WALLET_ADDRESS, CALIBRATION_WALLET_PRIVATE_KEY +# MAINNET_WALLET_ADDRESS, MAINNET_WALLET_PRIVATE_KEY +# +# Each active network must provide either `_WALLET_PRIVATE_KEY` or +# `_SESSION_KEY_PRIVATE_KEY`. When both are set, the session key +# takes precedence (see docs/runbooks/wallet-and-session-keys.md). + +# --- Required for calibration --- +CALIBRATION_WALLET_ADDRESS= +CALIBRATION_WALLET_PRIVATE_KEY= +# Optional: use a session key instead of the raw wallet key. +# CALIBRATION_SESSION_KEY_PRIVATE_KEY= + +# --- Required for mainnet (only if `mainnet` is in NETWORKS) --- +# MAINNET_WALLET_ADDRESS= +# MAINNET_WALLET_PRIVATE_KEY= +# MAINNET_SESSION_KEY_PRIVATE_KEY= + # # Optional (only if using an external DB or a non-default password) # DATABASE_PASSWORD= diff --git a/apps/backend/.env.example b/apps/backend/.env.example index 6815a66f..847d39aa 100644 --- a/apps/backend/.env.example +++ b/apps/backend/.env.example @@ -1,83 +1,158 @@ -# Node Environment +# ============================================================================= +# Dealbot backend — environment configuration +# ============================================================================= +# See docs/environment-variables.md for the full reference. +# +# Multi-network primer +# -------------------- +# `NETWORKS` is a comma-separated list of Filecoin networks the instance should +# drive (supported: `calibration`, `mainnet`). Every network-scoped variable is +# namespaced with the UPPERCASE network name: +# +# NETWORKS=calibration,mainnet +# CALIBRATION_WALLET_PRIVATE_KEY=0x... +# MAINNET_WALLET_PRIVATE_KEY=0x... +# +# Globals (database, jobs, timeouts, HTTP ports, etc.) remain unprefixed and +# apply to all networks. Variables for INACTIVE networks are ignored; only +# active networks are validated at startup. +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Application +# ----------------------------------------------------------------------------- NODE_ENV=development -# Specify port for dealbot service (optional) -DEALBOT_PORT=8080 +# api | worker | both (default: both) +DEALBOT_RUN_MODE=both -# Specify host for dealbot service (optional) +DEALBOT_PORT=8080 DEALBOT_HOST=localhost +# Metrics-only HTTP server (used when DEALBOT_RUN_MODE=worker) +DEALBOT_METRICS_PORT=9090 +DEALBOT_METRICS_HOST=0.0.0.0 + # Comma-separated list of allowed origins for CORS (for web dev) DEALBOT_ALLOWED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 -# Database Configuration +# Enables /api/dev/* endpoints for manual deal/retrieval testing. +# NEVER enable in production. +ENABLE_DEV_MODE=false + +# ----------------------------------------------------------------------------- +# Database +# ----------------------------------------------------------------------------- DATABASE_HOST=localhost DATABASE_PORT=5432 DATABASE_USER=dealbot DATABASE_PASSWORD=dealbot_password DATABASE_NAME=filecoin_dealbot - -# Blockchain Configuration -NETWORK=calibration # or mainnet -WALLET_ADDRESS=0x0000000000000000000000000000000000000000 -WALLET_PRIVATE_KEY=your_private_key_here -CHECK_DATASET_CREATION_FEES=true -USE_ONLY_APPROVED_PROVIDERS=true -PDP_SUBGRAPH_ENDPOINT=https://api.thegraph.com/subgraphs/filecoin/pdp - -# Minimum number of datasets per SP (default: 1). When > 1, a separate data_set_creation job provisions extra datasets. -MIN_NUM_DATASETS_FOR_CHECKS=1 - -# Dataset Versioning (optional) -# Uncomment and set to enable dataset versioning (e.g., "dealbot-v1", "dealbot-v2") -# This allows creating new logical datasets without changing wallet addresses -# DEALBOT_DATASET_VERSION=dealbot-v1 - -# Scheduling Configuration -# Intervals: How often jobs run (in seconds) -PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 # Run providers refresh every 4 hours -DATA_RETENTION_POLL_INTERVAL_SECONDS=3600 # Run data retention poll every 60 minutes - -# Prometheus Metrics Configuration -# Cache TTL for wallet balance collection (in seconds) -PROMETHEUS_WALLET_BALANCE_TTL_SECONDS=3600 # Refresh wallet balance every 1 hour -PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS=60 # Wait 1 minute before retry after error - -# Maintenance windows (UTC) -DEALBOT_MAINTENANCE_WINDOWS_UTC=07:00,22:00 -DEALBOT_MAINTENANCE_WINDOW_MINUTES=20 - -# pg-boss (DB-backed jobs) Configuration -# See docs/environment-variables.md for details, limits, and fractional examples. -DEALS_PER_SP_PER_HOUR=2 -DATASET_CREATIONS_PER_SP_PER_HOUR=1 -RETRIEVALS_PER_SP_PER_HOUR=1 +DATABASE_POOL_MAX=1 + +# ----------------------------------------------------------------------------- +# Network selection +# ----------------------------------------------------------------------------- +# Comma-separated. Default: calibration. +NETWORKS=calibration + +# ----------------------------------------------------------------------------- +# Per-network configuration — CALIBRATION +# ----------------------------------------------------------------------------- +# At least one of WALLET_PRIVATE_KEY or SESSION_KEY_PRIVATE_KEY is required +# for each ACTIVE network. Session key (if present) takes precedence. +CALIBRATION_WALLET_ADDRESS=0x0000000000000000000000000000000000000000 +CALIBRATION_WALLET_PRIVATE_KEY=your_calibration_private_key_here +# CALIBRATION_SESSION_KEY_PRIVATE_KEY= + +# Optional: custom RPC (authenticated endpoints avoid public rate limits) +# CALIBRATION_RPC_URL=https://filecoin.chain.love/rpc/v1?token=YOUR_API_KEY + +CALIBRATION_CHECK_DATASET_CREATION_FEES=true +CALIBRATION_USE_ONLY_APPROVED_PROVIDERS=true +CALIBRATION_PDP_SUBGRAPH_ENDPOINT= + +# Minimum number of datasets per SP. When > 1, a separate data_set_creation +# job provisions extra datasets. +CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS=1 + +# Optional: dataset versioning (e.g. "dealbot-v1", "dealbot-v2"). +# CALIBRATION_DEALBOT_DATASET_VERSION=dealbot-v1 + +# Per-network scheduling rates (per hour; fractional values supported) +CALIBRATION_DEALS_PER_SP_PER_HOUR=2 +CALIBRATION_DATASET_CREATIONS_PER_SP_PER_HOUR=1 +CALIBRATION_RETRIEVALS_PER_SP_PER_HOUR=1 + +# Per-network job intervals (seconds) +CALIBRATION_PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 # 4 hours +CALIBRATION_DATA_RETENTION_POLL_INTERVAL_SECONDS=3600 # 1 hour + +# Per-network maintenance windows (UTC) +CALIBRATION_MAINTENANCE_WINDOWS_UTC=07:00,22:00 +CALIBRATION_MAINTENANCE_WINDOW_MINUTES=20 + +# Per-network SP blocklists +# CALIBRATION_BLOCKED_SP_IDS=1234,5678 +# CALIBRATION_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234... + +# ----------------------------------------------------------------------------- +# Per-network configuration — MAINNET (uncomment when adding to NETWORKS) +# ----------------------------------------------------------------------------- +# MAINNET_WALLET_ADDRESS= +# MAINNET_WALLET_PRIVATE_KEY= +# MAINNET_SESSION_KEY_PRIVATE_KEY= +# MAINNET_RPC_URL= +# MAINNET_CHECK_DATASET_CREATION_FEES=true +# MAINNET_USE_ONLY_APPROVED_PROVIDERS=true +# MAINNET_PDP_SUBGRAPH_ENDPOINT= +# MAINNET_MIN_NUM_DATASETS_FOR_CHECKS=1 +# MAINNET_DEALBOT_DATASET_VERSION= +# MAINNET_DEALS_PER_SP_PER_HOUR=2 +# MAINNET_DATASET_CREATIONS_PER_SP_PER_HOUR=1 +# MAINNET_RETRIEVALS_PER_SP_PER_HOUR=1 +# MAINNET_METRICS_PER_HOUR=0.5 +# MAINNET_PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 +# MAINNET_DATA_RETENTION_POLL_INTERVAL_SECONDS=3600 +# MAINNET_MAINTENANCE_WINDOWS_UTC=07:00,22:00 +# MAINNET_MAINTENANCE_WINDOW_MINUTES=20 +# MAINNET_BLOCKED_SP_IDS=1234,5678 +# MAINNET_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234... + +# ----------------------------------------------------------------------------- +# Jobs (pg-boss) +# ----------------------------------------------------------------------------- +# See docs/environment-variables.md for details, limits, and sizing guidance. PG_BOSS_LOCAL_CONCURRENCY=20 JOB_SCHEDULER_POLL_SECONDS=300 JOB_WORKER_POLL_SECONDS=60 JOB_CATCHUP_MAX_ENQUEUE=10 JOB_SCHEDULE_PHASE_SECONDS=0 JOB_ENQUEUE_JITTER_SECONDS=0 -DEAL_JOB_TIMEOUT_SECONDS=360 # 6m: Max runtime for deal jobs (TODO: reduce default to 3m) -RETRIEVAL_JOB_TIMEOUT_SECONDS=60 # 1m: Max runtime for retrieval jobs (TODO: reduce default to 30s) -IPFS_BLOCK_FETCH_CONCURRENCY=6 # Parallel block fetches when validating IPFS DAGs +DEAL_JOB_TIMEOUT_SECONDS=360 # 6m: max runtime for deal jobs +RETRIEVAL_JOB_TIMEOUT_SECONDS=60 # 1m: max runtime for retrieval jobs +DATA_SET_CREATION_JOB_TIMEOUT_SECONDS=300 # 5m: max runtime for dataset-creation jobs +IPFS_BLOCK_FETCH_CONCURRENCY=6 # parallel block fetches during IPFS DAG validation DEALBOT_PGBOSS_POOL_MAX=1 DEALBOT_PGBOSS_SCHEDULER_ENABLED=true -# Dataset Configuration -DEALBOT_LOCAL_DATASETS_PATH=./datasets -KAGGLE_DATASET_TOTAL_PAGES=500 - -# Proxy Configuration -PROXY_LIST=http://username:password@host:port,http://username:password@host:port -PROXY_LOCATIONS=l1,l2 - -# Timeout Configuration (in milliseconds) -CONNECT_TIMEOUT_MS=10000 # 10s: Initial connection timeout -HTTP_REQUEST_TIMEOUT_MS=240000 # 4m: Total transfer timeout for HTTP/1.1 (10MiB @ 170KB/s + overhead) -HTTP2_REQUEST_TIMEOUT_MS=240000 # 4m: Total transfer timeout for HTTP/2 (10MiB @ 170KB/s + overhead) - -# SP Blocklists configuration -# BLOCKED_SP_IDS=1234,5678 -# BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234... +# ----------------------------------------------------------------------------- +# Prometheus metrics +# ----------------------------------------------------------------------------- +PROMETHEUS_WALLET_BALANCE_TTL_SECONDS=3600 # refresh cache every 1h +PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS=60 # cooldown after fetch failure +# ----------------------------------------------------------------------------- +# Dataset generation +# ----------------------------------------------------------------------------- +DEALBOT_LOCAL_DATASETS_PATH=./datasets +RANDOM_PIECE_SIZES=10485760 # 10 MiB (comma-separated list supported) + +# ----------------------------------------------------------------------------- +# HTTP timeouts (milliseconds) +# ----------------------------------------------------------------------------- +CONNECT_TIMEOUT_MS=10000 # 10s: initial connection timeout +HTTP_REQUEST_TIMEOUT_MS=240000 # 4m: total transfer timeout (HTTP/1.1) +HTTP2_REQUEST_TIMEOUT_MS=240000 # 4m: total transfer timeout (HTTP/2) +IPNI_VERIFICATION_TIMEOUT_MS=60000 # 60s: IPNI propagation wait +IPNI_VERIFICATION_POLLING_MS=2000 # 2s: IPNI polling interval diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 9aff8c1e..227b71b7 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -2,23 +2,46 @@ This document provides a comprehensive guide to all environment variables used by the Dealbot. Understanding these variables is essential for proper configuration in development, testing, and production environments. +## Multi-Network Support + +Dealbot drives one or more Filecoin networks from a **single process**. The active set is controlled by the [`NETWORKS`](#networks) variable, and every network-scoped variable is namespaced with the UPPERCASE network name. + +```bash +# Run Dealbot against both networks from the same instance +NETWORKS=calibration,mainnet + +CALIBRATION_WALLET_PRIVATE_KEY=0xabc... +CALIBRATION_RPC_URL=https://api.calibration.node.glif.io/rpc/v1 +CALIBRATION_DEALS_PER_SP_PER_HOUR=2 + +MAINNET_WALLET_PRIVATE_KEY=0xdef... +MAINNET_RPC_URL=https://api.node.glif.io/rpc/v1 +MAINNET_DEALS_PER_SP_PER_HOUR=1 +``` + +**Rules** + +- **Global vs. per-network.** Unprefixed variables (database, HTTP ports, job timeouts, etc.) apply to the whole process. Prefixed variables configure a specific network. +- **Active-network validation.** Only networks listed in `NETWORKS` are validated at startup. Variables for inactive networks are ignored, so you can keep a `MAINNET_*` block commented out until you are ready. +- **Wallet vs. session key.** Each active network must provide either `_WALLET_PRIVATE_KEY` or `_SESSION_KEY_PRIVATE_KEY`. When both are present the session key takes precedence (see [`docs/runbooks/wallet-and-session-keys.md`](./runbooks/wallet-and-session-keys.md)). +- **Supported prefixes.** `CALIBRATION_*`, `MAINNET_*`. Additional networks can be added by extending `SUPPORTED_NETWORKS` in the codebase. + ## Quick Reference | Category | Variables | | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | | [Application](#application-configuration) | `NODE_ENV`, `DEALBOT_PORT`, `DEALBOT_HOST`, `DEALBOT_RUN_MODE`, `DEALBOT_METRICS_PORT`, `DEALBOT_METRICS_HOST`, `DEALBOT_ALLOWED_ORIGINS`, `ENABLE_DEV_MODE` | | [Database](#database-configuration) | `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_POOL_MAX`, `DATABASE_USER`, `DATABASE_PASSWORD`, `DATABASE_NAME` | -| [Blockchain](#blockchain-configuration) | `NETWORK`, `RPC_URL`, `WALLET_ADDRESS`, `WALLET_PRIVATE_KEY`, `SESSION_KEY_PRIVATE_KEY`, `CHECK_DATASET_CREATION_FEES`, `USE_ONLY_APPROVED_PROVIDERS`, `PDP_SUBGRAPH_ENDPOINT` | -| [Dataset Versioning](#dataset-versioning) | `DEALBOT_DATASET_VERSION` | -| [Scheduling](#scheduling-configuration) | `PROVIDERS_REFRESH_INTERVAL_SECONDS`, `DATA_RETENTION_POLL_INTERVAL_SECONDS`, `DEALBOT_MAINTENANCE_WINDOWS_UTC`, `DEALBOT_MAINTENANCE_WINDOW_MINUTES` | -| [Jobs (pg-boss)](#jobs-pg-boss) | `DEALBOT_PGBOSS_SCHEDULER_ENABLED`, `DEALBOT_PGBOSS_POOL_MAX`, `DEALS_PER_SP_PER_HOUR`, `DATASET_CREATIONS_PER_SP_PER_HOUR`, `RETRIEVALS_PER_SP_PER_HOUR`, `JOB_SCHEDULER_POLL_SECONDS`, `JOB_WORKER_POLL_SECONDS`, `PG_BOSS_LOCAL_CONCURRENCY`, `JOB_CATCHUP_MAX_ENQUEUE`, `JOB_SCHEDULE_PHASE_SECONDS`, `JOB_ENQUEUE_JITTER_SECONDS`, `DEAL_JOB_TIMEOUT_SECONDS`, `RETRIEVAL_JOB_TIMEOUT_SECONDS`, `IPFS_BLOCK_FETCH_CONCURRENCY` | +| [Per-Network](#per-network-configuration) | `_WALLET_ADDRESS`, `_WALLET_PRIVATE_KEY`, `_SESSION_KEY_PRIVATE_KEY`, `_RPC_URL`, `_CHECK_DATASET_CREATION_FEES`, `_USE_ONLY_APPROVED_PROVIDERS`, `_PDP_SUBGRAPH_ENDPOINT`, `_DEALBOT_DATASET_VERSION`, `_MIN_NUM_DATASETS_FOR_CHECKS` | +| [Per-Network Scheduling](#per-network-scheduling) | `_DEALS_PER_SP_PER_HOUR`, `_RETRIEVALS_PER_SP_PER_HOUR`, `_DATASET_CREATIONS_PER_SP_PER_HOUR`, `_METRICS_PER_HOUR`, `_PROVIDERS_REFRESH_INTERVAL_SECONDS`, `_DATA_RETENTION_POLL_INTERVAL_SECONDS`, `_MAINTENANCE_WINDOWS_UTC`, `_MAINTENANCE_WINDOW_MINUTES`, `_BLOCKED_SP_IDS`, `_BLOCKED_SP_ADDRESSES`,`_MAX_DATASET_STORAGE_SIZE_BYTES`, `_TARGET_DATASET_STORAGE_SIZE_BYTES`, `_JOB_PIECE_CLEANUP_PER_SP_PER_HOUR`, `_MAX_PIECE_CLEANUP_RUNTIME_SECONDS` | +| [Jobs (pg-boss)](#jobs-pg-boss) | `DEALBOT_PGBOSS_SCHEDULER_ENABLED`, `DEALBOT_PGBOSS_POOL_MAX`, `JOB_SCHEDULER_POLL_SECONDS`, `JOB_WORKER_POLL_SECONDS`, `PG_BOSS_LOCAL_CONCURRENCY`, `JOB_CATCHUP_MAX_ENQUEUE`, `JOB_SCHEDULE_PHASE_SECONDS`, `JOB_ENQUEUE_JITTER_SECONDS`, `DEAL_JOB_TIMEOUT_SECONDS`, `RETRIEVAL_JOB_TIMEOUT_SECONDS`, `IPFS_BLOCK_FETCH_CONCURRENCY` | | [Dataset](#dataset-configuration) | `DEALBOT_LOCAL_DATASETS_PATH`, `RANDOM_PIECE_SIZES` | | [Timeouts](#timeout-configuration) | `CONNECT_TIMEOUT_MS`, `HTTP_REQUEST_TIMEOUT_MS`, `HTTP2_REQUEST_TIMEOUT_MS`, `IPNI_VERIFICATION_TIMEOUT_MS`, `IPNI_VERIFICATION_POLLING_MS` | -| [Piece Cleanup](#piece-cleanup) | `MAX_DATASET_STORAGE_SIZE_BYTES`, `TARGET_DATASET_STORAGE_SIZE_BYTES`, `JOB_PIECE_CLEANUP_PER_SP_PER_HOUR`, `MAX_PIECE_CLEANUP_RUNTIME_SECONDS` | -| [SP Blocklist](#sp-blocklist-configuration) | `BLOCKED_SP_IDS`, `BLOCKED_SP_ADDRESSES` | | [Prometheus Metrics](#prometheus-metrics-configuration) | `PROMETHEUS_WALLET_BALANCE_TTL_SECONDS`, `PROMETHEUS_WALLET_BALANCE_ERROR_COOLDOWN_SECONDS` | | [Web Frontend](#web-frontend) | `VITE_API_BASE_URL`, `VITE_PLAUSIBLE_DATA_DOMAIN`, `DEALBOT_API_BASE_URL` | +> **Legend.** `` is the uppercase network name (`CALIBRATION`, `MAINNET`). + --- ## Application Configuration @@ -291,347 +314,382 @@ DATABASE_POOL_MAX=1 --- -## Blockchain Configuration +## Network Selection -### `NETWORK` +### `NETWORKS` -- **Type**: `string` +- **Type**: `string` (comma-separated list) - **Required**: No - **Default**: `calibration` -- **Valid values**: `mainnet`, `calibration` - -**Role**: Determines which Filecoin network to interact with. This affects contract addresses, RPC endpoints, and token economics. - -**When to update**: - -- Set to `calibration` for testing with test FIL/USDFC tokens -- Set to `mainnet` for production deployments with real FIL/USDFC - -**⚠️ Warning**: Switching to `mainnet` will use real FIL/USDFC tokens. Ensure your wallet is funded and you understand the costs involved. - ---- +- **Valid values (per entry)**: `mainnet`, `calibration` -### `RPC_URL` +**Role**: Selects which Filecoin networks the instance drives. Every active network is validated independently at startup; inactive networks have their `_*` variables ignored entirely. -- **Type**: `string` (HTTP/HTTPS URL) -- **Required**: No -- **Default**: Uses the default public RPC for the configured network - -**Role**: Custom Filecoin RPC endpoint URL. When set, all on-chain calls (Synapse SDK, viem) use this endpoint instead of the default public RPC. Use an authenticated endpoint to avoid rate limiting on shared public infrastructure. - -Providers like Glif/Chain.Love support passing the API key as a query parameter: +**Examples**: ```bash -RPC_URL=https://filecoin.chain.love/rpc/v1?token=YOUR_API_KEY -``` +# Single network (default) +NETWORKS=calibration -**When to update**: - -- When DealBot is hitting 429 rate limits on the default public RPC -- When switching RPC providers -- When rotating API keys +# Multi-network instance +NETWORKS=calibration,mainnet +``` -**Security**: Treat as a secret if the URL contains an API key. +**⚠️ Warning**: Adding `mainnet` to `NETWORKS` will cause the instance to spend real FIL/USDFC for every scheduled job. Ensure the configured wallet is funded and rate limits are reviewed before enabling. --- -### `WALLET_ADDRESS` +## Per-Network Configuration -- **Type**: `string` (Ethereum-style address) -- **Required**: Yes -- **Security**: Public, but should match `WALLET_PRIVATE_KEY` +Every variable below must be prefixed with the uppercase network name (e.g. `CALIBRATION_`, `MAINNET_`). Values only apply when the corresponding network is listed in [`NETWORKS`](#networks). -**Role**: The Ethereum/FEVM address used for signing transactions and paying for storage deals. +### `_WALLET_ADDRESS` -**When to update**: +- **Type**: `string` (Ethereum-style address) +- **Required**: No (defaults to the zero address) +- **Security**: Public, but should match `_WALLET_PRIVATE_KEY` or be the signer registered against `_SESSION_KEY_PRIVATE_KEY`. -- When switching to a different wallet -- When setting up a new Dealbot instance -- When rotating keys for security +**Role**: The FEVM address used for signing transactions and paying for storage deals on the given network. **Example**: ```bash -WALLET_ADDRESS=0x1234567890abcdef1234567890abcdef12345678 +CALIBRATION_WALLET_ADDRESS=0x1234567890abcdef1234567890abcdef12345678 ``` --- -### `WALLET_PRIVATE_KEY` +### `_WALLET_PRIVATE_KEY` -- **Type**: `string` -- **Required**: Yes -- **Security**: **HIGHLY SENSITIVE** - Never commit to version control, use secrets management +- **Type**: `string` (0x-prefixed hex) +- **Required**: One of `_WALLET_PRIVATE_KEY` or `_SESSION_KEY_PRIVATE_KEY` is required for every active network. Both may be set, in which case the session key takes precedence. +- **Security**: **HIGHLY SENSITIVE** — never commit to version control; use Kubernetes Secrets or an equivalent secrets manager. -**Role**: Private key for signing blockchain transactions. Required in direct key mode. Not needed when `SESSION_KEY_PRIVATE_KEY` is set (session key mode), since the session key handles all signing. If both are set, `SESSION_KEY_PRIVATE_KEY` takes precedence and `WALLET_PRIVATE_KEY` is ignored. +**Role**: Private key for signing blockchain transactions on this network. Required in direct-key mode. Ignored when a session key is provided. -**When to update**: +--- -- When rotating keys for security -- When setting up a new Dealbot instance -- When switching wallets +### `_SESSION_KEY_PRIVATE_KEY` -**Security best practices**: +- **Type**: `string` (0x-prefixed hex) +- **Required**: See `_WALLET_PRIVATE_KEY` above. +- **Security**: **HIGHLY SENSITIVE**. -- Use Kubernetes Secrets or a secrets manager (Vault, AWS Secrets Manager) -- Never log or expose this value +**Role**: When set, Dealbot uses session-key authentication on this network. The session key must be registered on the `SessionKeyRegistry` contract from `_WALLET_ADDRESS` (typically a Safe multisig). Storage operations (create dataset, add pieces) are signed with this key instead of `_WALLET_PRIVATE_KEY`. + +Session keys are scoped (only storage operations, not deposits or withdrawals) and time-limited (expiry set during registration). See [runbooks/wallet-and-session-keys.md](runbooks/wallet-and-session-keys.md) for the full setup process. --- -### `SESSION_KEY_PRIVATE_KEY` +### `_RPC_URL` -- **Type**: `string` (0x-prefixed hex private key) +- **Type**: `string` (HTTP/HTTPS URL) - **Required**: No -- **Security**: **HIGHLY SENSITIVE** - Treat like `WALLET_PRIVATE_KEY` +- **Default**: Empty (SDK falls back to its built-in default public RPC for the network) -**Role**: When set, DealBot uses session key authentication. The session key must be registered on the SessionKeyRegistry contract from the `WALLET_ADDRESS` (typically a Safe multisig). Storage operations (create dataset, add pieces) are signed with this key instead of `WALLET_PRIVATE_KEY`. +**Role**: Custom Filecoin RPC endpoint. When set, all on-chain calls (Synapse SDK, viem) for this network use the configured endpoint. Use an authenticated endpoint to avoid rate-limiting on shared public infrastructure. -Session keys are scoped (only storage operations, not deposits or withdrawals) and time-limited (expiry set during registration). See [runbooks/wallet-and-session-keys.md](runbooks/wallet-and-session-keys.md) for the full setup process. +Providers like Glif/Chain.Love accept the API key as a query parameter: -**When to update**: +```bash +CALIBRATION_RPC_URL=https://filecoin.chain.love/rpc/v1?token=YOUR_API_KEY +``` -- When rotating session keys -- When switching to session key mode from direct key mode -- When the session key has been compromised +**Security**: Treat as a secret if the URL contains an API key. --- -### `CHECK_DATASET_CREATION_FEES` +### `_CHECK_DATASET_CREATION_FEES` - **Type**: `boolean` - **Required**: No - **Default**: `true` -**Role**: When enabled, validates that the wallet has sufficient balance to cover dataset creation fees + 100 GiB of storage costs. +**Role**: When enabled, validates that the network's wallet has sufficient balance to cover dataset-creation fees plus 100 GiB of storage costs before creating a new dataset. **When to update**: -- Set to `false` to skip addition of dataset creation fees into storage costs. +- Set to `false` to skip the balance check (e.g. for CI/test environments where insufficient balance is expected). --- -### `USE_ONLY_APPROVED_PROVIDERS` +### `_USE_ONLY_APPROVED_PROVIDERS` - **Type**: `boolean` - **Required**: No - **Default**: `true` -**Role**: Restricts deal-making to only Filecoin Warm Storage Service (FWSS) approved storage providers. This ensures deals are made with approved providers. +**Role**: Restricts deal-making to Filecoin Warm Storage Service (FWSS) approved storage providers for the network. **When to update**: -- Set to `false` to test with any storage provider available for testing (providers that support "PDP" product in ServiceProviderRegistry) +- Set to `false` to test against any provider that supports the `PDP` product in `ServiceProviderRegistry`. --- -### `PDP_SUBGRAPH_ENDPOINT` +### `_PDP_SUBGRAPH_ENDPOINT` - **Type**: `string` (URL) - **Required**: No -- **Default**: Empty string (feature disabled) - -**Role**: The Graph API endpoint for querying PDP (Proof of Data Possession) subgraph data. This endpoint is used to retrieve data retention info for provider data. - -**When to update**: +- **Default**: Empty (feature disabled for this network) -- When switching between different Graph API endpoints +**Role**: The Graph API endpoint for querying PDP (Proof of Data Possession) subgraph data for this network. Used to retrieve data-retention information for provider datasets. **Example**: ```bash -PDP_SUBGRAPH_ENDPOINT=https://api.thegraph.com/subgraphs/filecoin/pdp +CALIBRATION_PDP_SUBGRAPH_ENDPOINT=https://api.thegraph.com/subgraphs/filecoin/pdp ``` --- -## Dataset Versioning +### `_MIN_NUM_DATASETS_FOR_CHECKS` -### `DEALBOT_DATASET_VERSION` +- **Type**: `number` (integer, ≥ 1) +- **Required**: No +- **Default**: `1` + +**Role**: Minimum number of datasets provisioned per storage provider before running checks on this network. When > 1, the `data_set_creation` job is responsible for provisioning any additional datasets. + +--- + +### `_DEALBOT_DATASET_VERSION` - **Type**: `string` - **Required**: No - **Default**: Not set (no versioning) -**Role**: Creates versioning for datasets, allowing multiple dataset versions without changing wallet addresses. Useful for separating test data from production data or managing dataset migrations. - -**When to update**: +**Role**: Tags newly created datasets with a version label, enabling multiple generations of datasets on the same wallet. Useful for separating test data from production data or managing dataset migrations per network. -- When you want to create a fresh set of datasets -- When separating environments (e.g., `dealbot-v1`, `dealbot-staging`) - -**Example scenario**: Creating a new dataset version for testing: +**Example**: ```bash -DEALBOT_DATASET_VERSION=dealbot-v2 +CALIBRATION_DEALBOT_DATASET_VERSION=dealbot-v2 ``` --- -## Scheduling Configuration +## Per-Network Scheduling -These variables control when and how often the Dealbot runs its automated jobs. +Scheduling rates and intervals are configured per network, so each network can be tuned independently (e.g. an aggressive calibration cadence with a conservative mainnet cadence). Every variable below must be prefixed with the uppercase network name. -**Note**: Dealbot uses pg-boss for rate-based scheduling (see [Jobs (pg-boss)](#jobs-pg-boss)). +Dealbot uses pg-boss for rate-based scheduling — see [Jobs (pg-boss)](#jobs-pg-boss) for global worker/timeout settings. -### `PROVIDERS_REFRESH_INTERVAL_SECONDS` +### `_DEALS_PER_SP_PER_HOUR` - **Type**: `number` - **Required**: No -- **Default**: `14400` (4 hours, recommended) +- **Default**: `4` +- **Limits**: capped at `20` to avoid excessive on-chain activity. -**Role**: How often the providers refresh job runs, in seconds. +**Role**: Target deal creation rate per storage provider on this network. -**When to update**: +**Notes**: Fractional values are supported (e.g. `0.25` ⇒ one deal every 4 hours per SP). -- Increase for less frequent providers refresh (reduces costs, slower testing) -- Decrease for more aggressive testing (higher costs, faster feedback) +--- -**Example scenario**: Running providers refresh every 4 hours (default): +### `_RETRIEVALS_PER_SP_PER_HOUR` -```bash -PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 -``` +- **Type**: `number` +- **Required**: No +- **Default**: `2` +- **Limits**: capped at `20`. + +**Role**: Target retrieval test rate per storage provider on this network. --- -### `DATA_RETENTION_POLL_INTERVAL_SECONDS` +### `_DATASET_CREATIONS_PER_SP_PER_HOUR` - **Type**: `number` - **Required**: No -- **Default**: `3600` (1 hour) +- **Default**: `1` +- **Limits**: capped at `20`. -**Role**: How often the data retention polling job runs, in seconds. This job checks and manages data retention stats of providers for stored datasets. +**Role**: Target dataset-creation rate per storage provider on this network. -**When to update**: +--- -- Increase for less frequent data retention checks -- Decrease for more frequent monitoring of data retention policies +### `_METRICS_PER_HOUR` -**Example scenario**: Running data retention checks every 2 hours: +- **Type**: `number` +- **Required**: No +- **Default**: `0.1` +- **Limits**: capped at `3` to limit database load from materialized-view refreshes. -```bash -DATA_RETENTION_POLL_INTERVAL_SECONDS=7200 -``` +**Role**: Frequency of metrics aggregation runs per hour on this network. --- -### `DEAL_START_OFFSET_SECONDS` +### `_PROVIDERS_REFRESH_INTERVAL_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `0` +- **Default**: `14400` (4 hours) -**Role**: Delay before the first deal creation job runs after startup. - -**When to update**: - -- Increase to allow other services to initialize first -- Keep at `0` for immediate deal creation on startup +**Role**: How often the providers-refresh job runs for this network. --- -### `RETRIEVAL_START_OFFSET_SECONDS` +### `_DATA_RETENTION_POLL_INTERVAL_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `600` (10 minutes) / `300` (5 minutes in .env.example) - -**Role**: Delay before the first retrieval test runs after startup. This offset prevents retrieval tests from running concurrently with deal creation. - -**When to update**: +- **Default**: `3600` (1 hour) -- Adjust to stagger job execution and prevent resource contention -- Increase if deal creation takes longer than expected +**Role**: How often the data-retention polling job runs for this network. The job checks and updates data-retention stats of providers for stored datasets. --- -### `DEALBOT_MAINTENANCE_WINDOWS_UTC` +### `_MAINTENANCE_WINDOWS_UTC` -- **Type**: `string` (comma-separated HH:MM times in UTC) +- **Type**: `string` (comma-separated `HH:MM` times in UTC) - **Required**: No - **Default**: `07:00,22:00` -**Role**: Daily maintenance windows (UTC) during which deal creation and retrieval checks are skipped. - -**Notes**: - -- Times must be in 24-hour `HH:MM` format. -- Applies to both cron and pg-boss modes. +**Role**: Daily maintenance windows (UTC) during which deal-creation and retrieval checks are skipped for this network. Different networks can have different schedules. **Example**: ```bash -DEALBOT_MAINTENANCE_WINDOWS_UTC=06:30,21:30 +CALIBRATION_MAINTENANCE_WINDOWS_UTC=06:30,21:30 +MAINNET_MAINTENANCE_WINDOWS_UTC=05:00,17:00 ``` --- -### `DEALBOT_MAINTENANCE_WINDOW_MINUTES` +### `_MAINTENANCE_WINDOW_MINUTES` - **Type**: `number` - **Required**: No - **Default**: `20` - **Minimum**: `20` -- **Maximum**: `360` (6 hours). With two daily windows, this keeps maintenance time ≤ runtime. +- **Maximum**: `360` (6 hours) + +**Role**: Duration (minutes) of each maintenance window in `_MAINTENANCE_WINDOWS_UTC`. + +--- + +### `_BLOCKED_SP_IDS` + +- **Type**: `string` (comma-separated provider IDs) +- **Required**: No +- **Default**: `""` (empty — no providers blocked) + +**Role**: Global blocklist by provider numeric ID. Providers listed here are excluded from **all** scheduled +check types (data-storage, retrieval, and data-retention). + +**Example**: `_BLOCKED_SP_IDS=1234,5678` + +--- + +### `_BLOCKED_SP_ADDRESSES` + +- **Type**: `string` (comma-separated provider Ethereum addresses) +- **Required**: No +- **Default**: `""` (empty — no providers blocked) + +**Role**: Global blocklist by provider address. Providers listed here are excluded from **all** scheduled +check types (data-storage, retrieval, and data-retention). Matching is case-insensitive. + +**Example**: `_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...` -**Role**: Duration (minutes) of each maintenance window in `DEALBOT_MAINTENANCE_WINDOWS_UTC`. +--- + +### `_MAX_DATASET_STORAGE_SIZE_BYTES` + +- **Type**: `number` (integer, bytes) +- **Required**: No +- **Default**: `25769803776` (24 GiB) +- **Minimum**: `1` + +**Role**: **High-water mark.** Maximum total stored data per SP (in bytes) before cleanup kicks in. When live storage for a provider exceeds this value, the cleanup job triggers and deletes the oldest pieces until usage drops below `_TARGET_DATASET_STORAGE_SIZE_BYTES` (the low-water mark). + +**When to update**: + +- Increase for longer runway before cleanup kicks in (e.g. months vs weeks) +- Decrease if SP storage is constrained or costs are a concern **Example**: ```bash -DEALBOT_MAINTENANCE_WINDOW_MINUTES=30 +_MAX_DATASET_STORAGE_SIZE_BYTES=12884901888 # 12 GiB per SP ``` --- -## Jobs (pg-boss) +### `_TARGET_DATASET_STORAGE_SIZE_BYTES` -In this mode, scheduling is -rate-based (per hour) and persisted in Postgres so restarts do not reset timing. +- **Type**: `number` (integer, bytes) +- **Required**: No +- **Default**: `21474836480` (20 GiB) +- **Minimum**: `1` +**Role**: **Low-water mark.** When cleanup triggers (live usage exceeds `_MAX_DATASET_STORAGE_SIZE_BYTES`), pieces are deleted until usage drops below this target. The gap between MAX and TARGET creates headroom so cleanup doesn't re-trigger immediately. -### `DEALS_PER_SP_PER_HOUR` +**Headroom math**: At 4 deals/SP/hour × 10 MiB = ~960 MiB/day growth. With 4 GiB headroom (24 GiB MAX − 20 GiB TARGET), cleanup provides ~4 days of breathing room per run, which aligns with the daily default cadence. -- **Type**: `number` -- **Required**: No -- **Default**: `4` +**When to update**: -**Role**: Target deal creation rate per storage provider. +- Decrease for more aggressive cleanup (larger gap = more headroom) +- Increase toward MAX for minimal cleanup (smaller gap = less headroom) +- Must be less than `_MAX_DATASET_STORAGE_SIZE_BYTES` for cleanup to have effect -**Limits**: Config schema caps this at 20 to avoid excessive on-chain activity. +**Example**: -**Notes**: Fractional values are supported. For example, `0.25` means one deal every 4 hours per storage provider. +```bash +_TARGET_DATASET_STORAGE_SIZE_BYTES=16106127360 # 15 GiB per SP (9 GiB headroom) +``` --- -### `RETRIEVALS_PER_SP_PER_HOUR` +### `_PIECE_CLEANUP_PER_SP_PER_HOUR` - **Type**: `number` -- **Required**: No -- **Default**: `2` +- **Required**: No +- **Default**: `0.0417` (~1/24, approximately once per day) +- **Minimum**: `0.001` +- **Maximum**: `20` -**Role**: Target retrieval test rate per storage provider. +**Role**: Target number of piece cleanup runs per storage provider per hour. Controls how frequently the cleanup job runs for each SP. The rate is converted to an interval internally (e.g. 1/hr = every 3600s, 1/24/hr ≈ every 86400s = once per day). -**Limits**: Config schema caps this at 20 to avoid overloading providers. +**When to update**: -**Notes**: Fractional values are supported. For example, `0.25` means one retrieval every 4 hours per storage provider. +- Increase to run cleanup more frequently when SPs are frequently over quota +- Decrease to reduce scheduling overhead + +**Example**: + +```bash +# Once per hour (more aggressive) +_PIECE_CLEANUP_PER_SP_PER_HOUR=1 + +# Once per week (very conservative) +_PIECE_CLEANUP_PER_SP_PER_HOUR=0.006 +``` --- -### `DATASET_CREATIONS_PER_SP_PER_HOUR` +### `_MAX_PIECE_CLEANUP_RUNTIME_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `1` +- **Default**: `300` (5 minutes) +- **Minimum**: `60` -**Role**: Target dataset creation rate per storage provider. +**Role**: Maximum runtime for a cleanup job before forced abort via `AbortController`. Prevents stuck cleanup jobs from blocking the SP work queue. -**Limits**: Config schema caps this at 20 to avoid excessive dataset generation. +**When to update**: -**Notes**: Fractional values are supported. For example, `0.5` means one dataset creation every 2 hours per storage provider. +- Increase if piece deletion calls to the Synapse SDK are known to be slow +- Decrease for faster abort detection on stuck jobs --- +## Jobs (pg-boss) + +These variables are **global** (not per-network) and control the shared pg-boss worker runtime. Scheduling is rate-based (per hour, per network) and persisted in Postgres so restarts do not reset timing — see [Per-Network Scheduling](#per-network-scheduling) for the rate/interval knobs. + ### `JOB_SCHEDULER_POLL_SECONDS` - **Type**: `number` @@ -784,125 +842,40 @@ Use this to stagger multiple dealbot deployments that are not sharing a database **Note**: This is independent of HTTP-level timeouts. The job timeout enforces end-to-end execution time of a Retrieval Check job. --- -### `IPFS_BLOCK_FETCH_CONCURRENCY` + +### `DATA_SET_CREATION_JOB_TIMEOUT_SECONDS` - **Type**: `number` - **Required**: No -- **Default**: `6` -- **Minimum**: `1` +- **Default**: `300` (5 minutes) +- **Minimum**: `60` - **Enforced**: Yes (config validation) -**Role**: Maximum number of parallel block fetches when validating IPFS retrievals via DAG traversal. - -**When to update**: - -- Increase to speed up validation on fast networks and responsive gateways -- Decrease to reduce pressure on slower storage providers or constrained environments - -**Note**: This affects the number of concurrent `/ipfs/` requests per retrieval. - ---- - -## Piece Cleanup - -These variables control the automatic cleanup of old pieces from storage providers to prevent -unbounded data growth. Cleanup runs as a periodic pg-boss job per SP. - -The cleanup flow checks **live provider data** first (via `filecoin-pin`'s `calculateActualStorage()`) to determine how much data an SP is storing. When usage exceeds the high-water mark (`MAX_DATASET_STORAGE_SIZE_BYTES`), the cleanup job deletes the oldest pieces until usage drops below the low-water mark (`TARGET_DATASET_STORAGE_SIZE_BYTES`). This high-water/low-water approach prevents thrashing near the threshold. - -If the live query fails, cleanup falls back to DB-based `SUM(piece_size)` for the quota decision. Deal creation continues regardless of cleanup state. - -### `MAX_DATASET_STORAGE_SIZE_BYTES` - -- **Type**: `number` (integer, bytes) -- **Required**: No -- **Default**: `25769803776` (24 GiB) -- **Minimum**: `1` - -**Role**: **High-water mark.** Maximum total stored data per SP (in bytes) before cleanup kicks in. When live storage for a provider exceeds this value, the cleanup job triggers and deletes the oldest pieces until usage drops below `TARGET_DATASET_STORAGE_SIZE_BYTES` (the low-water mark). +**Role**: Maximum runtime for dataset-creation jobs before forced abort. When a dataset-creation job exceeds this timeout, it is actively cancelled using `AbortController`. **When to update**: -- Increase for longer runway before cleanup kicks in (e.g. months vs weeks) -- Decrease if SP storage is constrained or costs are a concern - -**Example**: - -```bash -MAX_DATASET_STORAGE_SIZE_BYTES=12884901888 # 12 GiB per SP -``` - ---- - -### `TARGET_DATASET_STORAGE_SIZE_BYTES` - -- **Type**: `number` (integer, bytes) -- **Required**: No -- **Default**: `21474836480` (20 GiB) -- **Minimum**: `1` - -**Role**: **Low-water mark.** When cleanup triggers (live usage exceeds `MAX_DATASET_STORAGE_SIZE_BYTES`), pieces are deleted until usage drops below this target. The gap between MAX and TARGET creates headroom so cleanup doesn't re-trigger immediately. - -**Headroom math**: At 4 deals/SP/hour × 10 MiB = ~960 MiB/day growth. With 4 GiB headroom (24 GiB MAX − 20 GiB TARGET), cleanup provides ~4 days of breathing room per run, which aligns with the daily default cadence. - -**When to update**: - -- Decrease for more aggressive cleanup (larger gap = more headroom) -- Increase toward MAX for minimal cleanup (smaller gap = less headroom) -- Must be less than `MAX_DATASET_STORAGE_SIZE_BYTES` for cleanup to have effect - -**Example**: - -```bash -TARGET_DATASET_STORAGE_SIZE_BYTES=16106127360 # 15 GiB per SP (9 GiB headroom) -``` +- Increase if dataset creation consistently takes longer than the default (e.g. slow networks or large initial piece uploads). +- Decrease to fail faster on stuck provider interactions. --- -### `JOB_PIECE_CLEANUP_PER_SP_PER_HOUR` +### `IPFS_BLOCK_FETCH_CONCURRENCY` - **Type**: `number` - **Required**: No -- **Default**: `0.0417` (~1/24, approximately once per day) -- **Minimum**: `0.001` -- **Maximum**: `20` - -**Role**: Target number of piece cleanup runs per storage provider per hour. Controls how frequently the cleanup job runs for each SP. The rate is converted to an interval internally (e.g. 1/hr = every 3600s, 1/24/hr ≈ every 86400s = once per day). +- **Default**: `6` +- **Minimum**: `1` +- **Enforced**: Yes (config validation) -Only used when `DEALBOT_JOBS_MODE=pgboss`. +**Role**: Maximum number of parallel block fetches when validating IPFS retrievals via DAG traversal. **When to update**: -- Increase to run cleanup more frequently when SPs are frequently over quota -- Decrease to reduce scheduling overhead - -**Example**: - -```bash -# Once per hour (more aggressive) -JOB_PIECE_CLEANUP_PER_SP_PER_HOUR=1 - -# Once per week (very conservative) -JOB_PIECE_CLEANUP_PER_SP_PER_HOUR=0.006 -``` - ---- - -### `MAX_PIECE_CLEANUP_RUNTIME_SECONDS` - -- **Type**: `number` -- **Required**: No -- **Default**: `300` (5 minutes) -- **Minimum**: `60` - -**Role**: Maximum runtime for a cleanup job before forced abort via `AbortController`. Prevents stuck cleanup jobs from blocking the SP work queue. - -Only used when `DEALBOT_JOBS_MODE=pgboss`. - -**When to update**: +- Increase to speed up validation on fast networks and responsive gateways +- Decrease to reduce pressure on slower storage providers or constrained environments -- Increase if piece deletion calls to the Synapse SDK are known to be slow -- Decrease for faster abort detection on stuck jobs +**Note**: This affects the number of concurrent `/ipfs/` requests per retrieval. --- @@ -1027,43 +1000,6 @@ RANDOM_PIECE_SIZES=1024,10240,102400 --- -## SP Blocklist Configuration - -Both variables are **optional** and default to an empty list (no providers blocked). Values are -comma-separated lists of provider IDs or addresses. Addresses are matched case-insensitively. - -A blocked provider is excluded from **all** scheduled check types: data-storage, retrieval, and -data-retention. Blocking applies to **scheduled automation only** — manual/dev-triggered checks -(via dev-tools endpoints) are not affected. - ---- - -### `BLOCKED_SP_IDS` - -- **Type**: `string` (comma-separated provider IDs) -- **Required**: No -- **Default**: `""` (empty — no providers blocked) - -**Role**: Global blocklist by provider numeric ID. Providers listed here are excluded from **all** scheduled -check types (data-storage, retrieval, and data-retention). - -**Example**: `BLOCKED_SP_IDS=1234,5678` - ---- - -### `BLOCKED_SP_ADDRESSES` - -- **Type**: `string` (comma-separated provider Ethereum addresses) -- **Required**: No -- **Default**: `""` (empty — no providers blocked) - -**Role**: Global blocklist by provider address. Providers listed here are excluded from **all** scheduled -check types (data-storage, retrieval, and data-retention). Matching is case-insensitive. - -**Example**: `BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234...` - ---- - ## Prometheus Metrics Configuration ### `PROMETHEUS_WALLET_BALANCE_TTL_SECONDS` From 83fa9400b691ffa2b7619f1bbdf62a5b2ae2d61b Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Thu, 23 Apr 2026 12:32:39 +0530 Subject: [PATCH 4/5] chore: fix typecheck --- apps/backend/src/jobs/jobs.service.spec.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/backend/src/jobs/jobs.service.spec.ts b/apps/backend/src/jobs/jobs.service.spec.ts index 1ed1bafc..eafbcf4a 100644 --- a/apps/backend/src/jobs/jobs.service.spec.ts +++ b/apps/backend/src/jobs/jobs.service.spec.ts @@ -1437,7 +1437,13 @@ describe("JobsService schedule rows", () => { "calibration", 5, {}, - { providerAddress: "0xaaa", jobId: "job-ds-4", providerId: 1n, providerName: "test-provider" }, + { + network: "calibration", + providerAddress: "0xaaa", + jobId: "job-ds-4", + providerId: 1n, + providerName: "test-provider", + }, controller.signal, ), ).rejects.toThrow("Job timed out"); From 98bcb09adddc74779c807e127960902090164804 Mon Sep 17 00:00:00 2001 From: silent-cipher Date: Thu, 23 Apr 2026 21:24:04 +0530 Subject: [PATCH 5/5] chore: update env example --- apps/backend/.env.example | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/backend/.env.example b/apps/backend/.env.example index 847d39aa..a4dc1b47 100644 --- a/apps/backend/.env.example +++ b/apps/backend/.env.example @@ -80,9 +80,10 @@ CALIBRATION_MIN_NUM_DATASETS_FOR_CHECKS=1 # CALIBRATION_DEALBOT_DATASET_VERSION=dealbot-v1 # Per-network scheduling rates (per hour; fractional values supported) -CALIBRATION_DEALS_PER_SP_PER_HOUR=2 +CALIBRATION_DEALS_PER_SP_PER_HOUR=4 CALIBRATION_DATASET_CREATIONS_PER_SP_PER_HOUR=1 -CALIBRATION_RETRIEVALS_PER_SP_PER_HOUR=1 +CALIBRATION_RETRIEVALS_PER_SP_PER_HOUR=2 +CALIBRATION_PIECE_CLEANUP_PER_SP_PER_HOUR=0.05 # Per-network job intervals (seconds) CALIBRATION_PROVIDERS_REFRESH_INTERVAL_SECONDS=14400 # 4 hours @@ -96,6 +97,11 @@ CALIBRATION_MAINTENANCE_WINDOW_MINUTES=20 # CALIBRATION_BLOCKED_SP_IDS=1234,5678 # CALIBRATION_BLOCKED_SP_ADDRESSES=0xAbCd...,0x1234... +# Per-network piece cleanup configuration +CALIBRATION_MAX_DATASET_STORAGE_SIZE_BYTES=25769803776 +CALIBRATION_TARGET_DATASET_STORAGE_SIZE_BYTES=21474836480 +CALIBRATION_MAX_PIECE_CLEANUP_RUNTIME_SECONDS=3000 + # ----------------------------------------------------------------------------- # Per-network configuration — MAINNET (uncomment when adding to NETWORKS) # -----------------------------------------------------------------------------