diff --git a/packages/agent/src/dkg-agent-base.ts b/packages/agent/src/dkg-agent-base.ts index e65192931..d8bc5cf28 100644 --- a/packages/agent/src/dkg-agent-base.ts +++ b/packages/agent/src/dkg-agent-base.ts @@ -164,6 +164,10 @@ import { resolveCorePublicSyncBatchSize, type CorePublicSyncCoverageStatus, } from './sync/core-public-coverage-scheduler.js'; +import { + SyncCapacityRuntime, + type SyncCapacityStatus, +} from './sync/capacity-runtime.js'; import { bindRandomSampling, type RandomSamplingDisabledReason, type RandomSamplingHandle, type RandomSamplingStatus } from './random-sampling-bind.js'; import { connectToMultiaddr, ensurePeerConnected as ensurePeerConnectedAtom, primeCatchupConnections as primeCatchupConnectionsAtom } from './p2p/peer-connect.js'; import { Messenger, type SloProtocolStats } from './p2p/messenger.js'; @@ -1060,6 +1064,8 @@ export class DKGAgentBase { * capped by this scheduler; Edge nodes never register automatic coverage. */ protected readonly corePublicSyncCoverageScheduler: CorePublicSyncCoverageScheduler; + /** Role-aware requester admission and Core automatic-coverage capacity. */ + protected readonly syncCapacityRuntime: SyncCapacityRuntime; protected started = false; /** * One OT-RFC-64 persistence owner for the inventory lease and every resource @@ -1624,12 +1630,18 @@ export class DKGAgentBase { publicSnapshotStore?: WorkspacePublicSnapshotStore, ) { this.config = config; + const resolvedCoverageBatch = resolveCorePublicSyncBatchSize( + config.syncCorePublicBatchSize, + ); this.corePublicSyncCoverageScheduler = new CorePublicSyncCoverageScheduler( - resolveCorePublicSyncBatchSize(config.syncCorePublicBatchSize), + resolvedCoverageBatch, ); this.wallet = wallet; this.node = node; this.store = store; + this.syncCapacityRuntime = SyncCapacityRuntime.create(config, store, { + resolvedCoverageBatch, + }); this.contextGraphMetaProjection = new ContextGraphMetaProjection(store); this.publisher = publisher; this.queryEngine = queryEngine; @@ -1716,13 +1728,15 @@ export class DKGAgentBase { /** Build one canonical peer-round scope with named snapshot/live phases. */ protected planCorePublicSyncPeerRound(remotePeer: string): PeerSyncScope { const selected = [...(this.config.syncContextGraphs ?? [])]; - const automaticContextGraphIds = (this.config.nodeRole ?? 'edge') === 'core' - ? this.corePublicSyncCoverageScheduler.planAutomaticCoverage( - selected, - this.config.syncContextGraphPriorities, - remotePeer, - ) - : []; + let automaticContextGraphIds: string[] = []; + if ((this.config.nodeRole ?? 'edge') === 'core') { + automaticContextGraphIds = this.corePublicSyncCoverageScheduler + .planAutomaticCoverageWithOptions(selected, { + priorities: this.config.syncContextGraphPriorities, + planningLane: remotePeer, + effectiveBatchSize: this.syncCapacityRuntime.getEffectiveCoverageBatch(), + }); + } const initialDurableContextGraphIds = [...new Set([ ...selected, ...automaticContextGraphIds, @@ -1743,6 +1757,10 @@ export class DKGAgentBase { ); } + getSyncCapacityStatus(): SyncCapacityStatus { + return this.syncCapacityRuntime.getStatus(); + } + /** * Acquire the RFC-64 inventory, finish bounded stale-candidate cleanup, and * open the inherited-owner control-object tree before network consumers. diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 0c2320947..dd67d334c 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -303,11 +303,9 @@ import { type DurableSyncAccumulator, } from './sync/durable-progress.js'; import { - getSyncBackpressureSnapshot, getSyncBackpressureBusyError, resolveBooleanSwitch, resolveNonNegativeIntegerSwitch, - resolveSyncGlobalBackpressure, withGlobalSyncBackpressure, } from './sync/backpressure.js'; import { @@ -1233,9 +1231,10 @@ export class LifecycleSyncMethods extends DKGAgentBase { operationSignal, ); try { + const capacityAdmission = this.syncCapacityRuntime.getAdmissionOptions(); return await withGlobalSyncBackpressure( { - policy: resolveSyncGlobalBackpressure(this.config), + ...capacityAdmission, ctx, label, contextGraphId, @@ -2021,7 +2020,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { ); }, onDecline: (details) => { - const syncPressure = getSyncBackpressureSnapshot(resolveSyncGlobalBackpressure(this.config)); + const syncPressure = this.syncCapacityRuntime.getBackpressureSnapshot(); const syncPressureLabel = `syncGlobalInflight=${syncPressure.inflight} ` + `syncGlobalQueued=${syncPressure.queued} ` + @@ -2544,15 +2543,19 @@ export class LifecycleSyncMethods extends DKGAgentBase { process.env, (message) => this.log.warn(ctx, message), ); - const syncGlobalPolicy = resolveSyncGlobalBackpressure(this.config); + const syncGlobalPolicy = this.syncCapacityRuntime.getResolvedPolicyStatus(); + const syncCapacity = this.syncCapacityRuntime.getStatus(); const configuredPriorityCounts = countSyncPriorityClasses(this.config.syncContextGraphPriorities); this.log.info(ctx, `Resolved sync policy ${JSON.stringify({ snapshotGlobalRows: snapshotPolicy.budget.maxRows, snapshotGlobalBytesEstimate: snapshotPolicy.budget.maxBytesEstimate, snapshotLocalRows: snapshotPolicy.budget.maxSnapshotRows, snapshotLocalBytesEstimate: snapshotPolicy.budget.maxSnapshotBytesEstimate, - syncGlobalInflightLimit: syncGlobalPolicy.limit ?? 0, + syncGlobalInflightLimit: syncGlobalPolicy.inflightLimit ?? 0, syncGlobalQueueLimit: syncGlobalPolicy.queueLimit ?? 0, + syncCapacityMode: syncCapacity.mode, + syncCapacityCurrentInflight: syncCapacity.currentInflight ?? 0, + syncCapacityCoverageBatch: syncCapacity.currentCoverageBatch, configuredPriorities: configuredPriorityCounts, snapshotLocalClamped: snapshotPolicy.localRowsClamped || snapshotPolicy.localBytesEstimateClamped, })}`); @@ -3249,6 +3252,19 @@ export class LifecycleSyncMethods extends DKGAgentBase { this.log.warn(ctx, `Skipping periodic sync reconciler startup (DKG_SYNC_RECONCILER_ENABLED=0)`); } + this.syncCapacityRuntime.startSampling({ + hasSupplementalDemand: () => ( + this.corePublicSyncCoverageScheduler.hasAutomaticCoverageBacklog( + this.config.syncContextGraphs ?? [], + this.syncCapacityRuntime.getEffectiveCoverageBatch(), + ) + ), + onError: (error) => { + const message = error instanceof Error ? error.message : String(error); + this.log.warn(ctx, `Adaptive sync capacity sample failed: ${message}`); + }, + }); + // A.4-lite+: keep a small set of Core nodes warm (connection pinned + // auto-redialed by libp2p) so catch-up / chain reconciliation never pays // a cold circuit-relay dial to reach a Core. Opt-in via diff --git a/packages/agent/src/dkg-agent-types.ts b/packages/agent/src/dkg-agent-types.ts index 92faf5a04..834de6ed1 100644 --- a/packages/agent/src/dkg-agent-types.ts +++ b/packages/agent/src/dkg-agent-types.ts @@ -1160,6 +1160,23 @@ export interface Rfc64PublicCatalogBootstrapConfigV1 { readonly retryIntervalMs?: number; } +/** + * Optional bounds for role-aware adaptive sync concurrency. + * + * These values are operator policy, not instantaneous capacity: + * `minInflight` is the floor the controller may reduce to and `maxInflight` is + * the ceiling it may grow to. Omitting the whole block preserves role-aware + * defaults; `enabled` can explicitly opt in or out without restating bounds. + */ +export interface SyncAdaptiveCapacityConfig { + /** Explicitly enable or disable adaptive sync capacity. Omit for the role-aware default. */ + readonly enabled?: boolean; + /** Minimum effective global sync concurrency the controller may select. */ + readonly minInflight?: number; + /** Maximum effective global sync concurrency the controller may select. */ + readonly maxInflight?: number; +} + export interface DKGAgentConfig { name: string; /** Selected genesis document. Defaults to the compatibility Base testnet genesis. */ @@ -1272,6 +1289,11 @@ export interface DKGAgentConfig { syncGlobalLimit?: number; /** Max sync jobs waiting behind the global cap. Defaults to 2x the inflight cap. */ syncGlobalQueueLimit?: number; + /** + * Optional role-aware adaptive sync-concurrency policy. Omission preserves + * role-aware defaults and is distinct from an explicit disable. + */ + syncAdaptiveCapacity?: SyncAdaptiveCapacityConfig; /** * Maximum automatically discovered public CGs a Core adds to one peer-sync * round. Explicitly selected CGs are not capped. Defaults to 8; 0 disables diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index faf5e20df..89ce17c18 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -361,6 +361,7 @@ import { type DurableSyncResult, type SharedMemorySyncResult, type DKGAgentConfig, + type SyncAdaptiveCapacityConfig, type Rfc64CatalogAccessPolicyAuthorityConfigV1, type DKGAgentACKTransportOptions, type ImportedArtifactByteStore, @@ -473,6 +474,7 @@ export type { SharedMemorySyncDiagnostics, CatchupSyncDiagnostics, DKGAgentConfig, + SyncAdaptiveCapacityConfig, Rfc64CatalogAccessPolicyAuthorityConfigV1, DKGAgentACKTransportOptions, ImportedArtifactByteStore, @@ -1720,6 +1722,7 @@ export class DKGAgent extends DKGAgentBase { async stop(): Promise { if (!this.started) return; + this.syncCapacityRuntime.stopSampling(); if (this.chainPoller) { // Await so any in-flight poll (and its HTTP keep-alive socket) settles // BEFORE we tear down the chain adapter — otherwise the RPC connection diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 7ad98d3f8..07005727c 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -194,6 +194,7 @@ export { InvalidContentError, StaleSenderKeyTargetError, type DKGAgentConfig, + type SyncAdaptiveCapacityConfig, type Rfc64CatalogAccessPolicyAuthorityConfigV1, type DKGAgentACKTransportOptions, type ContextGraphSub, diff --git a/packages/agent/src/sync/backpressure.ts b/packages/agent/src/sync/backpressure.ts index 9186f8a53..2d102fa40 100644 --- a/packages/agent/src/sync/backpressure.ts +++ b/packages/agent/src/sync/backpressure.ts @@ -313,13 +313,16 @@ export function resolveExplicitSyncGlobalLimit( ?? nonNegativeInteger(config.syncGlobalLimit); } -/** Resolve one node-wide policy; the optional live limit is shared by every admission. */ -export function resolveSyncGlobalBackpressure( - config: SyncGlobalBackpressureConfig, +/** Build a policy from a final hard limit without re-reading global-limit env inputs. */ +export function createSyncGlobalBackpressurePolicy( + resolvedLimit: number, + configuredQueueLimit?: number, currentLimit?: SyncBackpressureCurrentLimit, ): SyncGlobalBackpressurePolicy { - const limit = resolveExplicitSyncGlobalLimit(config) - ?? DEFAULT_SYNC_GLOBAL_MAX_INFLIGHT; + const limit = nonNegativeInteger(resolvedLimit); + if (limit === undefined) { + throw new TypeError('resolved sync global limit must be a non-negative integer'); + } if (limit === 0) { return Object.freeze({ limit: undefined, @@ -327,8 +330,10 @@ export function resolveSyncGlobalBackpressure( }) as SyncGlobalBackpressurePolicy; } + // Queue capacity remains an independent operator control. Only the already- + // resolved hard limit is protected from a second global-limit env lookup. const queueLimit = nonNegativeInteger(parseIntegerEnv('DKG_SYNC_GLOBAL_QUEUE_LIMIT')) - ?? nonNegativeInteger(config.syncGlobalQueueLimit) + ?? nonNegativeInteger(configuredQueueLimit) ?? limit * DEFAULT_SYNC_GLOBAL_QUEUE_LIMIT_MULTIPLIER; return Object.freeze({ limit, @@ -337,6 +342,20 @@ export function resolveSyncGlobalBackpressure( }) as SyncGlobalBackpressurePolicy; } +/** Resolve one node-wide policy; the optional live limit is shared by every admission. */ +export function resolveSyncGlobalBackpressure( + config: SyncGlobalBackpressureConfig, + currentLimit?: SyncBackpressureCurrentLimit, +): SyncGlobalBackpressurePolicy { + const limit = resolveExplicitSyncGlobalLimit(config) + ?? DEFAULT_SYNC_GLOBAL_MAX_INFLIGHT; + return createSyncGlobalBackpressurePolicy( + limit, + config.syncGlobalQueueLimit, + currentLimit, + ); +} + export function getSyncBackpressureSnapshot( policy?: SyncGlobalBackpressurePolicy, now = Date.now(), diff --git a/packages/agent/src/sync/capacity-runtime.ts b/packages/agent/src/sync/capacity-runtime.ts new file mode 100644 index 000000000..399a68c1d --- /dev/null +++ b/packages/agent/src/sync/capacity-runtime.ts @@ -0,0 +1,276 @@ +import type { TripleStore } from '@origintrail-official/dkg-storage'; +import type { SyncAdaptiveCapacityConfig } from '../dkg-agent-types.js'; +import { + AdaptiveCapacityController, + MAX_SYNC_ADAPTIVE_INFLIGHT, + resolveAdaptiveCapacityBounds, + type AdaptiveCapacityDecision, + type AdaptiveCapacityState, +} from './adaptive-capacity.js'; +import { + AdaptiveCapacitySampler, + deriveAdaptiveInflightHardMax, + type AdaptiveCapacitySamplerDependencies, +} from './adaptive-capacity-sampler.js'; +import { + createSyncGlobalBackpressurePolicy, + getSyncBackpressureSnapshot, + notifyGlobalSyncBackpressureCapacityChanged, + parseBooleanEnv, + resolveExplicitSyncGlobalLimit, + resolveSyncGlobalBackpressure, + type SyncBackpressureSnapshot, + type SyncGlobalBackpressureConfig, + type SyncGlobalBackpressurePolicy, +} from './backpressure.js'; + +export const DEFAULT_SYNC_CAPACITY_SAMPLE_INTERVAL_MS = 5_000; + +export interface SyncCapacityRuntimeConfig extends SyncGlobalBackpressureConfig { + nodeRole?: 'core' | 'edge'; + syncAdaptiveCapacity?: SyncAdaptiveCapacityConfig; +} + +export interface SyncCapacityStatus { + mode: 'static' | 'adaptive'; + state: AdaptiveCapacityState | 'healthy'; + currentInflight: number | null; + minInflight: number | null; + maxInflight: number | null; + currentCoverageBatch: number; + configuredCoverageBatch: number; + storePressureTelemetryAvailable: boolean; + lastDecision: Pick< + AdaptiveCapacityDecision, + 'action' | 'reason' | 'atMs' + > | null; +} + +export interface SyncCapacityRuntimeOptions { + /** Agent-boundary value shared with the Core coverage scheduler. */ + resolvedCoverageBatch: number; + parallelism?: number; + samplerDependencies?: AdaptiveCapacitySamplerDependencies; + now?: () => number; +} + +export interface SyncCapacitySamplingOptions { + /** Additional Core work that does not currently occupy requester admission. */ + hasSupplementalDemand?: () => boolean; + intervalMs?: number; + onError?: (error: unknown) => void; +} + +export interface SyncCapacityPolicyStatus { + inflightLimit: number | null; + queueLimit: number | null; +} + +function readStorePressure(store: TripleStore) { + try { + return store.getPressureSnapshot?.(); + } catch { + return undefined; + } +} + +function requirePositiveInteger(value: unknown, label: string): number { + const parsed = typeof value === 'string' ? Number(value.trim()) : value; + if (typeof parsed !== 'number' || !Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${label} must be a positive integer`); + } + return parsed; +} + +function resolveAdaptivePositiveInteger( + configValue: number | undefined, + envName: string, + configName: string, +): number | undefined { + const envValue = process.env[envName]; + if (envValue !== undefined) return requirePositiveInteger(envValue, envName); + if (configValue !== undefined) { + return requirePositiveInteger(configValue, `syncAdaptiveCapacity.${configName}`); + } + return undefined; +} + +/** One agent-owned capacity policy; static callers keep the exact old path. */ +export class SyncCapacityRuntime { + private readonly controller?: AdaptiveCapacityController; + private readonly sampler?: AdaptiveCapacitySampler; + private samplingTimer: ReturnType | undefined; + + private constructor( + private readonly policy: SyncGlobalBackpressurePolicy, + private readonly configuredCoverageBatch: number, + controller?: AdaptiveCapacityController, + sampler?: AdaptiveCapacitySampler, + ) { + this.controller = controller; + this.sampler = sampler; + } + + static create( + config: SyncCapacityRuntimeConfig, + store: TripleStore, + options: SyncCapacityRuntimeOptions, + ): SyncCapacityRuntime { + const configuredCoverageBatch = options.resolvedCoverageBatch; + const staticPolicy = resolveSyncGlobalBackpressure(config); + const explicitGlobalLimit = resolveExplicitSyncGlobalLimit(config); + const explicitlyEnabled = parseBooleanEnv('DKG_SYNC_ADAPTIVE_CAPACITY_ENABLED') + ?? config.syncAdaptiveCapacity?.enabled; + const adaptive = (config.nodeRole ?? 'edge') === 'core' + && staticPolicy.limit !== undefined + && (explicitlyEnabled ?? explicitGlobalLimit === undefined); + if (!adaptive) { + return new SyncCapacityRuntime(staticPolicy, configuredCoverageBatch); + } + + const requestedMax = resolveAdaptivePositiveInteger( + config.syncAdaptiveCapacity?.maxInflight, + 'DKG_SYNC_ADAPTIVE_MAX_INFLIGHT', + 'maxInflight', + ); + const minInflight = resolveAdaptivePositiveInteger( + config.syncAdaptiveCapacity?.minInflight, + 'DKG_SYNC_ADAPTIVE_MIN_INFLIGHT', + 'minInflight', + ); + const hardMax = deriveAdaptiveInflightHardMax({ + operatorMax: Math.min( + requestedMax ?? MAX_SYNC_ADAPTIVE_INFLIGHT, + explicitGlobalLimit ?? MAX_SYNC_ADAPTIVE_INFLIGHT, + ), + parallelism: options.parallelism, + storePressure: readStorePressure(store), + }); + const bounds = resolveAdaptiveCapacityBounds({ + initialInflight: Math.min(2, hardMax), + minInflight, + maxInflight: hardMax, + configuredCoverageBatch, + }); + if (bounds.mode === 'unbounded') { + return new SyncCapacityRuntime(staticPolicy, configuredCoverageBatch); + } + + const controller = new AdaptiveCapacityController(bounds, { now: options.now }); + const adaptivePolicy = createSyncGlobalBackpressurePolicy( + // The controller can never exceed hardMax, so use that same ceiling for + // queue sizing and admission observability. Basing the queue on a larger + // explicit operator limit would allow a backlog that the adaptive Core + // can never drain at the advertised policy capacity. + hardMax, + config.syncGlobalQueueLimit, + () => controller.getCurrentInflight(), + ); + return new SyncCapacityRuntime( + adaptivePolicy, + configuredCoverageBatch, + controller, + new AdaptiveCapacitySampler(store, options.samplerDependencies), + ); + } + + isAdaptive(): boolean { + return this.controller !== undefined; + } + + /** Stable admission contract; callers do not need to branch on capacity mode. */ + getAdmissionOptions(): { policy: SyncGlobalBackpressurePolicy } { + return { policy: this.policy }; + } + + /** Runtime-owned requester-pressure view for lifecycle diagnostics and sampling. */ + getBackpressureSnapshot(): SyncBackpressureSnapshot { + return getSyncBackpressureSnapshot(this.policy); + } + + /** Stable resolved ceilings without exposing the branded admission policy. */ + getResolvedPolicyStatus(): SyncCapacityPolicyStatus { + return { + inflightLimit: this.policy.limit ?? null, + queueLimit: this.policy.queueLimit ?? null, + }; + } + + getEffectiveCoverageBatch(): number { + return this.controller?.getEffectiveCoverageBatch() ?? this.configuredCoverageBatch; + } + + sample(demand: boolean): void { + if (!this.controller || !this.sampler) return; + const previousInflight = this.controller.getCurrentInflight(); + this.controller.observe(this.sampler.sample(demand)); + if (this.controller.getCurrentInflight() !== previousInflight) { + notifyGlobalSyncBackpressureCapacityChanged(); + } + } + + /** + * Own the adaptive sampling lifecycle and requester-demand calculation. + * Static runtimes deliberately no-op so lifecycle callers stay mode-agnostic. + */ + startSampling(options: SyncCapacitySamplingOptions = {}): boolean { + if (!this.controller || !this.sampler || this.samplingTimer) return false; + const intervalMs = requirePositiveInteger( + options.intervalMs ?? DEFAULT_SYNC_CAPACITY_SAMPLE_INTERVAL_MS, + 'sync capacity sample interval', + ); + this.samplingTimer = setInterval(() => { + try { + const pressure = this.getBackpressureSnapshot(); + const demand = pressure.inflight > 0 + || pressure.queued > 0 + || (options.hasSupplementalDemand?.() ?? false); + this.sample(demand); + } catch (error) { + options.onError?.(error); + } + }, intervalMs); + this.samplingTimer.unref?.(); + return true; + } + + stopSampling(): boolean { + if (!this.samplingTimer) return false; + clearInterval(this.samplingTimer); + this.samplingTimer = undefined; + return true; + } + + getStatus(): SyncCapacityStatus { + if (!this.controller) { + const currentInflight = this.policy.limit ?? null; + return { + mode: 'static', + state: 'healthy', + currentInflight, + minInflight: currentInflight, + maxInflight: currentInflight, + currentCoverageBatch: this.configuredCoverageBatch, + configuredCoverageBatch: this.configuredCoverageBatch, + storePressureTelemetryAvailable: false, + lastDecision: null, + }; + } + const status = this.controller.getStatus(); + return { + mode: 'adaptive', + state: status.state, + currentInflight: status.currentInflight, + minInflight: status.minInflight, + maxInflight: status.maxInflight, + currentCoverageBatch: status.effectiveCoverageBatch, + configuredCoverageBatch: status.configuredCoverageBatch, + storePressureTelemetryAvailable: status.storePressureTelemetryAvailable, + lastDecision: { + action: status.lastDecision.action, + reason: status.lastDecision.reason, + atMs: status.lastDecision.atMs, + }, + }; + } +} diff --git a/packages/agent/src/sync/core-public-coverage-scheduler.ts b/packages/agent/src/sync/core-public-coverage-scheduler.ts index 2b295f650..12cf1b106 100644 --- a/packages/agent/src/sync/core-public-coverage-scheduler.ts +++ b/packages/agent/src/sync/core-public-coverage-scheduler.ts @@ -135,7 +135,7 @@ export class CorePublicSyncCoverageScheduler { }); } - /** Named adaptive planning boundary; existing static callers keep their exact call shape. */ + /** Canonical named planning boundary; the positional wrapper preserves outside call shapes. */ planAutomaticCoverageWithOptions( selectedContextGraphIds: readonly string[], options: Readonly = {}, @@ -194,6 +194,25 @@ export class CorePublicSyncCoverageScheduler { return scheduledCoverage; } + /** Pure demand check for automatic public coverage beyond the current batch. */ + hasAutomaticCoverageBacklog( + selectedContextGraphIds: readonly string[], + effectiveBatchSize?: number, + ): boolean { + if (this.batchSize === 0) return false; + const batchSize = resolveEffectiveBatchSize(this.batchSize, effectiveBatchSize); + const selected = new Set( + selectedContextGraphIds.map((id) => id.trim()).filter(Boolean), + ); + let candidates = 0; + for (const contextGraphId of this.tracked) { + if (selected.has(contextGraphId)) continue; + candidates += 1; + if (candidates > batchSize) return true; + } + return false; + } + getStatus(enabled: boolean): CorePublicSyncCoverageStatus { return { enabled: enabled && this.batchSize > 0, diff --git a/packages/agent/test/core-public-coverage-scheduler.test.ts b/packages/agent/test/core-public-coverage-scheduler.test.ts index 489fd5407..341872e43 100644 --- a/packages/agent/test/core-public-coverage-scheduler.test.ts +++ b/packages/agent/test/core-public-coverage-scheduler.test.ts @@ -137,6 +137,7 @@ describe('Core public Context Graph coverage scheduler', () => { scheduler.register('public-a'); expect(scheduler.planAutomaticCoverage(['selected'])).toEqual([]); + expect(scheduler.hasAutomaticCoverageBacklog(['selected'], 0)).toBe(false); expect(scheduler.getStatus(true)).toMatchObject({ enabled: false, batchSize: 0, @@ -189,6 +190,17 @@ describe('Core public Context Graph coverage scheduler', () => { expect(third).toEqual(['cg:d']); }); + it('computes automatic coverage demand from current selections without a prior plan', () => { + const scheduler = new CorePublicSyncCoverageScheduler(4); + for (const contextGraphId of ['cg:a', 'cg:b', 'cg:c', 'cg:d', 'cg:e']) { + scheduler.register(contextGraphId); + } + + expect(scheduler.hasAutomaticCoverageBacklog([], 2)).toBe(true); + expect(scheduler.hasAutomaticCoverageBacklog(['cg:a', 'cg:b'], 2)).toBe(true); + expect(scheduler.hasAutomaticCoverageBacklog(['cg:a', 'cg:b'], 3)).toBe(false); + }); + it('rejects invalid live automatic-coverage batches', () => { const scheduler = new CorePublicSyncCoverageScheduler(3); scheduler.register('cg:a'); diff --git a/packages/agent/test/discovery-subscription-boundary.test.ts b/packages/agent/test/discovery-subscription-boundary.test.ts index 08327d858..491c4a686 100644 --- a/packages/agent/test/discovery-subscription-boundary.test.ts +++ b/packages/agent/test/discovery-subscription-boundary.test.ts @@ -19,6 +19,7 @@ import { type ContextGraphSubscriptionRecord, } from '../src/index.js'; import { normalizeLegacyContextGraphSubscriptionInput } from '../src/context-graph-subscription-policy.js'; +import { SyncCapacityRuntime } from '../src/sync/capacity-runtime.js'; describe('Context Graph discovery/subscription boundary', () => { it('normalizes the legacy public subscription shape before it enters live state', () => { @@ -402,15 +403,51 @@ describe('Context Graph discovery/subscription boundary', () => { } }, 30_000); - it('bounds store-discovered public coverage without capping explicit scope', async () => { - const publicIds = ['store-public-a', 'store-public-b', 'store-public-c']; + it('keeps static Core peer-round planning on the configured batch', async () => { const agent = await DKGAgent.create({ - name: 'BoundedCoreStoreDiscovery', + name: 'StaticCoreCoverageBoundary', listenHost: '127.0.0.1', nodeRole: 'core', chainAdapter: new MockChainAdapter(), syncContextGraphs: ['explicit-selection'], syncCorePublicBatchSize: 2, + syncGlobalMaxInflight: 3, + }); + + try { + await agent.start(); + expect(agent.getSyncCapacityStatus()).toMatchObject({ mode: 'static' }); + const scheduler = (agent as any).corePublicSyncCoverageScheduler; + for (const id of ['static-public-a', 'static-public-b', 'static-public-c']) { + scheduler.register(id); + } + + const scope = (agent as any).planCorePublicSyncPeerRound('static-peer'); + expect(scope.initialDurableContextGraphIds[0]).toBe('explicit-selection'); + expect(scope.automaticContextGraphIds).toHaveLength(2); + expect(scope.automaticContextGraphIds.every((id: string) => id.startsWith('static-public-'))) + .toBe(true); + } finally { + await agent.stop().catch(() => {}); + } + }, 30_000); + + it('applies the live adaptive coverage batch without capping explicit scope', async () => { + const publicIds = [ + 'store-public-a', + 'store-public-b', + 'store-public-c', + 'store-public-d', + 'store-public-e', + 'store-public-f', + ]; + const agent = await DKGAgent.create({ + name: 'BoundedCoreStoreDiscovery', + listenHost: '127.0.0.1', + nodeRole: 'core', + chainAdapter: new MockChainAdapter(), + syncContextGraphs: ['explicit-selection'], + syncCorePublicBatchSize: 8, }); try { @@ -436,15 +473,38 @@ describe('Context Graph discovery/subscription boundary', () => { }, ]))); - expect(await agent.discoverContextGraphsFromStore()).toBe(3); + expect(await agent.discoverContextGraphsFromStore()).toBe(6); expect((agent as any).config.syncContextGraphs).toEqual(['explicit-selection']); - expect(agent.getCorePublicSyncCoverageStatus().trackedContextGraphs).toBe(3); + expect(agent.getCorePublicSyncCoverageStatus().trackedContextGraphs).toBe(6); + + let cpuIdle = 0; + let cpuTotal = 0; + const constrainedRuntime = SyncCapacityRuntime.create({ + nodeRole: 'core', + }, agent.store, { + resolvedCoverageBatch: 8, + parallelism: 16, + samplerDependencies: { + readCpuTimes: () => { + cpuIdle += 80; + cpuTotal += 100; + return { idle: cpuIdle, total: cpuTotal }; + }, + readEventLoopUtilization: () => ({ idle: 1, active: 1, utilization: 0.5 }), + eventLoopUtilizationDelta: () => 0.2, + readHeapRatio: () => 0.82, + }, + }); + constrainedRuntime.sample(true); + expect(constrainedRuntime.getStatus().currentCoverageBatch).toBe(4); + (agent as any).syncCapacityRuntime.stopSampling(); + (agent as any).syncCapacityRuntime = constrainedRuntime; const scope = (agent as any).planCorePublicSyncPeerRound('store-peer'); const planned = scope.initialDurableContextGraphIds; expect(planned[0]).toBe('explicit-selection'); - expect(planned.slice(1)).toHaveLength(2); - expect(new Set(planned.slice(1)).size).toBe(2); + expect(planned.slice(1)).toHaveLength(4); + expect(new Set(planned.slice(1)).size).toBe(4); expect(planned.slice(1).every((id: string) => publicIds.includes(id))).toBe(true); } finally { await agent.stop().catch(() => {}); diff --git a/packages/agent/test/rfc64-agent-inventory-lifecycle.test.ts b/packages/agent/test/rfc64-agent-inventory-lifecycle.test.ts index 14035787f..723501009 100644 --- a/packages/agent/test/rfc64-agent-inventory-lifecycle.test.ts +++ b/packages/agent/test/rfc64-agent-inventory-lifecycle.test.ts @@ -10,7 +10,10 @@ import { type SignedControlEnvelopeV1, type UnsignedControlEnvelopeV1, } from '@origintrail-official/dkg-core'; -import { verifyControlEnvelopeIssuerSignatureV1 } from '@origintrail-official/dkg-chain'; +import { + MockChainAdapter, + verifyControlEnvelopeIssuerSignatureV1, +} from '@origintrail-official/dkg-chain'; import { ethers } from 'ethers'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -153,6 +156,7 @@ function minimalStartedAgent( }); Object.assign(agent, { started: true, + syncCapacityRuntime: { stopSampling: vi.fn() }, chainPoller: null, coreHostRecordingsClosed: false, drainCoreHostRecordings: vi.fn(async () => {}), @@ -386,6 +390,46 @@ describe('DKGAgent RFC-64 inventory lifecycle', () => { expect(agent.started).toBe(false); }); + it('starts adaptive-capacity sampling through the real agent lifecycle', async () => { + const agent = await DKGAgent.create({ + name: 'AdaptiveCapacityLifecycle', + listenHost: '127.0.0.1', + listenPort: 0, + chainAdapter: new MockChainAdapter(), + nodeRole: 'core', + syncContextGraphs: ['explicit-cg'], + syncAdaptiveCapacity: { enabled: true }, + }); + const runtime = (agent as any).syncCapacityRuntime; + const scheduler = (agent as any).corePublicSyncCoverageScheduler; + const startSampling = vi.spyOn(runtime, 'startSampling').mockReturnValue(true); + const getEffectiveCoverageBatch = vi.spyOn( + runtime, + 'getEffectiveCoverageBatch', + ).mockReturnValue(4); + const hasAutomaticCoverageBacklog = vi.spyOn( + scheduler, + 'hasAutomaticCoverageBacklog', + ).mockReturnValue(true); + + try { + await agent.start(); + + expect(runtime.isAdaptive()).toBe(true); + expect(startSampling).toHaveBeenCalledOnce(); + const samplingOptions = startSampling.mock.calls[0]?.[0]; + expect(samplingOptions).toBeDefined(); + expect(samplingOptions.hasSupplementalDemand()).toBe(true); + expect(getEffectiveCoverageBatch).toHaveBeenCalledOnce(); + expect(hasAutomaticCoverageBacklog).toHaveBeenCalledWith( + ['explicit-cg'], + 4, + ); + } finally { + await agent.stop().catch(() => {}); + } + }, 30_000); + it.runIf(process.platform !== 'win32')( 'releases inventory ownership when control-store topology is unsafe', async () => { @@ -504,6 +548,7 @@ describe('DKGAgent RFC-64 inventory lifecycle', () => { 'store', ]); expect(inventoryClose).toHaveBeenCalledOnce(); + expect(agent.syncCapacityRuntime.stopSampling).toHaveBeenCalledOnce(); expect(agent.finalizationRuntime.getRecoveryStore()).toBeUndefined(); expect(agent.rfc64PersistenceV1).toBeUndefined(); expect(agent.started).toBe(false); diff --git a/packages/agent/test/sync-adaptive-capacity-public.typecheck.ts b/packages/agent/test/sync-adaptive-capacity-public.typecheck.ts new file mode 100644 index 000000000..5ad1d1a24 --- /dev/null +++ b/packages/agent/test/sync-adaptive-capacity-public.typecheck.ts @@ -0,0 +1,23 @@ +import type { + DKGAgentConfig, + SyncAdaptiveCapacityConfig, +} from '@origintrail-official/dkg-agent'; + +// Keep the nested adaptive-capacity knobs available to package-root consumers. +// Real literals make every public field name load-bearing for this typecheck. +const adaptiveCapacity: SyncAdaptiveCapacityConfig = { + enabled: true, + minInflight: 2, + maxInflight: 8, +}; + +const agentConfig: Pick = { + syncAdaptiveCapacity: adaptiveCapacity, +}; + +// Omission is a distinct, supported configuration state: it preserves the +// role-aware default instead of manufacturing an empty policy object. +const omittedAgentConfig: Pick = {}; + +void agentConfig; +void omittedAgentConfig; diff --git a/packages/agent/test/sync-backpressure.test.ts b/packages/agent/test/sync-backpressure.test.ts index b590c534e..c60676367 100644 --- a/packages/agent/test/sync-backpressure.test.ts +++ b/packages/agent/test/sync-backpressure.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { backpressureRegistry, createOperationContext, @@ -396,10 +396,14 @@ describe('sync global backpressure', () => { // while every real admission reports `durable:unspecified` on // /api/diagnostics/backpressure, which is the attribution issue #2006 had to // reconstruct from daemon logs. + const config = { syncGlobalMaxInflight: 1, syncGlobalQueueLimit: 1 }; const agentLike = { - config: { syncGlobalMaxInflight: 1, syncGlobalQueueLimit: 1 }, + config, node: { stopSignal: undefined }, log: { info: () => {}, warn: () => {}, debug: () => {} }, + syncCapacityRuntime: { + getAdmissionOptions: () => ({ policy: resolveSyncGlobalBackpressure(config) }), + }, }; let releaseWork!: () => void; @@ -434,6 +438,52 @@ describe('sync global backpressure', () => { } }); + it('uses the runtime-owned live admission policy at the lifecycle boundary', async () => { + const config = { syncGlobalMaxInflight: 2, syncGlobalQueueLimit: 2 }; + const runtimePolicy = resolveSyncGlobalBackpressure(config, () => 1); + const getAdmissionOptions = vi.fn(() => ({ policy: runtimePolicy })); + const agentLike = { + config, + node: { stopSignal: undefined }, + log: { info: () => {}, warn: () => {}, debug: () => {} }, + syncCapacityRuntime: { getAdmissionOptions }, + }; + let releaseFirst!: () => void; + const first = LifecycleSyncMethods.prototype.runContextGraphSyncWithBackpressure.call( + agentLike as never, + createOperationContext('sync'), + 'runtime-policy-first', + 'durable' as never, + 'durable:runtime-policy-first', + () => new Promise((resolve) => { releaseFirst = resolve; }), + ); + await tick(); + + let secondRan = false; + const second = LifecycleSyncMethods.prototype.runContextGraphSyncWithBackpressure.call( + agentLike as never, + createOperationContext('sync'), + 'runtime-policy-second', + 'durable' as never, + 'durable:runtime-policy-second', + async () => { secondRan = true; }, + ); + await tick(); + + try { + expect(getAdmissionOptions).toHaveBeenCalledTimes(2); + expect(secondRan).toBe(false); + expect(getSyncBackpressureSnapshot(runtimePolicy)).toMatchObject({ + inflight: 1, + queued: 1, + limit: 1, + }); + } finally { + releaseFirst(); + await Promise.all([first, second]); + } + }); + it('removes CG and peer correlation identifiers from node-wide pressure diagnostics', async () => { const ctx = createOperationContext('sync'); const policy = resolveSyncGlobalBackpressure({ diff --git a/packages/agent/test/sync-capacity-runtime.test.ts b/packages/agent/test/sync-capacity-runtime.test.ts new file mode 100644 index 000000000..1dbd06f8b --- /dev/null +++ b/packages/agent/test/sync-capacity-runtime.test.ts @@ -0,0 +1,388 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createOperationContext } from '@origintrail-official/dkg-core'; +import type { StorePressureSnapshot, TripleStore } from '@origintrail-official/dkg-storage'; +import { + SyncCapacityRuntime, + type SyncCapacityRuntimeOptions, +} from '../src/sync/capacity-runtime.js'; +import { + CorePublicSyncCoverageScheduler, + DEFAULT_CORE_PUBLIC_SYNC_BATCH_SIZE, + resolveCorePublicSyncBatchSize, +} from '../src/sync/core-public-coverage-scheduler.js'; +import { withGlobalSyncBackpressure } from '../src/sync/backpressure.js'; + +function fakeStore(pressure?: StorePressureSnapshot): TripleStore { + return { + getPressureSnapshot: () => pressure, + } as unknown as TripleStore; +} + +function capacityOptions( + overrides: Partial = {}, +): SyncCapacityRuntimeOptions { + return { + resolvedCoverageBatch: DEFAULT_CORE_PUBLIC_SYNC_BATCH_SIZE, + ...overrides, + }; +} + +const STORE_PRESSURE: StorePressureSnapshot = { + ackInflight: 0, + healthInflight: 0, + normalInflight: 0, + backgroundInflight: 0, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 0, + maxConcurrent: 5, + ackReservedSlots: 1, + healthReservedSlots: 1, +}; + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllEnvs(); +}); + +describe('sync capacity runtime resolution', () => { + it('keeps Edge nodes on the exact static policy even if adaptive is requested', () => { + const runtime = SyncCapacityRuntime.create({ + nodeRole: 'edge', + syncGlobalMaxInflight: 4, + syncAdaptiveCapacity: { enabled: true }, + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 16 })); + + expect(runtime.isAdaptive()).toBe(false); + expect(runtime.getResolvedPolicyStatus()).toMatchObject({ inflightLimit: 4 }); + expect(runtime.getAdmissionOptions()).toHaveProperty('policy'); + expect(runtime.getStatus()).toMatchObject({ mode: 'static', currentInflight: 4 }); + }); + + it('defaults Core nodes without an explicit global limit to adaptive mode', () => { + const runtime = SyncCapacityRuntime.create({ + nodeRole: 'core', + }, fakeStore(STORE_PRESSURE), capacityOptions({ + resolvedCoverageBatch: 7, + parallelism: 16, + now: () => 100, + })); + + expect(runtime.isAdaptive()).toBe(true); + expect(runtime.getResolvedPolicyStatus()).toMatchObject({ + inflightLimit: 3, + queueLimit: 6, + }); + expect(runtime.getStatus()).toMatchObject({ + mode: 'adaptive', + currentInflight: 2, + minInflight: 1, + maxInflight: 3, + currentCoverageBatch: 7, + configuredCoverageBatch: 7, + }); + expect(runtime.getAdmissionOptions()).toHaveProperty('policy'); + expect(runtime.getBackpressureSnapshot().limit).toBe(2); + }); + + it('shares one boundary-resolved coverage batch across scheduler and runtime', () => { + const config = { nodeRole: 'core' as const, syncCorePublicBatchSize: 7 }; + vi.stubEnv('DKG_SYNC_CORE_PUBLIC_BATCH_SIZE', '3'); + const resolvedCoverageBatch = resolveCorePublicSyncBatchSize( + config.syncCorePublicBatchSize, + ); + const scheduler = new CorePublicSyncCoverageScheduler(resolvedCoverageBatch); + + // A later environment change would expose any second owner that re-resolved + // the original config instead of consuming the agent-boundary value. + vi.stubEnv('DKG_SYNC_CORE_PUBLIC_BATCH_SIZE', '5'); + const runtime = SyncCapacityRuntime.create( + config, + fakeStore(STORE_PRESSURE), + capacityOptions({ resolvedCoverageBatch }), + ); + + expect(resolveCorePublicSyncBatchSize(config.syncCorePublicBatchSize)).toBe(5); + expect(scheduler.getStatus(true).batchSize).toBe(3); + expect(runtime.getStatus()).toMatchObject({ + configuredCoverageBatch: 3, + currentCoverageBatch: 3, + }); + }); + + it('owns sampling and restores constrained coverage from supplemental Core demand', () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + let heapRatio = 0.82; + let cpuIdle = 0; + let cpuTotal = 0; + const runtime = SyncCapacityRuntime.create({ + nodeRole: 'core', + }, fakeStore(STORE_PRESSURE), capacityOptions({ + parallelism: 16, + samplerDependencies: { + readCpuTimes: () => { + cpuIdle += 80; + cpuTotal += 100; + return { idle: cpuIdle, total: cpuTotal }; + }, + readEventLoopUtilization: () => ({ idle: 1, active: 1, utilization: 0.5 }), + eventLoopUtilizationDelta: () => 0.2, + readHeapRatio: () => heapRatio, + }, + })); + const coverage = new CorePublicSyncCoverageScheduler(8); + for (const contextGraphId of ['cg:a', 'cg:b', 'cg:c', 'cg:d', 'cg:e']) { + coverage.register(contextGraphId); + } + + runtime.sample(true); + expect(runtime.getStatus()).toMatchObject({ + currentInflight: 1, + currentCoverageBatch: 4, + lastDecision: { action: 'halve', reason: 'critical_heap' }, + }); + + heapRatio = 0.3; + expect(runtime.startSampling({ + hasSupplementalDemand: () => coverage.hasAutomaticCoverageBacklog( + [], + runtime.getEffectiveCoverageBatch(), + ), + intervalMs: 5_000, + onError: (error) => { throw error; }, + })).toBe(true); + vi.advanceTimersByTime(30_000); + + expect(runtime.getStatus()).toMatchObject({ + currentInflight: 2, + currentCoverageBatch: 5, + lastDecision: { action: 'increase', reason: 'healthy_hysteresis' }, + }); + expect(runtime.stopSampling()).toBe(true); + expect(runtime.stopSampling()).toBe(false); + }); + + it('pumps queued requester work when a healthy sample raises live capacity', async () => { + let now = 0; + let heapRatio = 0.82; + let cpuIdle = 0; + let cpuTotal = 0; + const runtime = SyncCapacityRuntime.create( + { nodeRole: 'core' }, + fakeStore(STORE_PRESSURE), + capacityOptions({ + parallelism: 16, + now: () => now, + samplerDependencies: { + readCpuTimes: () => { + cpuIdle += 80; + cpuTotal += 100; + return { idle: cpuIdle, total: cpuTotal }; + }, + readEventLoopUtilization: () => ({ idle: 1, active: 1, utilization: 0.5 }), + eventLoopUtilizationDelta: () => 0.2, + readHeapRatio: () => heapRatio, + }, + }), + ); + runtime.sample(true); + expect(runtime.getStatus().currentInflight).toBe(1); + + let releaseFirst!: () => void; + let markFirstStarted!: () => void; + const firstBlock = new Promise((resolve) => { releaseFirst = resolve; }); + const firstStarted = new Promise((resolve) => { markFirstStarted = resolve; }); + const ctx = createOperationContext('test'); + const first = withGlobalSyncBackpressure({ + ...runtime.getAdmissionOptions(), + ctx, + label: 'durable:first', + }, async () => { + markFirstStarted(); + await firstBlock; + }); + await firstStarted; + + let markSecondStarted!: () => void; + const secondStarted = new Promise((resolve) => { markSecondStarted = resolve; }); + let secondRan = false; + const second = withGlobalSyncBackpressure({ + ...runtime.getAdmissionOptions(), + ctx, + label: 'durable:second', + }, async () => { + secondRan = true; + markSecondStarted(); + }); + await Promise.resolve(); + expect(secondRan).toBe(false); + + heapRatio = 0.3; + for (let sample = 1; sample <= 6; sample += 1) { + now = sample * 5_000; + runtime.sample(true); + } + await secondStarted; + expect(runtime.getStatus().currentInflight).toBe(2); + expect(secondRan).toBe(true); + + releaseFirst(); + await Promise.all([first, second]); + }); + + it('keeps an explicit Core limit static unless adaptation is explicitly enabled', () => { + const staticRuntime = SyncCapacityRuntime.create({ + nodeRole: 'core', + syncGlobalMaxInflight: 6, + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 16 })); + const adaptiveRuntime = SyncCapacityRuntime.create({ + nodeRole: 'core', + syncGlobalMaxInflight: 6, + syncAdaptiveCapacity: { enabled: true }, + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 16 })); + + expect(staticRuntime.isAdaptive()).toBe(false); + expect(staticRuntime.getResolvedPolicyStatus().inflightLimit).toBe(6); + expect(adaptiveRuntime.isAdaptive()).toBe(true); + expect(adaptiveRuntime.getResolvedPolicyStatus()).toEqual({ + inflightLimit: 3, + queueLimit: 6, + }); + expect(adaptiveRuntime.getStatus().maxInflight).toBe(3); + }); + + it('sizes the default adaptive queue from the effective hard maximum', () => { + const runtime = SyncCapacityRuntime.create({ + nodeRole: 'core', + syncGlobalMaxInflight: 100, + syncAdaptiveCapacity: { enabled: true }, + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 64 })); + + expect(runtime.getResolvedPolicyStatus()).toEqual({ + inflightLimit: 3, + queueLimit: 6, + }); + expect(runtime.getStatus()).toMatchObject({ + mode: 'adaptive', + currentInflight: 2, + maxInflight: 3, + }); + }); + + it('does not reapply a larger global-limit env after deriving the adaptive hard maximum', () => { + vi.stubEnv('DKG_SYNC_GLOBAL_MAX_INFLIGHT', '100'); + const runtime = SyncCapacityRuntime.create({ + nodeRole: 'core', + syncAdaptiveCapacity: { enabled: true }, + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 64 })); + + expect(runtime.getResolvedPolicyStatus()).toEqual({ + inflightLimit: 3, + queueLimit: 6, + }); + expect(runtime.getStatus()).toMatchObject({ + mode: 'adaptive', + currentInflight: 2, + maxInflight: 3, + }); + }); + + it('honors the config-level adaptive opt-out', () => { + const runtime = SyncCapacityRuntime.create({ + nodeRole: 'core', + syncAdaptiveCapacity: { enabled: false }, + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 16 })); + + expect(runtime.isAdaptive()).toBe(false); + expect(runtime.getStatus()).toMatchObject({ mode: 'static', currentInflight: 2 }); + }); + + it('applies valid config-level adaptive min and max bounds', () => { + const runtime = SyncCapacityRuntime.create({ + nodeRole: 'core', + syncAdaptiveCapacity: { minInflight: 1, maxInflight: 2 }, + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 16 })); + + expect(runtime.isAdaptive()).toBe(true); + expect(runtime.getResolvedPolicyStatus()).toEqual({ + inflightLimit: 2, + queueLimit: 4, + }); + expect(runtime.getStatus()).toMatchObject({ + mode: 'adaptive', + currentInflight: 2, + minInflight: 1, + maxInflight: 2, + }); + }); + + it('uses the legacy environment limit ahead of newer config for adaptive policy and status', () => { + vi.stubEnv('DKG_SYNC_GLOBAL_LIMIT', '1'); + const runtime = SyncCapacityRuntime.create({ + nodeRole: 'core', + syncGlobalMaxInflight: 6, + syncAdaptiveCapacity: { enabled: true }, + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 16 })); + + expect(runtime.isAdaptive()).toBe(true); + expect(runtime.getResolvedPolicyStatus().inflightLimit).toBe(1); + expect(runtime.getStatus()).toMatchObject({ + mode: 'adaptive', + currentInflight: 1, + minInflight: 1, + maxInflight: 1, + }); + }); + + it.each([0, -1, 1.5])('rejects invalid configured adaptive max %s', (maxInflight) => { + expect(() => SyncCapacityRuntime.create({ + nodeRole: 'core', + syncAdaptiveCapacity: { maxInflight }, + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 16 }))).toThrow( + 'syncAdaptiveCapacity.maxInflight must be a positive integer', + ); + }); + + it('rejects an invalid adaptive max environment override instead of dropping a valid config ceiling', () => { + vi.stubEnv('DKG_SYNC_ADAPTIVE_MAX_INFLIGHT', '0'); + expect(() => SyncCapacityRuntime.create({ + nodeRole: 'core', + syncAdaptiveCapacity: { maxInflight: 1 }, + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 16 }))).toThrow( + 'DKG_SYNC_ADAPTIVE_MAX_INFLIGHT must be a positive integer', + ); + }); + + it('rejects an invalid adaptive minimum instead of silently widening the controller bounds', () => { + vi.stubEnv('DKG_SYNC_ADAPTIVE_MIN_INFLIGHT', '1.5'); + expect(() => SyncCapacityRuntime.create({ + nodeRole: 'core', + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 16 }))).toThrow( + 'DKG_SYNC_ADAPTIVE_MIN_INFLIGHT must be a positive integer', + ); + }); + + it('retains explicit zero as unbounded and disables adaptation', () => { + const runtime = SyncCapacityRuntime.create({ + nodeRole: 'core', + syncGlobalMaxInflight: 0, + syncAdaptiveCapacity: { enabled: true }, + }, fakeStore(STORE_PRESSURE), capacityOptions({ parallelism: 16 })); + + expect(runtime.isAdaptive()).toBe(false); + expect(runtime.getResolvedPolicyStatus().inflightLimit).toBeNull(); + expect(runtime.getStatus().currentInflight).toBeNull(); + }); + + it('honors the adaptive environment disable over inferred Core defaults', () => { + vi.stubEnv('DKG_SYNC_ADAPTIVE_CAPACITY_ENABLED', '0'); + const runtime = SyncCapacityRuntime.create( + { nodeRole: 'core' }, + fakeStore(STORE_PRESSURE), + capacityOptions(), + ); + expect(runtime.isAdaptive()).toBe(false); + }); +}); diff --git a/packages/agent/vitest.unit.config.ts b/packages/agent/vitest.unit.config.ts index e1e677ed5..4c2ed0ee4 100644 --- a/packages/agent/vitest.unit.config.ts +++ b/packages/agent/vitest.unit.config.ts @@ -88,6 +88,7 @@ export default defineConfig({ "test/core-public-coverage-scheduler.test.ts", "test/adaptive-capacity.test.ts", "test/adaptive-capacity-sampler.test.ts", + "test/sync-capacity-runtime.test.ts", "test/sync-requester-progress.test.ts", "test/rootless-durable-bounded-progress.test.ts", "test/rootless-durable-skips-legacy-partition.test.ts", diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts index 181762c9c..e4bb2d8ab 100644 --- a/packages/cli/src/config.ts +++ b/packages/cli/src/config.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url'; import yaml from 'js-yaml'; import type { DKGAgentConfig, + SyncAdaptiveCapacityConfig, SyncContextGraphPriorityConfig, SyncResponderSnapshotLimitsConfig, } from '@origintrail-official/dkg-agent'; @@ -628,6 +629,11 @@ export interface DkgConfig { syncGlobalLimit?: number; /** Max sync jobs waiting behind the global cap. Defaults to 2x the inflight cap. */ syncGlobalQueueLimit?: number; + /** + * Optional role-aware adaptive sync-concurrency policy forwarded unchanged + * to the agent. Omit the block to preserve role-aware defaults. + */ + syncAdaptiveCapacity?: SyncAdaptiveCapacityConfig; /** * Public-CG coverage admitted per Core peer-sync round. Explicit Edge/Core * selections are never capped. Default 8; 0 disables automatic Core catch-up. diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index f76f0b108..86446156f 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -1777,6 +1777,9 @@ export async function runDaemonInner( syncGlobalMaxInflight: config.syncGlobalMaxInflight, syncGlobalLimit: config.syncGlobalLimit, syncGlobalQueueLimit: config.syncGlobalQueueLimit, + ...(config.syncAdaptiveCapacity === undefined + ? {} + : { syncAdaptiveCapacity: config.syncAdaptiveCapacity }), syncCorePublicBatchSize: config.syncCorePublicBatchSize, syncResponderSnapshotLimits: config.syncResponderSnapshotLimits, syncContextGraphPriorities: config.syncContextGraphPriorities, diff --git a/packages/cli/src/daemon/routes/status.ts b/packages/cli/src/daemon/routes/status.ts index 3a8264138..2ff56f61d 100644 --- a/packages/cli/src/daemon/routes/status.ts +++ b/packages/cli/src/daemon/routes/status.ts @@ -718,6 +718,7 @@ export async function handleStatusRoutes(ctx: RequestContext): Promise { trackedContextGraphs: 0, planningLanes: 0, }; + const syncCapacity = agent.getSyncCapacityStatus(); return jsonResponse(res, 200, { name: config.name, version: nodeVersion, @@ -783,6 +784,7 @@ export async function handleStatusRoutes(ctx: RequestContext): Promise { diagnosticsAvailable: '/api/diagnostics/backpressure', }, corePublicSyncCoverage, + syncCapacity, connectedPeers: uniquePeers.size, connections: { total: allConns.length, diff --git a/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts b/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts index c6e0f74f8..ba1d99885 100644 --- a/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts +++ b/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts @@ -162,6 +162,23 @@ describe('runDaemonInner wires sync options into DKGAgent.create', () => { expect(createArg.syncGlobalQueueLimit).toBe(0); }); + it('passes the nested adaptive-capacity policy through unchanged', async () => { + const syncAdaptiveCapacity = { + enabled: true, + minInflight: 2, + maxInflight: 7, + } as const; + const createArg = await captureCreateArg({ syncAdaptiveCapacity }); + + expect(createArg.syncAdaptiveCapacity).toBe(syncAdaptiveCapacity); + }); + + it('keeps adaptive-capacity policy omitted when the operator omits it', async () => { + const createArg = await captureCreateArg(); + + expect(createArg).not.toHaveProperty('syncAdaptiveCapacity'); + }); + it('passes the Core public-CG peer-round batch size through unchanged', async () => { const createArg = await captureCreateArg({ syncCorePublicBatchSize: 13 }); diff --git a/packages/cli/test/status-route-rpc.test.ts b/packages/cli/test/status-route-rpc.test.ts index ee4e21bd5..2840dba9e 100644 --- a/packages/cli/test/status-route-rpc.test.ts +++ b/packages/cli/test/status-route-rpc.test.ts @@ -49,6 +49,17 @@ const DISABLED_PUBLISHER_STATE: RequestContext['publisherState'] = { operatorActionRequired: true, }, }; +const STATIC_SYNC_CAPACITY = { + mode: 'static' as const, + state: 'healthy' as const, + currentInflight: 2, + minInflight: 2, + maxInflight: 2, + currentCoverageBatch: 8, + configuredCoverageBatch: 8, + storePressureTelemetryAvailable: false, + lastDecision: null, +}; async function requestStatusWithAgent( agentOverrides: Record, @@ -76,6 +87,7 @@ async function requestStatusWithAgent( getRelayStats: () => null, }, publisher: { getIdentityId: () => 0n }, + getSyncCapacityStatus: () => STATIC_SYNC_CAPACITY, ...agentOverrides, }, nodeVersion: '0.0.0-test', @@ -235,12 +247,29 @@ describe('/api/status Core public synchronization coverage', () => { totalContextGraphs: 10, }, }; + const syncCapacity = { + mode: 'adaptive' as const, + state: 'cooldown' as const, + currentInflight: 3, + minInflight: 1, + maxInflight: 6, + currentCoverageBatch: 4, + configuredCoverageBatch: 8, + storePressureTelemetryAvailable: true, + lastDecision: { + action: 'increase' as const, + reason: 'healthy_hysteresis' as const, + atMs: 123, + }, + }; const response = await requestStatusWithAgent({ getCorePublicSyncCoverageStatus: () => corePublicSyncCoverage, + getSyncCapacityStatus: () => syncCapacity, }); expect(response.status).toBe(200); expect(response.body.corePublicSyncCoverage).toEqual(corePublicSyncCoverage); + expect(response.body.syncCapacity).toEqual(syncCapacity); }); }); @@ -273,6 +302,7 @@ describe('/api/status selected overlay details', () => { getRelayStats: () => null, }, publisher: { getIdentityId: () => 0n }, + getSyncCapacityStatus: () => STATIC_SYNC_CAPACITY, }, nodeVersion: '0.0.0-test', nodeCommit: '', @@ -341,6 +371,7 @@ describe('/api/status selected overlay details', () => { multiaddrs: [], node: { libp2p: { getConnections: () => [] }, getRelayStats: () => null }, publisher: { getIdentityId: () => 0n }, + getSyncCapacityStatus: () => STATIC_SYNC_CAPACITY, }, nodeVersion: '0.0.0-test', nodeCommit: '', diff --git a/packages/cli/test/status-route-store-quads.test.ts b/packages/cli/test/status-route-store-quads.test.ts index 2a07beec1..959692e48 100644 --- a/packages/cli/test/status-route-store-quads.test.ts +++ b/packages/cli/test/status-route-store-quads.test.ts @@ -16,6 +16,17 @@ const DISABLED_PUBLISHER_STATE: RequestContext['publisherState'] = { operatorActionRequired: true, }, }; +const STATIC_SYNC_CAPACITY = { + mode: 'static' as const, + state: 'healthy' as const, + currentInflight: 2, + minInflight: 2, + maxInflight: 2, + currentCoverageBatch: 8, + configuredCoverageBatch: 8, + storePressureTelemetryAvailable: false, + lastDecision: null, +}; interface Deferred { promise: Promise; @@ -65,6 +76,7 @@ async function startStatusServer(query: () => Promise): Promise<{ getRelayStats: () => null, }, publisher: { getIdentityId: () => 0n }, + getSyncCapacityStatus: () => STATIC_SYNC_CAPACITY, }, nodeVersion: '0.0.0-test', nodeCommit: '',