From 010760f9bb103d47fdfe710c1d47a04fe147aa14 Mon Sep 17 00:00:00 2001 From: RunTimeAdmin <248247024+RunTimeAdmin@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:22:09 -0600 Subject: [PATCH 1/6] Cut a stalled request body after 10 s, and hold /attest bodies to 32 KB A client could send all but the last byte of a body and stop. readBody kept every chunk until the body ended, and the server set no timeouts, so Node's five-minute requestTimeout applied: at the write limit of 60 requests a minute, one address could hold about 300 MB of the oracle, and three could pass the container's 768 MiB. - readBody drops a body still arriving after 10 s, with its connection, and the route answers 408. - POST /attest, the public write route, takes at most 32 KB. The largest real body, a payer outcome with a 4,096-byte ERC-1271 signature in hex, is about 10 KB. The loopback routes keep 1 MB. - The server allows 10 s for headers and 15 s for a whole request, as the signer already does. Neither bounds an answer, so a long /evidence is not cut. Co-Authored-By: Claude Opus 5.5 --- oracle/http-helpers.js | 39 +++++++++++++++++++++++++++++++++---- oracle/http-helpers.test.js | 31 +++++++++++++++++++++++++++++ oracle/http.test.js | 9 +++++++++ oracle/index.js | 10 +++++++--- 4 files changed, 82 insertions(+), 7 deletions(-) diff --git a/oracle/http-helpers.js b/oracle/http-helpers.js index d4f0430a..8e7b0267 100644 --- a/oracle/http-helpers.js +++ b/oracle/http-helpers.js @@ -8,6 +8,16 @@ const net = require('net'); const MAX_BODY_SIZE = 1024 * 1024; // 1 MB +// POST /attest is public, so its bodies are held to what a real one needs. The largest is a payer +// outcome: its signature (payer-outcome.js MAX_SIGNATURE_BYTES) in hex plus a few hundred bytes of +// fields. A megabyte let each request a client left unfinished hold a megabyte of this process. +const ATTEST_MAX_BODY_SIZE = 32 * 1024; + +// How long a request has to deliver its body. Node's requestTimeout is five minutes, and a client that +// sent all but the last byte and stopped kept everything it had sent in memory for that long: at the +// write limit of 60 requests a minute, about 300 MB from one address. +const BODY_DEADLINE_MS = 10_000; + // Past this many bytes the connection is cut. Up to it, the rest of an oversize body is read and // thrown away, so a client that is still writing can finish and read the 413. Destroying the // request at the limit left it with a reset and no answer. @@ -24,21 +34,37 @@ function bodyTooLarge() { return Object.assign(new Error('Request body too large'), { code: 'body_too_large' }); } -// What a route answers when readBody rejects: 413 for an oversize body, 400 for anything else. +function bodyTooSlow() { + return Object.assign(new Error('Request body too slow'), { code: 'body_timeout' }); +} + +// What a route answers when readBody rejects: 413 for an oversize body, 408 for one that did not +// arrive in time, 400 for anything else. function bodyErrorStatus(err) { - return err && err.code === 'body_too_large' ? 413 : 400; + if (err && err.code === 'body_too_large') return 413; + if (err && err.code === 'body_timeout') return 408; + return 400; } // req only needs to be an EventEmitter emitting 'data' | 'end' | 'error' and // exposing destroy() — a real http.IncomingMessage satisfies this, and so // does a plain fake in tests. A declared content-length is read from // req.headers when it has them. -async function readBody(req, maxBodySize = MAX_BODY_SIZE) { +async function readBody(req, maxBodySize = MAX_BODY_SIZE, deadlineMs = BODY_DEADLINE_MS) { return new Promise((resolve, reject) => { const chunks = []; let size = 0; let refused = false; const refuse = () => { refused = true; chunks.length = 0; reject(bodyTooLarge()); }; + // A body still arriving at the deadline is dropped, and the connection with it, so what was sent + // is let go. The timer does not hold the process open on its own. + const deadline = setTimeout(() => { + refused = true; + chunks.length = 0; + reject(bodyTooSlow()); + req.destroy(); + }, deadlineMs); + deadline.unref?.(); // A declared length over the limit is refused before any of the body is read. if (Number(req.headers && req.headers['content-length']) > maxBodySize) refuse(); req.on('data', c => { @@ -51,10 +77,14 @@ async function readBody(req, maxBodySize = MAX_BODY_SIZE) { chunks.push(c); }); req.on('end', () => { + clearTimeout(deadline); try { resolve(JSON.parse(Buffer.concat(chunks).toString() || '{}')); } catch { reject(new Error('Invalid JSON')); } }); - req.on('error', reject); + req.on('error', (err) => { + clearTimeout(deadline); + reject(err); + }); }); } @@ -425,6 +455,7 @@ module.exports = { SCOPES, MIN_TOKEN_LENGTH, MAX_BODY_SIZE, + ATTEST_MAX_BODY_SIZE, RATE_WINDOW_MS, RATE_MAX, json, diff --git a/oracle/http-helpers.test.js b/oracle/http-helpers.test.js index f0f1abc3..00234d6d 100644 --- a/oracle/http-helpers.test.js +++ b/oracle/http-helpers.test.js @@ -200,6 +200,37 @@ test('bodyErrorStatus: 413 for an oversized body, 400 for anything else', async assert.equal(bodyErrorStatus(undefined), 400); }); +test('readBody: a body still arriving at the deadline is dropped with its connection, and answered 408', async () => { + // Before the deadline a client could send all but the last byte and stop, and everything it had sent + // stayed in memory until Node's five-minute requestTimeout. + const req = makeFakeRequest(); + let destroyed = false; + req.on('destroyed', () => { destroyed = true; }); + req.headers = { 'content-length': '1000' }; + const promise = readBody(req, 1000, 50); + req.emit('data', Buffer.from('x'.repeat(999))); + + const outcome = await Promise.race([ + promise.then(() => 'resolved', (err) => err), + new Promise((r) => setTimeout(() => r('still waiting'), 1_000)), + ]); + assert.equal(outcome.code, 'body_timeout', `readBody ended as: ${outcome}`); + assert.equal(bodyErrorStatus(outcome), 408); + assert.equal(destroyed, true, 'the connection is cut, which lets go of what it sent'); +}); + +test('readBody: a body that arrives in time is not cut by the deadline', async () => { + const req = makeFakeRequest(); + let destroyed = false; + req.on('destroyed', () => { destroyed = true; }); + const promise = readBody(req, 1000, 50); + req.emit('data', Buffer.from('{"a":1}')); + req.emit('end'); + assert.deepEqual(await promise, { a: 1 }); + await new Promise((r) => setTimeout(r, 100)); + assert.equal(destroyed, false); +}); + test('readBody: chunks are reassembled in order', async () => { const req = makeFakeRequest(); const promise = readBody(req); diff --git a/oracle/http.test.js b/oracle/http.test.js index ae5be11d..38482166 100644 --- a/oracle/http.test.js +++ b/oracle/http.test.js @@ -407,6 +407,15 @@ test('a body over the 1 MB limit is answered 413 on every route that reads one', } }); +test('POST /attest takes at most 32 KB (the loopback routes keep 1 MB, tested below)', async () => { + // /attest is public. At a megabyte, every request a client left unfinished held a megabyte of the + // oracle until Node's five-minute request timeout. + const room = 32 * 1024 - JSON.stringify({ didHash: DID, success: true, attester: 'counteraudit', pad: '' }).length; + const over = await oracle.call('POST', '/attest', { didHash: DID, success: true, attester: 'counteraudit', pad: 'x'.repeat(room + 1) }, AUDITOR); + assert.equal(over.status, 413, over.text.slice(0, 120)); + assert.equal(over.body.error, 'Request body too large'); +}); + test('the limit is 1 MB to the byte', async () => { const unflagged = `0x${'12'.repeat(32)}`; const room = MAX_BODY - JSON.stringify({ didHash: unflagged, pad: '' }).length; diff --git a/oracle/index.js b/oracle/index.js index db55b647..422d241f 100644 --- a/oracle/index.js +++ b/oracle/index.js @@ -31,7 +31,7 @@ const { numberSetting } = require('./config'); const { json, readBody, bodyErrorStatus, readCredentials, identifyCaller, mayAttestUnauthenticated, parseScorePath, parseDidHash, rateLimited, clientKey, adminTokenPolicyError, runningCommit, - parseTrustedProxies, parseRequestTarget, readScopes, mayUse, weakCredentials, MIN_TOKEN_LENGTH, + parseTrustedProxies, parseRequestTarget, readScopes, mayUse, weakCredentials, MIN_TOKEN_LENGTH, ATTEST_MAX_BODY_SIZE, } = require('./http-helpers'); const payments = require('./payments'); const merkle = require('./merkle'); @@ -3300,7 +3300,7 @@ async function handleRequest(req, res) { if (req.method === 'POST' && pathname === '/attest') { if (overLimit(req, res, 'write')) return; try { - const body = await readBody(req); + const body = await readBody(req, ATTEST_MAX_BODY_SIZE); const { success, packetId } = body; // An outcome the payer signed (docs/adr/0006-payer-signed-outcomes.md). The signature is @@ -4080,8 +4080,12 @@ async function handleRequest(req, res) { * restart policy brought the container back, and the compose files re-cloned `main` each * time (they now fetch a pinned commit). The error is logged with the request line quoted, so a hostile target cannot forge * a log line, and the caller is told nothing about it. + * + * Node's defaults gave a client 60 s to send its headers and five minutes to send the whole request, + * and a request still arriving holds what it has sent. Nothing this server accepts needs more than a + * few seconds to send. These bound receiving only: a slow answer, such as a long /evidence, is not cut. */ -const server = http.createServer((req, res) => { +const server = http.createServer({ headersTimeout: 10_000, requestTimeout: 15_000 }, (req, res) => { handleRequest(req, res).catch((err) => { metrics.inc('httpHandlerErrors'); console.error( From a5c8ef3358b69be75bd329845e5037372cf58f66 Mon Sep 17 00:00:00 2001 From: RunTimeAdmin <248247024+RunTimeAdmin@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:38:44 -0600 Subject: [PATCH 2/6] Read payer standings once a day at the snapshot block, share reads in flight, and keep /score from starting long reads Payer standings were read at the latest block and the cache was emptied at every epoch, so every payer was read again every hour. Reads in flight were not shared, and each GET /score read every payer it found missing when it started: measured, 30 requests 300 ms apart for an agent with 300 payers made 4,650 reads in 45 s, against the 6.7 a second the epoch keeps to. /health's epochRunning let the burst be timed to land on the epoch, whose reads then failed and whose agents were skipped. - Standings are read at the snapshot block, the start of the UTC day containing the score's time, as the external factor's reviewers already are, and kept until the snapshot moves (today's and yesterday's). A checker auditing a proposal made the same day reads what its primary read. This changes what a standing is, so both operators deploy it together. - One read per payer at a time, shared by the epoch and every request, and the cache is checked as each chunk is made up. - GET /score reads standings for itself only between epochs, one chunk a request, and no faster than the epoch reads (two chunks of burst, refilled at its pace). Past that it answers 503 standings_pending with Retry-After, and the epoch reads the rest. The two pacing tests now run through an epoch, since /score no longer reads 25 payers in one request. Co-Authored-By: Claude Opus 5.5 --- oracle/chain.js | 17 +++-- oracle/chain.test.js | 27 +++++++ oracle/index.js | 141 +++++++++++++++++++++++++++++++------ oracle/payer-reads.test.js | 67 ++++++++++++++---- 4 files changed, 211 insertions(+), 41 deletions(-) diff --git a/oracle/chain.js b/oracle/chain.js index 6b32bea5..572ccf47 100644 --- a/oracle/chain.js +++ b/oracle/chain.js @@ -452,9 +452,13 @@ async function getReputationData(didHash) { }; } -async function getAgentScore(address) { +/** + * A counterparty's standing, at `blockTag` when given (the scoring snapshot's block) and at the latest + * block otherwise. + */ +async function getAgentScore(address, blockTag) { try { - return await readWithBackoff('payer standing', () => readStanding(address)); + return await readWithBackoff('payer standing', () => readStanding(address, blockTag)); } catch (err) { // A failure of the node says nothing about the counterparty. Read as "no standing" it lowers the // score this oracle proposes, and the checker, whose own reads succeeded, then records a divergence @@ -466,7 +470,8 @@ async function getAgentScore(address) { } } -async function readStanding(address) { +async function readStanding(address, blockTag) { + const at = blockTag === undefined ? [] : [{ blockTag }]; // Was three serial round trips per counterparty. The first is now arithmetic when // the derivation has been verified, so it is two, and those two are genuinely // dependent. At ~124 ms a call on Arc that is a third off every payer lookup, and @@ -475,7 +480,7 @@ async function readStanding(address) { ? didHashOf(address, localDidHash.chainId) : await identityContract.computeDidHash(address); - const id = await identityContract.getIdentity(didHash); + const id = await identityContract.getIdentity(didHash, ...at); if (Number(id.registeredAt) === 0) return 0; if (Number(id.status) === STATUS_SLASHED) return 0; @@ -492,8 +497,8 @@ async function readStanding(address) { // propagate it the same epoch, and a handover restarts maturity, so bought standing // is not instantly spendable through counterparties either. const [rep, matured] = await Promise.all([ - reputationContract.getReputation(didHash), - reputationContract.getTotalScore(didHash), + reputationContract.getReputation(didHash, ...at), + reputationContract.getTotalScore(didHash, ...at), ]); return Math.min(Number(rep.externalScore), Number(matured)); } diff --git a/oracle/chain.test.js b/oracle/chain.test.js index c70da46c..b89b3e0f 100644 --- a/oracle/chain.test.js +++ b/oracle/chain.test.js @@ -867,6 +867,33 @@ test('getAgentScore: capped by the matured total, so linked standing is not inst assert.equal(await chain.getAgentScore('0xPayer'), 4); }); +test('getAgentScore: every read is made at the block it is given, and at the latest without one', async () => { + // The epoch reads standings at the day's snapshot block, where they cannot change, and keeps them for + // the day. A read that went to the latest block instead would be cached as the snapshot's. + const asked = []; + const note = (name, value) => async (...args) => { + const last = args[args.length - 1]; + asked.push([name, last && typeof last === 'object' ? last.blockTag : 'latest']); + return value; + }; + chain.init(CFG, { + provider: makeFakeProvider(1), + identityContract: { + computeDidHash: async () => '0xdeadbeef', + getIdentity: note('getIdentity', { registeredAt: 1n, status: 0 }), + }, + reputationContract: { + getReputation: note('getReputation', { externalScore: 20n }), + getTotalScore: note('getTotalScore', 60n), + }, + }); + assert.equal(await chain.getAgentScore('0xPayer', 4242), 20); + assert.deepEqual(asked.sort(), [['getIdentity', 4242], ['getReputation', 4242], ['getTotalScore', 4242]]); + asked.length = 0; + await chain.getAgentScore('0xPayer'); + assert.deepEqual(asked.sort(), [['getIdentity', 'latest'], ['getReputation', 'latest'], ['getTotalScore', 'latest']]); +}); + test('getAgentScore: an unregistered counterparty has no standing', async () => { chain.init(CFG, { provider: makeFakeProvider(1), ...makeStandingContracts({ registeredAt: 0, externalScore: 25, total: 90 }) }); assert.equal(await chain.getAgentScore('0xStranger'), 0); diff --git a/oracle/index.js b/oracle/index.js index 422d241f..0f6c336b 100644 --- a/oracle/index.js +++ b/oracle/index.js @@ -685,10 +685,59 @@ async function rereadCredits(agentInfos, deadline) { if (updates.size) persistState(); } -// Counterparty standing, for the web of trust. Resolved once per epoch per address: -// an agent's payers repeat across epochs and within one, and each lookup is three -// chain reads. Cleared at the start of every epoch so standings cannot go stale. -let payerScoreCache = new Map(); +// Counterparty standing, for the web of trust: each payer's standing as of the snapshot, the start of the +// UTC day containing the score's time (reviewers.snapshotOf), read at the snapshot's block as the external +// factor's reviewers are. A standing read there cannot change, so it is kept until the snapshot moves, and +// a checker auditing a proposal made the same day reads exactly what its primary read. Each lookup is two +// or three chain reads. Standings used to be read at the latest block and cleared at every epoch, so every +// payer was read again every hour, and 20,000 one-time payers added most of an hour to every epoch. +const standingsBySnapshot = new Map(); // snapshot block => Map(lowercased payer => standing) +// Today's, and yesterday's for a checker auditing a proposal made just before midnight. +const STANDING_SNAPSHOTS_KEPT = 2; +// Reads in flight, by snapshot block and payer, so the epoch and every concurrent GET /score share one +// read per payer. Each caller used to read every payer missing when it started, so 30 requests for an +// agent with 300 payers made 4,650 reads in 45 s, against the 6.7 a second the epoch keeps to. +const standingReads = new Map(); + +function standingsAt(block) { + let standings = standingsBySnapshot.get(block); + if (!standings) { + standings = new Map(); + standingsBySnapshot.set(block, standings); + while (standingsBySnapshot.size > STANDING_SNAPSHOTS_KEPT) { + standingsBySnapshot.delete(Math.min(...standingsBySnapshot.keys())); + } + } + return standings; +} + +async function standingsBlockFor(nowMs) { + return chain.blockAtOrBefore(reviewers.snapshotOf(Math.floor(nowMs / 1000))); +} + +function standingOf(block, key, address) { + const standings = standingsAt(block); + if (standings.has(key)) return Promise.resolve(standings.get(key)); + const id = `${block}|${key}`; + let read = standingReads.get(id); + if (!read) { + read = chain.getAgentScore(address, block) + .then((standing) => { standings.set(key, standing); return standing; }) + .finally(() => standingReads.delete(id)); + standingReads.set(id, read); + } + return read; +} + +/** An agent's payers, each once: lowercased key => the address as recorded. */ +function payersOf(events) { + const unique = new Map(); + for (const e of events) { + const key = String(e.payer || '').toLowerCase(); + if (!unique.has(key)) unique.set(key, e.payer); + } + return unique; +} // A checker's memory of empty slots (decideCheckerAction): when it first found each agent's // slot empty, in chain seconds, and which agents it is covering because it finalized their @@ -716,16 +765,14 @@ const checkerSlots = { const PAYER_READ_CHUNK = 10; const PAYER_READ_PACE_MS = 1_500; -async function payerScoresFor(didHash, events = getPaymentEvents(didHash)) { +async function payerScoresFor(didHash, events, nowMs) { if (!payments.required(paymentCfg)) return null; // Unique payers first: an address recurs across an agent's events, and this way it is // neither looked up nor awaited twice. - const unique = new Map(); // lowercased key => the address as recorded - for (const e of events) { - const key = String(e.payer || '').toLowerCase(); - if (!unique.has(key)) unique.set(key, e.payer); - } + const unique = payersOf(events); + const block = await standingsBlockFor(nowMs); + const standings = standingsAt(block); // Resolve the cache misses together, a chunk at a time, rather than one at a time. // @@ -739,21 +786,65 @@ async function payerScoresFor(didHash, events = getPaymentEvents(didHash)) { // read is kept either way, so the next try asks only for what is missing. The rejection reaches the // agent's own handler, which skips the agent for this epoch, so the rest are not asked for: that // would only press a node that is already failing. - const misses = [...unique].filter(([key]) => !payerScoreCache.has(key)); - for (let i = 0; i < misses.length; i += PAYER_READ_CHUNK) { - if (i > 0) await new Promise((r) => setTimeout(r, PAYER_READ_PACE_MS)); - const chunk = misses.slice(i, i + PAYER_READ_CHUNK); - const settled = await Promise.allSettled(chunk.map(([, addr]) => chain.getAgentScore(addr))); - chunk.forEach(([key], j) => { if (settled[j].status === 'fulfilled') payerScoreCache.set(key, settled[j].value); }); + // + // The cache is checked as each chunk is made up, not once before the first, so a payer another caller + // has read in the meantime is not read again; a read another caller has in flight is shared. + const out = {}; + const all = [...unique]; + let first = true; + for (let i = 0; i < all.length;) { + const chunk = []; + for (; i < all.length && chunk.length < PAYER_READ_CHUNK; i += 1) { + const [key] = all[i]; + if (standings.has(key)) out[key] = standings.get(key); + else chunk.push(all[i]); + } + if (!chunk.length) break; + if (!first) await new Promise((r) => setTimeout(r, PAYER_READ_PACE_MS)); + first = false; + const settled = await Promise.allSettled(chunk.map(([key, addr]) => standingOf(block, key, addr))); + chunk.forEach(([key], j) => { if (settled[j].status === 'fulfilled') out[key] = settled[j].value; }); const failed = settled.find((s) => s.status === 'rejected'); if (failed) throw failed.reason; } - - const out = {}; - for (const [key] of unique) out[key] = payerScoreCache.get(key); return out; } +// GET /score reads payers' standings for itself only between epochs, at most a chunk a request, and all +// requests together no faster than the epoch reads (a bucket of two chunks, refilled at the epoch's +// pace). A request that finds more than a chunk unread reads one chunk and asks to be tried again, so +// each request moves the reads on and no burst of requests outruns the epoch. Requests used to read +// their whole list each, and could be timed, by /health's epochRunning, to land on the epoch, whose +// reads then failed and whose agents were skipped. +const PREVIEW_READ_BURST = 2 * PAYER_READ_CHUNK; +let previewReads = { tokens: PREVIEW_READ_BURST, at: 0 }; + +function takePreviewReads(n) { + const now = Date.now(); + const refilled = previewReads.tokens + ((now - previewReads.at) * PAYER_READ_CHUNK) / PAYER_READ_PACE_MS; + const tokens = Math.min(PREVIEW_READ_BURST, refilled); + if (tokens < n) { + previewReads = { tokens, at: now }; + return false; + } + previewReads = { tokens: tokens - n, at: now }; + return true; +} + +async function previewStandingsReady(events, nowMs) { + if (!payments.required(paymentCfg)) return { ready: true }; + const block = await standingsBlockFor(nowMs); + const standings = standingsAt(block); + const unread = [...payersOf(events)].filter(([key]) => !standings.has(key) && !standingReads.has(`${block}|${key}`)); + if (!unread.length) return { ready: true }; + if (epochRunning) return { ready: false, retryAfter: 60 }; + const chunk = unread.slice(0, PAYER_READ_CHUNK); + if (!takePreviewReads(chunk.length)) return { ready: false, retryAfter: 2 }; + if (unread.length === chunk.length) return { ready: true }; // scoreAt reads them, as one chunk + await Promise.allSettled(chunk.map(([key, addr]) => standingOf(block, key, addr))); + return { ready: false, retryAfter: 2 }; +} + // An agent's external factor, kept per snapshot. Read at one block with the same evidence it cannot // change, so the epoch, the checker's audit and every GET /score can share one reading, and feedback // posted later waits for the next snapshot as it always did. A registry on another chain is read at @@ -913,7 +1004,7 @@ async function scoreAt(didHash, { registeredAt, att, flagWeight, externalScore } // The evidence this score reads, once: the payers looked up are the ones it scores. By default // what counted by the cut-off for a score made now (payment-chains.cutFor), as the epoch uses. const held = events ?? evidenceAt(didHash, paymentChains.cutFor(nowMs, paymentChainsCfg)); - const measured = measuredFactorsFor(didHash, nowMs, await payerScoresFor(didHash, held), held); + const measured = measuredFactorsFor(didHash, nowMs, await payerScoresFor(didHash, held, nowMs), held); const scores = computeScore({ registeredAt, attestations: measured.measuredAttestations ?? att, @@ -1389,7 +1480,6 @@ async function evidenceCutFor() { async function runEpochInner() { const start = Date.now(); - payerScoreCache = new Map(); // This epoch's own, so one that stops early does not leave the last one's showing. lastEpochHold = null; console.log(`[oracle] epoch start — ${new Date().toISOString()}`); @@ -4022,9 +4112,16 @@ async function handleRequest(req, res) { // // One extra chain read per request, on an endpoint that already makes several. const scoringNowMs = (await chain.getLatestBlockTimestamp()) * 1000; + const held = evidenceAt(didHash, paymentChains.cutFor(scoringNowMs, paymentChainsCfg)); + const standingsReady = await previewStandingsReady(held, scoringNowMs); + if (!standingsReady.ready) { + metrics.inc('scoreStandingsPending'); + res.setHeader('Retry-After', String(standingsReady.retryAfter)); + return json(res, 503, { error: 'payer standings have not been read yet', code: 'standings_pending' }); + } const { score: externalScore, agentId: linkedId } = await externalScoreOf(didHash, { operator, agentAddress }, scoringNowMs); - const { measured, scores } = await scoreAt(didHash, { registeredAt, att, flagWeight, externalScore }, scoringNowMs); + const { measured, scores } = await scoreAt(didHash, { registeredAt, att, flagWeight, externalScore }, scoringNowMs, held); return json(res, 200, { didHash, status, diff --git a/oracle/payer-reads.test.js b/oracle/payer-reads.test.js index 1bd084af..339bc536 100644 --- a/oracle/payer-reads.test.js +++ b/oracle/payer-reads.test.js @@ -68,14 +68,15 @@ test('an agent whose payer the node could not read is skipped for the epoch, and // ---- how fast the standing of many payers is asked for ------------------------------------ -/** An agent paid by `n` different wallets in one transaction, each leg a payment, on a running oracle. */ +/** A scored agent paid by `n` different wallets in one transaction, each leg a payment, on a running oracle. */ async function paidByMany(n) { const world = new World(); - const agent = world.addAgent(); + const agent = world.addRegisteredAgent(); const payers = Array.from({ length: n }, () => wallet()); const tx = world.addTx(payers.map((from) => ({ from, to: agent.agentAddress, value: 5n * USDC }))); world.set('standingReadMs', 20); oracle = await startOracle({ world, env: ENV }); + await waitForEpochs(1); const r = await oracle.call('POST', '/attest', { didHash: agent.didHash, success: true, payment: { txHash: tx } }, ADMIN); assert.equal(r.status, 200, r.text); return { world, agent, payers }; @@ -85,10 +86,9 @@ const standingReads = () => oracle.calls().filter((c) => c.name === 'standingRea test('payers are asked about a few at a time: a node takes only so many reads in one batch', async () => { // All at once, a same-tick burst of more than about 80 went past what the backoff could catch up with // on Arc's public RPC, and the agent went unscored for having customers. - const { agent } = await paidByMany(25); + await paidByMany(25); const started = Date.now(); - const r = await oracle.call('GET', `/score/${agent.didHash}`); - assert.equal(r.status, 200, r.text); + await runEpoch(2); const reads = standingReads(); assert.equal(reads.length, 25, 'each of them, once'); assert.ok(Math.max(...reads) <= 10, `at most ten at once, not ${Math.max(...reads)}`); @@ -98,25 +98,66 @@ test('payers are asked about a few at a time: a node takes only so many reads in }); test('a node that cannot answer is not asked for the rest, and what it did answer is kept', async () => { - const { world, agent, payers } = await paidByMany(25); + const { world, payers } = await paidByMany(25); const failing = (list) => Object.fromEntries(list.map((p) => [p.toLowerCase(), true])); - // None of them can be read: it stops after the first chunk. + // None of them can be read: it stops after the first chunk, and nothing is proposed on it. world.set('standingFails', failing(payers)); - const none = await oracle.call('GET', `/score/${agent.didHash}`); - assert.equal(none.status, 502, none.text); - assert.equal(none.body.code, 'rpc_error'); + await runEpoch(2); assert.equal(standingReads().length, 10); + assert.equal(proposals(), 1, 'only the first epoch\'s, made before the payments'); // One of them cannot: evidence is not held in the order paid, so which chunk it falls in varies. world.set('standingFails', failing([payers[0]])); - assert.equal((await oracle.call('GET', `/score/${agent.didHash}`)).status, 502); + await runEpoch(3); const before = standingReads().length - 10; assert.ok(before % 10 === 0 || before === 25, `stopped at the end of a chunk, after ${before}`); // Now it can: everything read so far is kept, so only that one and those not reached are asked for. world.set('standingFails', {}); - const ok = await oracle.call('GET', `/score/${agent.didHash}`); - assert.equal(ok.status, 200, ok.text); + await runEpoch(4); assert.equal(standingReads().length - 10 - before, 25 - (before - 1)); + assert.equal(proposals(), 2, 'and the agent is scored'); +}); + +test('a payer\'s standing is read once a snapshot, not again every epoch', async () => { + // Standings were read at the latest block and cleared at every epoch, so every payer was read again + // every hour: 20,000 one-time payers added most of an hour to every epoch. They are read at the day's + // snapshot block now, where they cannot change. + const { world } = await paidByMany(5); + await runEpoch(2); + assert.equal(standingReads().length, 5); + await runEpoch(3); + assert.equal(standingReads().length, 5, 'the next epoch of the same day reads none of them again'); + world.set('snapshotBlock', 1_000_001); // the next UTC day's snapshot + await runEpoch(4); + assert.equal(standingReads().length, 10, 'a new snapshot reads them all again'); +}); + +// ---- GET /score's own reads ------------------------------------------------------------------- + +test('requests for the same agent share each payer\'s read, instead of each reading all of them', async () => { + // Each request used to read every payer missing when it started: 30 requests for an agent with 300 + // payers made 4,650 reads in 45 s, against the 6.7 a second the epoch keeps to. + const { agent } = await paidByMany(8); + const answers = await Promise.all(Array.from({ length: 30 }, () => oracle.call('GET', `/score/${agent.didHash}`))); + for (const a of answers) { + assert.ok(a.status === 200 || (a.status === 503 && a.body.code === 'standings_pending'), a.text); + } + assert.ok(answers.some((a) => a.status === 200), 'and at least one is answered'); + assert.equal(standingReads().length, 8, 'each payer read once'); +}); + +test('/score reads at most a chunk of standings itself, and leaves the rest to the epoch', async () => { + const { agent } = await paidByMany(25); + const first = await oracle.call('GET', `/score/${agent.didHash}`); + assert.equal(first.status, 503, first.text); + assert.equal(first.body.code, 'standings_pending'); + assert.equal(standingReads().length, 10, 'one chunk, not all 25'); + + await runEpoch(2); + assert.equal(standingReads().length, 25, 'the epoch read the other 15, and kept the first 10'); + const after = await oracle.call('GET', `/score/${agent.didHash}`); + assert.equal(after.status, 200, after.text); + assert.equal(standingReads().length, 25, 'and the preview then needed no read of its own'); }); From f741a1caaacd7ef6e5ad4eb6a895923a7670141d Mon Sep 17 00:00:00 2001 From: RunTimeAdmin <248247024+RunTimeAdmin@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:43:18 -0600 Subject: [PATCH 3/6] Confirm a payment chain's payments eight blocks at a time The payment-chain scan confirmed each credited payment with two round trips, one after the other: the recipient's code at that block, then the block's time. A step of 1,000 reads took about 150 s at Base Sepolia's 145 ms, so payments in more blocks an hour than the 10-minute budget could confirm (roughly 1,000 to 3,000 of Base Sepolia's 18,000) put the scan behind, and a chain behind the epoch's cut-off holds every agent (ADR 0009). Paying your own agent the 0.01 USDC minimum gets the money back, so on Base Sepolia that cost only free gas. Each block's time and its recipients' code now go out together, eight blocks at a time, which ethers sends as one JSON-RPC batch, paced as the log reads are. The endpoint the operators use (publicnode) answered batches of 100 such reads in 140 to 330 ms on 15 Nov 2026; eight blocks ask about sixteen. A block's time is also read when all its payments turn out to be to a contract, since it rides in the same batch. What is credited, and when, is unchanged. This raises the rate needed to trip the hold several times over; the rule that a lagging chain holds everyone stays. Co-Authored-By: Claude Opus 5.5 --- oracle/payment-chains.js | 57 ++++++++++++++++++++++++----------- oracle/payment-chains.test.js | 31 +++++++++++++++++++ 2 files changed, 71 insertions(+), 17 deletions(-) diff --git a/oracle/payment-chains.js b/oracle/payment-chains.js index 6ad282a4..d120c615 100644 --- a/oracle/payment-chains.js +++ b/oracle/payment-chains.js @@ -64,9 +64,16 @@ const DEFAULT_PACE_MS = 500; // advances a bounded step per epoch instead of retrying an ever larger range after one failed call. const DEFAULT_MAX_BLOCKS = 36_000; // Reads one scan spends on its payments at most: a getCode for each agent and block, and a getBlock for each -// block, one at a time. A chain that is on is scanned inside the epoch, so without this a step of many -// payments could hold an epoch for hours with nothing saved. +// block. A chain that is on is scanned inside the epoch, so without this a step of many payments could hold +// an epoch for hours with nothing saved. const DEFAULT_MAX_READS = 1_000; +// Blocks whose reads go out together: each block's time and its recipients' code, asked in one tick, which +// ethers sends as one JSON-RPC batch, paced as the log reads are. They went one round trip at a time, so a +// step of 1,000 reads took about 150 s at Base Sepolia's 145 ms, and payments in more blocks an hour than +// the 10-minute budget could confirm put the scan behind, which holds every agent (ADR 0009). The endpoint +// the operators use answered batches of 100 such reads on 15 Nov 2026; eight blocks ask about sixteen. +const BLOCKS_PER_READ_BATCH = 8; +const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); // How long one read of a payment chain may take. A chain that is on is scanned inside the epoch, before // scoring, and each read is tried four times, so ethers' default of 5 minutes would let one hung // endpoint hold an epoch for 20 minutes. At 30 seconds it costs about 2. @@ -599,28 +606,44 @@ async function scanChain(chain, deps) { } return scoredCode.get(a.didHash); }; + + // With the code, the block's own time, as on the scored chain: crediting with the clock would make an + // old payment look fresh and defeat the decay. Both are asked for BLOCKS_PER_READ_BATCH blocks at a + // time. A block's time is read even when every payment in it turns out to be to a contract: it rides in + // the same batch, so it costs no round trip. + const byBlock = new Map(); for (const c of plan.credits) { - const a = byDid.get(c.didHash); - if (!a?.agentAddress || codeOk.has(codeKey(c))) continue; - const [here, scored] = await Promise.all([ - read('payment chain getCode', () => provider.getCode(a.agentAddress, c.blockNumber)), - scoredCodeOf(a), - ]); - codeOk.set(codeKey(c), recipientCodeOk(here) && recipientCodeOk(scored)); + if (!byBlock.has(c.blockNumber)) byBlock.set(c.blockNumber, []); + byBlock.get(c.blockNumber).push(c); + } + const blocks = [...byBlock.keys()].sort((x, y) => x - y); + const times = new Map(); + const pause = deps.sleep ?? wait; + for (let i = 0; i < blocks.length; i += BLOCKS_PER_READ_BATCH) { + if (i > 0) await pause(chain.paceMs); + await Promise.all(blocks.slice(i, i + BLOCKS_PER_READ_BATCH).map(async (n) => { + const asking = new Map(); // codeKey => agent, once each + for (const c of byBlock.get(n)) { + const a = byDid.get(c.didHash); + if (a?.agentAddress) asking.set(codeKey(c), a); + } + const keys = [...asking.keys()]; + const [block, ...codes] = await Promise.all([ + read('payment chain getBlock', () => provider.getBlock(n)), + ...keys.map((k) => Promise.all([ + read('payment chain getCode', () => provider.getCode(asking.get(k).agentAddress, n)), + scoredCodeOf(asking.get(k)), + ])), + ]); + if (block) times.set(n, Number(block.timestamp) * 1000); + keys.forEach((k, j) => codeOk.set(k, recipientCodeOk(codes[j][0]) && recipientCodeOk(codes[j][1]))); + })); } const kept = []; for (const c of plan.credits) { if (codeOk.get(codeKey(c)) === false) skip(c, 'contract_recipient'); else kept.push(c); } - - // The block's own time, as on the scored chain: crediting with the clock would make an old - // payment look fresh and defeat the decay. - const times = new Map(); - for (const n of new Set(kept.map((c) => c.blockNumber))) { - const block = await read('payment chain getBlock', () => provider.getBlock(n)); - if (block) times.set(n, Number(block.timestamp) * 1000); - } const registered = new Map(); const named = new Map(); for (const did of new Set(kept.map((c) => c.didHash))) { diff --git a/oracle/payment-chains.test.js b/oracle/payment-chains.test.js index 835d1266..434212f9 100644 --- a/oracle/payment-chains.test.js +++ b/oracle/payment-chains.test.js @@ -613,6 +613,37 @@ test('a scan of many payments stops at MAX_READS, saves its place, and the next assert.throws(() => config({ [`PAYMENT_CHAIN_${BASE_SEPOLIA}_MAX_READS`]: '0' }), /MAX_READS/); }); +test('a scan asks for its payments\' block times and code eight blocks at a time, not a round trip each', async () => { + // One at a time, a step of 1,000 reads took about 150 s at Base Sepolia's 145 ms, so payments in more + // blocks an hour than the 10-minute budget could confirm put the scan behind, and that holds every agent. + const a = agent(); + const payer = ethers.Wallet.createRandom().address; + const logs = Array.from({ length: 40 }, (_, i) => ( + transferLog({ from: payer, to: a.agentAddress, amount: 1_000_000n, block: 1100 + i }) + )); + const chain = fakeChain({ logs }); + let inFlight = 0; + let most = 0; + const slow = (fn) => async (...args) => { + inFlight += 1; + most = Math.max(most, inFlight); + await new Promise((r) => setTimeout(r, 5)); + inFlight -= 1; + return fn(...args); + }; + const provider = { + ...chain, + getCode: slow(chain.getCode), + getBlock: async (tag) => (Number.isInteger(tag) ? slow(chain.getBlock)(tag) : chain.getBlock(tag)), + }; + const { deps: d, credited } = deps({ provider, agents: [a] }); + + await pc.scanChain(config().chains[0], d); + assert.equal(credited.length, 40); + assert.ok(credited.every((c) => c.ts === (1_700_000_000 + c.blockNumber) * 1000), 'each at its own block\'s time'); + assert.equal(most, 16, 'eight blocks\' time and code in one go'); +}); + test('a scan keeps a block whole, and moves however many payments one block holds', async () => { const agents = [agent(), agent(), agent(), agent(), agent()]; const payer = ethers.Wallet.createRandom().address; From c16c2f0ab421204f29407782d1bbd73962592279 Mon Sep 17 00:00:00 2001 From: RunTimeAdmin <248247024+RunTimeAdmin@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:45:19 -0600 Subject: [PATCH 4/6] Read the Arc payment scan's block times ten at a time The scored chain's payment scan read each credited block's time one round trip at a time, with no pace. Past Arc's bucket of about 60 requests the reads crawled on backoff, so payments in enough blocks could keep the scan behind, and a scan far enough behind holds every agent, the same way a payment chain's does. The reads now go ten blocks at a time, half of the 20 calls Arc answers in one batch, paced as the scan's log reads are. What is credited, and when, is unchanged. Co-Authored-By: Claude Opus 5.5 --- oracle/index.js | 18 +++++++++++++++--- oracle/scan-checkpoint.test.js | 24 ++++++++++++++++++++++++ oracle/test-support/stub-chain.js | 17 +++++++++++++---- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/oracle/index.js b/oracle/index.js index 0f6c336b..399d58ee 100644 --- a/oracle/index.js +++ b/oracle/index.js @@ -1161,6 +1161,9 @@ async function runScanCanary(byAddress, provider) { } } +// Half of the 20 calls Arc's public RPC answers in one batch (measured 4 Nov 2026), as for payers' standing. +const SCAN_BLOCKS_PER_BATCH = 10; + async function runPaymentScan(agents) { if (!paymentScanCfg.enabled) return null; if (!paymentCfg.asset) { @@ -1226,10 +1229,19 @@ async function runPaymentScan(agents) { // The settlement time is the block's, not now. Crediting a back-scanned payment with // the current clock would make an old settlement look fresh and defeat the decay that // every volume figure depends on. + // + // Asked SCAN_BLOCKS_PER_BATCH blocks at a time, which ethers sends as one batch, paced as the log reads + // are. They went one round trip at a time with no pace, which ran through Arc's bucket of about 60 + // requests and then crawled on backoff, so payments in enough blocks could keep the scan behind, and a + // scan far enough behind holds every agent. const times = new Map(); - for (const n of new Set(credits.map((c) => c.blockNumber))) { - const block = await chain.readWithBackoff('payment-scan getBlock', () => provider.getBlock(n)); - if (block) times.set(n, Number(block.timestamp) * 1000); + const blocks = [...new Set(credits.map((c) => c.blockNumber))]; + for (let i = 0; i < blocks.length; i += SCAN_BLOCKS_PER_BATCH) { + if (i > 0) await new Promise((r) => setTimeout(r, paymentScanCfg.paceMs)); + await Promise.all(blocks.slice(i, i + SCAN_BLOCKS_PER_BATCH).map(async (n) => { + const block = await chain.readWithBackoff('payment-scan getBlock', () => provider.getBlock(n)); + if (block) times.set(n, Number(block.timestamp) * 1000); + })); } let credited = 0; diff --git a/oracle/scan-checkpoint.test.js b/oracle/scan-checkpoint.test.js index 89cae137..e6a4de02 100644 --- a/oracle/scan-checkpoint.test.js +++ b/oracle/scan-checkpoint.test.js @@ -177,3 +177,27 @@ test('an operator handed over past the registry scan\'s reach cannot pay its own assert.match(oracle.logs(), /"self_payment":1/); assert.deepEqual(await evidenceTxs(agent.didHash), [], 'refused as the operator paying its own agent'); }); + +test('the scan reads its payments\' block times ten at a time, not a round trip each', async () => { + // One at a time and unpaced, the reads ran through Arc's bucket of about 60 requests and then crawled on + // backoff, so payments in enough blocks could keep the scan behind, which holds every agent. + const world = new World(); + const agent = world.addRegisteredAgent({ registeredBlock: HEAD - 900 }); + const payer = wallet(); + const log = (block) => ({ + address: ASSET, + topics: [TRANSFER_TOPIC, ethers.zeroPadValue(payer, 32), ethers.zeroPadValue(agent.agentAddress, 32)], + data: ethers.toBeHex(5n * USDC, 32), + blockNumber: block, + transactionHash: ethers.id(`batched-${block}`), + logIndex: 0, + removed: false, + }); + world.set('scanLogs', Array.from({ length: 30 }, (_, i) => log(HEAD - 800 + i))); + world.set('blockReadMs', 5); + oracle = await startOracle({ world, env: ENV }); + await waitForEpochs(1); + assert.match(oracle.logs(), /payment scan \d+-\d+: 30 credited/); + const inFlight = oracle.calls().filter((c) => c.name === 'blockRead').map((c) => c.args[0]); + assert.equal(Math.max(...inFlight), 10, 'ten blocks\' times in one go'); +}); diff --git a/oracle/test-support/stub-chain.js b/oracle/test-support/stub-chain.js index b2a7c30a..7ffd90e8 100644 --- a/oracle/test-support/stub-chain.js +++ b/oracle/test-support/stub-chain.js @@ -126,6 +126,8 @@ const step = (name) => { moved += world().clockSteps?.[name] ?? 0; }; const registryUnreadable = () => Object.assign(new Error('could not read the ERC-8004 registry (8004 owner)'), { code: 'NETWORK_ERROR' }); // Standing reads in flight, recorded when world.standingReadMs gives them time to overlap. let standingReads = 0; +// Numbered block reads in flight, recorded the same way when world.blockReadMs is set. +let blockReads = 0; const hostNow = Date.now; let hostAhead = 0; Date.now = () => hostNow() + hostAhead; @@ -145,13 +147,20 @@ const provider = { // 'finalized' is the head unless world.finalized holds it back (ADR 0012). A numbered block lists its transactions // only where world.blockTxs[n] says what they are: a world holds payments it has no block for, and a block without its // list reads as unsettled rather than as one that lost them. - getBlock: async (n) => (n === 'finalized' - ? { number: world().finalized ?? world().head + moved, timestamp: world().nowSec + moved - 60 } - : { + getBlock: async (n) => { + if (n === 'finalized') return { number: world().finalized ?? world().head + moved, timestamp: world().nowSec + moved - 60 }; + if (world().blockReadMs && Number.isInteger(n)) { + blockReads += 1; + record('blockRead', blockReads); + await new Promise((r) => setTimeout(r, world().blockReadMs)); + blockReads -= 1; + } + return { number: n, timestamp: world().blockTimes?.[n] ?? world().nowSec + moved - 60, ...(world().blockTxs?.[n] ? { transactions: world().blockTxs[n] } : {}), - }), + }; + }, getLogs: async (filter) => { if (filter.fromBlock === filter.toBlock) return world().canaryLogs ?? []; return (world().scanLogs ?? []).filter((l) => l.blockNumber >= filter.fromBlock && l.blockNumber <= filter.toBlock); From b644ddab66aa92ea8f2d7805ce9554b19f3ec391 Mon Sep 17 00:00:00 2001 From: RunTimeAdmin <248247024+RunTimeAdmin@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:55:50 -0600 Subject: [PATCH 5/6] Load the state a piece at a time, write each agent's ledger in slices, and hold signatures to 1,024 bytes load() read the state file into one string and parsed it while the string was alive, so a load peaked at about 2.2 times the file. V8's heap in the 768 MiB container is 432 MB (measured on the primary), and the caps had been sized against the 768. A state at the caps where every payment carries an outcome (182 MB) ended the process inside JSON.parse, and a payer could get there with far fewer payments by padding its outcomes with 4,096-byte ERC-1271 signatures: about 24,000 of them broke a restart. The checker imports outcomes, so both operators failed, an out-of-memory abort is not an exception, so the .bak was never tried, and the restart policy started each into the same failure. - The file is read as bytes, outside V8's heap, and parsed a piece of at most 1 MB at a time (parseJsonBuffer): a container larger than that is walked member by member, and runs of small members are parsed together. Same result as JSON.parse, refuses what it refuses, and the format is unchanged, so an older build reads what this one writes. - Each agent's ledger is written in slices of 256, the same bytes as before. It was one string: 11 MB at the caps, 151 MB with padded outcomes. - A payer outcome's signature is held to 1,024 bytes. The longest held on the primary on 15 Nov 2026 was 640, a smart account's. Both operators change this together, or one refuses imports the other accepted. Measured under a 432 MB heap, synthetic state at the caps: today's mix (157 MB) loads with 207 MB of heap instead of 346 MB, in 2.0 s instead of 1.2 s; every payment with an outcome (182 MB) loads with 220 MB and 458 MB RSS, where it ran out of heap before. Not changed: the caps still count payments, not bytes, so a ledger filled to 150,000 payments that all carry 1,024-byte signatures is still too large for the heap. That takes about 150,000 payments from an ERC-1271 payer and needs caps that weigh bytes, which changes what both operators drop. Co-Authored-By: Claude Opus 5.5 --- oracle/payer-outcome.js | 7 +- oracle/payer-outcome.test.js | 11 +++ oracle/state-serialize.js | 164 ++++++++++++++++++++++++++++++++- oracle/state-serialize.test.js | 84 +++++++++++++++++ oracle/store.js | 6 +- 5 files changed, 263 insertions(+), 9 deletions(-) diff --git a/oracle/payer-outcome.js b/oracle/payer-outcome.js index b2d2bba3..81e41b56 100644 --- a/oracle/payer-outcome.js +++ b/oracle/payer-outcome.js @@ -31,8 +31,11 @@ const MAX_FUTURE_SKEW_SECONDS = 300; // How far signedAt may be after the time of the block it is checked at. A confirmed block is a // little behind the head, and the head a little behind real time; an hour is far more than either. const MAX_CHECK_LAG_SECONDS = 3_600; -// An ERC-1271 signature can be longer than 65 bytes (a multisig's, say), but not unboundedly. -const MAX_SIGNATURE_BYTES = 4096; +// An ERC-1271 signature can be longer than 65 bytes (a multisig's, say), but not unboundedly. The longest held +// on 15 Nov 2026 was 640 bytes, a smart account's. 4,096 let a payer make each payment it holds 15 times the size of +// an ordinary one, and the caps count payments, not bytes: about 24,000 such outcomes took a state past what a +// restart could load. Both operators change this together, or one refuses imports the other accepted. +const MAX_SIGNATURE_BYTES = 1024; const ERC1271_MAGIC = '0x1626ba7e'; const ERC1271 = new ethers.Interface(['function isValidSignature(bytes32 hash, bytes signature) view returns (bytes4)']); // Gas for one isValidSignature call. Enough for a passkey (P-256) account without a precompile, diff --git a/oracle/payer-outcome.test.js b/oracle/payer-outcome.test.js index 716a78b9..eda72d2f 100644 --- a/oracle/payer-outcome.test.js +++ b/oracle/payer-outcome.test.js @@ -1219,3 +1219,14 @@ test('refuses to start taking payer outcomes without the chain id they are signe assert.equal(code, 1); assert.match(log, /PAYER_OUTCOMES=positive needs EXPECTED_CHAIN_ID/); }); + +test('a signature is held to 1,024 bytes: room for a smart account\'s, not for padding', () => { + // 4,096 let a payer make each payment it holds 15 times the size of an ordinary one, and the caps count + // payments, not bytes: about 24,000 such outcomes took a state past what a restart could load. + const base = { payer: `0x${'44'.repeat(20)}`, outcome: payerOutcome.DELIVERED, signedAt: 1_700_000_000, deadline: 1_700_003_600 }; + assert.equal(payerOutcome.parse({ ...base, signature: `0x${'ab'.repeat(640)}` }).ok, true, 'the longest held on 15 Nov'); + assert.equal(payerOutcome.parse({ ...base, signature: `0x${'ab'.repeat(1024)}` }).ok, true); + const over = payerOutcome.parse({ ...base, signature: `0x${'ab'.repeat(1025)}` }); + assert.equal(over.ok, false); + assert.match(over.error, /signature/); +}); diff --git a/oracle/state-serialize.js b/oracle/state-serialize.js index 79d34cd3..ab64576f 100644 --- a/oracle/state-serialize.js +++ b/oracle/state-serialize.js @@ -23,6 +23,9 @@ const isMap = (value) => value !== null && typeof value === 'object' && !Array.i // A top-level array is written in slices this long. usedPaymentTxs is one entry per credited payment, so it is the // longest list in the file. const SLICE = 4096; +// And an entry's list, such as one agent's payments, in slices this long. An agent's whole ledger was one string: +// 11 MB at the caps, and 151 MB when a payer padded its outcomes with the longest signatures it was allowed. +const ENTRY_SLICE = 256; /** * @param {object} state in the order the keys should be written, as JSON.stringify would write them @@ -37,14 +40,24 @@ function* stateChunks(state, { pieceChars = DEFAULT_PIECE_CHARS } = {}) { if (piece.length >= pieceChars) { ready.push(piece); piece = ''; } }; function* drain() { while (ready.length) yield ready.shift(); } - // A map, one entry at a time. + // A map, one entry at a time, and a long list in an entry a slice at a time. function* entries(map) { put('{'); let firstEntry = true; for (const [k, v] of Object.entries(map)) { - const text = JSON.stringify(v); - if (text === undefined) continue; - put((firstEntry ? '' : ',') + JSON.stringify(k) + ':' + text); + if (Array.isArray(v) && v.length > ENTRY_SLICE) { + put((firstEntry ? '' : ',') + JSON.stringify(k) + ':['); + for (let i = 0; i < v.length; i += ENTRY_SLICE) { + // The inside of the list's own brackets, so the slices join with commas into one list. + put((i ? ',' : '') + JSON.stringify(v.slice(i, i + ENTRY_SLICE)).slice(1, -1)); + yield* drain(); + } + put(']'); + } else { + const text = JSON.stringify(v); + if (text === undefined) continue; + put((firstEntry ? '' : ',') + JSON.stringify(k) + ':' + text); + } firstEntry = false; yield* drain(); } @@ -94,4 +107,145 @@ function* stateChunks(state, { pieceChars = DEFAULT_PIECE_CHARS } = {}) { yield* drain(); } -module.exports = { stateChunks, DEFAULT_PIECE_CHARS }; +// ---- Reading it back --------------------------------------------------------------------------------------- +// +// load() used to read the file into one string and parse it while the string was alive, so a load peaked at about +// 2.2 times the file: at the caps, 351 MB of the 432 MB heap V8 allows in the 768 MiB container, and a file of +// about 165 MB, or one where a payer padded its outcomes, ended the process inside JSON.parse. An out-of-memory +// abort is not an exception, so the .bak was never tried, and the restart policy started it into the same failure. +// +// parseJsonBuffer gives what JSON.parse gives for the file's text, but reads the file's bytes, which sit outside +// V8's heap, and turns only pieces of about `limit` bytes into strings at a time: a container larger than that is +// walked member by member, and runs of its small members are parsed together. Its peak is the parsed state plus one +// piece. The format is unchanged, so a build before this one reads what this one writes. + +const WHITESPACE = new Set([0x20, 0x09, 0x0a, 0x0d]); +const QUOTE = 0x22; +const BACKSLASH = 0x5c; +const COMMA = 0x2c; +const COLON = 0x3a; +const OPEN_OBJECT = 0x7b; +const CLOSE_OBJECT = 0x7d; +const OPEN_LIST = 0x5b; +const CLOSE_LIST = 0x5d; + +const unexpected = (at) => new SyntaxError(`Unexpected token in JSON at position ${at}`); + +function skipSpace(buf, i, end) { + while (i < end && WHITESPACE.has(buf[i])) i += 1; + return i; +} + +// Where the string whose opening quote is at `i` ends, past its closing quote. No byte of a multi-byte UTF-8 +// character is a quote or a backslash, so the bytes can be read one at a time. +function stringEnd(buf, i, end) { + for (let j = i + 1; j < end; j += 1) { + if (buf[j] === BACKSLASH) j += 1; + else if (buf[j] === QUOTE) return j + 1; + } + throw unexpected(i); +} + +// Where the value starting at `i` ends. Brackets are only counted here: the pieces that are parsed check their own +// syntax, and the containers that are walked check theirs as they are walked. +function valueEnd(buf, i, end) { + const c = buf[i]; + if (c === QUOTE) return stringEnd(buf, i, end); + if (c === OPEN_OBJECT || c === OPEN_LIST) { + let depth = 0; + for (let j = i; j < end; j += 1) { + const b = buf[j]; + if (b === QUOTE) j = stringEnd(buf, j, end) - 1; + else if (b === OPEN_OBJECT || b === OPEN_LIST) depth += 1; + else if ((b === CLOSE_OBJECT || b === CLOSE_LIST) && --depth === 0) return j + 1; + } + throw unexpected(i); + } + let j = i; + while (j < end && !WHITESPACE.has(buf[j]) && buf[j] !== COMMA && buf[j] !== CLOSE_OBJECT && buf[j] !== CLOSE_LIST) j += 1; + if (j === i) throw unexpected(i); + return j; +} + +// The members of the container at [start, end): [keyStart, keyEnd, valueStart, valueEnd] for an object, and +// [valueStart, valueEnd] for a list. +function* membersOf(buf, start, end, keyed) { + const close = end - 1; + let i = skipSpace(buf, start + 1, close); + if (i === close) return; + for (;;) { + let keyStart; + let keyEnd; + if (keyed) { + if (buf[i] !== QUOTE) throw unexpected(i); + keyStart = i; + keyEnd = stringEnd(buf, i, close); + i = skipSpace(buf, keyEnd, close); + if (buf[i] !== COLON) throw unexpected(i); + i = skipSpace(buf, i + 1, close); + } + if (i >= close) throw unexpected(i); + const valueStop = valueEnd(buf, i, close); + yield keyed ? [keyStart, keyEnd, i, valueStop] : [i, valueStop]; + i = skipSpace(buf, valueStop, close); + if (i === close) return; + if (buf[i] !== COMMA) throw unexpected(i); + i = skipSpace(buf, i + 1, close); + } +} + +function parseRange(buf, start, end, limit) { + if (end - start <= limit) return JSON.parse(buf.toString('utf8', start, end)); + const open = buf[start]; + if (open === OPEN_OBJECT && buf[end - 1] === CLOSE_OBJECT) { + const out = {}; + for (const [ks, ke, vs, ve] of membersOf(buf, start, end, true)) { + // An own property, as JSON.parse makes, even for a key named __proto__. + Object.defineProperty(out, JSON.parse(buf.toString('utf8', ks, ke)), { + value: parseRange(buf, vs, ve, limit), enumerable: true, writable: true, configurable: true, + }); + } + return out; + } + if (open === OPEN_LIST && buf[end - 1] === CLOSE_LIST) { + const out = []; + let runStart = -1; + let runEnd = -1; + const flush = () => { + if (runStart < 0) return; + for (const v of JSON.parse(`[${buf.toString('utf8', runStart, runEnd)}]`)) out.push(v); + runStart = -1; + }; + for (const [vs, ve] of membersOf(buf, start, end, false)) { + if (ve - vs > limit) { + flush(); + out.push(parseRange(buf, vs, ve, limit)); + continue; + } + if (runStart >= 0 && ve - runStart > limit) flush(); + if (runStart < 0) runStart = vs; + runEnd = ve; + } + flush(); + return out; + } + // A container whose last byte does not close it is not JSON, and is refused without making a string of it. + if (open === OPEN_OBJECT || open === OPEN_LIST) throw unexpected(end - 1); + // A string, a number or a literal. + return JSON.parse(buf.toString('utf8', start, end)); +} + +/** + * JSON.parse for text held in a Buffer, without making one string of it. + * @param {Buffer} buf + * @param {{ limit?: number }} [options] the largest piece made into a string at once, in bytes + */ +function parseJsonBuffer(buf, { limit = DEFAULT_PIECE_CHARS } = {}) { + const start = skipSpace(buf, 0, buf.length); + let end = buf.length; + while (end > start && WHITESPACE.has(buf[end - 1])) end -= 1; + if (start === end) throw unexpected(0); + return parseRange(buf, start, end, limit); +} + +module.exports = { stateChunks, parseJsonBuffer, DEFAULT_PIECE_CHARS }; diff --git a/oracle/state-serialize.test.js b/oracle/state-serialize.test.js index ad006062..3f581608 100644 --- a/oracle/state-serialize.test.js +++ b/oracle/state-serialize.test.js @@ -132,3 +132,87 @@ test('persist then load: what one process saves, the next restores exactly', () fsx.rmSync(dir, { recursive: true, force: true }); } }); + +// ---- an agent's ledger in slices, and reading the file back a piece at a time ------------------------------- + +const { parseJsonBuffer } = require('./state-serialize'); +const BS = String.fromCharCode(92); // a backslash, written so no tool between here and the file can eat it + +test('stateChunks: an agent\'s long ledger is written in slices, the same bytes as JSON.stringify', () => { + // It was one string: 11 MB at the caps, and 151 MB when a payer padded its outcomes. + const events = Array.from({ length: 2_000 }, (_, i) => ({ + txHash: `0x${i.toString(16).padStart(64, '0')}`, ts: 1_700_000_000_000 + i, amount: '5', payer: `0x${'22'.repeat(20)}`, + success: i % 2 === 0, + })); + const state = sampleState({ paymentEvents: { '0xaa': events } }); + const pieces = Array.from(stateChunks(state, { pieceChars: 16_384 })); + assert.equal(pieces.join(''), JSON.stringify(state)); + const ledger = JSON.stringify(events).length; + assert.ok(Math.max(...pieces.map((p) => p.length)) < ledger / 4, `largest piece ${Math.max(...pieces.map((p) => p.length))} of a ${ledger}-char ledger`); +}); + +test('parseJsonBuffer: what JSON.parse gives, however small the pieces', () => { + const docs = [ + sampleState(), + { a: [1, -2.5e-3, true, false, null, 'x', [], {}, [[[]]], { b: { c: [] } }] }, + [{ 'k"ey': `va${BS}lue "quoted"`, '\u00e9\u4e2d\u{1F600}': '\u{1F600}\u4e2d\u00e9', '': '' }, BS, '"', `${BS}"`, `${BS}${BS}`], + { __proto__: 1, constructor: 2, toString: 3 }, + JSON.parse('{"__proto__":{"polluted":true},"x":1}'), + { dup: 1 }, + ]; + for (const doc of docs) { + const text = JSON.stringify(doc); + for (const limit of [1, 2, 7, 64, 1 << 20]) { + assert.deepEqual(parseJsonBuffer(Buffer.from(text), { limit }), JSON.parse(text), `limit ${limit}: ${text.slice(0, 60)}`); + } + } + // Whitespace anywhere JSON allows it, as a file edited by hand may have, and a key given twice (the last wins). + const spaced = ' \n{ "a" :\t[ 1 , 2 ,\r\n 3 ] , "b" : { "c" : "d" } , "a" : [ 4 ] }\n '; + assert.deepEqual(parseJsonBuffer(Buffer.from(spaced), { limit: 3 }), JSON.parse(spaced)); + const own = parseJsonBuffer(Buffer.from('{"__proto__":{"polluted":true}}'), { limit: 1 }); + assert.equal(Object.getPrototypeOf(own), Object.prototype, 'a __proto__ key is an own property, not the prototype'); + assert.deepEqual(Object.keys(own), ['__proto__']); + assert.equal({}.polluted, undefined); +}); + +test('parseJsonBuffer: refuses what JSON.parse refuses', () => { + const bad = [ + '', ' ', '{', '{"a":1', '{"a":1}}', '{"a":1} x', '{"a" 1}', '{"a":1 "b":2}', '{a:1}', '[1,]', '[1 2]', '[1,2}', + '{"a":[1,2}', '{"a":"unterminated}', '{"a":tru}', '{"a":1,}', '{,}', '[,1]', '{"a":1}{"b":2}', `"${BS}`, 'nul', + `{"a":"${BS}"}`, `["${BS}x"]`, + ]; + for (const text of bad) { + assert.throws(() => JSON.parse(text), SyntaxError, `JSON.parse should refuse ${text}`); + for (const limit of [1, 3, 1 << 20]) { + assert.throws(() => parseJsonBuffer(Buffer.from(text), { limit }), SyntaxError, `limit ${limit}: ${JSON.stringify(text)}`); + } + } +}); + +test('load: a state twice the size of the heap\'s room still loads, as the file is parsed a piece at a time', () => { + // Reading the file into one string and parsing it while the string was alive peaked at about 2.2 times the file. + // At the caps that was 351 MB of the 432 MB heap V8 allows in the 768 MiB container, and an out-of-memory abort + // is not an exception, so the .bak was never tried and the restart policy started it into the same failure. + const dir = fsx.mkdtempSync(pathx.join(os.tmpdir(), 'sigvara-bigload-')); + const statePath = pathx.join(dir, 'oracle-state.json'); + const run = (script, flags = []) => execFileSync(process.execPath, [...flags, '-e', script], { + env: { ...process.env, ORACLE_STATE_PATH: statePath }, cwd: __dirname, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], + }); + try { + run(` + const s = require('./store'); + const did = '0x' + 'a1'.repeat(32); + for (let i = 0; i < 220000; i++) s.usedPaymentTxs.add('0x' + i.toString(16).padStart(64, '0') + '|' + did + '|0x' + '22'.repeat(20)); + if (!s.persist()) throw new Error('not saved'); + `); + assert.ok(fsx.statSync(statePath).size > 38e6, 'a file of about 39 MB'); + const loaded = run(` + const s = require('./store'); + s.load(); + console.log(s.usedPaymentTxs.size); + `, ['--max-old-space-size=80']); + assert.equal(loaded.trim().split('\n').pop(), '220000'); + } finally { + fsx.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/oracle/store.js b/oracle/store.js index c5a8bcc2..5e14cd67 100644 --- a/oracle/store.js +++ b/oracle/store.js @@ -12,7 +12,7 @@ const fs = require('fs'); const path = require('path'); -const { stateChunks } = require('./state-serialize'); +const { stateChunks, parseJsonBuffer } = require('./state-serialize'); // The decay curve lives in payments.js. Imported rather than reimplemented so flags and // payments age on exactly the same function; payments.js requires only ethers, so this // direction of dependency is not circular. @@ -464,8 +464,10 @@ function applyState(next) { stateChainId = next.chainId; } +// The file's bytes, parsed a piece at a time (state-serialize.js, parseJsonBuffer). Reading it as one string and +// parsing that peaked at about 2.2 times the file, which a state near the caps took past V8's heap. function readStateFile(file) { - return parseState(JSON.parse(fs.readFileSync(file, 'utf8'))); + return parseState(parseJsonBuffer(fs.readFileSync(file))); } /** From a20d296317f902310036cf67be1b7693fce6e537 Mon Sep 17 00:00:00 2001 From: RunTimeAdmin <248247024+RunTimeAdmin@users.noreply.github.com> Date: Sun, 15 Nov 2026 11:12:31 -0600 Subject: [PATCH 6/6] Declare the /score standings metric, answer batches in a test RPC, and count 1514 oracle tests - scoreStandingsPending is declared and exported, as every metric the source increments must be. - The answer-size tests' fake endpoint answers a JSON-RPC batch call by call. The payment-chain scan now asks for its blocks' times and code together, and ethers sends those as one batch; the real endpoint answers batches (measured). - README and whitepaper: 1514 oracle tests, from 1500. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/whitepaper.md | 2 +- oracle/metrics.js | 8 ++++++ oracle/payment-chain-answer-size.test.js | 33 +++++++++++++++--------- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index d1a5e761..07738a9d 100644 --- a/README.md +++ b/README.md @@ -601,7 +601,7 @@ Everything below runs offline, with no deployed contracts, except three fork sui | Suite | Count | Command | |---|---|---| | Contracts: unit, fuzz, E2E, upgrade, deploy-script, handover and 22 invariants | 796 | `forge test` | -| Oracle | 1500 | `cd oracle && node --test` | +| Oracle | 1514 | `cd oracle && node --test` | | SDK | 381 | `cd packages/sdk && npx vitest run` | Counts are individual tests, so each of the 22 invariants counts once. The contracts were counted on 10 November 2026 and have not changed since; the SDK on 13 November and the oracle on 14 November. Without a fork URL, 784 of the contract tests run and 12 skip. Run after `npm run build`, 372 of the SDK's 381 pass and 9 skip: the 8 integration tests, which need a live deployment, and one check that runs only in CI. Without a build, the 9 built-package tests skip too. Vitest strips types without checking them, so `npm run typecheck` in `packages/sdk` checks the tests against the SDK's types as well as the sources. [`scripts/mutation`](scripts/mutation/README.md) holds the harnesses that measure whether these tests would catch a deliberate break in the contracts or deploy scripts. diff --git a/docs/whitepaper.md b/docs/whitepaper.md index 45df0fe3..3c93f7e1 100644 --- a/docs/whitepaper.md +++ b/docs/whitepaper.md @@ -635,7 +635,7 @@ apply to a fresh deployment, not to the testnet. **Test coverage**, as of 14 November 2026: 796 contract tests across 36 Foundry test files including fuzz and invariant tests (the two slash drill suites fork Arc testnet themselves, and -the twelve in `SplitAuthorityFork` skip without `--fork-url`), 1500 oracle tests and 381 in the SDK (9 skip +the twelve in `SplitAuthorityFork` skip without `--fork-url`), 1514 oracle tests and 381 in the SDK (9 skip outside CI and without a live deployment; every test that runs passes). CI runs Foundry, the oracle and SDK suites, and Slither static analysis on every push, plus a documentation check that fails when the site states something the contracts do not. diff --git a/oracle/metrics.js b/oracle/metrics.js index fd57487a..2942d0e3 100644 --- a/oracle/metrics.js +++ b/oracle/metrics.js @@ -45,6 +45,9 @@ const counters = { writesNotDurable: 0, scoreRpcErrors: 0, scoreErrors: 0, + // /score previews answered 503 because the payers' standings were not read yet. A steady count means requests + // are reaching for agents the epoch does not score, or are arriving faster than the epoch reads. + scoreStandingsPending: 0, // Badges are embedded on pages this oracle does not control, so request volume is // set by an agent's audience. The hit ratio is the thing to watch: a falling one // means the cache is being missed and every reader is costing two RPC calls. @@ -292,6 +295,11 @@ function toPrometheusText(extra = {}) { lines.push('# TYPE sigvara_oracle_score_errors_total counter'); lines.push(`sigvara_oracle_score_errors_total ${counters.scoreErrors}`); + lines.push(''); + lines.push('# HELP sigvara_oracle_score_standings_pending_total /score previews whose payers\' standings were not read yet, answered 503'); + lines.push('# TYPE sigvara_oracle_score_standings_pending_total counter'); + lines.push(`sigvara_oracle_score_standings_pending_total ${counters.scoreStandingsPending}`); + lines.push(''); lines.push('# HELP sigvara_oracle_badge_requests_total Badge SVG requests'); lines.push('# TYPE sigvara_oracle_badge_requests_total counter'); diff --git a/oracle/payment-chain-answer-size.test.js b/oracle/payment-chain-answer-size.test.js index 8f1dce9a..32ee84fd 100644 --- a/oracle/payment-chain-answer-size.test.js +++ b/oracle/payment-chain-answer-size.test.js @@ -40,35 +40,44 @@ function transferLog(to, amount, n, i = 0) { /** * Base Sepolia at 127.0.0.1. `logsAnswer(from, to)` is the JSON a getLogs answers with, and `answer[method]` - * the headers and raw body for anything else it should answer oddly. `asked` holds every span asked for, and - * `head.finalized` can be moved on. + * the headers and raw body for anything else it should answer oddly, when asked on its own. `asked` holds every + * span asked for, and `head.finalized` can be moved on. A batch, as ethers sends the calls made in one tick (the + * scan asks for its blocks' times and code together), is answered call by call. */ async function endpoint({ logsAnswer = () => '[]', answer = {} } = {}) { const asked = []; const head = { finalized: 1199 }; const reply = (r, result) => `{"jsonrpc":"2.0","id":${JSON.stringify(r.id)},"result":${result}}`; + const one = (r) => { + let result = 'null'; + if (r.method === 'eth_chainId') result = JSON.stringify(hex(BASE_SEPOLIA)); + if (r.method === 'eth_getBlockByNumber') result = JSON.stringify(block(r.params[0] === 'finalized' ? head.finalized : Number(r.params[0]))); + if (r.method === 'eth_getCode') result = '"0x"'; + if (r.method === 'eth_getLogs') { + const [from, to] = [Number(r.params[0].fromBlock), Number(r.params[0].toBlock)]; + asked.push([from, to]); + result = logsAnswer(from, to); + } + return reply(r, result); + }; const server = http.createServer((req, res) => { let body = ''; req.on('data', (d) => { body += d; }); req.on('end', () => { const r = JSON.parse(body); + if (Array.isArray(r)) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(`[${r.map(one).join(',')}]`); + return; + } if (answer[r.method]) { const [headers, out] = answer[r.method](r); res.writeHead(200, { 'Content-Type': 'application/json', ...headers }); res.end(out); return; } - let result = 'null'; - if (r.method === 'eth_chainId') result = JSON.stringify(hex(BASE_SEPOLIA)); - if (r.method === 'eth_getBlockByNumber') result = JSON.stringify(block(r.params[0] === 'finalized' ? head.finalized : Number(r.params[0]))); - if (r.method === 'eth_getCode') result = '"0x"'; - if (r.method === 'eth_getLogs') { - const [from, to] = [Number(r.params[0].fromBlock), Number(r.params[0].toBlock)]; - asked.push([from, to]); - result = logsAnswer(from, to); - } res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(reply(r, result)); + res.end(one(r)); }); }); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));