diff --git a/README.md b/README.md index d1a5e76..07738a9 100644 --- a/README.md +++ b/README.md @@ -601,7 +601,7 @@ Everything below runs offline, with no deployed contracts, except three fork sui | Suite | Count | Command | |---|---|---| | Contracts: unit, fuzz, E2E, upgrade, deploy-script, handover and 22 invariants | 796 | `forge test` | -| Oracle | 1500 | `cd oracle && node --test` | +| Oracle | 1514 | `cd oracle && node --test` | | SDK | 381 | `cd packages/sdk && npx vitest run` | Counts are individual tests, so each of the 22 invariants counts once. The contracts were counted on 10 November 2026 and have not changed since; the SDK on 13 November and the oracle on 14 November. Without a fork URL, 784 of the contract tests run and 12 skip. Run after `npm run build`, 372 of the SDK's 381 pass and 9 skip: the 8 integration tests, which need a live deployment, and one check that runs only in CI. Without a build, the 9 built-package tests skip too. Vitest strips types without checking them, so `npm run typecheck` in `packages/sdk` checks the tests against the SDK's types as well as the sources. [`scripts/mutation`](scripts/mutation/README.md) holds the harnesses that measure whether these tests would catch a deliberate break in the contracts or deploy scripts. diff --git a/docs/whitepaper.md b/docs/whitepaper.md index 45df0fe..3c93f7e 100644 --- a/docs/whitepaper.md +++ b/docs/whitepaper.md @@ -635,7 +635,7 @@ apply to a fresh deployment, not to the testnet. **Test coverage**, as of 14 November 2026: 796 contract tests across 36 Foundry test files including fuzz and invariant tests (the two slash drill suites fork Arc testnet themselves, and -the twelve in `SplitAuthorityFork` skip without `--fork-url`), 1500 oracle tests and 381 in the SDK (9 skip +the twelve in `SplitAuthorityFork` skip without `--fork-url`), 1514 oracle tests and 381 in the SDK (9 skip outside CI and without a live deployment; every test that runs passes). CI runs Foundry, the oracle and SDK suites, and Slither static analysis on every push, plus a documentation check that fails when the site states something the contracts do not. diff --git a/oracle/chain.js b/oracle/chain.js index 6b32bea..572ccf4 100644 --- a/oracle/chain.js +++ b/oracle/chain.js @@ -452,9 +452,13 @@ async function getReputationData(didHash) { }; } -async function getAgentScore(address) { +/** + * A counterparty's standing, at `blockTag` when given (the scoring snapshot's block) and at the latest + * block otherwise. + */ +async function getAgentScore(address, blockTag) { try { - return await readWithBackoff('payer standing', () => readStanding(address)); + return await readWithBackoff('payer standing', () => readStanding(address, blockTag)); } catch (err) { // A failure of the node says nothing about the counterparty. Read as "no standing" it lowers the // score this oracle proposes, and the checker, whose own reads succeeded, then records a divergence @@ -466,7 +470,8 @@ async function getAgentScore(address) { } } -async function readStanding(address) { +async function readStanding(address, blockTag) { + const at = blockTag === undefined ? [] : [{ blockTag }]; // Was three serial round trips per counterparty. The first is now arithmetic when // the derivation has been verified, so it is two, and those two are genuinely // dependent. At ~124 ms a call on Arc that is a third off every payer lookup, and @@ -475,7 +480,7 @@ async function readStanding(address) { ? didHashOf(address, localDidHash.chainId) : await identityContract.computeDidHash(address); - const id = await identityContract.getIdentity(didHash); + const id = await identityContract.getIdentity(didHash, ...at); if (Number(id.registeredAt) === 0) return 0; if (Number(id.status) === STATUS_SLASHED) return 0; @@ -492,8 +497,8 @@ async function readStanding(address) { // propagate it the same epoch, and a handover restarts maturity, so bought standing // is not instantly spendable through counterparties either. const [rep, matured] = await Promise.all([ - reputationContract.getReputation(didHash), - reputationContract.getTotalScore(didHash), + reputationContract.getReputation(didHash, ...at), + reputationContract.getTotalScore(didHash, ...at), ]); return Math.min(Number(rep.externalScore), Number(matured)); } diff --git a/oracle/chain.test.js b/oracle/chain.test.js index c70da46..b89b3e0 100644 --- a/oracle/chain.test.js +++ b/oracle/chain.test.js @@ -867,6 +867,33 @@ test('getAgentScore: capped by the matured total, so linked standing is not inst assert.equal(await chain.getAgentScore('0xPayer'), 4); }); +test('getAgentScore: every read is made at the block it is given, and at the latest without one', async () => { + // The epoch reads standings at the day's snapshot block, where they cannot change, and keeps them for + // the day. A read that went to the latest block instead would be cached as the snapshot's. + const asked = []; + const note = (name, value) => async (...args) => { + const last = args[args.length - 1]; + asked.push([name, last && typeof last === 'object' ? last.blockTag : 'latest']); + return value; + }; + chain.init(CFG, { + provider: makeFakeProvider(1), + identityContract: { + computeDidHash: async () => '0xdeadbeef', + getIdentity: note('getIdentity', { registeredAt: 1n, status: 0 }), + }, + reputationContract: { + getReputation: note('getReputation', { externalScore: 20n }), + getTotalScore: note('getTotalScore', 60n), + }, + }); + assert.equal(await chain.getAgentScore('0xPayer', 4242), 20); + assert.deepEqual(asked.sort(), [['getIdentity', 4242], ['getReputation', 4242], ['getTotalScore', 4242]]); + asked.length = 0; + await chain.getAgentScore('0xPayer'); + assert.deepEqual(asked.sort(), [['getIdentity', 'latest'], ['getReputation', 'latest'], ['getTotalScore', 'latest']]); +}); + test('getAgentScore: an unregistered counterparty has no standing', async () => { chain.init(CFG, { provider: makeFakeProvider(1), ...makeStandingContracts({ registeredAt: 0, externalScore: 25, total: 90 }) }); assert.equal(await chain.getAgentScore('0xStranger'), 0); diff --git a/oracle/http-helpers.js b/oracle/http-helpers.js index d4f0430..8e7b026 100644 --- a/oracle/http-helpers.js +++ b/oracle/http-helpers.js @@ -8,6 +8,16 @@ const net = require('net'); const MAX_BODY_SIZE = 1024 * 1024; // 1 MB +// POST /attest is public, so its bodies are held to what a real one needs. The largest is a payer +// outcome: its signature (payer-outcome.js MAX_SIGNATURE_BYTES) in hex plus a few hundred bytes of +// fields. A megabyte let each request a client left unfinished hold a megabyte of this process. +const ATTEST_MAX_BODY_SIZE = 32 * 1024; + +// How long a request has to deliver its body. Node's requestTimeout is five minutes, and a client that +// sent all but the last byte and stopped kept everything it had sent in memory for that long: at the +// write limit of 60 requests a minute, about 300 MB from one address. +const BODY_DEADLINE_MS = 10_000; + // Past this many bytes the connection is cut. Up to it, the rest of an oversize body is read and // thrown away, so a client that is still writing can finish and read the 413. Destroying the // request at the limit left it with a reset and no answer. @@ -24,21 +34,37 @@ function bodyTooLarge() { return Object.assign(new Error('Request body too large'), { code: 'body_too_large' }); } -// What a route answers when readBody rejects: 413 for an oversize body, 400 for anything else. +function bodyTooSlow() { + return Object.assign(new Error('Request body too slow'), { code: 'body_timeout' }); +} + +// What a route answers when readBody rejects: 413 for an oversize body, 408 for one that did not +// arrive in time, 400 for anything else. function bodyErrorStatus(err) { - return err && err.code === 'body_too_large' ? 413 : 400; + if (err && err.code === 'body_too_large') return 413; + if (err && err.code === 'body_timeout') return 408; + return 400; } // req only needs to be an EventEmitter emitting 'data' | 'end' | 'error' and // exposing destroy() — a real http.IncomingMessage satisfies this, and so // does a plain fake in tests. A declared content-length is read from // req.headers when it has them. -async function readBody(req, maxBodySize = MAX_BODY_SIZE) { +async function readBody(req, maxBodySize = MAX_BODY_SIZE, deadlineMs = BODY_DEADLINE_MS) { return new Promise((resolve, reject) => { const chunks = []; let size = 0; let refused = false; const refuse = () => { refused = true; chunks.length = 0; reject(bodyTooLarge()); }; + // A body still arriving at the deadline is dropped, and the connection with it, so what was sent + // is let go. The timer does not hold the process open on its own. + const deadline = setTimeout(() => { + refused = true; + chunks.length = 0; + reject(bodyTooSlow()); + req.destroy(); + }, deadlineMs); + deadline.unref?.(); // A declared length over the limit is refused before any of the body is read. if (Number(req.headers && req.headers['content-length']) > maxBodySize) refuse(); req.on('data', c => { @@ -51,10 +77,14 @@ async function readBody(req, maxBodySize = MAX_BODY_SIZE) { chunks.push(c); }); req.on('end', () => { + clearTimeout(deadline); try { resolve(JSON.parse(Buffer.concat(chunks).toString() || '{}')); } catch { reject(new Error('Invalid JSON')); } }); - req.on('error', reject); + req.on('error', (err) => { + clearTimeout(deadline); + reject(err); + }); }); } @@ -425,6 +455,7 @@ module.exports = { SCOPES, MIN_TOKEN_LENGTH, MAX_BODY_SIZE, + ATTEST_MAX_BODY_SIZE, RATE_WINDOW_MS, RATE_MAX, json, diff --git a/oracle/http-helpers.test.js b/oracle/http-helpers.test.js index f0f1abc..00234d6 100644 --- a/oracle/http-helpers.test.js +++ b/oracle/http-helpers.test.js @@ -200,6 +200,37 @@ test('bodyErrorStatus: 413 for an oversized body, 400 for anything else', async assert.equal(bodyErrorStatus(undefined), 400); }); +test('readBody: a body still arriving at the deadline is dropped with its connection, and answered 408', async () => { + // Before the deadline a client could send all but the last byte and stop, and everything it had sent + // stayed in memory until Node's five-minute requestTimeout. + const req = makeFakeRequest(); + let destroyed = false; + req.on('destroyed', () => { destroyed = true; }); + req.headers = { 'content-length': '1000' }; + const promise = readBody(req, 1000, 50); + req.emit('data', Buffer.from('x'.repeat(999))); + + const outcome = await Promise.race([ + promise.then(() => 'resolved', (err) => err), + new Promise((r) => setTimeout(() => r('still waiting'), 1_000)), + ]); + assert.equal(outcome.code, 'body_timeout', `readBody ended as: ${outcome}`); + assert.equal(bodyErrorStatus(outcome), 408); + assert.equal(destroyed, true, 'the connection is cut, which lets go of what it sent'); +}); + +test('readBody: a body that arrives in time is not cut by the deadline', async () => { + const req = makeFakeRequest(); + let destroyed = false; + req.on('destroyed', () => { destroyed = true; }); + const promise = readBody(req, 1000, 50); + req.emit('data', Buffer.from('{"a":1}')); + req.emit('end'); + assert.deepEqual(await promise, { a: 1 }); + await new Promise((r) => setTimeout(r, 100)); + assert.equal(destroyed, false); +}); + test('readBody: chunks are reassembled in order', async () => { const req = makeFakeRequest(); const promise = readBody(req); diff --git a/oracle/http.test.js b/oracle/http.test.js index ae5be11..3848216 100644 --- a/oracle/http.test.js +++ b/oracle/http.test.js @@ -407,6 +407,15 @@ test('a body over the 1 MB limit is answered 413 on every route that reads one', } }); +test('POST /attest takes at most 32 KB (the loopback routes keep 1 MB, tested below)', async () => { + // /attest is public. At a megabyte, every request a client left unfinished held a megabyte of the + // oracle until Node's five-minute request timeout. + const room = 32 * 1024 - JSON.stringify({ didHash: DID, success: true, attester: 'counteraudit', pad: '' }).length; + const over = await oracle.call('POST', '/attest', { didHash: DID, success: true, attester: 'counteraudit', pad: 'x'.repeat(room + 1) }, AUDITOR); + assert.equal(over.status, 413, over.text.slice(0, 120)); + assert.equal(over.body.error, 'Request body too large'); +}); + test('the limit is 1 MB to the byte', async () => { const unflagged = `0x${'12'.repeat(32)}`; const room = MAX_BODY - JSON.stringify({ didHash: unflagged, pad: '' }).length; diff --git a/oracle/index.js b/oracle/index.js index db55b64..399d58e 100644 --- a/oracle/index.js +++ b/oracle/index.js @@ -31,7 +31,7 @@ const { numberSetting } = require('./config'); const { json, readBody, bodyErrorStatus, readCredentials, identifyCaller, mayAttestUnauthenticated, parseScorePath, parseDidHash, rateLimited, clientKey, adminTokenPolicyError, runningCommit, - parseTrustedProxies, parseRequestTarget, readScopes, mayUse, weakCredentials, MIN_TOKEN_LENGTH, + parseTrustedProxies, parseRequestTarget, readScopes, mayUse, weakCredentials, MIN_TOKEN_LENGTH, ATTEST_MAX_BODY_SIZE, } = require('./http-helpers'); const payments = require('./payments'); const merkle = require('./merkle'); @@ -685,10 +685,59 @@ async function rereadCredits(agentInfos, deadline) { if (updates.size) persistState(); } -// Counterparty standing, for the web of trust. Resolved once per epoch per address: -// an agent's payers repeat across epochs and within one, and each lookup is three -// chain reads. Cleared at the start of every epoch so standings cannot go stale. -let payerScoreCache = new Map(); +// Counterparty standing, for the web of trust: each payer's standing as of the snapshot, the start of the +// UTC day containing the score's time (reviewers.snapshotOf), read at the snapshot's block as the external +// factor's reviewers are. A standing read there cannot change, so it is kept until the snapshot moves, and +// a checker auditing a proposal made the same day reads exactly what its primary read. Each lookup is two +// or three chain reads. Standings used to be read at the latest block and cleared at every epoch, so every +// payer was read again every hour, and 20,000 one-time payers added most of an hour to every epoch. +const standingsBySnapshot = new Map(); // snapshot block => Map(lowercased payer => standing) +// Today's, and yesterday's for a checker auditing a proposal made just before midnight. +const STANDING_SNAPSHOTS_KEPT = 2; +// Reads in flight, by snapshot block and payer, so the epoch and every concurrent GET /score share one +// read per payer. Each caller used to read every payer missing when it started, so 30 requests for an +// agent with 300 payers made 4,650 reads in 45 s, against the 6.7 a second the epoch keeps to. +const standingReads = new Map(); + +function standingsAt(block) { + let standings = standingsBySnapshot.get(block); + if (!standings) { + standings = new Map(); + standingsBySnapshot.set(block, standings); + while (standingsBySnapshot.size > STANDING_SNAPSHOTS_KEPT) { + standingsBySnapshot.delete(Math.min(...standingsBySnapshot.keys())); + } + } + return standings; +} + +async function standingsBlockFor(nowMs) { + return chain.blockAtOrBefore(reviewers.snapshotOf(Math.floor(nowMs / 1000))); +} + +function standingOf(block, key, address) { + const standings = standingsAt(block); + if (standings.has(key)) return Promise.resolve(standings.get(key)); + const id = `${block}|${key}`; + let read = standingReads.get(id); + if (!read) { + read = chain.getAgentScore(address, block) + .then((standing) => { standings.set(key, standing); return standing; }) + .finally(() => standingReads.delete(id)); + standingReads.set(id, read); + } + return read; +} + +/** An agent's payers, each once: lowercased key => the address as recorded. */ +function payersOf(events) { + const unique = new Map(); + for (const e of events) { + const key = String(e.payer || '').toLowerCase(); + if (!unique.has(key)) unique.set(key, e.payer); + } + return unique; +} // A checker's memory of empty slots (decideCheckerAction): when it first found each agent's // slot empty, in chain seconds, and which agents it is covering because it finalized their @@ -716,16 +765,14 @@ const checkerSlots = { const PAYER_READ_CHUNK = 10; const PAYER_READ_PACE_MS = 1_500; -async function payerScoresFor(didHash, events = getPaymentEvents(didHash)) { +async function payerScoresFor(didHash, events, nowMs) { if (!payments.required(paymentCfg)) return null; // Unique payers first: an address recurs across an agent's events, and this way it is // neither looked up nor awaited twice. - const unique = new Map(); // lowercased key => the address as recorded - for (const e of events) { - const key = String(e.payer || '').toLowerCase(); - if (!unique.has(key)) unique.set(key, e.payer); - } + const unique = payersOf(events); + const block = await standingsBlockFor(nowMs); + const standings = standingsAt(block); // Resolve the cache misses together, a chunk at a time, rather than one at a time. // @@ -739,21 +786,65 @@ async function payerScoresFor(didHash, events = getPaymentEvents(didHash)) { // read is kept either way, so the next try asks only for what is missing. The rejection reaches the // agent's own handler, which skips the agent for this epoch, so the rest are not asked for: that // would only press a node that is already failing. - const misses = [...unique].filter(([key]) => !payerScoreCache.has(key)); - for (let i = 0; i < misses.length; i += PAYER_READ_CHUNK) { - if (i > 0) await new Promise((r) => setTimeout(r, PAYER_READ_PACE_MS)); - const chunk = misses.slice(i, i + PAYER_READ_CHUNK); - const settled = await Promise.allSettled(chunk.map(([, addr]) => chain.getAgentScore(addr))); - chunk.forEach(([key], j) => { if (settled[j].status === 'fulfilled') payerScoreCache.set(key, settled[j].value); }); + // + // The cache is checked as each chunk is made up, not once before the first, so a payer another caller + // has read in the meantime is not read again; a read another caller has in flight is shared. + const out = {}; + const all = [...unique]; + let first = true; + for (let i = 0; i < all.length;) { + const chunk = []; + for (; i < all.length && chunk.length < PAYER_READ_CHUNK; i += 1) { + const [key] = all[i]; + if (standings.has(key)) out[key] = standings.get(key); + else chunk.push(all[i]); + } + if (!chunk.length) break; + if (!first) await new Promise((r) => setTimeout(r, PAYER_READ_PACE_MS)); + first = false; + const settled = await Promise.allSettled(chunk.map(([key, addr]) => standingOf(block, key, addr))); + chunk.forEach(([key], j) => { if (settled[j].status === 'fulfilled') out[key] = settled[j].value; }); const failed = settled.find((s) => s.status === 'rejected'); if (failed) throw failed.reason; } - - const out = {}; - for (const [key] of unique) out[key] = payerScoreCache.get(key); return out; } +// GET /score reads payers' standings for itself only between epochs, at most a chunk a request, and all +// requests together no faster than the epoch reads (a bucket of two chunks, refilled at the epoch's +// pace). A request that finds more than a chunk unread reads one chunk and asks to be tried again, so +// each request moves the reads on and no burst of requests outruns the epoch. Requests used to read +// their whole list each, and could be timed, by /health's epochRunning, to land on the epoch, whose +// reads then failed and whose agents were skipped. +const PREVIEW_READ_BURST = 2 * PAYER_READ_CHUNK; +let previewReads = { tokens: PREVIEW_READ_BURST, at: 0 }; + +function takePreviewReads(n) { + const now = Date.now(); + const refilled = previewReads.tokens + ((now - previewReads.at) * PAYER_READ_CHUNK) / PAYER_READ_PACE_MS; + const tokens = Math.min(PREVIEW_READ_BURST, refilled); + if (tokens < n) { + previewReads = { tokens, at: now }; + return false; + } + previewReads = { tokens: tokens - n, at: now }; + return true; +} + +async function previewStandingsReady(events, nowMs) { + if (!payments.required(paymentCfg)) return { ready: true }; + const block = await standingsBlockFor(nowMs); + const standings = standingsAt(block); + const unread = [...payersOf(events)].filter(([key]) => !standings.has(key) && !standingReads.has(`${block}|${key}`)); + if (!unread.length) return { ready: true }; + if (epochRunning) return { ready: false, retryAfter: 60 }; + const chunk = unread.slice(0, PAYER_READ_CHUNK); + if (!takePreviewReads(chunk.length)) return { ready: false, retryAfter: 2 }; + if (unread.length === chunk.length) return { ready: true }; // scoreAt reads them, as one chunk + await Promise.allSettled(chunk.map(([key, addr]) => standingOf(block, key, addr))); + return { ready: false, retryAfter: 2 }; +} + // An agent's external factor, kept per snapshot. Read at one block with the same evidence it cannot // change, so the epoch, the checker's audit and every GET /score can share one reading, and feedback // posted later waits for the next snapshot as it always did. A registry on another chain is read at @@ -913,7 +1004,7 @@ async function scoreAt(didHash, { registeredAt, att, flagWeight, externalScore } // The evidence this score reads, once: the payers looked up are the ones it scores. By default // what counted by the cut-off for a score made now (payment-chains.cutFor), as the epoch uses. const held = events ?? evidenceAt(didHash, paymentChains.cutFor(nowMs, paymentChainsCfg)); - const measured = measuredFactorsFor(didHash, nowMs, await payerScoresFor(didHash, held), held); + const measured = measuredFactorsFor(didHash, nowMs, await payerScoresFor(didHash, held, nowMs), held); const scores = computeScore({ registeredAt, attestations: measured.measuredAttestations ?? att, @@ -1070,6 +1161,9 @@ async function runScanCanary(byAddress, provider) { } } +// Half of the 20 calls Arc's public RPC answers in one batch (measured 4 Nov 2026), as for payers' standing. +const SCAN_BLOCKS_PER_BATCH = 10; + async function runPaymentScan(agents) { if (!paymentScanCfg.enabled) return null; if (!paymentCfg.asset) { @@ -1135,10 +1229,19 @@ async function runPaymentScan(agents) { // The settlement time is the block's, not now. Crediting a back-scanned payment with // the current clock would make an old settlement look fresh and defeat the decay that // every volume figure depends on. + // + // Asked SCAN_BLOCKS_PER_BATCH blocks at a time, which ethers sends as one batch, paced as the log reads + // are. They went one round trip at a time with no pace, which ran through Arc's bucket of about 60 + // requests and then crawled on backoff, so payments in enough blocks could keep the scan behind, and a + // scan far enough behind holds every agent. const times = new Map(); - for (const n of new Set(credits.map((c) => c.blockNumber))) { - const block = await chain.readWithBackoff('payment-scan getBlock', () => provider.getBlock(n)); - if (block) times.set(n, Number(block.timestamp) * 1000); + const blocks = [...new Set(credits.map((c) => c.blockNumber))]; + for (let i = 0; i < blocks.length; i += SCAN_BLOCKS_PER_BATCH) { + if (i > 0) await new Promise((r) => setTimeout(r, paymentScanCfg.paceMs)); + await Promise.all(blocks.slice(i, i + SCAN_BLOCKS_PER_BATCH).map(async (n) => { + const block = await chain.readWithBackoff('payment-scan getBlock', () => provider.getBlock(n)); + if (block) times.set(n, Number(block.timestamp) * 1000); + })); } let credited = 0; @@ -1389,7 +1492,6 @@ async function evidenceCutFor() { async function runEpochInner() { const start = Date.now(); - payerScoreCache = new Map(); // This epoch's own, so one that stops early does not leave the last one's showing. lastEpochHold = null; console.log(`[oracle] epoch start — ${new Date().toISOString()}`); @@ -3300,7 +3402,7 @@ async function handleRequest(req, res) { if (req.method === 'POST' && pathname === '/attest') { if (overLimit(req, res, 'write')) return; try { - const body = await readBody(req); + const body = await readBody(req, ATTEST_MAX_BODY_SIZE); const { success, packetId } = body; // An outcome the payer signed (docs/adr/0006-payer-signed-outcomes.md). The signature is @@ -4022,9 +4124,16 @@ async function handleRequest(req, res) { // // One extra chain read per request, on an endpoint that already makes several. const scoringNowMs = (await chain.getLatestBlockTimestamp()) * 1000; + const held = evidenceAt(didHash, paymentChains.cutFor(scoringNowMs, paymentChainsCfg)); + const standingsReady = await previewStandingsReady(held, scoringNowMs); + if (!standingsReady.ready) { + metrics.inc('scoreStandingsPending'); + res.setHeader('Retry-After', String(standingsReady.retryAfter)); + return json(res, 503, { error: 'payer standings have not been read yet', code: 'standings_pending' }); + } const { score: externalScore, agentId: linkedId } = await externalScoreOf(didHash, { operator, agentAddress }, scoringNowMs); - const { measured, scores } = await scoreAt(didHash, { registeredAt, att, flagWeight, externalScore }, scoringNowMs); + const { measured, scores } = await scoreAt(didHash, { registeredAt, att, flagWeight, externalScore }, scoringNowMs, held); return json(res, 200, { didHash, status, @@ -4080,8 +4189,12 @@ async function handleRequest(req, res) { * restart policy brought the container back, and the compose files re-cloned `main` each * time (they now fetch a pinned commit). The error is logged with the request line quoted, so a hostile target cannot forge * a log line, and the caller is told nothing about it. + * + * Node's defaults gave a client 60 s to send its headers and five minutes to send the whole request, + * and a request still arriving holds what it has sent. Nothing this server accepts needs more than a + * few seconds to send. These bound receiving only: a slow answer, such as a long /evidence, is not cut. */ -const server = http.createServer((req, res) => { +const server = http.createServer({ headersTimeout: 10_000, requestTimeout: 15_000 }, (req, res) => { handleRequest(req, res).catch((err) => { metrics.inc('httpHandlerErrors'); console.error( diff --git a/oracle/metrics.js b/oracle/metrics.js index fd57487..2942d0e 100644 --- a/oracle/metrics.js +++ b/oracle/metrics.js @@ -45,6 +45,9 @@ const counters = { writesNotDurable: 0, scoreRpcErrors: 0, scoreErrors: 0, + // /score previews answered 503 because the payers' standings were not read yet. A steady count means requests + // are reaching for agents the epoch does not score, or are arriving faster than the epoch reads. + scoreStandingsPending: 0, // Badges are embedded on pages this oracle does not control, so request volume is // set by an agent's audience. The hit ratio is the thing to watch: a falling one // means the cache is being missed and every reader is costing two RPC calls. @@ -292,6 +295,11 @@ function toPrometheusText(extra = {}) { lines.push('# TYPE sigvara_oracle_score_errors_total counter'); lines.push(`sigvara_oracle_score_errors_total ${counters.scoreErrors}`); + lines.push(''); + lines.push('# HELP sigvara_oracle_score_standings_pending_total /score previews whose payers\' standings were not read yet, answered 503'); + lines.push('# TYPE sigvara_oracle_score_standings_pending_total counter'); + lines.push(`sigvara_oracle_score_standings_pending_total ${counters.scoreStandingsPending}`); + lines.push(''); lines.push('# HELP sigvara_oracle_badge_requests_total Badge SVG requests'); lines.push('# TYPE sigvara_oracle_badge_requests_total counter'); diff --git a/oracle/payer-outcome.js b/oracle/payer-outcome.js index b2d2bba..81e41b5 100644 --- a/oracle/payer-outcome.js +++ b/oracle/payer-outcome.js @@ -31,8 +31,11 @@ const MAX_FUTURE_SKEW_SECONDS = 300; // How far signedAt may be after the time of the block it is checked at. A confirmed block is a // little behind the head, and the head a little behind real time; an hour is far more than either. const MAX_CHECK_LAG_SECONDS = 3_600; -// An ERC-1271 signature can be longer than 65 bytes (a multisig's, say), but not unboundedly. -const MAX_SIGNATURE_BYTES = 4096; +// An ERC-1271 signature can be longer than 65 bytes (a multisig's, say), but not unboundedly. The longest held +// on 15 Nov 2026 was 640 bytes, a smart account's. 4,096 let a payer make each payment it holds 15 times the size of +// an ordinary one, and the caps count payments, not bytes: about 24,000 such outcomes took a state past what a +// restart could load. Both operators change this together, or one refuses imports the other accepted. +const MAX_SIGNATURE_BYTES = 1024; const ERC1271_MAGIC = '0x1626ba7e'; const ERC1271 = new ethers.Interface(['function isValidSignature(bytes32 hash, bytes signature) view returns (bytes4)']); // Gas for one isValidSignature call. Enough for a passkey (P-256) account without a precompile, diff --git a/oracle/payer-outcome.test.js b/oracle/payer-outcome.test.js index 716a78b..eda72d2 100644 --- a/oracle/payer-outcome.test.js +++ b/oracle/payer-outcome.test.js @@ -1219,3 +1219,14 @@ test('refuses to start taking payer outcomes without the chain id they are signe assert.equal(code, 1); assert.match(log, /PAYER_OUTCOMES=positive needs EXPECTED_CHAIN_ID/); }); + +test('a signature is held to 1,024 bytes: room for a smart account\'s, not for padding', () => { + // 4,096 let a payer make each payment it holds 15 times the size of an ordinary one, and the caps count + // payments, not bytes: about 24,000 such outcomes took a state past what a restart could load. + const base = { payer: `0x${'44'.repeat(20)}`, outcome: payerOutcome.DELIVERED, signedAt: 1_700_000_000, deadline: 1_700_003_600 }; + assert.equal(payerOutcome.parse({ ...base, signature: `0x${'ab'.repeat(640)}` }).ok, true, 'the longest held on 15 Nov'); + assert.equal(payerOutcome.parse({ ...base, signature: `0x${'ab'.repeat(1024)}` }).ok, true); + const over = payerOutcome.parse({ ...base, signature: `0x${'ab'.repeat(1025)}` }); + assert.equal(over.ok, false); + assert.match(over.error, /signature/); +}); diff --git a/oracle/payer-reads.test.js b/oracle/payer-reads.test.js index 1bd084a..339bc53 100644 --- a/oracle/payer-reads.test.js +++ b/oracle/payer-reads.test.js @@ -68,14 +68,15 @@ test('an agent whose payer the node could not read is skipped for the epoch, and // ---- how fast the standing of many payers is asked for ------------------------------------ -/** An agent paid by `n` different wallets in one transaction, each leg a payment, on a running oracle. */ +/** A scored agent paid by `n` different wallets in one transaction, each leg a payment, on a running oracle. */ async function paidByMany(n) { const world = new World(); - const agent = world.addAgent(); + const agent = world.addRegisteredAgent(); const payers = Array.from({ length: n }, () => wallet()); const tx = world.addTx(payers.map((from) => ({ from, to: agent.agentAddress, value: 5n * USDC }))); world.set('standingReadMs', 20); oracle = await startOracle({ world, env: ENV }); + await waitForEpochs(1); const r = await oracle.call('POST', '/attest', { didHash: agent.didHash, success: true, payment: { txHash: tx } }, ADMIN); assert.equal(r.status, 200, r.text); return { world, agent, payers }; @@ -85,10 +86,9 @@ const standingReads = () => oracle.calls().filter((c) => c.name === 'standingRea test('payers are asked about a few at a time: a node takes only so many reads in one batch', async () => { // All at once, a same-tick burst of more than about 80 went past what the backoff could catch up with // on Arc's public RPC, and the agent went unscored for having customers. - const { agent } = await paidByMany(25); + await paidByMany(25); const started = Date.now(); - const r = await oracle.call('GET', `/score/${agent.didHash}`); - assert.equal(r.status, 200, r.text); + await runEpoch(2); const reads = standingReads(); assert.equal(reads.length, 25, 'each of them, once'); assert.ok(Math.max(...reads) <= 10, `at most ten at once, not ${Math.max(...reads)}`); @@ -98,25 +98,66 @@ test('payers are asked about a few at a time: a node takes only so many reads in }); test('a node that cannot answer is not asked for the rest, and what it did answer is kept', async () => { - const { world, agent, payers } = await paidByMany(25); + const { world, payers } = await paidByMany(25); const failing = (list) => Object.fromEntries(list.map((p) => [p.toLowerCase(), true])); - // None of them can be read: it stops after the first chunk. + // None of them can be read: it stops after the first chunk, and nothing is proposed on it. world.set('standingFails', failing(payers)); - const none = await oracle.call('GET', `/score/${agent.didHash}`); - assert.equal(none.status, 502, none.text); - assert.equal(none.body.code, 'rpc_error'); + await runEpoch(2); assert.equal(standingReads().length, 10); + assert.equal(proposals(), 1, 'only the first epoch\'s, made before the payments'); // One of them cannot: evidence is not held in the order paid, so which chunk it falls in varies. world.set('standingFails', failing([payers[0]])); - assert.equal((await oracle.call('GET', `/score/${agent.didHash}`)).status, 502); + await runEpoch(3); const before = standingReads().length - 10; assert.ok(before % 10 === 0 || before === 25, `stopped at the end of a chunk, after ${before}`); // Now it can: everything read so far is kept, so only that one and those not reached are asked for. world.set('standingFails', {}); - const ok = await oracle.call('GET', `/score/${agent.didHash}`); - assert.equal(ok.status, 200, ok.text); + await runEpoch(4); assert.equal(standingReads().length - 10 - before, 25 - (before - 1)); + assert.equal(proposals(), 2, 'and the agent is scored'); +}); + +test('a payer\'s standing is read once a snapshot, not again every epoch', async () => { + // Standings were read at the latest block and cleared at every epoch, so every payer was read again + // every hour: 20,000 one-time payers added most of an hour to every epoch. They are read at the day's + // snapshot block now, where they cannot change. + const { world } = await paidByMany(5); + await runEpoch(2); + assert.equal(standingReads().length, 5); + await runEpoch(3); + assert.equal(standingReads().length, 5, 'the next epoch of the same day reads none of them again'); + world.set('snapshotBlock', 1_000_001); // the next UTC day's snapshot + await runEpoch(4); + assert.equal(standingReads().length, 10, 'a new snapshot reads them all again'); +}); + +// ---- GET /score's own reads ------------------------------------------------------------------- + +test('requests for the same agent share each payer\'s read, instead of each reading all of them', async () => { + // Each request used to read every payer missing when it started: 30 requests for an agent with 300 + // payers made 4,650 reads in 45 s, against the 6.7 a second the epoch keeps to. + const { agent } = await paidByMany(8); + const answers = await Promise.all(Array.from({ length: 30 }, () => oracle.call('GET', `/score/${agent.didHash}`))); + for (const a of answers) { + assert.ok(a.status === 200 || (a.status === 503 && a.body.code === 'standings_pending'), a.text); + } + assert.ok(answers.some((a) => a.status === 200), 'and at least one is answered'); + assert.equal(standingReads().length, 8, 'each payer read once'); +}); + +test('/score reads at most a chunk of standings itself, and leaves the rest to the epoch', async () => { + const { agent } = await paidByMany(25); + const first = await oracle.call('GET', `/score/${agent.didHash}`); + assert.equal(first.status, 503, first.text); + assert.equal(first.body.code, 'standings_pending'); + assert.equal(standingReads().length, 10, 'one chunk, not all 25'); + + await runEpoch(2); + assert.equal(standingReads().length, 25, 'the epoch read the other 15, and kept the first 10'); + const after = await oracle.call('GET', `/score/${agent.didHash}`); + assert.equal(after.status, 200, after.text); + assert.equal(standingReads().length, 25, 'and the preview then needed no read of its own'); }); diff --git a/oracle/payment-chain-answer-size.test.js b/oracle/payment-chain-answer-size.test.js index 8f1dce9..32ee84f 100644 --- a/oracle/payment-chain-answer-size.test.js +++ b/oracle/payment-chain-answer-size.test.js @@ -40,35 +40,44 @@ function transferLog(to, amount, n, i = 0) { /** * Base Sepolia at 127.0.0.1. `logsAnswer(from, to)` is the JSON a getLogs answers with, and `answer[method]` - * the headers and raw body for anything else it should answer oddly. `asked` holds every span asked for, and - * `head.finalized` can be moved on. + * the headers and raw body for anything else it should answer oddly, when asked on its own. `asked` holds every + * span asked for, and `head.finalized` can be moved on. A batch, as ethers sends the calls made in one tick (the + * scan asks for its blocks' times and code together), is answered call by call. */ async function endpoint({ logsAnswer = () => '[]', answer = {} } = {}) { const asked = []; const head = { finalized: 1199 }; const reply = (r, result) => `{"jsonrpc":"2.0","id":${JSON.stringify(r.id)},"result":${result}}`; + const one = (r) => { + let result = 'null'; + if (r.method === 'eth_chainId') result = JSON.stringify(hex(BASE_SEPOLIA)); + if (r.method === 'eth_getBlockByNumber') result = JSON.stringify(block(r.params[0] === 'finalized' ? head.finalized : Number(r.params[0]))); + if (r.method === 'eth_getCode') result = '"0x"'; + if (r.method === 'eth_getLogs') { + const [from, to] = [Number(r.params[0].fromBlock), Number(r.params[0].toBlock)]; + asked.push([from, to]); + result = logsAnswer(from, to); + } + return reply(r, result); + }; const server = http.createServer((req, res) => { let body = ''; req.on('data', (d) => { body += d; }); req.on('end', () => { const r = JSON.parse(body); + if (Array.isArray(r)) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(`[${r.map(one).join(',')}]`); + return; + } if (answer[r.method]) { const [headers, out] = answer[r.method](r); res.writeHead(200, { 'Content-Type': 'application/json', ...headers }); res.end(out); return; } - let result = 'null'; - if (r.method === 'eth_chainId') result = JSON.stringify(hex(BASE_SEPOLIA)); - if (r.method === 'eth_getBlockByNumber') result = JSON.stringify(block(r.params[0] === 'finalized' ? head.finalized : Number(r.params[0]))); - if (r.method === 'eth_getCode') result = '"0x"'; - if (r.method === 'eth_getLogs') { - const [from, to] = [Number(r.params[0].fromBlock), Number(r.params[0].toBlock)]; - asked.push([from, to]); - result = logsAnswer(from, to); - } res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(reply(r, result)); + res.end(one(r)); }); }); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); diff --git a/oracle/payment-chains.js b/oracle/payment-chains.js index 6ad282a..d120c61 100644 --- a/oracle/payment-chains.js +++ b/oracle/payment-chains.js @@ -64,9 +64,16 @@ const DEFAULT_PACE_MS = 500; // advances a bounded step per epoch instead of retrying an ever larger range after one failed call. const DEFAULT_MAX_BLOCKS = 36_000; // Reads one scan spends on its payments at most: a getCode for each agent and block, and a getBlock for each -// block, one at a time. A chain that is on is scanned inside the epoch, so without this a step of many -// payments could hold an epoch for hours with nothing saved. +// block. A chain that is on is scanned inside the epoch, so without this a step of many payments could hold +// an epoch for hours with nothing saved. const DEFAULT_MAX_READS = 1_000; +// Blocks whose reads go out together: each block's time and its recipients' code, asked in one tick, which +// ethers sends as one JSON-RPC batch, paced as the log reads are. They went one round trip at a time, so a +// step of 1,000 reads took about 150 s at Base Sepolia's 145 ms, and payments in more blocks an hour than +// the 10-minute budget could confirm put the scan behind, which holds every agent (ADR 0009). The endpoint +// the operators use answered batches of 100 such reads on 15 Nov 2026; eight blocks ask about sixteen. +const BLOCKS_PER_READ_BATCH = 8; +const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); // How long one read of a payment chain may take. A chain that is on is scanned inside the epoch, before // scoring, and each read is tried four times, so ethers' default of 5 minutes would let one hung // endpoint hold an epoch for 20 minutes. At 30 seconds it costs about 2. @@ -599,28 +606,44 @@ async function scanChain(chain, deps) { } return scoredCode.get(a.didHash); }; + + // With the code, the block's own time, as on the scored chain: crediting with the clock would make an + // old payment look fresh and defeat the decay. Both are asked for BLOCKS_PER_READ_BATCH blocks at a + // time. A block's time is read even when every payment in it turns out to be to a contract: it rides in + // the same batch, so it costs no round trip. + const byBlock = new Map(); for (const c of plan.credits) { - const a = byDid.get(c.didHash); - if (!a?.agentAddress || codeOk.has(codeKey(c))) continue; - const [here, scored] = await Promise.all([ - read('payment chain getCode', () => provider.getCode(a.agentAddress, c.blockNumber)), - scoredCodeOf(a), - ]); - codeOk.set(codeKey(c), recipientCodeOk(here) && recipientCodeOk(scored)); + if (!byBlock.has(c.blockNumber)) byBlock.set(c.blockNumber, []); + byBlock.get(c.blockNumber).push(c); + } + const blocks = [...byBlock.keys()].sort((x, y) => x - y); + const times = new Map(); + const pause = deps.sleep ?? wait; + for (let i = 0; i < blocks.length; i += BLOCKS_PER_READ_BATCH) { + if (i > 0) await pause(chain.paceMs); + await Promise.all(blocks.slice(i, i + BLOCKS_PER_READ_BATCH).map(async (n) => { + const asking = new Map(); // codeKey => agent, once each + for (const c of byBlock.get(n)) { + const a = byDid.get(c.didHash); + if (a?.agentAddress) asking.set(codeKey(c), a); + } + const keys = [...asking.keys()]; + const [block, ...codes] = await Promise.all([ + read('payment chain getBlock', () => provider.getBlock(n)), + ...keys.map((k) => Promise.all([ + read('payment chain getCode', () => provider.getCode(asking.get(k).agentAddress, n)), + scoredCodeOf(asking.get(k)), + ])), + ]); + if (block) times.set(n, Number(block.timestamp) * 1000); + keys.forEach((k, j) => codeOk.set(k, recipientCodeOk(codes[j][0]) && recipientCodeOk(codes[j][1]))); + })); } const kept = []; for (const c of plan.credits) { if (codeOk.get(codeKey(c)) === false) skip(c, 'contract_recipient'); else kept.push(c); } - - // The block's own time, as on the scored chain: crediting with the clock would make an old - // payment look fresh and defeat the decay. - const times = new Map(); - for (const n of new Set(kept.map((c) => c.blockNumber))) { - const block = await read('payment chain getBlock', () => provider.getBlock(n)); - if (block) times.set(n, Number(block.timestamp) * 1000); - } const registered = new Map(); const named = new Map(); for (const did of new Set(kept.map((c) => c.didHash))) { diff --git a/oracle/payment-chains.test.js b/oracle/payment-chains.test.js index 835d126..434212f 100644 --- a/oracle/payment-chains.test.js +++ b/oracle/payment-chains.test.js @@ -613,6 +613,37 @@ test('a scan of many payments stops at MAX_READS, saves its place, and the next assert.throws(() => config({ [`PAYMENT_CHAIN_${BASE_SEPOLIA}_MAX_READS`]: '0' }), /MAX_READS/); }); +test('a scan asks for its payments\' block times and code eight blocks at a time, not a round trip each', async () => { + // One at a time, a step of 1,000 reads took about 150 s at Base Sepolia's 145 ms, so payments in more + // blocks an hour than the 10-minute budget could confirm put the scan behind, and that holds every agent. + const a = agent(); + const payer = ethers.Wallet.createRandom().address; + const logs = Array.from({ length: 40 }, (_, i) => ( + transferLog({ from: payer, to: a.agentAddress, amount: 1_000_000n, block: 1100 + i }) + )); + const chain = fakeChain({ logs }); + let inFlight = 0; + let most = 0; + const slow = (fn) => async (...args) => { + inFlight += 1; + most = Math.max(most, inFlight); + await new Promise((r) => setTimeout(r, 5)); + inFlight -= 1; + return fn(...args); + }; + const provider = { + ...chain, + getCode: slow(chain.getCode), + getBlock: async (tag) => (Number.isInteger(tag) ? slow(chain.getBlock)(tag) : chain.getBlock(tag)), + }; + const { deps: d, credited } = deps({ provider, agents: [a] }); + + await pc.scanChain(config().chains[0], d); + assert.equal(credited.length, 40); + assert.ok(credited.every((c) => c.ts === (1_700_000_000 + c.blockNumber) * 1000), 'each at its own block\'s time'); + assert.equal(most, 16, 'eight blocks\' time and code in one go'); +}); + test('a scan keeps a block whole, and moves however many payments one block holds', async () => { const agents = [agent(), agent(), agent(), agent(), agent()]; const payer = ethers.Wallet.createRandom().address; diff --git a/oracle/scan-checkpoint.test.js b/oracle/scan-checkpoint.test.js index 89cae13..e6a4de0 100644 --- a/oracle/scan-checkpoint.test.js +++ b/oracle/scan-checkpoint.test.js @@ -177,3 +177,27 @@ test('an operator handed over past the registry scan\'s reach cannot pay its own assert.match(oracle.logs(), /"self_payment":1/); assert.deepEqual(await evidenceTxs(agent.didHash), [], 'refused as the operator paying its own agent'); }); + +test('the scan reads its payments\' block times ten at a time, not a round trip each', async () => { + // One at a time and unpaced, the reads ran through Arc's bucket of about 60 requests and then crawled on + // backoff, so payments in enough blocks could keep the scan behind, which holds every agent. + const world = new World(); + const agent = world.addRegisteredAgent({ registeredBlock: HEAD - 900 }); + const payer = wallet(); + const log = (block) => ({ + address: ASSET, + topics: [TRANSFER_TOPIC, ethers.zeroPadValue(payer, 32), ethers.zeroPadValue(agent.agentAddress, 32)], + data: ethers.toBeHex(5n * USDC, 32), + blockNumber: block, + transactionHash: ethers.id(`batched-${block}`), + logIndex: 0, + removed: false, + }); + world.set('scanLogs', Array.from({ length: 30 }, (_, i) => log(HEAD - 800 + i))); + world.set('blockReadMs', 5); + oracle = await startOracle({ world, env: ENV }); + await waitForEpochs(1); + assert.match(oracle.logs(), /payment scan \d+-\d+: 30 credited/); + const inFlight = oracle.calls().filter((c) => c.name === 'blockRead').map((c) => c.args[0]); + assert.equal(Math.max(...inFlight), 10, 'ten blocks\' times in one go'); +}); diff --git a/oracle/state-serialize.js b/oracle/state-serialize.js index 79d34cd..ab64576 100644 --- a/oracle/state-serialize.js +++ b/oracle/state-serialize.js @@ -23,6 +23,9 @@ const isMap = (value) => value !== null && typeof value === 'object' && !Array.i // A top-level array is written in slices this long. usedPaymentTxs is one entry per credited payment, so it is the // longest list in the file. const SLICE = 4096; +// And an entry's list, such as one agent's payments, in slices this long. An agent's whole ledger was one string: +// 11 MB at the caps, and 151 MB when a payer padded its outcomes with the longest signatures it was allowed. +const ENTRY_SLICE = 256; /** * @param {object} state in the order the keys should be written, as JSON.stringify would write them @@ -37,14 +40,24 @@ function* stateChunks(state, { pieceChars = DEFAULT_PIECE_CHARS } = {}) { if (piece.length >= pieceChars) { ready.push(piece); piece = ''; } }; function* drain() { while (ready.length) yield ready.shift(); } - // A map, one entry at a time. + // A map, one entry at a time, and a long list in an entry a slice at a time. function* entries(map) { put('{'); let firstEntry = true; for (const [k, v] of Object.entries(map)) { - const text = JSON.stringify(v); - if (text === undefined) continue; - put((firstEntry ? '' : ',') + JSON.stringify(k) + ':' + text); + if (Array.isArray(v) && v.length > ENTRY_SLICE) { + put((firstEntry ? '' : ',') + JSON.stringify(k) + ':['); + for (let i = 0; i < v.length; i += ENTRY_SLICE) { + // The inside of the list's own brackets, so the slices join with commas into one list. + put((i ? ',' : '') + JSON.stringify(v.slice(i, i + ENTRY_SLICE)).slice(1, -1)); + yield* drain(); + } + put(']'); + } else { + const text = JSON.stringify(v); + if (text === undefined) continue; + put((firstEntry ? '' : ',') + JSON.stringify(k) + ':' + text); + } firstEntry = false; yield* drain(); } @@ -94,4 +107,145 @@ function* stateChunks(state, { pieceChars = DEFAULT_PIECE_CHARS } = {}) { yield* drain(); } -module.exports = { stateChunks, DEFAULT_PIECE_CHARS }; +// ---- Reading it back --------------------------------------------------------------------------------------- +// +// load() used to read the file into one string and parse it while the string was alive, so a load peaked at about +// 2.2 times the file: at the caps, 351 MB of the 432 MB heap V8 allows in the 768 MiB container, and a file of +// about 165 MB, or one where a payer padded its outcomes, ended the process inside JSON.parse. An out-of-memory +// abort is not an exception, so the .bak was never tried, and the restart policy started it into the same failure. +// +// parseJsonBuffer gives what JSON.parse gives for the file's text, but reads the file's bytes, which sit outside +// V8's heap, and turns only pieces of about `limit` bytes into strings at a time: a container larger than that is +// walked member by member, and runs of its small members are parsed together. Its peak is the parsed state plus one +// piece. The format is unchanged, so a build before this one reads what this one writes. + +const WHITESPACE = new Set([0x20, 0x09, 0x0a, 0x0d]); +const QUOTE = 0x22; +const BACKSLASH = 0x5c; +const COMMA = 0x2c; +const COLON = 0x3a; +const OPEN_OBJECT = 0x7b; +const CLOSE_OBJECT = 0x7d; +const OPEN_LIST = 0x5b; +const CLOSE_LIST = 0x5d; + +const unexpected = (at) => new SyntaxError(`Unexpected token in JSON at position ${at}`); + +function skipSpace(buf, i, end) { + while (i < end && WHITESPACE.has(buf[i])) i += 1; + return i; +} + +// Where the string whose opening quote is at `i` ends, past its closing quote. No byte of a multi-byte UTF-8 +// character is a quote or a backslash, so the bytes can be read one at a time. +function stringEnd(buf, i, end) { + for (let j = i + 1; j < end; j += 1) { + if (buf[j] === BACKSLASH) j += 1; + else if (buf[j] === QUOTE) return j + 1; + } + throw unexpected(i); +} + +// Where the value starting at `i` ends. Brackets are only counted here: the pieces that are parsed check their own +// syntax, and the containers that are walked check theirs as they are walked. +function valueEnd(buf, i, end) { + const c = buf[i]; + if (c === QUOTE) return stringEnd(buf, i, end); + if (c === OPEN_OBJECT || c === OPEN_LIST) { + let depth = 0; + for (let j = i; j < end; j += 1) { + const b = buf[j]; + if (b === QUOTE) j = stringEnd(buf, j, end) - 1; + else if (b === OPEN_OBJECT || b === OPEN_LIST) depth += 1; + else if ((b === CLOSE_OBJECT || b === CLOSE_LIST) && --depth === 0) return j + 1; + } + throw unexpected(i); + } + let j = i; + while (j < end && !WHITESPACE.has(buf[j]) && buf[j] !== COMMA && buf[j] !== CLOSE_OBJECT && buf[j] !== CLOSE_LIST) j += 1; + if (j === i) throw unexpected(i); + return j; +} + +// The members of the container at [start, end): [keyStart, keyEnd, valueStart, valueEnd] for an object, and +// [valueStart, valueEnd] for a list. +function* membersOf(buf, start, end, keyed) { + const close = end - 1; + let i = skipSpace(buf, start + 1, close); + if (i === close) return; + for (;;) { + let keyStart; + let keyEnd; + if (keyed) { + if (buf[i] !== QUOTE) throw unexpected(i); + keyStart = i; + keyEnd = stringEnd(buf, i, close); + i = skipSpace(buf, keyEnd, close); + if (buf[i] !== COLON) throw unexpected(i); + i = skipSpace(buf, i + 1, close); + } + if (i >= close) throw unexpected(i); + const valueStop = valueEnd(buf, i, close); + yield keyed ? [keyStart, keyEnd, i, valueStop] : [i, valueStop]; + i = skipSpace(buf, valueStop, close); + if (i === close) return; + if (buf[i] !== COMMA) throw unexpected(i); + i = skipSpace(buf, i + 1, close); + } +} + +function parseRange(buf, start, end, limit) { + if (end - start <= limit) return JSON.parse(buf.toString('utf8', start, end)); + const open = buf[start]; + if (open === OPEN_OBJECT && buf[end - 1] === CLOSE_OBJECT) { + const out = {}; + for (const [ks, ke, vs, ve] of membersOf(buf, start, end, true)) { + // An own property, as JSON.parse makes, even for a key named __proto__. + Object.defineProperty(out, JSON.parse(buf.toString('utf8', ks, ke)), { + value: parseRange(buf, vs, ve, limit), enumerable: true, writable: true, configurable: true, + }); + } + return out; + } + if (open === OPEN_LIST && buf[end - 1] === CLOSE_LIST) { + const out = []; + let runStart = -1; + let runEnd = -1; + const flush = () => { + if (runStart < 0) return; + for (const v of JSON.parse(`[${buf.toString('utf8', runStart, runEnd)}]`)) out.push(v); + runStart = -1; + }; + for (const [vs, ve] of membersOf(buf, start, end, false)) { + if (ve - vs > limit) { + flush(); + out.push(parseRange(buf, vs, ve, limit)); + continue; + } + if (runStart >= 0 && ve - runStart > limit) flush(); + if (runStart < 0) runStart = vs; + runEnd = ve; + } + flush(); + return out; + } + // A container whose last byte does not close it is not JSON, and is refused without making a string of it. + if (open === OPEN_OBJECT || open === OPEN_LIST) throw unexpected(end - 1); + // A string, a number or a literal. + return JSON.parse(buf.toString('utf8', start, end)); +} + +/** + * JSON.parse for text held in a Buffer, without making one string of it. + * @param {Buffer} buf + * @param {{ limit?: number }} [options] the largest piece made into a string at once, in bytes + */ +function parseJsonBuffer(buf, { limit = DEFAULT_PIECE_CHARS } = {}) { + const start = skipSpace(buf, 0, buf.length); + let end = buf.length; + while (end > start && WHITESPACE.has(buf[end - 1])) end -= 1; + if (start === end) throw unexpected(0); + return parseRange(buf, start, end, limit); +} + +module.exports = { stateChunks, parseJsonBuffer, DEFAULT_PIECE_CHARS }; diff --git a/oracle/state-serialize.test.js b/oracle/state-serialize.test.js index ad00606..3f58160 100644 --- a/oracle/state-serialize.test.js +++ b/oracle/state-serialize.test.js @@ -132,3 +132,87 @@ test('persist then load: what one process saves, the next restores exactly', () fsx.rmSync(dir, { recursive: true, force: true }); } }); + +// ---- an agent's ledger in slices, and reading the file back a piece at a time ------------------------------- + +const { parseJsonBuffer } = require('./state-serialize'); +const BS = String.fromCharCode(92); // a backslash, written so no tool between here and the file can eat it + +test('stateChunks: an agent\'s long ledger is written in slices, the same bytes as JSON.stringify', () => { + // It was one string: 11 MB at the caps, and 151 MB when a payer padded its outcomes. + const events = Array.from({ length: 2_000 }, (_, i) => ({ + txHash: `0x${i.toString(16).padStart(64, '0')}`, ts: 1_700_000_000_000 + i, amount: '5', payer: `0x${'22'.repeat(20)}`, + success: i % 2 === 0, + })); + const state = sampleState({ paymentEvents: { '0xaa': events } }); + const pieces = Array.from(stateChunks(state, { pieceChars: 16_384 })); + assert.equal(pieces.join(''), JSON.stringify(state)); + const ledger = JSON.stringify(events).length; + assert.ok(Math.max(...pieces.map((p) => p.length)) < ledger / 4, `largest piece ${Math.max(...pieces.map((p) => p.length))} of a ${ledger}-char ledger`); +}); + +test('parseJsonBuffer: what JSON.parse gives, however small the pieces', () => { + const docs = [ + sampleState(), + { a: [1, -2.5e-3, true, false, null, 'x', [], {}, [[[]]], { b: { c: [] } }] }, + [{ 'k"ey': `va${BS}lue "quoted"`, '\u00e9\u4e2d\u{1F600}': '\u{1F600}\u4e2d\u00e9', '': '' }, BS, '"', `${BS}"`, `${BS}${BS}`], + { __proto__: 1, constructor: 2, toString: 3 }, + JSON.parse('{"__proto__":{"polluted":true},"x":1}'), + { dup: 1 }, + ]; + for (const doc of docs) { + const text = JSON.stringify(doc); + for (const limit of [1, 2, 7, 64, 1 << 20]) { + assert.deepEqual(parseJsonBuffer(Buffer.from(text), { limit }), JSON.parse(text), `limit ${limit}: ${text.slice(0, 60)}`); + } + } + // Whitespace anywhere JSON allows it, as a file edited by hand may have, and a key given twice (the last wins). + const spaced = ' \n{ "a" :\t[ 1 , 2 ,\r\n 3 ] , "b" : { "c" : "d" } , "a" : [ 4 ] }\n '; + assert.deepEqual(parseJsonBuffer(Buffer.from(spaced), { limit: 3 }), JSON.parse(spaced)); + const own = parseJsonBuffer(Buffer.from('{"__proto__":{"polluted":true}}'), { limit: 1 }); + assert.equal(Object.getPrototypeOf(own), Object.prototype, 'a __proto__ key is an own property, not the prototype'); + assert.deepEqual(Object.keys(own), ['__proto__']); + assert.equal({}.polluted, undefined); +}); + +test('parseJsonBuffer: refuses what JSON.parse refuses', () => { + const bad = [ + '', ' ', '{', '{"a":1', '{"a":1}}', '{"a":1} x', '{"a" 1}', '{"a":1 "b":2}', '{a:1}', '[1,]', '[1 2]', '[1,2}', + '{"a":[1,2}', '{"a":"unterminated}', '{"a":tru}', '{"a":1,}', '{,}', '[,1]', '{"a":1}{"b":2}', `"${BS}`, 'nul', + `{"a":"${BS}"}`, `["${BS}x"]`, + ]; + for (const text of bad) { + assert.throws(() => JSON.parse(text), SyntaxError, `JSON.parse should refuse ${text}`); + for (const limit of [1, 3, 1 << 20]) { + assert.throws(() => parseJsonBuffer(Buffer.from(text), { limit }), SyntaxError, `limit ${limit}: ${JSON.stringify(text)}`); + } + } +}); + +test('load: a state twice the size of the heap\'s room still loads, as the file is parsed a piece at a time', () => { + // Reading the file into one string and parsing it while the string was alive peaked at about 2.2 times the file. + // At the caps that was 351 MB of the 432 MB heap V8 allows in the 768 MiB container, and an out-of-memory abort + // is not an exception, so the .bak was never tried and the restart policy started it into the same failure. + const dir = fsx.mkdtempSync(pathx.join(os.tmpdir(), 'sigvara-bigload-')); + const statePath = pathx.join(dir, 'oracle-state.json'); + const run = (script, flags = []) => execFileSync(process.execPath, [...flags, '-e', script], { + env: { ...process.env, ORACLE_STATE_PATH: statePath }, cwd: __dirname, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], + }); + try { + run(` + const s = require('./store'); + const did = '0x' + 'a1'.repeat(32); + for (let i = 0; i < 220000; i++) s.usedPaymentTxs.add('0x' + i.toString(16).padStart(64, '0') + '|' + did + '|0x' + '22'.repeat(20)); + if (!s.persist()) throw new Error('not saved'); + `); + assert.ok(fsx.statSync(statePath).size > 38e6, 'a file of about 39 MB'); + const loaded = run(` + const s = require('./store'); + s.load(); + console.log(s.usedPaymentTxs.size); + `, ['--max-old-space-size=80']); + assert.equal(loaded.trim().split('\n').pop(), '220000'); + } finally { + fsx.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/oracle/store.js b/oracle/store.js index c5a8bcc..5e14cd6 100644 --- a/oracle/store.js +++ b/oracle/store.js @@ -12,7 +12,7 @@ const fs = require('fs'); const path = require('path'); -const { stateChunks } = require('./state-serialize'); +const { stateChunks, parseJsonBuffer } = require('./state-serialize'); // The decay curve lives in payments.js. Imported rather than reimplemented so flags and // payments age on exactly the same function; payments.js requires only ethers, so this // direction of dependency is not circular. @@ -464,8 +464,10 @@ function applyState(next) { stateChainId = next.chainId; } +// The file's bytes, parsed a piece at a time (state-serialize.js, parseJsonBuffer). Reading it as one string and +// parsing that peaked at about 2.2 times the file, which a state near the caps took past V8's heap. function readStateFile(file) { - return parseState(JSON.parse(fs.readFileSync(file, 'utf8'))); + return parseState(parseJsonBuffer(fs.readFileSync(file))); } /** diff --git a/oracle/test-support/stub-chain.js b/oracle/test-support/stub-chain.js index b2a7c30..7ffd90e 100644 --- a/oracle/test-support/stub-chain.js +++ b/oracle/test-support/stub-chain.js @@ -126,6 +126,8 @@ const step = (name) => { moved += world().clockSteps?.[name] ?? 0; }; const registryUnreadable = () => Object.assign(new Error('could not read the ERC-8004 registry (8004 owner)'), { code: 'NETWORK_ERROR' }); // Standing reads in flight, recorded when world.standingReadMs gives them time to overlap. let standingReads = 0; +// Numbered block reads in flight, recorded the same way when world.blockReadMs is set. +let blockReads = 0; const hostNow = Date.now; let hostAhead = 0; Date.now = () => hostNow() + hostAhead; @@ -145,13 +147,20 @@ const provider = { // 'finalized' is the head unless world.finalized holds it back (ADR 0012). A numbered block lists its transactions // only where world.blockTxs[n] says what they are: a world holds payments it has no block for, and a block without its // list reads as unsettled rather than as one that lost them. - getBlock: async (n) => (n === 'finalized' - ? { number: world().finalized ?? world().head + moved, timestamp: world().nowSec + moved - 60 } - : { + getBlock: async (n) => { + if (n === 'finalized') return { number: world().finalized ?? world().head + moved, timestamp: world().nowSec + moved - 60 }; + if (world().blockReadMs && Number.isInteger(n)) { + blockReads += 1; + record('blockRead', blockReads); + await new Promise((r) => setTimeout(r, world().blockReadMs)); + blockReads -= 1; + } + return { number: n, timestamp: world().blockTimes?.[n] ?? world().nowSec + moved - 60, ...(world().blockTxs?.[n] ? { transactions: world().blockTxs[n] } : {}), - }), + }; + }, getLogs: async (filter) => { if (filter.fromBlock === filter.toBlock) return world().canaryLogs ?? []; return (world().scanLogs ?? []).filter((l) => l.blockNumber >= filter.fromBlock && l.blockNumber <= filter.toBlock);