From 57a5f0553d1cc7d172dcc58a723234abcad5936e Mon Sep 17 00:00:00 2001 From: RunTimeAdmin <248247024+RunTimeAdmin@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:27:51 -0600 Subject: [PATCH] Fix the oracle's part of the minor findings from the 15 November performance review - 13: /health sums an outcome count kept per agent list instead of recounting every payment, and asks an unreachable ERC-8004 feed again after 30 s at most, not on every request. - 15: an epoch's per-agent steps, /score and /evidence reuse a chain-clock reading up to 5 s old. - 23: the signer answers a failed chain read "chain read failed", logs the node's words without the RPC URL, and gives a request 8 s. - 24: the watcher reads at most 1 MiB of the checker's or a webhook's answer. - 25: /attest, /flag, /flag/resolve and /link answer an error their checks did not expect "bad request", and log its message. Oracle tests: 1542. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/whitepaper.md | 2 +- oracle/chain.js | 10 ++++++++-- oracle/chain.test.js | 15 +++++++++++++++ oracle/external.js | 10 ++++++++-- oracle/external.test.js | 19 +++++++++++++++++++ oracle/http-helpers.js | 12 +++++++++++- oracle/http-helpers.test.js | 24 ++++++++++++++++++++++++ oracle/index.js | 31 +++++++++++++++++++++---------- oracle/signer.js | 21 ++++++++++++++++++--- oracle/signer.test.js | 24 ++++++++++++++++++++++-- oracle/store.js | 16 +++++++++++++++- oracle/store.test.js | 29 +++++++++++++++++++++++++++++ oracle/watcher.js | 11 ++++++++--- oracle/watcher.test.js | 9 +++++++++ 15 files changed, 209 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index b049826..dc870bd 100644 --- a/README.md +++ b/README.md @@ -601,7 +601,7 @@ Everything below runs offline, with no deployed contracts, except three fork sui | Suite | Count | Command | |---|---|---| | Contracts: unit, fuzz, E2E, upgrade, deploy-script, handover and 22 invariants | 799 | `forge test` | -| Oracle | 1532 | `cd oracle && node --test` | +| Oracle | 1542 | `cd oracle && node --test` | | SDK | 385 | `cd packages/sdk && npx vitest run` | Counts are individual tests, so each of the 22 invariants counts once. The contracts were counted on 15 November 2026, the SDK on 15 November and the oracle on 15 November. Without a fork URL, 787 of the contract tests run and 12 skip. Run after `npm run build`, 376 of the SDK's 385 pass and 9 skip: the 8 integration tests, which need a live deployment, and one check that runs only in CI. Without a build, the 9 built-package tests skip too. Vitest strips types without checking them, so `npm run typecheck` in `packages/sdk` checks the tests against the SDK's types as well as the sources. [`scripts/mutation`](scripts/mutation/README.md) holds the harnesses that measure whether these tests would catch a deliberate break in the contracts or deploy scripts. diff --git a/docs/whitepaper.md b/docs/whitepaper.md index 80d9bfc..2a3bf28 100644 --- a/docs/whitepaper.md +++ b/docs/whitepaper.md @@ -635,7 +635,7 @@ apply to a fresh deployment, not to the testnet. **Test coverage**, as of 15 November 2026: 799 contract tests across 36 Foundry test files including fuzz and invariant tests (the two slash drill suites fork Arc testnet themselves, and -the twelve in `SplitAuthorityFork` skip without `--fork-url`), 1532 oracle tests and 385 in the SDK (9 skip +the twelve in `SplitAuthorityFork` skip without `--fork-url`), 1542 oracle tests and 385 in the SDK (9 skip outside CI and without a live deployment; every test that runs passes). CI runs Foundry, the oracle and SDK suites, and Slither static analysis on every push, plus a documentation check that fails when the site states something the contracts do not. diff --git a/oracle/chain.js b/oracle/chain.js index 6b58d37..4339fa2 100644 --- a/oracle/chain.js +++ b/oracle/chain.js @@ -1061,9 +1061,15 @@ async function getChallengeWindow() { // would trigger premature finalize attempts that just revert and waste gas). // Retried like every other read: an epoch reads this before anything else, so one throttled // answer used to cost the whole epoch. -async function getLatestBlockTimestamp() { +// The latest block's time and when it was read. A caller that can use a reading a few seconds old passes `maxAgeMs`: +// an epoch's per-agent steps and the read routes each asked the chain one round trip at a time. +let latestBlockTime = null; + +async function getLatestBlockTimestamp({ maxAgeMs = 0 } = {}) { + if (maxAgeMs > 0 && latestBlockTime && Date.now() - latestBlockTime.atMs <= maxAgeMs) return latestBlockTime.seconds; const block = await readWithBackoff('latest block', () => provider.getBlock('latest')); - return Number(block.timestamp); + latestBlockTime = { seconds: Number(block.timestamp), atMs: Date.now() }; + return latestBlockTime.seconds; } // ---- Epoch-fee gating (opt-in) -------------------------------------------- diff --git a/oracle/chain.test.js b/oracle/chain.test.js index b89b3e0..c4fedff 100644 --- a/oracle/chain.test.js +++ b/oracle/chain.test.js @@ -999,3 +999,18 @@ test('isRateLimited and isSlotTaken do not overlap', () => { assert.equal(chain.isSlotTaken(taken), true); assert.equal(chain.isRateLimited(taken), false); }); + +// An epoch's per-agent steps, /score and /evidence each read the chain's clock one round trip at a time. +test('getLatestBlockTimestamp reuses a reading up to maxAgeMs old, and reads afresh without one', async () => { + let reads = 0; + chain.init(CFG, { + provider: { getBlock: async (tag) => { assert.equal(tag, 'latest'); reads += 1; return { timestamp: 1000 + reads }; } }, + identityContract: makeFakeIdentityContract({ events: [] }), + reputationContract: makeFakeReputationContract(), + }); + assert.equal(await chain.getLatestBlockTimestamp(), 1001); + assert.equal(await chain.getLatestBlockTimestamp({ maxAgeMs: 5_000 }), 1001, 'a reading under 5 s old is reused'); + assert.equal(reads, 1); + assert.equal(await chain.getLatestBlockTimestamp(), 1002, 'without maxAgeMs the chain is read'); + assert.equal(reads, 2); +}); diff --git a/oracle/external.js b/oracle/external.js index 6ed493d..36fa952 100644 --- a/oracle/external.js +++ b/oracle/external.js @@ -117,6 +117,8 @@ let chainIdError = null; // The check in flight, which every caller shares, and when the last one finished (ms). let checking = null; let checkedAt = 0; +// How long a feed that could not be reached is left before a caller that accepts a cached answer asks again. +const UNREACHABLE_RECHECK_MS = 30_000; // deps lets tests inject fake contracts. Production (index.js) calls init(cfg). @@ -127,9 +129,10 @@ function init(cfg, deps = {}) { idContract = deps.idContract ?? new ethers.Contract(cfg.externalIdentity, ID_ABI, provider); repContract = deps.repContract ?? new ethers.Contract(cfg.externalReputation, REP_ABI, provider); // A re-init points at a different chain, so anything cached about the old one is a - // lie until proven otherwise. + // lie until proven otherwise, and the new one is asked at once. chainId = null; chainIdError = null; + checkedAt = 0; } /** @@ -142,10 +145,13 @@ function init(cfg, deps = {}) { * A caller arriving while a check is in flight waits for it rather than reading the cache. * A cached id younger than `maxAgeMs` is returned without asking; the default asks every * time. An id that cannot be read again is dropped, so a failed re-check reads as unreachable. + * A feed that could not be reached is asked again after at most UNREACHABLE_RECHECK_MS for a + * caller that accepts a cached answer: /health asked a down feed on every request. */ async function refreshChainId(maxAgeMs = 0, nowMs = Date.now()) { if (!enabled) return chainId; - if (!checking && (chainId === null || nowMs - checkedAt >= maxAgeMs)) { + const ageLimit = chainId === null ? Math.min(maxAgeMs, UNREACHABLE_RECHECK_MS) : maxAgeMs; + if (!checking && nowMs - checkedAt >= ageLimit) { checking = checkChainId().finally(() => { checkedAt = Date.now(); checking = null; }); } return checking ?? chainId; diff --git a/oracle/external.test.js b/oracle/external.test.js index a4cdd69..c7d6d98 100644 --- a/oracle/external.test.js +++ b/oracle/external.test.js @@ -575,3 +575,22 @@ test('readLinkLogs: filters on the key, and returns each log\'s agent, value and assert.deepEqual(calls, [['queryFilter', 'sigvara', 100, 199]]); external.init({ externalRpc: '', externalIdentity: '', externalReputation: '' }); }); + +// /health asked a feed it could not reach on every request, each ask up to READ_TIMEOUT_MS. +test('refreshChainId: an unreachable feed is asked again after 30 s for a caller that takes a cached answer, at once for one that does not', async () => { + let calls = 0; + external.init(CFG, { + provider: { send: async () => { calls += 1; throw new Error('ECONNREFUSED'); } }, + idContract: {}, repContract: {}, + }); + const FIVE_MIN = 5 * 60_000; + const t = Date.now(); + assert.strictEqual(await external.refreshChainId(FIVE_MIN, t), null); + assert.strictEqual(calls, 1); + assert.strictEqual(await external.refreshChainId(FIVE_MIN, t + 10_000), null); + assert.strictEqual(calls, 1, 'not asked again within 30 s'); + assert.strictEqual(await external.refreshChainId(FIVE_MIN, t + 31_000), null); + assert.strictEqual(calls, 2, 'asked again after 30 s'); + assert.strictEqual(await external.refreshChainId(0, t + 31_001), null); + assert.strictEqual(calls, 3, 'a caller that takes no cached answer is always asked for'); +}); diff --git a/oracle/http-helpers.js b/oracle/http-helpers.js index 8e7b026..353f071 100644 --- a/oracle/http-helpers.js +++ b/oracle/http-helpers.js @@ -46,6 +46,15 @@ function bodyErrorStatus(err) { return 400; } +// What a route answers for an error its own checks did not catch, and what it logs. A body readBody could not take is +// said as it is. Anything else is answered `bad request`, and its words go only to the log: they can name what a +// caller should not see, such as a failing node's URL. +const READ_ERRORS = new Set(['body_too_large', 'body_timeout', 'bad_json']); +function uncaughtReply(err) { + if (err && READ_ERRORS.has(err.code)) return { status: bodyErrorStatus(err), body: { error: err.message }, log: null }; + return { status: 400, body: { error: 'bad request' }, log: String((err && err.message) || err) }; +} + // req only needs to be an EventEmitter emitting 'data' | 'end' | 'error' and // exposing destroy() — a real http.IncomingMessage satisfies this, and so // does a plain fake in tests. A declared content-length is read from @@ -79,7 +88,7 @@ async function readBody(req, maxBodySize = MAX_BODY_SIZE, deadlineMs = BODY_DEAD req.on('end', () => { clearTimeout(deadline); try { resolve(JSON.parse(Buffer.concat(chunks).toString() || '{}')); } - catch { reject(new Error('Invalid JSON')); } + catch { reject(Object.assign(new Error('Invalid JSON'), { code: 'bad_json' })); } }); req.on('error', (err) => { clearTimeout(deadline); @@ -461,6 +470,7 @@ module.exports = { json, readBody, bodyErrorStatus, + uncaughtReply, readCredentials, identifyCaller, mayAttestUnauthenticated, diff --git a/oracle/http-helpers.test.js b/oracle/http-helpers.test.js index 00234d6..a634722 100644 --- a/oracle/http-helpers.test.js +++ b/oracle/http-helpers.test.js @@ -655,3 +655,27 @@ test('weakCredentials: names the credentials shorter than 32 characters, never t const weak = weakCredentials(new Map([['admin', 'x'.repeat(31)], ['ok', 'y'.repeat(32)], ['tiny', 'z']])); assert.deepEqual(weak, ['admin', 'tiny']); }); + +// /attest and the token routes answered an error none of their checks expected with its raw message, which can name +// what a caller should not see. +test('uncaughtReply: a body that could not be read is said as it is; anything else is a bad request, its words kept for the log', () => { + const { uncaughtReply } = require('./http-helpers'); + const tooBig = Object.assign(new Error('Request body too large'), { code: 'body_too_large' }); + assert.deepEqual(uncaughtReply(tooBig), { status: 413, body: { error: 'Request body too large' }, log: null }); + const badJson = Object.assign(new Error('Invalid JSON'), { code: 'bad_json' }); + assert.deepEqual(uncaughtReply(badJson), { status: 400, body: { error: 'Invalid JSON' }, log: null }); + const leak = new Error('request to https://rpc.example.invalid/v2/very-secret-key failed'); + assert.deepEqual(uncaughtReply(leak), { status: 400, body: { error: 'bad request' }, log: leak.message }); +}); + +test('readBody: invalid JSON is refused as a body error, with its own words', async () => { + const { readBody, uncaughtReply } = require('./http-helpers'); + const { EventEmitter } = require('node:events'); + const req = Object.assign(new EventEmitter(), { headers: {}, destroy() {} }); + const reading = readBody(req); + req.emit('data', Buffer.from('{')); + req.emit('end'); + const err = await reading.catch((e) => e); + assert.equal(err.code, 'bad_json'); + assert.deepEqual(uncaughtReply(err).body, { error: 'Invalid JSON' }); +}); diff --git a/oracle/index.js b/oracle/index.js index 305b955..0ec48a7 100644 --- a/oracle/index.js +++ b/oracle/index.js @@ -29,7 +29,7 @@ const { const { createBondWatch } = require('./bond-watch'); const { numberSetting } = require('./config'); const { - json, readBody, bodyErrorStatus, readCredentials, identifyCaller, mayAttestUnauthenticated, parseScorePath, parseDidHash, + json, readBody, uncaughtReply, readCredentials, identifyCaller, mayAttestUnauthenticated, parseScorePath, parseDidHash, rateLimited, clientKey, adminTokenPolicyError, runningCommit, parseTrustedProxies, parseRequestTarget, readScopes, mayUse, weakCredentials, MIN_TOKEN_LENGTH, ATTEST_MAX_BODY_SIZE, } = require('./http-helpers'); @@ -435,6 +435,10 @@ const NOBODY_NAMED = () => undefined; // A transaction hash as a caller may send it, in either case. const TX_HASH = /^0x[0-9a-fA-F]{64}$/; +// How old a reading of the chain's clock an epoch's per-agent steps and the read routes take: each read it one round +// trip at a time, against a margin of minutes. The epoch's own reading, at its start, is always fresh. +const CHAIN_TIME_REUSE_MS = 5_000; + /** * How the checker reads a proposal's evidence (evidence-as-of.js). With a payment chain on, at each * whole-minute cut-off in the half hour before it (payment-chains.auditCuts), since the proposer chose its @@ -1967,7 +1971,7 @@ async function runEpochInner() { // One that starts to count after this evidence time and before the proposal can land: the checker reads the // evidence at the proposal's own time, so it would count it where this does not. Proposed next epoch. if (!cfg.checkerMode && chosen.some(negativeHold.isPayerNegative) && negativeHold.startingBetween( - chosen, evidenceCutMs, (await chain.getLatestBlockTimestamp()) * 1000 + proposeMarginMs, negativeHoldCfg, + chosen, evidenceCutMs, (await chain.getLatestBlockTimestamp({ maxAgeMs: CHAIN_TIME_REUSE_MS })) * 1000 + proposeMarginMs, negativeHoldCfg, ).length) { console.log(`[oracle] ${didHash.slice(0, 10)}… a payer negative starts to count before a proposal could land, skipping`); agentsHeldByNegative++; @@ -1979,7 +1983,7 @@ async function runEpochInner() { // landing after midnight, it was audited against a reading it was never given. So a linked agent // whose snapshot would change before its proposal could land is proposed next epoch instead. if (!cfg.checkerMode && external.configured() && hasLink(didHash)) { - const landsByMs = (await chain.getLatestBlockTimestamp()) * 1000 + proposeMarginMs; + const landsByMs = (await chain.getLatestBlockTimestamp({ maxAgeMs: CHAIN_TIME_REUSE_MS })) * 1000 + proposeMarginMs; if (reviewers.snapshotOf(Math.floor(scoringNowMs / 1000)) !== reviewers.snapshotOf(Math.floor(landsByMs / 1000))) { console.log(`[oracle] ${didHash.slice(0, 10)}… its external factor's snapshot changes before a proposal could land; proposing next epoch`); continue; @@ -2297,7 +2301,7 @@ async function runEpochInner() { console.log(`[oracle] ${didHash.slice(0, 10)}… a transaction from this wallet has not been seen mined; proposing next epoch`); continue; } - if (cutTooOld((await chain.getLatestBlockTimestamp()) * 1000, evidenceCutMs)) { + if (cutTooOld((await chain.getLatestBlockTimestamp({ maxAgeMs: CHAIN_TIME_REUSE_MS })) * 1000, evidenceCutMs)) { console.log(`[oracle] ${didHash.slice(0, 10)}… its evidence was chosen too long ago to propose; proposing next epoch`); continue; } @@ -2517,6 +2521,13 @@ function saveSoon() { // ---- HTTP API -------------------------------------------------------------- +// The answer to an error a write route's own checks did not catch (http-helpers.uncaughtReply), with its words logged. +function answerUncaught(res, err, route) { + const reply = uncaughtReply(err); + if (reply.log) console.warn(`[oracle] ${route} failed: ${reply.log}`); + return json(res, reply.status, reply.body); +} + // 32MB of serialized evidence responses. The revision token is the payment-event array // itself: the store replaces it on every change, so nothing has to remember to // invalidate this, and a case it somehow missed degrades to a cache miss rather than to @@ -3853,7 +3864,7 @@ async function handleRequest(req, res) { }); } catch (err) { metrics.inc('attestRejectedOther'); - return json(res, bodyErrorStatus(err), { error: err.message }); + return answerUncaught(res, err, '/attest'); } } @@ -3965,7 +3976,7 @@ async function handleRequest(req, res) { let nowMs = Date.now(); if (getPaymentEvents(didHash).some(negativeHold.isPayerNegative)) { try { - nowMs = (await chain.getLatestBlockTimestamp()) * 1000; + nowMs = (await chain.getLatestBlockTimestamp({ maxAgeMs: CHAIN_TIME_REUSE_MS })) * 1000; } catch { /* this host's clock */ } } const events = evidenceAt(didHash, paymentChains.cutFor(nowMs, paymentChainsCfg)); @@ -4025,7 +4036,7 @@ async function handleRequest(req, res) { if (replyKey) rememberReply(caller, replyKey, { didHash, flags: raised }); return savedReply(res, durable, { didHash, flags: raised }); } catch (err) { - return json(res, bodyErrorStatus(err), { error: err.message }); + return answerUncaught(res, err, '/flag'); } } @@ -4070,7 +4081,7 @@ async function handleRequest(req, res) { if (replyKey) rememberReply(caller, replyKey, { didHash, ...result }); return savedReply(res, durable, { didHash, ...result }); } catch (err) { - return json(res, bodyErrorStatus(err), { error: err.message }); + return answerUncaught(res, err, '/flag/resolve'); } } @@ -4118,7 +4129,7 @@ async function handleRequest(req, res) { metrics.inc('linksCreated'); return savedReply(res, durable, { didHash, agentId: String(agentId), operator, linked: true }); } catch (err) { - return json(res, bodyErrorStatus(err), { error: err.message }); + return answerUncaught(res, err, '/link'); } } @@ -4249,7 +4260,7 @@ async function handleRequest(req, res) { // epoch path removes, in the place people look to explain it. // // One extra chain read per request, on an endpoint that already makes several. - const scoringNowMs = (await chain.getLatestBlockTimestamp()) * 1000; + const scoringNowMs = (await chain.getLatestBlockTimestamp({ maxAgeMs: CHAIN_TIME_REUSE_MS })) * 1000; const held = evidenceAt(didHash, paymentChains.cutFor(scoringNowMs, paymentChainsCfg)); const standingsReady = await previewStandingsReady(held, scoringNowMs); if (!standingsReady.ready) { diff --git a/oracle/signer.js b/oracle/signer.js index 47f9938..c10eb4b 100644 --- a/oracle/signer.js +++ b/oracle/signer.js @@ -33,6 +33,8 @@ const { ethers } = require('ethers'); const { HEADER, MAX_SKEW_SECONDS, authProblem, readSecret } = require('./signer-auth'); const { readWithBackoff } = require('./read-retry'); const { rpcProvider } = require('./rpc-provider'); +const { redact } = require('./event-index'); +const { wordsOf } = require('./log-spans'); const FEEDBACK = new ethers.Interface([ 'function giveFeedback(uint256 agentId, int128 value, uint8 valueDecimals, string tag1, string tag2, string endpoint, string feedbackURI, bytes32 feedbackHash)', @@ -160,7 +162,7 @@ function signedRecord(file) { * within MAX_SKEW_SECONDS of this clock either side, so one stamped ahead of the clock just before a * restart can come again until twice that after it. */ -function createSigner({ wallet, policy, secret, allowFrom, maxPerHour, chainId, chain, record, log = console.log, now = () => Math.floor(Date.now() / 1000), warmup = 2 * MAX_SKEW_SECONDS }) { +function createSigner({ wallet, policy, secret, allowFrom, maxPerHour, chainId, chain, record, log = console.log, now = () => Math.floor(Date.now() / 1000), warmup = 2 * MAX_SKEW_SECONDS, said = wordsOf }) { const seen = new Map(); const startedAt = now(); let busy = Promise.resolve(); @@ -172,7 +174,12 @@ function createSigner({ wallet, policy, secret, allowFrom, maxPerHour, chainId, async function sign(method, params, t) { if (t < startedAt + warmup) return { error: `starting; signing from ${warmup} s after start` }; if (record.since(t) >= maxPerHour) return { error: `no more than ${maxPerHour} signatures an hour` }; - const state = await chain(); + let state; + try { + state = await chain(); + } catch (err) { + throw Object.assign(new Error(said(err)), { code: 'chain_read' }); + } if (method === 'eth_signTransaction') { const tx = params[0] ?? {}; const why = policy.transaction(tx, state); @@ -225,6 +232,12 @@ function createSigner({ wallet, policy, secret, allowFrom, maxPerHour, chainId, const { result, error } = await done; return answer(res, id, result, error); } catch (err) { + // The node's own words can name the RPC, key and all, so they are logged shortened and without the URL, and the + // caller is told only that the read failed. It is the node's trouble, not the request's. + if (err.code === 'chain_read') { + log(`[signer] chain read failed: ${String(err.message).slice(0, 160)}`); + return answer(res, id, null, 'chain read failed'); + } return answer(res, id, null, `bad request: ${err.message}`); } } @@ -310,7 +323,8 @@ async function main(argv = process.argv.slice(2), env = process.env) { const password = fs.readFileSync(privateFile(cfg.passwordFile), 'utf8').trim(); const wallet = await ethers.Wallet.fromEncryptedJson(fs.readFileSync(privateFile(cfg.keystore), 'utf8'), password); const secret = readSecret(privateFile(cfg.secretFile)); - const provider = rpcProvider(cfg.rpcUrl, cfg.chainId); + // 8 s a request: a publisher waits on each signature, and a node that has not answered by then will not. + const provider = rpcProvider(cfg.rpcUrl, cfg.chainId, { timeoutMs: 8_000 }); // Retried here on a throttle or a timeout: the provider tries nothing again itself (rpc-provider.js). const chain = () => readWithBackoff('signer chain', async () => { const [pendingNonce, block] = await Promise.all([provider.getTransactionCount(wallet.address, 'pending'), provider.getBlock('latest')]); @@ -320,6 +334,7 @@ async function main(argv = process.argv.slice(2), env = process.env) { const policy = createPolicy({ ...cfg, address: wallet.address }); const handler = createSigner({ wallet, policy, secret, allowFrom: cfg.allowFrom, maxPerHour: cfg.maxPerHour, chainId: cfg.chainId, chain, record: signedRecord(cfg.stateFile), + said: (err) => redact(wordsOf(err), [cfg.rpcUrl]), }); createServer(handler, cfg.allowFrom).listen(cfg.port, cfg.host, () => { console.log(`[signer] ${wallet.address} on ${cfg.host}:${cfg.port}, from ${[...cfg.allowFrom].join(', ')}, at most ${cfg.maxPerHour} an hour`); diff --git a/oracle/signer.test.js b/oracle/signer.test.js index 3b511ca..581033c 100644 --- a/oracle/signer.test.js +++ b/oracle/signer.test.js @@ -137,12 +137,13 @@ test('signedRecord: an hour\'s window, kept across a restart', () => { // ---- the server, with the publisher and agent-wallet.js as its clients ------------------ -async function serve({ allowFrom = new Set(['127.0.0.1']), maxPerHour = 60, warmup = 0, pendingNonce = 4 } = {}) { +async function serve({ allowFrom = new Set(['127.0.0.1']), maxPerHour = 60, warmup = 0, pendingNonce = 4, chain = null, said = undefined } = {}) { const logs = []; const state = { pendingNonce }; const handler = createSigner({ wallet: KEY, policy, secret: SECRET, allowFrom, maxPerHour, chainId: CHAIN, warmup, record: signedRecord(null), log: (l) => logs.push(l), - chain: async () => ({ pendingNonce: state.pendingNonce, time: Math.floor(Date.now() / 1000) }), + chain: chain ?? (async () => ({ pendingNonce: state.pendingNonce, time: Math.floor(Date.now() / 1000) })), + ...(said ? { said } : {}), }); const server = createServer(handler, allowFrom); await new Promise((r) => server.listen(0, '127.0.0.1', r)); @@ -329,3 +330,22 @@ test('readConfig: a cap that does not parse is refused, not read as no cap', () assert.throws(() => readConfig({ ...base, SIGNER_ALLOW_FROM: '' }), /SIGNER_ALLOW_FROM is required/); assert.throws(() => readConfig({ ...base, SIGNER_AGENT_OWNER: '' }), /SIGNER_AGENT_OWNER is required/); }); + +// The signer answered a failed chain read with the provider's own message, which can hold the RPC URL and its key. +test('a failed chain read is answered "chain read failed", and logged without the RPC URL', async () => { + const { redact } = require('./event-index'); + const { wordsOf } = require('./log-spans'); + const RPC = 'https://rpc.example.invalid/v2/very-secret-key'; + const s = await serve({ + chain: async () => { throw new Error(`request to ${RPC} failed: socket hang up`); }, + said: (err) => redact(wordsOf(err), [RPC]), + }); + try { + const signer = client(s, fakeProvider(s)); + await assert.rejects(signer.send('revoke', 7, [7, 1]), (err) => /chain read failed/.test(err.message) && !err.message.includes('very-secret-key')); + assert.ok(s.logs.some((l) => l.startsWith('[signer] chain read failed')), s.logs.join(' | ')); + assert.ok(s.logs.every((l) => !l.includes('very-secret-key')), s.logs.join(' | ')); + } finally { + await s.close(); + } +}); diff --git a/oracle/store.js b/oracle/store.js index dc23af4..04bfbea 100644 --- a/oracle/store.js +++ b/oracle/store.js @@ -1333,10 +1333,24 @@ function paymentVolume(didHash) { /// not grow without bound. At a 90-day half-life and the default floor this keeps /// roughly the last three years, by which point an event contributes under a /// thousandth of its original value and cannot move an integer score. +// How many of a list's payments carry an outcome, counted once a list. The store replaces an agent's list on every +// change and never edits one, so a count cannot go stale. /health recounted every payment on every request: 4 to +// 19 ms a request at the caps (performance review, 15 November). +const outcomeCounts = new WeakMap(); +function outcomesIn(list) { + let n = outcomeCounts.get(list); + if (n === undefined) { + n = 0; + for (const e of list) if (hasOutcome(e)) n += 1; + outcomeCounts.set(list, n); + } + return n; +} + /** The ledger's size against its caps, and how many payments the caps dropped since this process started. */ const ledgerStatus = () => { let withOutcome = 0; - for (const list of paymentEvents.values()) for (const e of list) if (hasOutcome(e)) withOutcome += 1; + for (const list of paymentEvents.values()) withOutcome += outcomesIn(list); return { events: eventTotal, maxTotal: MAX_EVENTS_TOTAL, maxPerAgent: MAX_EVENTS_PER_AGENT, dropped: ledgerDrops, // What the caps also bound (boundDropped): the dropped payments' keys, and the agents that have lost some. diff --git a/oracle/store.test.js b/oracle/store.test.js index ed7b0f4..df7ece5 100644 --- a/oracle/store.test.js +++ b/oracle/store.test.js @@ -1288,3 +1288,32 @@ test('a page of outcomes costs its agents\' events once, not once an entry', () // About 1 s with a .find an entry on the machine this was written on, about 10 ms indexed. assert.ok(got.ms < 200, `a page of 500 took ${got.ms.toFixed(0)} ms`); }); + +// ---- /health ------------------------------------------------------------------------------------------------------ + +// /health recounted every payment's outcome on every request: 4 to 19 ms a request at the caps. +test('the ledger status counts outcomes once a list, and follows a list that changes', () => { + const out = inChildWithState(null, ` + const s = require('./store'); + const tx = (n) => '0x' + n.toString(16).padStart(64, '0'); + const did = (a) => '0x' + a.toString(16).padStart(64, '0'); + const payer = '0x' + '11'.repeat(20); + const credits = []; + for (let a = 1; a <= 50; a += 1) for (let n = 0; n < 1000; n += 1) { + credits.push({ didHash: did(a), txHash: tx(a * 100000 + n), amount: '5', payer, success: n % 2 === 0 ? true : null, ts: 1000 + n }); + } + s.creditPayments(credits); + const first = s.ledgerStatus().withOutcome; + const t = process.hrtime.bigint(); + for (let i = 0; i < 200; i += 1) s.ledgerStatus(); + const ms = Number(process.hrtime.bigint() - t) / 1e6; + const signed = { outcome: 1, signedAt: 1, deadline: 9, signature: '0x', signer: 'eoa', checkedAtBlock: 40, checkedAtMs: 4000 }; + s.recordPayerOutcome(did(7), tx(7 * 100000 + 1), payer, true, signed, { windowMs: 1 }); + console.log(JSON.stringify({ first, after: s.ledgerStatus().withOutcome, ms })); + `); + const got = JSON.parse(out.trim().split('\n').at(-1)); + assert.equal(got.first, 25_000); + assert.equal(got.after, 25_001, 'a payment given its outcome is counted'); + // Recounting 50,000 payments 200 times took about 200 ms where this was written; summing 50 counts, under 5. + assert.ok(got.ms < 50, `200 status reads took ${got.ms.toFixed(0)} ms`); +}); diff --git a/oracle/watcher.js b/oracle/watcher.js index 78e05cb..4fdb3bd 100644 --- a/oracle/watcher.js +++ b/oracle/watcher.js @@ -37,7 +37,12 @@ const { checkerStatus, classifyDivergence, triage, nextAlert, webhookPayload, describeDivergence, trackPollOutcome, } = require('./watcher-policy'); const { redact } = require('./event-index'); -const { wordsOf } = require('./log-spans'); +const { wordsOf, readAnswer } = require('./log-spans'); + +// The most of an answer this watcher reads, from the checker or a webhook: its /divergence page is 100 records at most. +// An answer was read whole however long it ran, so a broken or hostile server could fill this process's memory. +const MAX_ANSWER_BYTES = 1024 * 1024; +const readCapped = async (res) => (res.body ? (await readAnswer(res.body, MAX_ANSWER_BYTES)).toString('utf8') : ''); const { readWithBackoff } = require('./read-retry'); const { rpcProvider } = require('./rpc-provider'); @@ -106,7 +111,7 @@ async function getJson(path, { acceptStatus = [] } = {}) { signal: AbortSignal.timeout(20_000), redirect: 'error', headers: { 'User-Agent': WATCHER_AGENT }, }); if (!res.ok && !acceptStatus.includes(res.status)) throw new Error(`${path} returned ${res.status}`); - const body = await res.text(); + const body = await readCapped(res); // Not the parser's message, which quotes the body: what this server sends is not ours to repeat. try { return JSON.parse(body); } catch { throw new Error(`${path} did not return JSON`); } } @@ -133,7 +138,7 @@ async function notify(level, title, lines) { // payload or a revoked webhook counts as delivered. That is the exact failure this // component cannot afford: believing it raised an alarm that nobody heard. if (!res.ok) { - const detail = await res.text().catch(() => ''); + const detail = await readCapped(res).catch(() => ''); console.error(`[watcher] webhook rejected the alert: ${res.status} ${detail.slice(0, 200)}`); } } catch (err) { diff --git a/oracle/watcher.test.js b/oracle/watcher.test.js index 1eeff38..d22db40 100644 --- a/oracle/watcher.test.js +++ b/oracle/watcher.test.js @@ -282,3 +282,12 @@ test('watcher: a credit the checker removed after a reorganization is paged once assert.match(alerts.filter((a) => a.includes('credit the chain no longer holds'))[1], /holds 2 credit\(s\) removed after a reorganization, 1 new/); }); }); + +// The watcher read an answer whole however long it ran, so a broken or hostile checker could fill its memory. +test('watcher: an answer over 1 MiB is refused, not read whole', async () => { + const divergence = (res) => json(res, { mode: 'checker', seesProposals: true, divergences: [], pad: 'x'.repeat(2 * 1024 * 1024) }); + await withWatcher({ divergence }, async ({ alerts }) => { + await until(() => alerts.some((a) => a.includes('the checker is not checking'))); + assert.match(alerts.find((a) => a.includes('the checker is not checking')), /past 1048576 bytes/); + }); +});