diff --git a/README.md b/README.md index 4690f8a..90423f8 100644 --- a/README.md +++ b/README.md @@ -601,7 +601,7 @@ Everything below runs offline, with no deployed contracts, except three fork sui | Suite | Count | Command | |---|---|---| | Contracts: unit, fuzz, E2E, upgrade, deploy-script, handover and 22 invariants | 815 | `forge test` | -| Oracle | 1544 | `cd oracle && node --test` | +| Oracle | 1545 | `cd oracle && node --test` | | SDK | 392 | `cd packages/sdk && npx vitest run` | Counts are individual tests, so each of the 22 invariants counts once. The contracts were counted on 15 November 2026, the SDK on 15 November and the oracle on 15 November. Without a fork URL, 803 of the contract tests run and 12 skip. Run after `npm run build`, 383 of the SDK's 392 pass and 9 skip: the 8 integration tests, which need a live deployment, and one check that runs only in CI. Without a build, the 9 built-package tests skip too. Vitest strips types without checking them, so `npm run typecheck` in `packages/sdk` checks the tests against the SDK's types as well as the sources. [`scripts/mutation`](scripts/mutation/README.md) holds the harnesses that measure whether these tests would catch a deliberate break in the contracts or deploy scripts. diff --git a/docs/whitepaper.md b/docs/whitepaper.md index 7517824..d57ec91 100644 --- a/docs/whitepaper.md +++ b/docs/whitepaper.md @@ -635,7 +635,7 @@ apply to a fresh deployment, not to the testnet. **Test coverage**, as of 15 November 2026: 815 contract tests across 36 Foundry test files including fuzz and invariant tests (the two slash drill suites fork Arc testnet themselves, and -the twelve in `SplitAuthorityFork` skip without `--fork-url`), 1544 oracle tests and 392 in the SDK (9 skip +the twelve in `SplitAuthorityFork` skip without `--fork-url`), 1545 oracle tests and 392 in the SDK (9 skip outside CI and without a live deployment; every test that runs passes). CI runs Foundry, the oracle and SDK suites, and Slither static analysis on every push, plus a documentation check that fails when the site states something the contracts do not. diff --git a/oracle/chain.js b/oracle/chain.js index 4339fa2..586164d 100644 --- a/oracle/chain.js +++ b/oracle/chain.js @@ -14,7 +14,7 @@ const STAKE_VIEW_ABI = ['function hasMinimumStake(bytes32 didHash) view returns const { SCORE_FACTORS } = require('./epoch-policy'); const { readSplitting, spanRefused } = require('./log-spans'); -const { isTransientReadError, readWithBackoff } = require('./read-retry'); +const { isTransientReadError, readWithBackoff, isRateLimited } = require('./read-retry'); const { rpcProvider } = require('./rpc-provider'); const reviewers = require('./reviewers'); const { createFunderReader } = require('./funder-scan'); @@ -964,33 +964,6 @@ function isAlreadyCharged(err) { return namedError(err, 'AlreadyCharged'); } /// than anything being wrong. Matched as namedError does: decoded name, revert data, text. function isSlotTaken(err) { return namedError(err, 'ScoreAlreadyPending'); } -/** - * True when the node refused the request because we are being throttled, rather than - * because anything is wrong with the transaction. - * - * Reads already treat throttling as transient and retry, in readWithBackoff. Writes - * deliberately do not, and that is unchanged here: proposeScore and finalizeScore race - * other parties for one slot, and resending something that may already be in the mempool - * is how you pay twice for one epoch. This only names the cause. - * - * Naming it matters because the alternative was counting it as a propose failure. - * `proposeErrors` should mean this operator could not do its job; on the live checker it - * also meant the provider said slow down, which is a different problem with a different - * fix, and the log line was a wall of JSON-RPC payload rather than a sentence. - * - * -32005 is the JSON-RPC code providers use for this. ethers wraps it in "could not - * coalesce error" with the original nested inside, so the string match is doing real work - * and is not just belt and braces. - */ -function isRateLimited(err) { - if (!err) return false; - const nested = err.error || err.info?.error || {}; - if (nested.code === -32005 || err.code === -32005) return true; - return /rate limit|429|too many requests/i.test( - String(err.shortMessage || err.message || '') - ); -} - async function finalizeScore(didHash) { // Asked first, with retries, as canPropose asks before a proposal. Sending estimates gas and is not // retried, because a write must not be repeated, so a node refusing that estimate for its rate limit diff --git a/oracle/chain.test.js b/oracle/chain.test.js index c4fedff..a4188b6 100644 --- a/oracle/chain.test.js +++ b/oracle/chain.test.js @@ -1014,3 +1014,43 @@ test('getLatestBlockTimestamp reuses a reading up to maxAgeMs old, and reads afr assert.equal(await chain.getLatestBlockTimestamp(), 1002, 'without maxAgeMs the chain is read'); assert.equal(reads, 2); }); + +// The checker's node answers a burst with HTTP 429 and -32005. ethers used to retry that inside each try, for minutes; +// since rpc-provider.js stopped it, four tries over 3.5 s left 2 to 4 agents an hourly epoch unread (16 November 2026). +const throttled429 = () => Object.assign( + new Error('server response 429 Too Many Requests (responseBody="{\\"jsonrpc\\":\\"2.0\\",\\"error\\":{\\"code\\":-32005,\\"message\\":\\"rate limit exceeded\\"}}")'), + { code: 'SERVER_ERROR' }, +); + +test('readWithBackoff: a refusal for the rate limit is tried on a longer schedule, its waits jittered and capped', async () => { + const waits = []; + let calls = 0; + const out = await chain.readWithBackoff('x', async () => { + if (++calls < 7) throw throttled429(); + return 'read'; + }, 4, undefined, { pause: async (ms) => { waits.push(ms); }, random: () => 1 }); + assert.equal(out, 'read'); + assert.equal(calls, 7); + assert.deepEqual(waits, [500, 1000, 2000, 4000, 8000, 8000], 'doubling to 8 s'); + const low = []; + let tries = 0; + await chain.readWithBackoff('x', async () => { if (++tries < 3) throw throttled429(); return 1; }, + 4, undefined, { pause: async (ms) => { low.push(ms); }, random: () => 0 }); + assert.deepEqual(low, [250, 500], 'each wait at least half its step'); +}); + +test('readWithBackoff: a rate limit nested as -32005 counts, and the longer schedule still ends', async () => { + let calls = 0; + const nested = () => Object.assign(new Error('missing revert data'), { code: 'CALL_EXCEPTION', data: null, info: { error: { code: -32005, message: 'rate limit exceeded' } } }); + await assert.rejects( + chain.readWithBackoff('x', async () => { calls += 1; throw nested(); }, 4, undefined, { pause: async () => {} }), + /missing revert data/, + ); + assert.equal(calls, 7, 'seven tries, then the failure'); + let other = 0; + await assert.rejects( + chain.readWithBackoff('x', async () => { other += 1; throw Object.assign(new Error('boom'), { code: 'SERVER_ERROR' }); }, 4, undefined, { pause: async () => {} }), + /boom/, + ); + assert.equal(other, 4, 'any other failure keeps its four tries'); +}); diff --git a/oracle/keeper.test.js b/oracle/keeper.test.js index 524f871..3717ea2 100644 --- a/oracle/keeper.test.js +++ b/oracle/keeper.test.js @@ -638,7 +638,7 @@ async function withKeeperProcess(answer, run) { }); const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sigvara-keeper-')); const url = (s) => `http://127.0.0.1:${s.address().port}`; - const child = spawn(process.execPath, ['--require', path.join(__dirname, 'test-support', 'fast-interval.js'), path.join(__dirname, 'keeper.js')], { + const child = spawn(process.execPath, ['--require', path.join(__dirname, 'test-support', 'fast-interval.js'), '--require', path.join(__dirname, 'test-support', 'fast-retry.js'), path.join(__dirname, 'keeper.js')], { env: { ...process.env, RPC_URL: `${url(node)}/v2/${RPC_KEY}`, diff --git a/oracle/read-retry.js b/oracle/read-retry.js index 934a908..b12b5b9 100644 --- a/oracle/read-retry.js +++ b/oracle/read-retry.js @@ -47,19 +47,57 @@ function isTransientReadError(err) { (code === 'CALL_EXCEPTION' && (err.data === null || err.data === undefined)); } -async function readWithBackoff(label, fn, attempts = 4, retryable = isTransientReadError) { +/** + * True when the node refused the request because we are being throttled, rather than + * because anything is wrong with the transaction. + * + * Reads already treat throttling as transient and retry, in readWithBackoff. Writes + * deliberately do not, and that is unchanged here: proposeScore and finalizeScore race + * other parties for one slot, and resending something that may already be in the mempool + * is how you pay twice for one epoch. This only names the cause. + * + * Naming it matters because the alternative was counting it as a propose failure. + * `proposeErrors` should mean this operator could not do its job; on the live checker it + * also meant the provider said slow down, which is a different problem with a different + * fix, and the log line was a wall of JSON-RPC payload rather than a sentence. + * + * -32005 is the JSON-RPC code providers use for this. ethers wraps it in "could not + * coalesce error" with the original nested inside, so the string match is doing real work + * and is not just belt and braces. + */ +function isRateLimited(err) { + if (!err) return false; + const nested = err.error || err.info?.error || {}; + if (nested.code === -32005 || err.code === -32005) return true; + return /rate limit|429|too many requests/i.test( + String(err.shortMessage || err.message || '') + ); +} + +// A refusal for the node's rate limit is tried again on a longer schedule than other failures: up to +// RATE_LIMIT_ATTEMPTS tries, the wait doubling to at most RATE_LIMIT_MAX_WAIT_MS, about 24 s at most in all. Each wait is +// between half and all of its step, so reads refused together do not all come back together. ethers used to retry an +// HTTP 429 inside each try, for minutes (rpc-provider.js stopped that), and the checker's node answers a burst with one: +// four tries over 3.5 s left 2 to 4 agents an hourly epoch unread (16 November 2026). +const RATE_LIMIT_ATTEMPTS = 7; +const RATE_LIMIT_MAX_WAIT_MS = 8_000; + +async function readWithBackoff(label, fn, attempts = 4, retryable = isTransientReadError, { pause = sleep, random = Math.random } = {}) { let delay = 500; for (let i = 0; ; i++) { try { return await fn(); } catch (err) { const code = err && err.code; - if (!retryable(err) || i >= attempts - 1) throw err; - console.log(`[oracle] ${label} failed (${code || 'error'}), retrying in ${delay}ms`); - await sleep(delay); - delay *= 2; + if (!retryable(err)) throw err; + const limited = isRateLimited(err); + if (i >= (limited ? Math.max(attempts, RATE_LIMIT_ATTEMPTS) : attempts) - 1) throw err; + const wait = limited ? Math.round(delay / 2 + random() * (delay / 2)) : delay; + console.log(`[oracle] ${label} failed (${code || 'error'}), retrying in ${wait}ms`); + await pause(wait); + delay = Math.min(delay * 2, RATE_LIMIT_MAX_WAIT_MS); } } } -module.exports = { isTransientReadError, readWithBackoff, MAX_BATCH_CALLS }; +module.exports = { isTransientReadError, isRateLimited, readWithBackoff, MAX_BATCH_CALLS, RATE_LIMIT_ATTEMPTS }; diff --git a/oracle/test-support/fast-retry.js b/oracle/test-support/fast-retry.js new file mode 100644 index 0000000..5310083 --- /dev/null +++ b/oracle/test-support/fast-retry.js @@ -0,0 +1,7 @@ +'use strict'; + +// Loaded with --require beside fast-interval.js, for a process test that waits through several runs, each of which +// fails only after readWithBackoff's whole schedule (about 24 s for a node refusing for its rate limit). A wait of +// 250 ms or more is cut to 10 ms; shorter ones are left alone. +const realSetTimeout = global.setTimeout; +global.setTimeout = (fn, ms, ...args) => realSetTimeout(fn, ms >= 250 ? 10 : ms, ...args);