Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion packages/chain/src/evm-adapter-errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
/**
* EVM error decoding / classification helpers extracted from
* evm-adapter.ts. Covers low-level error-shape extraction
* (`errorMessage` / `errorCode` / `errorStatus`), custom-error ABI
* (`errorMessage` / `errorName` / `errorCode` / `errorStatus`), custom-error ABI
* decoding (`decodeEvmError` / `enrichEvmError` and the lazily-cached
* error/PCA interfaces), the Hub-stale revert markers, and the
* `TooLowAllowance` classifier. Bodies are a 1:1 move from the original
Expand All @@ -22,6 +22,18 @@ export function errorMessage(err: unknown): string {
try { return JSON.stringify(err); } catch { return String(err); }
}

/**
* Read the top-level error class name exposed by Error / DOMException shapes.
* Unlike status and code extraction, this deliberately does not traverse
* wrappers: a name describes the caught surface error, while nested transport
* metadata is decoded separately by {@link errorCode} / {@link errorStatus}.
*/
export function errorName(err: unknown): string {
if (!err || typeof err !== 'object') return '';
const { name } = err as { name?: unknown };
return typeof name === 'string' ? name : '';
}

export function errorCode(err: unknown): string {
return String((err as any)?.code ?? (err as any)?.error?.code ?? '').toUpperCase();
}
Expand Down
19 changes: 18 additions & 1 deletion packages/chain/src/evm-adapter-rpc.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,13 @@
* assertions. Bodies are a 1:1 move from the original module.
*/
import { ethers, FetchRequest } from 'ethers';
import { enrichEvmError, errorCode, errorMessage, errorStatus } from './evm-adapter-errors.js';
import {
enrichEvmError,
errorCode,
errorMessage,
errorName,
errorStatus,
} from './evm-adapter-errors.js';
import { createRpcTimeoutError } from './chain-rpc-transport-error.js';

/**
Expand Down Expand Up @@ -118,6 +124,7 @@ export function isRetryableRpcError(err: unknown): boolean {
const code = errorCode(err);
const status = errorStatus(err);
const msg = errorMessage(err).toLowerCase();
const name = errorName(err);

if (code === 'CALL_EXCEPTION' || code === 'INSUFFICIENT_FUNDS' || code === 'NONCE_EXPIRED'
|| code === 'RPC_RECEIPT_LOOKUP_FAILED'
Expand All @@ -132,6 +139,16 @@ export function isRetryableRpcError(err: unknown): boolean {
return false;
}

// Node/undici/ethers surface an aborted fetch as DOMException
// `{ name: 'AbortError' }` (and some Node paths also stamp `ABORT_ERR`).
// At the chain-RPC boundary this is a transport interruption, not a
// deterministic EVM result. Treat it like the timeout/network cases below so
// the SAME signed transaction or point read fails over to another endpoint.
// This is especially important after `eth_sendRawTransaction`: an RPC can
// accept the tx and then abort the client response, so fail-fast would report
// publish failure even though the mint is already on chain.
if (name === 'AbortError' || code === 'ABORT_ERR') return true;
Comment thread
branarakic marked this conversation as resolved.

if (status === 429 || (typeof status === 'number' && status >= 500)) return true;
if (code === 'TIMEOUT' || code === 'RPC_TIMEOUT' || code === 'TIMEOUT_ERROR' || code === 'SERVER_ERROR'
|| code === 'NETWORK_ERROR' || code === 'ECONNRESET' || code === 'ECONNREFUSED'
Expand Down
2 changes: 1 addition & 1 deletion packages/chain/src/rpc-failover-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -607,7 +607,7 @@ export class RpcFailoverClient {
throw new ChainRpcTransportError(
'RPC_ENDPOINTS_EXHAUSTED',
`${label} broadcast failed on all configured RPC endpoints for tx ${txHash}: ${errorMessage(lastRetryable)}`,
{ cause: lastRetryable, rpcUrls: canonical.map((e) => e.rpcUrl) },
{ cause: lastRetryable, rpcUrls: canonical.map((e) => e.rpcUrl), txHash },
);
} finally {
metrics.chainRpcDuration.record(Date.now() - startedAt, {
Expand Down
12 changes: 12 additions & 0 deletions packages/chain/test/evm-adapter-tip-read-carveouts.unit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,18 @@ describe('getBlockTimestamp: a null (unimported) receipt block fails over instea
expect(backupGetBlock.calls).toHaveLength(1);
});

it('primary aborts the receipt-block request → fails over to the backup timestamp', async () => {
const abort = new Error('This operation was aborted');
abort.name = 'AbortError';
const primaryGetBlock = recorder(async () => { throw abort; });
const backupGetBlock = recorder(async () => ({ timestamp: 42 }));
const a = makeTwoEndpointAdapter({ getBlock: primaryGetBlock }, { getBlock: backupGetBlock });

await expect(a.getBlockTimestamp(123n)).resolves.toBe(42);
expect(primaryGetBlock.calls).toHaveLength(1);
expect(backupGetBlock.calls).toHaveLength(1);
});

it('all endpoints lack the block → best-effort 0 (callers tolerate it)', async () => {
const a = makeTwoEndpointAdapter({ getBlock: recorder(async () => null) }, { getBlock: recorder(async () => null) });
expect(await a.getBlockTimestamp(123n)).toBe(0);
Expand Down
44 changes: 44 additions & 0 deletions packages/chain/test/rpc-failover-client.unit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,15 @@ const badDataError = () => {
return e;
};
const knownTxError = () => new Error('already known');
const abortedRpcError = () => {
const error = new Error('This operation was aborted');
error.name = 'AbortError';
return error;
};
const abortedRpcCodeError = () => Object.assign(
new Error('request aborted'),
{ code: 'ABORT_ERR' },
);

const URLS = ['https://primary.example', 'https://backup.example'];

Expand Down Expand Up @@ -210,6 +219,16 @@ describe('RpcFailoverClient.read / readContract — policy matrix applied + view
).resolves.toBe('BACKUP');
expect(backupView.calls).toHaveLength(1);
});

it('read: an AbortError is a retryable transport interruption and fails over', async () => {
const primary = { read: recorder(async () => { throw abortedRpcError(); }) };
const backup = { read: recorder(async () => 'BACKUP') };
const client = makeClient([primary, backup], URLS);

await expect(client.read('aborted point read', (p: any) => p.read())).resolves.toBe('BACKUP');
expect(primary.read.calls).toHaveLength(1);
expect(backup.read.calls).toHaveLength(1);
});
});

// ── broadcast (write transport) ──────────────────────────────────────────────
Expand Down Expand Up @@ -245,6 +264,7 @@ describe('RpcFailoverClient.broadcast — idempotent short-circuit + typed exhau
try { await client.broadcast('0xsigned', '0xDEADBEEF', 'unit write'); } catch (e) { thrown = e; }
expect(thrown.code).toBe('RPC_ENDPOINTS_EXHAUSTED'); // #1329: maps to a retryable 503, not a code-less 500
expect(thrown.rpcUrls).toEqual(URLS);
expect(thrown.txHash).toBe('0xDEADBEEF');
expect(thrown.message).toContain('0xDEADBEEF'); // names the tx
expect(thrown.message).not.toContain('https://'); // never a full URL
expect(primary.broadcastTransaction.calls).toHaveLength(1);
Expand All @@ -260,6 +280,30 @@ describe('RpcFailoverClient.broadcast — idempotent short-circuit + typed exhau
await expect(client.broadcast('0xsigned', '0xhash', 'unit write')).rejects.toBe(err);
expect(backup.broadcastTransaction.calls).toEqual([]);
});

it('an AbortError after submit is ambiguous, so the byte-identical signed tx fails over', async () => {
const primary = { broadcastTransaction: recorder(async () => { throw abortedRpcError(); }) };
const backup = { broadcastTransaction: recorder(async () => undefined) };
const client = makeClient([primary, backup], URLS);

await expect(client.broadcast('0xsigned', '0xhash', 'unit write')).resolves.toBeUndefined();
expect(primary.broadcastTransaction.calls).toHaveLength(1);
expect(backup.broadcastTransaction.calls).toHaveLength(1);
expect(primary.broadcastTransaction.calls[0][0]).toBe('0xsigned');
expect(backup.broadcastTransaction.calls[0][0]).toBe('0xsigned');
});

it('a code-only ABORT_ERR after submit also fails over with the byte-identical signed tx', async () => {
const primary = { broadcastTransaction: recorder(async () => { throw abortedRpcCodeError(); }) };
const backup = { broadcastTransaction: recorder(async () => undefined) };
const client = makeClient([primary, backup], URLS);

await expect(client.broadcast('0xsigned', '0xhash', 'unit write')).resolves.toBeUndefined();
expect(primary.broadcastTransaction.calls).toHaveLength(1);
expect(backup.broadcastTransaction.calls).toHaveLength(1);
expect(primary.broadcastTransaction.calls[0][0]).toBe('0xsigned');
expect(backup.broadcastTransaction.calls[0][0]).toBe('0xsigned');
});
});

// ── getReceipt (write transport) ─────────────────────────────────────────────
Expand Down
19 changes: 13 additions & 6 deletions packages/cli/src/daemon/http-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -284,28 +284,35 @@ export function classifyChainRpcTransportStatus(
const { code } = err;
const msg = sanitizeRpcMessage(typeof err.message === "string" ? err.message : "");
const txHash = typeof err.txHash === "string" && err.txHash ? err.txHash : "";
const transportBody = (error: string, responseCode: string): Record<string, unknown> => ({
error,
code: responseCode,
...(txHash ? { txHash } : {}),
});
// Exhaustive over ChainRpcTransportCode: a new code added to the boundary
// without a case here is a COMPILE error (the `never` default), so the
// classifier can never silently inherit timeout/504 semantics for a new code.
switch (code) {
case "RPC_ENDPOINTS_EXHAUSTED":
return { status: 503, body: { error: msg || "Configured chain RPC endpoints were exhausted.", code } };
return {
status: 503,
body: transportBody(msg || "Configured chain RPC endpoints were exhausted.", code),
};
case "RPC_RECEIPT_LOOKUP_FAILED":
return {
status: 503,
body: {
error: msg || "Transaction receipt lookup failed on all configured chain RPC endpoints.",
body: transportBody(
msg || "Transaction receipt lookup failed on all configured chain RPC endpoints.",
code,
...(txHash ? { txHash } : {}),
},
),
};
case "RPC_TIMEOUT":
// Internal, chain-namespaced timeout code. Expose the public/legacy
// `code: "TIMEOUT"` in the 504 body (clients key on that), keeping the
// wire contract stable while the boundary stays namespaced internally.
return {
status: 504,
body: { error: msg || "Chain transaction timed out.", code: "TIMEOUT", ...(txHash ? { txHash } : {}) },
body: transportBody(msg || "Chain transaction timed out.", "TIMEOUT"),
};
default: {
const _exhaustive: never = code;
Expand Down
10 changes: 10 additions & 0 deletions packages/cli/test/chain-rpc-transport-status.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,16 @@ describe('classifyChainRpcTransportStatus (W2 shared transport-status helper)',
).toEqual({ status: 503, body: { error: 'all endpoints failed', code: 'RPC_ENDPOINTS_EXHAUSTED' } });
});

it('preserves the known txHash on a broadcast RPC_ENDPOINTS_EXHAUSTED response', () => {
const r = classifyChainRpcTransportStatus({
code: 'RPC_ENDPOINTS_EXHAUSTED',
message: 'broadcast endpoints failed',
txHash: '0xabc',
});
expect(r?.status).toBe(503);
expect(r?.body).toMatchObject({ code: 'RPC_ENDPOINTS_EXHAUSTED', txHash: '0xabc' });
});

it('maps RPC_RECEIPT_LOOKUP_FAILED -> 503 (+code, +txHash)', () => {
const r = classifyChainRpcTransportStatus({ code: 'RPC_RECEIPT_LOOKUP_FAILED', message: 'm', txHash: '0xabc' });
expect(r?.status).toBe(503);
Expand Down
3 changes: 3 additions & 0 deletions packages/cli/vitest.unit.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ export default defineConfig({
'test/promote-async-routes.test.ts',
'test/promote-async-daemon-lifecycle.test.ts',
'test/daemon-ka-transport.test.ts',
// Pure HTTP classification coverage for chain transport failures,
// including preserving a known transaction hash on endpoint exhaustion.
'test/chain-rpc-transport-status.test.ts',
'test/async-promote-worker.test.ts',
'test/async-promote-queue-e2e.test.ts',
'test/knowledge-assets-1116-share-errors.test.ts',
Expand Down
Loading