From 603b9adf6635e4603c0ec4ba5b432833d9323449 Mon Sep 17 00:00:00 2001 From: liutingqiu Date: Mon, 19 Oct 2026 09:43:03 +0800 Subject: [PATCH] fix(accrual-keeper): keep accruing when state store is unclaimable --- .../src/accrual-keeper.test.ts | 29 ++++++++++++ .../stellar-sdk-helpers/src/accrual-keeper.ts | 44 ++++++++++++++----- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/packages/stellar-sdk-helpers/src/accrual-keeper.test.ts b/packages/stellar-sdk-helpers/src/accrual-keeper.test.ts index 6cb3cb9d..e5ec8d72 100644 --- a/packages/stellar-sdk-helpers/src/accrual-keeper.test.ts +++ b/packages/stellar-sdk-helpers/src/accrual-keeper.test.ts @@ -1593,6 +1593,35 @@ describe("runBlendAccrualKeeper cross-invocation dedup", () => { ).toHaveLength(1); }); + it("keeps accruing when the lease cannot be claimed either, instead of halting every vault", async () => { + // A full outage: the prior read *and* the lease claim fail. Before the + // fail-open covered the claim path, that error pushed the adapter into + // `skipped`, halting accrual for exactly the outage the documented + // fail-open exists for. + const inner = await store(); + const stateStore: KeeperStateStore = { + ...inner, + get: async () => { + throw new Error("KV unavailable"); + }, + claim: async () => { + throw new Error("KV unavailable"); + }, + }; + const server = makeServer(); + stellarMocks.getRpcServer.mockReturnValue(server); + const log = logger(); + + const result = await run(stateStore, log); + + expect(result.skipped).toEqual([]); + expect(result.successes).toHaveLength(1); + expect(server.sendTransaction).toHaveBeenCalledOnce(); + expect( + warningsMatching(log, "proceeding without a submission lease") + ).toHaveLength(1); + }); + it("releases the claim when nothing was ever signed", async () => { // A stale adapter aborts before the transaction is built; leaving the // claim behind would lock the adapter out for the claim's whole window diff --git a/packages/stellar-sdk-helpers/src/accrual-keeper.ts b/packages/stellar-sdk-helpers/src/accrual-keeper.ts index f7861338..b25671cd 100644 --- a/packages/stellar-sdk-helpers/src/accrual-keeper.ts +++ b/packages/stellar-sdk-helpers/src/accrual-keeper.ts @@ -37,6 +37,11 @@ export type { KeeperFailure, KeeperLogger } from "./keeper-retry"; const DEFAULT_MAX_ATTEMPTS = 3; const DEFAULT_BASE_DELAY_MS = 1_000; +// Matches the literal returned by SubmissionLease.acquire in keeper-state.ts +// when the store itself cannot be reached, as opposed to a live peer already +// holding the lease. Only the former may fail open (see the acquire call +// below); the latter still has to skip. +const STATE_STORE_UNAVAILABLE_ERROR = "submission state store unavailable"; // Capped at 10_000 to match tx.ts's own hardcoded Soroban RPC timeout // (SOROBAN_RPC_TIMEOUT_MS): discovery's simulate() calls go through // simulateView, which races every call against that fixed 10s ceiling @@ -480,18 +485,32 @@ export async function runBlendAccrualKeeper( logger, context: priorContext, }); + let lease: SubmissionLease | undefined; + let submissionHooks: KeeperSubmissionHooks; if ("error" in acquired) { - skipped.push({ - vaultId: adapter.vaultId, - vaultContractId: adapter.vaultContractId, - adapterId: adapter.adapterId, - protocol: adapter.protocol, - reason: `could not take the submission lease (${acquired.error}); skipped this run`, - }); - continue; + // A lease held by a live peer still has to be respected: skipping there + // is duplicate prevention working as intended. An unreachable store is + // different — it is the outage the fail-open above exists for, and + // skipping on it would halt accrual exactly when the store is down. + if (acquired.error !== STATE_STORE_UNAVAILABLE_ERROR) { + skipped.push({ + vaultId: adapter.vaultId, + vaultContractId: adapter.vaultContractId, + adapterId: adapter.adapterId, + protocol: adapter.protocol, + reason: `could not take the submission lease (${acquired.error}); skipped this run`, + }); + continue; + } + logger.warn( + "[accrual-keeper] proceeding without a submission lease; state store unavailable", + { ...priorContext, reason: acquired.error } + ); + submissionHooks = {}; + } else { + lease = acquired.lease; + submissionHooks = lease.hooks; } - const lease = acquired.lease; - const submissionHooks = lease.hooks; // In-run tracking (priorHash) still exists alongside the lease: it's // what keeps a retry inside this same run rechecking one hash instead of @@ -585,8 +604,9 @@ export async function runBlendAccrualKeeper( } finally { // A claim that never became a signed transaction (a stale adapter, a // simulation error) must not keep the next run out for the claim's - // full window. - await lease.releaseIfUnsent(); + // full window. There is no claim to release when the store was + // unreachable and we failed open. + if (lease) await lease.releaseIfUnsent(); } }