From 694b18e8fd8f46aad86ebeb14a61d22708d124d7 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Mon, 24 Aug 2026 22:19:37 +0200 Subject: [PATCH] LeiosDemoDb: BEGIN IMMEDIATE for writing transactions A bare BEGIN is BEGIN DEFERRED, so a transaction that reads before it writes has to upgrade its lock. In WAL mode that upgrade fails with SQLITE_BUSY_SNAPSHOT whenever another connection committed in between, and that status is neither serviced by the busy handler nor retryable at the statement level: the transaction's snapshot is stale for good, so re-stepping the same statement can never succeed. The retry loops in withDie and dbStepInsert spin to exhaustion and then throw, which arrives as ExceptionInLinkedThread and takes the node down. Observed on a twelve-node devnet under load: 297 LeiosDbException ErrorBusy across all twelve nodes, every single one from the same statement, the stMarkNotifiedEbs UPDATE in sqlInsertTxs. That transaction inserts only txs not already persisted, so under backlog the inserts are frequently empty, the SELECT loop takes the snapshot, and that UPDATE is the first write, hence the upgrade. It never failed when inserts had happened, because the write lock was already held. Taking the lock at BEGIN makes that upgrade impossible, so the only place a writer can now be told BUSY is the BEGIN itself. That is already retried, since dbExec goes through withDie, and retrying there is sound in a way the old failure was not: no transaction is open and no work has been done, so re-attempting the BEGIN re-attempts the whole transaction. No separate transaction-level retry is needed; with IMMEDIATE the two coincide. Splits the helper so readers keep BEGIN DEFERRED and do not exclude each other, and only the three writers take the lock up front. --- .../ouroboros-consensus/LeiosDemoDb/SQLite.hs | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/ouroboros-consensus/src/ouroboros-consensus/LeiosDemoDb/SQLite.hs b/ouroboros-consensus/src/ouroboros-consensus/LeiosDemoDb/SQLite.hs index ca39583f2e..b5b3ab28c4 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/LeiosDemoDb/SQLite.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/LeiosDemoDb/SQLite.hs @@ -269,7 +269,7 @@ sqlLookupEbBody conn ebHash = sqlInsertEbPoint :: Conn -> LeiosPoint -> BytesSize -> IO () sqlInsertEbPoint conn point ebBytesSize = - dbWithTransaction db $ useStmt stmt $ do + dbWithWriteTransaction db $ useStmt stmt $ do dbBindInt64 stmt 1 (fromIntegral $ unSlotNo point.pointSlotNo) dbBindBlob stmt 2 point.pointEbHash.ebHashBytes dbBindInt64 stmt 3 (fromIntegral ebBytesSize) @@ -289,7 +289,7 @@ sqlInsertEbBody :: sqlInsertEbBody tracer conn notify point eb = do when (null items) $ error "leiosDbInsertEbBody: empty EB body (programmer error)" - completedNow <- dbWithTransaction db $ do + completedNow <- dbWithWriteTransaction db $ do forM_ items $ \(txOffset, txHash, txBytesSize) -> useStmt stInsertEbTxsRow $ do dbBindBlob stInsertEbTxsRow 1 point.pointEbHash.ebHashBytes dbBindInt64 stInsertEbTxsRow 2 (fromIntegral txOffset) @@ -355,7 +355,7 @@ sqlInsertTxs _tracer conn notify txs = do -- hashes; attempting the INSERT and catching a constraint violation -- still pays the bind + PK-lookup + reset cost per row. missing <- Set.fromList <$> sqlFilterMissingTxs conn (map fst txs) - completed <- dbWithTransaction db $ do + completed <- dbWithWriteTransaction db $ do -- 'dbStepInsert' still handles the rare race where a concurrent -- writer inserted the same hash between the filter above and the -- INSERT below. @@ -687,12 +687,30 @@ dbPrepare :: HasCallStack => DB.Database -> DB.Utf8 -> IO DB.Statement dbPrepare db q = withDieJust db $ DB.prepare db q -- TODO: alternative: bind and use https://www.sqlite.org/c3ref/busy_handler.html + +-- | A read-only transaction: @BEGIN DEFERRED@, so readers do not exclude each +-- other. Any transaction that writes must use 'dbWithWriteTransaction'. dbWithTransaction :: HasCallStack => DB.Database -> IO a -> IO a -dbWithTransaction db k = +dbWithTransaction = dbWithTransactionAs "BEGIN" + +-- | A writing transaction: @BEGIN IMMEDIATE@, taking the write lock up front. +-- +-- A deferred transaction that reads before it writes has to upgrade its lock, +-- and in WAL mode that upgrade fails with @SQLITE_BUSY_SNAPSHOT@ whenever +-- another connection committed in between. That status is not serviced by the +-- busy handler and cannot be retried at the statement level, because the +-- transaction's snapshot is stale for good: the only remedy is to roll back and +-- start over. Taking the lock at BEGIN removes the upgrade, so contention +-- surfaces here instead, where waiting actually resolves it. +dbWithWriteTransaction :: HasCallStack => DB.Database -> IO a -> IO a +dbWithWriteTransaction = dbWithTransactionAs "BEGIN IMMEDIATE" + +dbWithTransactionAs :: HasCallStack => String -> DB.Database -> IO a -> IO a +dbWithTransactionAs begin db k = do fmap fst $ generalBracket - (dbExec db (fromString "BEGIN")) + (dbExec db (fromString begin)) ( \() -> \case MonadThrow.ExitCaseSuccess _ -> dbExec db (fromString "COMMIT") MonadThrow.ExitCaseException _ -> dbExec db (fromString "ROLLBACK") @@ -753,8 +771,15 @@ dbStepInsertOrTrace tracer table key stmt = do -- ** Error "handling" +-- | How many times a busy operation is re-attempted before it throws. +-- +-- The backoff below grows linearly, so the total wait grows with the square of +-- this: at 10000 it was about 83 minutes, which is indistinguishable from +-- hanging. 1000 gives roughly 50 s in total with a longest single sleep near +-- 100 ms, which outlasts any transient contention while still surfacing a real +-- deadlock inside a minute. maxBusyRetries :: Int -maxBusyRetries = 10000 +maxBusyRetries = 1000 -- | Execute a database action that may return an error. If the error is -- 'DB.ErrorBusy', retry up to 'maxBusyRetries' times with linear backoff and