merge main

src/logging/logging.h

@@ -315,6 +315,14 @@ class qLogger
     // custom log from smart contracts are not included in the digest computation
     inline static m256i digests[MAX_NUMBER_OF_TICKS_PER_EPOCH];
     inline static XKCP::KangarooTwelve_Instance k12;
+
+    // Framing of logEventState.db
+    static constexpr unsigned long long logStateVmSize = (LOG_BUFFER_PAGE_SIZE * sizeof(char) + 16)
+        + (PMAP_LOG_PAGE_SIZE * sizeof(BlobInfo) + 16)
+        + (IMAP_LOG_PAGE_SIZE * sizeof(TickBlobInfo) + 16);
+    static constexpr unsigned long long logStateTailSize = sizeof(k12) + 8 + 8 + 4 + 4 + 4 + 4;
+    static constexpr unsigned long long logStateBufferSize = LOG_BUFFER_PAGE_SIZE + PMAP_LOG_PAGE_SIZE * sizeof(BlobInfo)
+        + IMAP_LOG_PAGE_SIZE * sizeof(TickBlobInfo) + sizeof(digests) + 600;
 #endif
 
     inline static unsigned long long logBufferTail;
@@ -860,29 +868,57 @@ class qLogger
         return true;
     }
 
-    // This function is part of save/load feature and can only be called from main thread
-    void loadLastLoggingStates(CHAR16* dir)
+    // Checks the saved logging state without reading it, so that an unusable file can be rejected
+    // before any node state has been loaded
+    bool checkLastLoggingStates(CHAR16* dir, unsigned long long& savedDigestsSz)
     {
+        savedDigestsSz = 0;
 #if ENABLED_LOGGING
-        constexpr auto bufferSize = LOG_BUFFER_PAGE_SIZE + PMAP_LOG_PAGE_SIZE * sizeof(BlobInfo) + IMAP_LOG_PAGE_SIZE * sizeof(TickBlobInfo)
-            + sizeof(digests) + 600;
-        static_assert(defaultCommonBuffersSize >= bufferSize, "commonBuffer size is too small");
-        __ScopedScratchpad scratchpad(bufferSize, /*initZero=*/false);
-        unsigned char* buffer = (unsigned char*)scratchpad.ptr;
-        ASSERT(scratchpad.ptr);
         CHAR16 fileName[] = L"logEventState.db";
         const long long fileSz = getFileSize(fileName, dir);
-        if (fileSz == -1)
+        if (fileSz <= 0 || (unsigned long long)fileSz > logStateBufferSize)
         {
-            logToConsole(L"[1] Failed to load logging events");
-            return;
+            logToConsole(L"[1] Failed to load logging events: missing or oversized file");
+            return false;
+        }
+        if ((unsigned long long)fileSz < logStateVmSize + logStateTailSize)
+        {
+            logToConsole(L"[2] Failed to load logging events: file too small");
+            return false;
+        }
+        // digests length follows MAX_NUMBER_OF_TICKS_PER_EPOCH, so take it from the file, not from sizeof
+        savedDigestsSz = (unsigned long long)fileSz - logStateVmSize - logStateTailSize;
+        if (savedDigestsSz % sizeof(digests[0])
+            || (savedDigestsSz / sizeof(digests[0])) % NUMBER_OF_COMPUTORS)
+        {
+            logToConsole(L"[3] Failed to load logging events: bad digests section");
+            return false;
+        }
+#endif
+        return true;
+    }
+
+    // This function is part of save/load feature and can only be called from main thread
+    bool loadLastLoggingStates(CHAR16* dir)
+    {
+#if ENABLED_LOGGING
+        static_assert(defaultCommonBuffersSize >= logStateBufferSize, "commonBuffer size is too small");
+        unsigned long long savedDigestsSz = 0;
+        if (!checkLastLoggingStates(dir, savedDigestsSz))
+        {
+            return false;
         }
+        __ScopedScratchpad scratchpad(logStateBufferSize, /*initZero=*/false);
+        unsigned char* buffer = (unsigned char*)scratchpad.ptr;
+        ASSERT(scratchpad.ptr);
+        CHAR16 fileName[] = L"logEventState.db";
+        const unsigned long long fileSz = logStateVmSize + savedDigestsSz + logStateTailSize;
         unsigned long long sz = load(fileName, fileSz, buffer, dir);
         if (fileSz != sz)
         {
             // failed to load
-            logToConsole(L"[2] Failed to load logging events");
-            return;
+            logToConsole(L"[4] Failed to load logging events: short read");
+            return false;
         }
 
         unsigned long long readSz = 0;
@@ -901,10 +937,12 @@ class qLogger
         buffer += sz;
         readSz += sz;
 
-        // copy digests ~ 13MiB
-        copyMem(digests, buffer, sizeof(digests));
-        buffer += sizeof(digests);
-        readSz += sizeof(digests);
+        ASSERT(readSz == logStateVmSize);
+        const unsigned long long copyDigestsSz = savedDigestsSz < sizeof(digests) ? savedDigestsSz : sizeof(digests);
+        copyMem(digests, buffer, copyDigestsSz);
+        setMem((unsigned char*)digests + copyDigestsSz, sizeof(digests) - copyDigestsSz, 0);
+        buffer += savedDigestsSz;
+        readSz += savedDigestsSz;
 
         // copy k12 instance
         copyMem(&k12, buffer, sizeof(k12));
@@ -919,6 +957,7 @@ class qLogger
         currentTxId = *((unsigned int*)buffer); buffer += 4;
         currentTick = *((unsigned int*)buffer);
 #endif
+        return true;
     }
 
 #endif

src/public_settings.h

@@ -62,7 +62,7 @@
 
 // The tick duration used to calculate the size of memory buffers.
 // This determines the memory footprint of the application.
-#define TICK_DURATION_FOR_ALLOCATION_MS 350
+#define TICK_DURATION_FOR_ALLOCATION_MS 225
 #define TRANSACTION_SPARSENESS 10
 // Number of ticks that are stored in the pending txs pool. This also defines how many ticks in advance a tx can be registered.
 #define PENDING_TXS_POOL_NUM_TICKS (1000 * 60 * 3ULL / TICK_DURATION_FOR_ALLOCATION_MS) // 3 minutes
@@ -170,7 +170,7 @@ static constexpr long long NEURON_VALUE_LIMIT = 1LL;
   #ifdef LONG_RUN_LOCAL_TESTNET
   // 60 days at 1s ticks; see doc/long_run_local_testnet.md
   #ifndef LONG_RUN_EPOCH_TICK_CAPACITY
-  #define LONG_RUN_EPOCH_TICK_CAPACITY 5184000ULL
+  #define LONG_RUN_EPOCH_TICK_CAPACITY 5184244ULL
   #endif
   #define TESTNET_EPOCH_DURATION LONG_RUN_EPOCH_TICK_CAPACITY
   #else

src/qubic.cpp

@@ -5002,6 +5002,19 @@ static bool loadAllNodeStates()
         logToConsole(L"Found epoch snapshot directory. Using node states snapshot.");
     }
 
+#if ENABLED_LOGGING
+    // Reject an unusable logging state here: returning false after node state has been loaded
+    // would leave it partially applied
+    {
+        unsigned long long savedDigestsSz;
+        if (!logger.checkLastLoggingStates(directory, savedDigestsSz))
+        {
+            logToConsole(L"Skip using node states snapshot.");
+            return false;
+        }
+    }
+#endif
+
     if (ts.tryLoadFromFile(system.epoch, directory) != 0)
     {
         logToConsole(L"Failed to load tick storage");
@@ -5081,11 +5094,32 @@ static bool loadAllNodeStates()
     REVENUE_DATA_SNAPSHOT_FILE_NAME[sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME) / sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME[0]) - 4] = system.epoch / 100 + L'0';
     REVENUE_DATA_SNAPSHOT_FILE_NAME[sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME) / sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME[0]) - 3] = (system.epoch % 100) / 10 + L'0';
     REVENUE_DATA_SNAPSHOT_FILE_NAME[sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME) / sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME[0]) - 2] = system.epoch % 10 + L'0';
-    long long revenueDataSize = load(REVENUE_DATA_SNAPSHOT_FILE_NAME, sizeof(gEpochRevenueData), (unsigned char*)&gEpochRevenueData, directory);
-    if (revenueDataSize != sizeof(gEpochRevenueData))
+    const long long revenueFileSize = getFileSize(REVENUE_DATA_SNAPSHOT_FILE_NAME, directory);
+    bool revenueDataLoaded = false;
+    if (revenueFileSize == (long long)sizeof(gEpochRevenueData))
+    {
+        revenueDataLoaded = load(REVENUE_DATA_SNAPSHOT_FILE_NAME, sizeof(gEpochRevenueData), (unsigned char*)&gEpochRevenueData, directory)
+            == (long long)sizeof(gEpochRevenueData);
+    }
+    else if (revenueFileSize > 0 && revenueFileSize <= (long long)defaultCommonBuffersSize)
+    {
+        // Saved with a different MAX_NUMBER_OF_TICKS_PER_EPOCH
+        __ScopedScratchpad scratchpad(revenueFileSize, /*initZero=*/false);
+        revenueDataLoaded = scratchpad.ptr
+            && load(REVENUE_DATA_SNAPSHOT_FILE_NAME, revenueFileSize, (unsigned char*)scratchpad.ptr, directory) == revenueFileSize
+            && remapEpochRevenueData((unsigned char*)scratchpad.ptr, revenueFileSize, gEpochRevenueData);
+    }
+    if (!revenueDataLoaded)
     {
+#if USE_REVENUE_MULTI_DIMENSION
+        // v2Revenue is shadow when multi-dim is active, revenue uses gMultiDimRevenue.
+        // We mark zero to make flag out this data is invalid for analysis
+        logToConsole(L"Revenue data snapshot load/remap failed (v2 shadow mode), zeroing");
+        setMem(&gEpochRevenueData, sizeof(gEpochRevenueData), 0);
+#else
         logToConsole(L"Failed to load revenue data snapshot");
         return false;
+#endif
     }
 
     MULTIDIM_REVENUE_SNAPSHOT_FILE_NAME[sizeof(MULTIDIM_REVENUE_SNAPSHOT_FILE_NAME) / sizeof(MULTIDIM_REVENUE_SNAPSHOT_FILE_NAME[0]) - 4] = system.epoch / 100 + L'0';
@@ -5184,7 +5218,11 @@ static bool loadAllNodeStates()
 
 #if ENABLED_LOGGING
     logToConsole(L"Loading old logger...");
-    logger.loadLastLoggingStates(directory);
+    if (!logger.loadLastLoggingStates(directory))
+    {
+        logToConsole(L"Failed to load logging state, discarding snapshot");
+        return false;
+    }
 #endif
 
 #if !defined(NDEBUG)

src/revenue.h

@@ -421,6 +421,44 @@ struct EpochRevenueData
 
 static EpochRevenueData gEpochRevenueData;
 
+static constexpr unsigned long long revenueDataPerTickArrays = 5;
+static constexpr unsigned long long revenueDataFixedSize =
+    sizeof(EpochRevenueData) - revenueDataPerTickArrays * MAX_NUMBER_OF_TICKS_PER_EPOCH * sizeof(unsigned short);
+static_assert(revenueDataFixedSize == 8 + 4ULL * NUMBER_OF_COMPUTORS * sizeof(unsigned long long),
+    "EpochRevenueData has unexpected padding; the snapshot re-layout below relies on its exact layout");
+
+// Restore a revenue snapshot saved with a different MAX_NUMBER_OF_TICKS_PER_EPOCH.
+static bool remapEpochRevenueData(const unsigned char* saved, unsigned long long savedSize, EpochRevenueData& out)
+{
+    constexpr unsigned long long stride = revenueDataPerTickArrays * sizeof(unsigned short);
+    if (savedSize < revenueDataFixedSize || (savedSize - revenueDataFixedSize) % stride)
+    {
+        return false;
+    }
+    const unsigned long long savedTicks = (savedSize - revenueDataFixedSize) / stride;
+    if (savedTicks % NUMBER_OF_COMPUTORS)
+    {
+        return false;
+    }
+
+    // static_assert pins the sizes; this pins the field order the copies below rely on
+    ASSERT((const unsigned char*)out.perTickTxCount == (const unsigned char*)&out + revenueDataFixedSize);
+
+    copyMem(&out, saved, revenueDataFixedSize);
+
+    const unsigned long long copyTicks = savedTicks < MAX_NUMBER_OF_TICKS_PER_EPOCH ? savedTicks : MAX_NUMBER_OF_TICKS_PER_EPOCH;
+    unsigned short* dst[revenueDataPerTickArrays] = {
+        out.perTickTxCount, out.perTickProtocolTxCount, out.perTickContractTxCount,
+        out.perTickOtherTxCount, out.perTickTxTickLeaderCount };
+    const unsigned char* src = saved + revenueDataFixedSize;
+    for (unsigned long long i = 0; i < revenueDataPerTickArrays; i++)
+    {
+        copyMem(dst[i], src + i * savedTicks * sizeof(unsigned short), copyTicks * sizeof(unsigned short));
+        setMem(dst[i] + copyTicks, (MAX_NUMBER_OF_TICKS_PER_EPOCH - copyTicks) * sizeof(unsigned short), 0);
+    }
+    return true;
+}
+
 // Intermediate buffers for V2 revenue computation
 struct RevenueV2Buffers
 {