From ce021b148397e17df8f4097b412d1a57248ab389 Mon Sep 17 00:00:00 2001 From: dkat <39078779+krypdkat@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:40:15 +0700 Subject: [PATCH 1/3] patch: dynamically loading snapshot when changing MAX TICK on mid epoch (#965) --- src/logging/logging.h | 73 +++++++++++++++++++++++++++++++++---------- src/qubic.cpp | 44 ++++++++++++++++++++++++-- src/revenue.h | 38 ++++++++++++++++++++++ 3 files changed, 135 insertions(+), 20 deletions(-) diff --git a/src/logging/logging.h b/src/logging/logging.h index 73891996..d2cfac21 100644 --- a/src/logging/logging.h +++ b/src/logging/logging.h @@ -308,6 +308,14 @@ class qLogger // custom log from smart contracts are not included in the digest computation inline static m256i digests[MAX_NUMBER_OF_TICKS_PER_EPOCH]; inline static XKCP::KangarooTwelve_Instance k12; + + // Framing of logEventState.db + static constexpr unsigned long long logStateVmSize = (LOG_BUFFER_PAGE_SIZE * sizeof(char) + 16) + + (PMAP_LOG_PAGE_SIZE * sizeof(BlobInfo) + 16) + + (IMAP_LOG_PAGE_SIZE * sizeof(TickBlobInfo) + 16); + static constexpr unsigned long long logStateTailSize = sizeof(k12) + 8 + 8 + 4 + 4 + 4 + 4; + static constexpr unsigned long long logStateBufferSize = LOG_BUFFER_PAGE_SIZE + PMAP_LOG_PAGE_SIZE * sizeof(BlobInfo) + + IMAP_LOG_PAGE_SIZE * sizeof(TickBlobInfo) + sizeof(digests) + 600; #endif inline static unsigned long long logBufferTail; @@ -694,29 +702,57 @@ class qLogger return true; } - // This function is part of save/load feature and can only be called from main thread - void loadLastLoggingStates(CHAR16* dir) + // Checks the saved logging state without reading it, so that an unusable file can be rejected + // before any node state has been loaded + bool checkLastLoggingStates(CHAR16* dir, unsigned long long& savedDigestsSz) { + savedDigestsSz = 0; #if ENABLED_LOGGING - constexpr auto bufferSize = LOG_BUFFER_PAGE_SIZE + PMAP_LOG_PAGE_SIZE * sizeof(BlobInfo) + IMAP_LOG_PAGE_SIZE * sizeof(TickBlobInfo) - + sizeof(digests) + 600; - static_assert(defaultCommonBuffersSize >= bufferSize, "commonBuffer size is too small"); - __ScopedScratchpad scratchpad(bufferSize, /*initZero=*/false); - unsigned char* buffer = (unsigned char*)scratchpad.ptr; - ASSERT(scratchpad.ptr); CHAR16 fileName[] = L"logEventState.db"; const long long fileSz = getFileSize(fileName, dir); - if (fileSz == -1) + if (fileSz <= 0 || (unsigned long long)fileSz > logStateBufferSize) { - logToConsole(L"[1] Failed to load logging events"); - return; + logToConsole(L"[1] Failed to load logging events: missing or oversized file"); + return false; + } + if ((unsigned long long)fileSz < logStateVmSize + logStateTailSize) + { + logToConsole(L"[2] Failed to load logging events: file too small"); + return false; + } + // digests length follows MAX_NUMBER_OF_TICKS_PER_EPOCH, so take it from the file, not from sizeof + savedDigestsSz = (unsigned long long)fileSz - logStateVmSize - logStateTailSize; + if (savedDigestsSz % sizeof(digests[0]) + || (savedDigestsSz / sizeof(digests[0])) % NUMBER_OF_COMPUTORS) + { + logToConsole(L"[3] Failed to load logging events: bad digests section"); + return false; + } +#endif + return true; + } + + // This function is part of save/load feature and can only be called from main thread + bool loadLastLoggingStates(CHAR16* dir) + { +#if ENABLED_LOGGING + static_assert(defaultCommonBuffersSize >= logStateBufferSize, "commonBuffer size is too small"); + unsigned long long savedDigestsSz = 0; + if (!checkLastLoggingStates(dir, savedDigestsSz)) + { + return false; } + __ScopedScratchpad scratchpad(logStateBufferSize, /*initZero=*/false); + unsigned char* buffer = (unsigned char*)scratchpad.ptr; + ASSERT(scratchpad.ptr); + CHAR16 fileName[] = L"logEventState.db"; + const unsigned long long fileSz = logStateVmSize + savedDigestsSz + logStateTailSize; unsigned long long sz = load(fileName, fileSz, buffer, dir); if (fileSz != sz) { // failed to load - logToConsole(L"[2] Failed to load logging events"); - return; + logToConsole(L"[4] Failed to load logging events: short read"); + return false; } unsigned long long readSz = 0; @@ -735,10 +771,12 @@ class qLogger buffer += sz; readSz += sz; - // copy digests ~ 13MiB - copyMem(digests, buffer, sizeof(digests)); - buffer += sizeof(digests); - readSz += sizeof(digests); + ASSERT(readSz == logStateVmSize); + const unsigned long long copyDigestsSz = savedDigestsSz < sizeof(digests) ? savedDigestsSz : sizeof(digests); + copyMem(digests, buffer, copyDigestsSz); + setMem((unsigned char*)digests + copyDigestsSz, sizeof(digests) - copyDigestsSz, 0); + buffer += savedDigestsSz; + readSz += savedDigestsSz; // copy k12 instance copyMem(&k12, buffer, sizeof(k12)); @@ -753,6 +791,7 @@ class qLogger currentTxId = *((unsigned int*)buffer); buffer += 4; currentTick = *((unsigned int*)buffer); #endif + return true; } #endif diff --git a/src/qubic.cpp b/src/qubic.cpp index 4be60a67..ea041bb0 100644 --- a/src/qubic.cpp +++ b/src/qubic.cpp @@ -4463,6 +4463,19 @@ static bool loadAllNodeStates() logToConsole(L"Found epoch snapshot directory. Using node states snapshot."); } +#if ENABLED_LOGGING + // Reject an unusable logging state here: returning false after node state has been loaded + // would leave it partially applied + { + unsigned long long savedDigestsSz; + if (!logger.checkLastLoggingStates(directory, savedDigestsSz)) + { + logToConsole(L"Skip using node states snapshot."); + return false; + } + } +#endif + if (ts.tryLoadFromFile(system.epoch, directory) != 0) { logToConsole(L"Failed to load tick storage"); @@ -4542,11 +4555,32 @@ static bool loadAllNodeStates() REVENUE_DATA_SNAPSHOT_FILE_NAME[sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME) / sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME[0]) - 4] = system.epoch / 100 + L'0'; REVENUE_DATA_SNAPSHOT_FILE_NAME[sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME) / sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME[0]) - 3] = (system.epoch % 100) / 10 + L'0'; REVENUE_DATA_SNAPSHOT_FILE_NAME[sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME) / sizeof(REVENUE_DATA_SNAPSHOT_FILE_NAME[0]) - 2] = system.epoch % 10 + L'0'; - long long revenueDataSize = load(REVENUE_DATA_SNAPSHOT_FILE_NAME, sizeof(gEpochRevenueData), (unsigned char*)&gEpochRevenueData, directory); - if (revenueDataSize != sizeof(gEpochRevenueData)) + const long long revenueFileSize = getFileSize(REVENUE_DATA_SNAPSHOT_FILE_NAME, directory); + bool revenueDataLoaded = false; + if (revenueFileSize == (long long)sizeof(gEpochRevenueData)) + { + revenueDataLoaded = load(REVENUE_DATA_SNAPSHOT_FILE_NAME, sizeof(gEpochRevenueData), (unsigned char*)&gEpochRevenueData, directory) + == (long long)sizeof(gEpochRevenueData); + } + else if (revenueFileSize > 0 && revenueFileSize <= (long long)defaultCommonBuffersSize) + { + // Saved with a different MAX_NUMBER_OF_TICKS_PER_EPOCH + __ScopedScratchpad scratchpad(revenueFileSize, /*initZero=*/false); + revenueDataLoaded = scratchpad.ptr + && load(REVENUE_DATA_SNAPSHOT_FILE_NAME, revenueFileSize, (unsigned char*)scratchpad.ptr, directory) == revenueFileSize + && remapEpochRevenueData((unsigned char*)scratchpad.ptr, revenueFileSize, gEpochRevenueData); + } + if (!revenueDataLoaded) { +#if USE_REVENUE_MULTI_DIMENSION + // v2Revenue is shadow when multi-dim is active, revenue uses gMultiDimRevenue. + // We mark zero to make flag out this data is invalid for analysis + logToConsole(L"Revenue data snapshot load/remap failed (v2 shadow mode), zeroing"); + setMem(&gEpochRevenueData, sizeof(gEpochRevenueData), 0); +#else logToConsole(L"Failed to load revenue data snapshot"); return false; +#endif } MULTIDIM_REVENUE_SNAPSHOT_FILE_NAME[sizeof(MULTIDIM_REVENUE_SNAPSHOT_FILE_NAME) / sizeof(MULTIDIM_REVENUE_SNAPSHOT_FILE_NAME[0]) - 4] = system.epoch / 100 + L'0'; @@ -4644,7 +4678,11 @@ static bool loadAllNodeStates() #if ENABLED_LOGGING logToConsole(L"Loading old logger..."); - logger.loadLastLoggingStates(directory); + if (!logger.loadLastLoggingStates(directory)) + { + logToConsole(L"Failed to load logging state, discarding snapshot"); + return false; + } #endif #if !defined(NDEBUG) diff --git a/src/revenue.h b/src/revenue.h index 7d5031c6..fa063843 100644 --- a/src/revenue.h +++ b/src/revenue.h @@ -421,6 +421,44 @@ struct EpochRevenueData static EpochRevenueData gEpochRevenueData; +static constexpr unsigned long long revenueDataPerTickArrays = 5; +static constexpr unsigned long long revenueDataFixedSize = + sizeof(EpochRevenueData) - revenueDataPerTickArrays * MAX_NUMBER_OF_TICKS_PER_EPOCH * sizeof(unsigned short); +static_assert(revenueDataFixedSize == 8 + 4ULL * NUMBER_OF_COMPUTORS * sizeof(unsigned long long), + "EpochRevenueData has unexpected padding; the snapshot re-layout below relies on its exact layout"); + +// Restore a revenue snapshot saved with a different MAX_NUMBER_OF_TICKS_PER_EPOCH. +static bool remapEpochRevenueData(const unsigned char* saved, unsigned long long savedSize, EpochRevenueData& out) +{ + constexpr unsigned long long stride = revenueDataPerTickArrays * sizeof(unsigned short); + if (savedSize < revenueDataFixedSize || (savedSize - revenueDataFixedSize) % stride) + { + return false; + } + const unsigned long long savedTicks = (savedSize - revenueDataFixedSize) / stride; + if (savedTicks % NUMBER_OF_COMPUTORS) + { + return false; + } + + // static_assert pins the sizes; this pins the field order the copies below rely on + ASSERT((const unsigned char*)out.perTickTxCount == (const unsigned char*)&out + revenueDataFixedSize); + + copyMem(&out, saved, revenueDataFixedSize); + + const unsigned long long copyTicks = savedTicks < MAX_NUMBER_OF_TICKS_PER_EPOCH ? savedTicks : MAX_NUMBER_OF_TICKS_PER_EPOCH; + unsigned short* dst[revenueDataPerTickArrays] = { + out.perTickTxCount, out.perTickProtocolTxCount, out.perTickContractTxCount, + out.perTickOtherTxCount, out.perTickTxTickLeaderCount }; + const unsigned char* src = saved + revenueDataFixedSize; + for (unsigned long long i = 0; i < revenueDataPerTickArrays; i++) + { + copyMem(dst[i], src + i * savedTicks * sizeof(unsigned short), copyTicks * sizeof(unsigned short)); + setMem(dst[i] + copyTicks, (MAX_NUMBER_OF_TICKS_PER_EPOCH - copyTicks) * sizeof(unsigned short), 0); + } + return true; +} + // Intermediate buffers for V2 revenue computation struct RevenueV2Buffers { From 2ebf23eef96fda9d174257eabb983b3dcdb843de Mon Sep 17 00:00:00 2001 From: krypdkat <39078779+krypdkat@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:10:31 +0700 Subject: [PATCH 2/3] expand testnet duration --- src/public_settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/public_settings.h b/src/public_settings.h index 041fb343..2c0025e4 100644 --- a/src/public_settings.h +++ b/src/public_settings.h @@ -170,7 +170,7 @@ static constexpr long long NEURON_VALUE_LIMIT = 1LL; #ifdef LONG_RUN_LOCAL_TESTNET // 60 days at 1s ticks; see doc/long_run_local_testnet.md #ifndef LONG_RUN_EPOCH_TICK_CAPACITY - #define LONG_RUN_EPOCH_TICK_CAPACITY 5184000ULL + #define LONG_RUN_EPOCH_TICK_CAPACITY 5184244ULL #endif #define TESTNET_EPOCH_DURATION LONG_RUN_EPOCH_TICK_CAPACITY #else From 69a9164b171ec5ec11395562f010d4767a77f7cf Mon Sep 17 00:00:00 2001 From: feiyu Date: Tue, 28 Jul 2026 16:50:16 +0700 Subject: [PATCH 3/3] adjust TICK_DURATION_FOR_ALLOCATION_MS --- src/public_settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/public_settings.h b/src/public_settings.h index 2c0025e4..dd652f6a 100644 --- a/src/public_settings.h +++ b/src/public_settings.h @@ -62,7 +62,7 @@ // The tick duration used to calculate the size of memory buffers. // This determines the memory footprint of the application. -#define TICK_DURATION_FOR_ALLOCATION_MS 350 +#define TICK_DURATION_FOR_ALLOCATION_MS 225 #define TRANSACTION_SPARSENESS 10 // Number of ticks that are stored in the pending txs pool. This also defines how many ticks in advance a tx can be registered. #define PENDING_TXS_POOL_NUM_TICKS (1000 * 60 * 3ULL / TICK_DURATION_FOR_ALLOCATION_MS) // 3 minutes