diff --git a/debug.ini.sample b/debug.ini.sample index 38a0e41..a7daf32 100644 --- a/debug.ini.sample +++ b/debug.ini.sample @@ -31,3 +31,21 @@ LOG.RDK.FOO=INFO LOG.RDK.BAR = NONE #ALL logs will be disabled for this module + +########################################################################## +# Duplicate Log Suppression Configuration +# +# These settings control the duplicate log message suppression feature. +# When enabled, consecutive identical log messages are suppressed and +# a summary count is emitted when the suppression ends. +# +# LOG.RDK.DEDUP_ENABLED: 0 = disabled (default), 1 = enabled +# LOG.RDK.DEDUP_WINDOW_SEC: Time window in seconds before auto-flushing +# summary (default: 30) +# LOG.RDK.DEDUP_MAX_SUPPRESS: Maximum consecutive duplicates before +# forced flush (default: 1000) +########################################################################## + +#LOG.RDK.DEDUP_ENABLED = 0 +#LOG.RDK.DEDUP_WINDOW_SEC = 30 +#LOG.RDK.DEDUP_MAX_SUPPRESS = 1000 diff --git a/src/Makefile.am b/src/Makefile.am index 6ca2fd2..0707d0a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -19,7 +19,7 @@ AM_CFLAGS = -pthread -Wall -Werror lib_LTLIBRARIES = librdkloggers.la -librdkloggers_la_SOURCES = rdk_debug.c rdk_debug_priv.c rdk_dynamic_logger.c rdk_logger_init.c rdk_logger_milestone.c rdk_logger_onboard.c +librdkloggers_la_SOURCES = rdk_debug.c rdk_debug_priv.c rdk_dynamic_logger.c rdk_logger_init.c rdk_logger_milestone.c rdk_logger_onboard.c rdk_log_dedup.c librdkloggers_la_CFLAGS = $(LOG4C_CFLAGS) $(GLIB_CFLAGS) $(SYSTEMD_CFLAGS) -I$(top_srcdir)/include -I$(top_srcdir)/src/include -DDEBUG_CONF_FILE="\"debug.ini\"" librdkloggers_la_LDFLAGS = $(GLIB_LIBS) $(LOG4C_LIBS) $(SYSTEMD_LIBS) diff --git a/src/include/rdk_log_dedup.h b/src/include/rdk_log_dedup.h new file mode 100644 index 0000000..1a27d2c --- /dev/null +++ b/src/include/rdk_log_dedup.h @@ -0,0 +1,123 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2024 RDK Management + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +#if !defined(_RDK_LOG_DEDUP_H) +#define _RDK_LOG_DEDUP_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** Maximum number of tracked category slots for dedup state. */ +#define RDK_DEDUP_TABLE_SIZE 256 + +/** Default dedup window in seconds. */ +#define RDK_DEDUP_DEFAULT_WINDOW_SEC 30 + +/** Default max suppression count before forced flush. */ +#define RDK_DEDUP_DEFAULT_MAX_SUPPRESS 1000 + +/** + * Configuration for the dedup engine. + * Parsed from debug.ini keys: + * LOG.RDK.DEDUP_ENABLED = 0|1 + * LOG.RDK.DEDUP_WINDOW_SEC = + * LOG.RDK.DEDUP_MAX_SUPPRESS = + */ +typedef struct { + bool enabled; /**< Whether dedup is active */ + uint32_t window_sec; /**< Time window before auto-flush of summary */ + uint32_t max_suppress; /**< Max consecutive duplicates before forced flush */ +} rdk_dedup_config_t; + +/** + * Per-category dedup state slot. + */ +typedef struct { + const void *category_key; /**< log4c category pointer (used as hash key) */ + uint32_t last_msg_hash; /**< FNV-1a hash of last formatted message */ + uint32_t repeat_count; /**< Number of suppressed duplicates */ + time_t first_seen; /**< Timestamp when first duplicate was seen */ + int last_level; /**< Log level of suppressed message (log4c priority) */ +} rdk_dedup_slot_t; + +/** + * Initialize the dedup engine with default configuration. + * Must be called once during logger init (from rdk_dbg_priv_init). + */ +void rdk_dedup_init(void); + +/** + * Update dedup configuration. Called when debug.ini is parsed. + * @param config New configuration values. + */ +void rdk_dedup_set_config(const rdk_dedup_config_t *config); + +/** + * Get current dedup configuration (read-only). + */ +const rdk_dedup_config_t* rdk_dedup_get_config(void); + +/** + * Check if a message should be suppressed. + * + * MUST be called under gLoggingMutex. + * + * @param category_key Opaque pointer identifying the category (log4c_category_t*). + * @param log4c_prio The log4c priority level of the message. + * @param format The format string of the log message. + * @param args The va_list arguments (used for hashing the formatted message). + * @return true if the message should be suppressed (duplicate), false if it should be logged. + * + * Side effects: If returning false after a suppression run, emits a summary + * message via the provided category before returning. + */ +bool rdk_dedup_check(const void *category_key, int log4c_prio, + const char *format, va_list args); + +/** + * Flush any pending dedup summary for a category (e.g., on window expiry). + * MUST be called under gLoggingMutex. + * @param category_key The category to flush. + */ +void rdk_dedup_flush(const void *category_key); + +/** + * Flush all pending dedup summaries. Called during deinit. + * MUST be called under gLoggingMutex. + */ +void rdk_dedup_flush_all(void); + +/** + * Compute FNV-1a 32-bit hash of a string. + * @param str Null-terminated string to hash. + * @return 32-bit hash value. + */ +uint32_t rdk_dedup_fnv1a_hash(const char *str); + +#ifdef __cplusplus +} +#endif + +#endif /* _RDK_LOG_DEDUP_H */ diff --git a/src/rdk_debug_priv.c b/src/rdk_debug_priv.c index 5775c90..2298062 100644 --- a/src/rdk_debug_priv.c +++ b/src/rdk_debug_priv.c @@ -48,6 +48,7 @@ #include "rdk_debug_priv.h" #include "rdk_dynamic_logger.h" +#include "rdk_log_dedup.h" #include "log4c.h" #include #include @@ -307,6 +308,34 @@ rdk_Error rdk_logger_parse_config( const char * path) } } } + + /* Parse dedup configuration keys */ + if (strcmp("LOG.RDK.DEDUP_ENABLED", trimname) == 0) + { + rdk_dedup_config_t cfg = *rdk_dedup_get_config(); + cfg.enabled = (atoi(trimvalue) != 0); + rdk_dedup_set_config(&cfg); + } + else if (strcmp("LOG.RDK.DEDUP_WINDOW_SEC", trimname) == 0) + { + rdk_dedup_config_t cfg = *rdk_dedup_get_config(); + int val = atoi(trimvalue); + if (val > 0) + { + cfg.window_sec = (uint32_t)val; + rdk_dedup_set_config(&cfg); + } + } + else if (strcmp("LOG.RDK.DEDUP_MAX_SUPPRESS", trimname) == 0) + { + rdk_dedup_config_t cfg = *rdk_dedup_get_config(); + int val = atoi(trimvalue); + if (val > 0) + { + cfg.max_suppress = (uint32_t)val; + rdk_dedup_set_config(&cfg); + } + } } fclose( f); @@ -345,6 +374,9 @@ void rdk_dbg_priv_init(void) if (NULL != legacy) (void) log4c_layout_set_type(legacy, &log4c_layout_type_comcast_dated); + /* Initialize the duplicate log suppression engine */ + rdk_dedup_init(); + isInited = true; } @@ -649,7 +681,11 @@ void rdk_dbg_priv_log_msg(rdk_LogLevel level, const char *module_name, const cha int log4cPriority = rdk_logLevel_to_log4c_priority(level); if (log4c_category_is_priority_enabled(cat, log4cPriority)) { - log4c_category_vlog(cat, log4cPriority, format, args); + /* Check for duplicate suppression before dispatching */ + if (!rdk_dedup_check((const void *)cat, log4cPriority, format, args)) + { + log4c_category_vlog(cat, log4cPriority, format, args); + } } } pthread_mutex_unlock(&gLoggingMutex); diff --git a/src/rdk_log_dedup.c b/src/rdk_log_dedup.c new file mode 100644 index 0000000..2770050 --- /dev/null +++ b/src/rdk_log_dedup.c @@ -0,0 +1,252 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2024 RDK Management + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +/** + * @file rdk_log_dedup.c + * @brief Duplicate log message suppression engine for rdk_logger. + * + * Tracks consecutive identical log messages per category and suppresses + * duplicates, emitting a summary count when the suppression ends. + */ + +#include +#include +#include +#include + +#include "rdk_log_dedup.h" + +/** Static dedup configuration. */ +static rdk_dedup_config_t g_dedup_config = { + .enabled = false, + .window_sec = RDK_DEDUP_DEFAULT_WINDOW_SEC, + .max_suppress = RDK_DEDUP_DEFAULT_MAX_SUPPRESS +}; + +/** Static hash table of dedup slots. */ +static rdk_dedup_slot_t g_dedup_table[RDK_DEDUP_TABLE_SIZE]; + +/** FNV-1a constants */ +#define FNV1A_OFFSET_BASIS 0x811c9dc5u +#define FNV1A_PRIME 0x01000193u + +/** Maximum formatted message length for hashing. */ +#define DEDUP_MSG_BUF_SIZE 512 + +uint32_t rdk_dedup_fnv1a_hash(const char *str) +{ + uint32_t hash = FNV1A_OFFSET_BASIS; + if (!str) + return hash; + + while (*str) + { + hash ^= (uint32_t)(unsigned char)(*str); + hash *= FNV1A_PRIME; + str++; + } + return hash; +} + +/** + * Get the table slot index for a category pointer. + */ +static unsigned int dedup_slot_index(const void *category_key) +{ + /* Use pointer value as hash input */ + uintptr_t ptr = (uintptr_t)category_key; + return (unsigned int)(ptr >> 4) % RDK_DEDUP_TABLE_SIZE; +} + +/** + * Find or allocate a slot for the given category. + * Returns NULL if no slot available (all occupied by other categories). + */ +static rdk_dedup_slot_t* dedup_find_slot(const void *category_key) +{ + unsigned int idx = dedup_slot_index(category_key); + unsigned int i; + + /* Linear probing */ + for (i = 0; i < RDK_DEDUP_TABLE_SIZE; i++) + { + unsigned int probe = (idx + i) % RDK_DEDUP_TABLE_SIZE; + if (g_dedup_table[probe].category_key == category_key) + { + return &g_dedup_table[probe]; + } + if (g_dedup_table[probe].category_key == NULL) + { + /* Empty slot - allocate it */ + g_dedup_table[probe].category_key = category_key; + g_dedup_table[probe].last_msg_hash = 0; + g_dedup_table[probe].repeat_count = 0; + g_dedup_table[probe].first_seen = 0; + g_dedup_table[probe].last_level = 0; + return &g_dedup_table[probe]; + } + } + return NULL; /* Table full */ +} + +void rdk_dedup_init(void) +{ + memset(g_dedup_table, 0, sizeof(g_dedup_table)); + g_dedup_config.enabled = false; + g_dedup_config.window_sec = RDK_DEDUP_DEFAULT_WINDOW_SEC; + g_dedup_config.max_suppress = RDK_DEDUP_DEFAULT_MAX_SUPPRESS; +} + +void rdk_dedup_set_config(const rdk_dedup_config_t *config) +{ + if (config) + { + g_dedup_config.enabled = config->enabled; + g_dedup_config.window_sec = config->window_sec; + g_dedup_config.max_suppress = config->max_suppress; + } +} + +const rdk_dedup_config_t* rdk_dedup_get_config(void) +{ + return &g_dedup_config; +} + +bool rdk_dedup_check(const void *category_key, int log4c_prio, + const char *format, va_list args) +{ + rdk_dedup_slot_t *slot; + uint32_t msg_hash; + char msg_buf[DEDUP_MSG_BUF_SIZE]; + time_t now; + va_list args_copy; + + if (!g_dedup_config.enabled) + return false; /* Not suppressed */ + + if (!category_key || !format) + return false; + + /* Format the message for hashing */ + va_copy(args_copy, args); + vsnprintf(msg_buf, sizeof(msg_buf), format, args_copy); + va_end(args_copy); + + msg_hash = rdk_dedup_fnv1a_hash(msg_buf); + + slot = dedup_find_slot(category_key); + if (!slot) + return false; /* Table full, don't suppress */ + + now = time(NULL); + + /* Check if this is a duplicate of the previous message */ + if (slot->repeat_count > 0 && slot->last_msg_hash == msg_hash) + { + /* Check window expiry */ + if ((now - slot->first_seen) >= (time_t)g_dedup_config.window_sec) + { + /* Window expired - flush summary and allow this message */ + fprintf(stderr, "Previous message repeated %u times\n", slot->repeat_count); + slot->repeat_count = 0; + slot->first_seen = now; + slot->last_msg_hash = msg_hash; + slot->last_level = log4c_prio; + return false; /* Don't suppress - allow through after flush */ + } + + /* Check max suppression count */ + if (slot->repeat_count >= g_dedup_config.max_suppress) + { + /* Max reached - flush and allow */ + fprintf(stderr, "Previous message repeated %u times\n", slot->repeat_count); + slot->repeat_count = 0; + slot->first_seen = now; + slot->last_msg_hash = msg_hash; + slot->last_level = log4c_prio; + return false; /* Don't suppress */ + } + + /* Suppress this duplicate */ + slot->repeat_count++; + return true; + } + else + { + /* Different message or first message */ + if (slot->repeat_count > 0) + { + /* Emit summary for previous suppression run */ + fprintf(stderr, "Previous message repeated %u times\n", slot->repeat_count); + } + + /* Record new message state */ + slot->last_msg_hash = msg_hash; + slot->repeat_count = 1; + slot->first_seen = now; + slot->last_level = log4c_prio; + return false; /* Don't suppress first occurrence */ + } +} + +void rdk_dedup_flush(const void *category_key) +{ + rdk_dedup_slot_t *slot; + unsigned int idx; + unsigned int i; + + if (!category_key) + return; + + idx = dedup_slot_index(category_key); + + for (i = 0; i < RDK_DEDUP_TABLE_SIZE; i++) + { + unsigned int probe = (idx + i) % RDK_DEDUP_TABLE_SIZE; + if (g_dedup_table[probe].category_key == category_key) + { + slot = &g_dedup_table[probe]; + if (slot->repeat_count > 1) + { + fprintf(stderr, "Previous message repeated %u times\n", + slot->repeat_count - 1); + } + slot->repeat_count = 0; + return; + } + if (g_dedup_table[probe].category_key == NULL) + { + return; /* Not found */ + } + } +} + +void rdk_dedup_flush_all(void) +{ + unsigned int i; + for (i = 0; i < RDK_DEDUP_TABLE_SIZE; i++) + { + if (g_dedup_table[i].category_key != NULL && g_dedup_table[i].repeat_count > 1) + { + fprintf(stderr, "Previous message repeated %u times\n", + g_dedup_table[i].repeat_count - 1); + g_dedup_table[i].repeat_count = 0; + } + } +} diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 47563b0..8c6f72c 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -41,6 +41,7 @@ add_executable( rdkLoggerRotationTest.cpp rdkLoggerErrorTest.cpp rdkLoggerPerformanceTest.cpp + rdkLoggerDedupTest.cpp main.cpp ) diff --git a/unittests/rdkLoggerDedupTest.cpp b/unittests/rdkLoggerDedupTest.cpp new file mode 100644 index 0000000..6f855d3 --- /dev/null +++ b/unittests/rdkLoggerDedupTest.cpp @@ -0,0 +1,191 @@ +/* + * Copyright 2024 RDK Management + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +extern "C" { +#include "rdk_log_dedup.h" +} + +class RDKLoggerDedupTest : public ::testing::Test { +protected: + void SetUp() override { + rdk_dedup_init(); + } + + void TearDown() override { + rdk_dedup_flush_all(); + } + + /* Helper to call rdk_dedup_check with printf-style args */ + bool check_dedup(const void *cat, int prio, const char *fmt, ...) { + va_list args; + va_start(args, fmt); + bool result = rdk_dedup_check(cat, prio, fmt, args); + va_end(args); + return result; + } +}; + +/* Test: When dedup is disabled, no messages are suppressed */ +TEST_F(RDKLoggerDedupTest, DisabledModePassThrough) { + /* Dedup is disabled by default after init */ + const void *cat = (const void *)0x1000; + + /* Same message 10 times should all pass through */ + for (int i = 0; i < 10; i++) { + EXPECT_FALSE(check_dedup(cat, 300, "Error: connection failed\n")); + } +} + +/* Test: When enabled, consecutive duplicates are suppressed */ +TEST_F(RDKLoggerDedupTest, ConsecutiveDuplicateSuppression) { + rdk_dedup_config_t cfg = { true, 30, 1000 }; + rdk_dedup_set_config(&cfg); + + const void *cat = (const void *)0x2000; + + /* First message passes through */ + EXPECT_FALSE(check_dedup(cat, 300, "Error: connection failed\n")); + + /* Consecutive identical messages are suppressed */ + EXPECT_TRUE(check_dedup(cat, 300, "Error: connection failed\n")); + EXPECT_TRUE(check_dedup(cat, 300, "Error: connection failed\n")); + EXPECT_TRUE(check_dedup(cat, 300, "Error: connection failed\n")); +} + +/* Test: Different messages are not suppressed */ +TEST_F(RDKLoggerDedupTest, DifferentMessagesPassThrough) { + rdk_dedup_config_t cfg = { true, 30, 1000 }; + rdk_dedup_set_config(&cfg); + + const void *cat = (const void *)0x3000; + + EXPECT_FALSE(check_dedup(cat, 300, "Message A\n")); + EXPECT_FALSE(check_dedup(cat, 300, "Message B\n")); + EXPECT_FALSE(check_dedup(cat, 300, "Message A\n")); + EXPECT_FALSE(check_dedup(cat, 300, "Message B\n")); +} + +/* Test: Max suppression count forces flush */ +TEST_F(RDKLoggerDedupTest, MaxSuppressionCountFlush) { + rdk_dedup_config_t cfg = { true, 30, 5 }; + rdk_dedup_set_config(&cfg); + + const void *cat = (const void *)0x4000; + + /* First passes through */ + EXPECT_FALSE(check_dedup(cat, 300, "Repeated error\n")); + + /* Next 5 are suppressed (reaching max) */ + for (int i = 0; i < 5; i++) { + EXPECT_TRUE(check_dedup(cat, 300, "Repeated error\n")); + } + + /* After max reached, next occurrence passes through (flush) */ + EXPECT_FALSE(check_dedup(cat, 300, "Repeated error\n")); +} + +/* Test: Independent category tracking */ +TEST_F(RDKLoggerDedupTest, IndependentCategoryTracking) { + rdk_dedup_config_t cfg = { true, 30, 1000 }; + rdk_dedup_set_config(&cfg); + + const void *cat1 = (const void *)0x5000; + const void *cat2 = (const void *)0x6000; + + /* First message for each category passes */ + EXPECT_FALSE(check_dedup(cat1, 300, "Error in cat1\n")); + EXPECT_FALSE(check_dedup(cat2, 300, "Error in cat2\n")); + + /* Duplicates in cat1 are suppressed */ + EXPECT_TRUE(check_dedup(cat1, 300, "Error in cat1\n")); + + /* cat2 with same message is still suppressed (its own duplicate) */ + EXPECT_TRUE(check_dedup(cat2, 300, "Error in cat2\n")); + + /* New message in cat1 passes through */ + EXPECT_FALSE(check_dedup(cat1, 300, "Different message\n")); + + /* cat2 duplicate still suppressed */ + EXPECT_TRUE(check_dedup(cat2, 300, "Error in cat2\n")); +} + +/* Test: FNV-1a hash function */ +TEST_F(RDKLoggerDedupTest, FNV1aHashConsistency) { + uint32_t h1 = rdk_dedup_fnv1a_hash("test message"); + uint32_t h2 = rdk_dedup_fnv1a_hash("test message"); + uint32_t h3 = rdk_dedup_fnv1a_hash("different message"); + + EXPECT_EQ(h1, h2); + EXPECT_NE(h1, h3); +} + +/* Test: NULL inputs don't crash */ +TEST_F(RDKLoggerDedupTest, NullInputsSafe) { + rdk_dedup_config_t cfg = { true, 30, 1000 }; + rdk_dedup_set_config(&cfg); + + EXPECT_FALSE(check_dedup(NULL, 300, "test\n")); + EXPECT_FALSE(check_dedup((const void *)0x7000, 300, NULL)); +} + +/* Test: Config getter returns correct values */ +TEST_F(RDKLoggerDedupTest, ConfigGetterReturnsCorrectValues) { + rdk_dedup_config_t cfg = { true, 60, 500 }; + rdk_dedup_set_config(&cfg); + + const rdk_dedup_config_t *got = rdk_dedup_get_config(); + EXPECT_TRUE(got->enabled); + EXPECT_EQ(got->window_sec, 60u); + EXPECT_EQ(got->max_suppress, 500u); +} + +/* Test: After new message arrives, summary is emitted and new message passes */ +TEST_F(RDKLoggerDedupTest, SummaryEmittedOnNewMessage) { + rdk_dedup_config_t cfg = { true, 30, 1000 }; + rdk_dedup_set_config(&cfg); + + const void *cat = (const void *)0x8000; + + /* First passes */ + EXPECT_FALSE(check_dedup(cat, 300, "Error A\n")); + /* Duplicates suppressed */ + EXPECT_TRUE(check_dedup(cat, 300, "Error A\n")); + EXPECT_TRUE(check_dedup(cat, 300, "Error A\n")); + + /* New message passes through (triggers summary internally) */ + EXPECT_FALSE(check_dedup(cat, 300, "Error B\n")); +} + +/* Test: Format string with arguments */ +TEST_F(RDKLoggerDedupTest, FormatStringWithArgs) { + rdk_dedup_config_t cfg = { true, 30, 1000 }; + rdk_dedup_set_config(&cfg); + + const void *cat = (const void *)0x9000; + + /* Same format with same args = duplicate */ + EXPECT_FALSE(check_dedup(cat, 300, "Error code: %d\n", 42)); + EXPECT_TRUE(check_dedup(cat, 300, "Error code: %d\n", 42)); + + /* Same format with different args = not duplicate */ + EXPECT_FALSE(check_dedup(cat, 300, "Error code: %d\n", 43)); +} diff --git a/utils/Makefile.am b/utils/Makefile.am index 5b6ba11..b3e06b7 100644 --- a/utils/Makefile.am +++ b/utils/Makefile.am @@ -34,3 +34,8 @@ onboarding_log_SOURCES = rdk_logger_onboard_main.c onboarding_log_LDADD = $(top_builddir)/src/librdkloggers.la $(LOG4C_LIBS) onboarding_log_CFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/src/include -I./usr/include onboarding_log_LDFLAGS = -L/usr/lib + +bin_PROGRAMS += rdk_log_dedup_filter +rdk_log_dedup_filter_SOURCES = rdk_log_dedup_filter.c +rdk_log_dedup_filter_CFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/src/include +rdk_log_dedup_filter_LDFLAGS = -L/usr/lib diff --git a/utils/rdk_log_dedup_filter.c b/utils/rdk_log_dedup_filter.c new file mode 100644 index 0000000..ac5f721 --- /dev/null +++ b/utils/rdk_log_dedup_filter.c @@ -0,0 +1,232 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2024 RDK Management + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +/** + * @file rdk_log_dedup_filter.c + * @brief Standalone utility to filter duplicate log lines from log files. + * + * Used during nvram sync or log upload to suppress consecutive duplicate + * log messages, reducing storage and bandwidth usage. + * + * Usage: + * rdk_log_dedup_filter [--in-place] [input-file] + * + * If no input file is specified, reads from stdin. + * Output is written to stdout unless --in-place is used. + */ + +#include +#include +#include +#include +#include + +#define MAX_LINE_LEN 4096 +#define TIMESTAMP_PREFIX_LEN 22 /* "YYMMDD-HH:MM:SS.uuuuuu " */ + +/** FNV-1a constants */ +#define FNV1A_OFFSET_BASIS 0x811c9dc5u +#define FNV1A_PRIME 0x01000193u + +static uint32_t fnv1a_hash(const char *str) +{ + uint32_t hash = FNV1A_OFFSET_BASIS; + if (!str) + return hash; + while (*str) + { + hash ^= (uint32_t)(unsigned char)(*str); + hash *= FNV1A_PRIME; + str++; + } + return hash; +} + +/** + * Strip the timestamp prefix from a log line for comparison purposes. + * RDK log format: "YYMMDD-HH:MM:SS.uuuuuu LEVEL ..." + * Returns pointer to the content after timestamp, or the original line + * if no timestamp pattern is detected. + */ +static const char* strip_timestamp(const char *line) +{ + /* Check for pattern: YYMMDD-HH:MM:SS */ + if (strlen(line) < 15) + return line; + + /* Check digits and separators: DDDDDD-DD:DD:DD */ + if (line[6] == '-' && line[9] == ':' && line[12] == ':') + { + /* Skip past timestamp and any trailing whitespace/microseconds */ + const char *p = line + 13; /* past HH:MM:SS */ + /* Skip optional microseconds (.uuuuuu) */ + if (*p == '.') + { + p++; + while (*p >= '0' && *p <= '9') + p++; + } + /* Skip whitespace */ + while (*p == ' ') + p++; + return p; + } + + return line; +} + +static void print_usage(const char *prog) +{ + fprintf(stderr, "Usage: %s [--in-place] [input-file]\n", prog); + fprintf(stderr, " Filters consecutive duplicate log lines.\n"); + fprintf(stderr, " If no input file, reads from stdin.\n"); + fprintf(stderr, " Output goes to stdout unless --in-place is used.\n"); +} + +int main(int argc, char *argv[]) +{ + FILE *input = stdin; + FILE *output = stdout; + FILE *tmp_output = NULL; + bool in_place = false; + const char *input_path = NULL; + char *tmp_path = NULL; + + char line[MAX_LINE_LEN]; + char first_line[MAX_LINE_LEN]; + uint32_t prev_hash = 0; + uint32_t repeat_count = 0; + bool has_prev = false; + + /* Parse arguments */ + for (int i = 1; i < argc; i++) + { + if (strcmp(argv[i], "--in-place") == 0) + { + in_place = true; + } + else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) + { + print_usage(argv[0]); + return 0; + } + else + { + input_path = argv[i]; + } + } + + if (in_place && !input_path) + { + fprintf(stderr, "Error: --in-place requires an input file path.\n"); + return 1; + } + + if (input_path) + { + input = fopen(input_path, "r"); + if (!input) + { + fprintf(stderr, "Error: Cannot open input file: %s\n", input_path); + return 1; + } + } + + if (in_place) + { + size_t tmp_path_len = strlen(input_path) + 8; + tmp_path = malloc(tmp_path_len); + if (!tmp_path) + { + fprintf(stderr, "Error: Memory allocation failed\n"); + fclose(input); + return 1; + } + snprintf(tmp_path, tmp_path_len, "%s.dedup", input_path); + tmp_output = fopen(tmp_path, "w"); + if (!tmp_output) + { + fprintf(stderr, "Error: Cannot create temp file: %s\n", tmp_path); + free(tmp_path); + fclose(input); + return 1; + } + output = tmp_output; + } + + /* Process lines */ + while (fgets(line, sizeof(line), input) != NULL) + { + const char *content = strip_timestamp(line); + uint32_t hash = fnv1a_hash(content); + + if (has_prev && hash == prev_hash) + { + /* Duplicate line - suppress */ + repeat_count++; + } + else + { + /* Different line - flush any pending duplicates */ + if (has_prev) + { + fputs(first_line, output); + if (repeat_count > 0) + { + fprintf(output, "... repeated %u times\n", repeat_count); + } + } + + /* Store this as the new reference line */ + strncpy(first_line, line, sizeof(first_line) - 1); + first_line[sizeof(first_line) - 1] = '\0'; + prev_hash = hash; + repeat_count = 0; + has_prev = true; + } + } + + /* Flush final pending line */ + if (has_prev) + { + fputs(first_line, output); + if (repeat_count > 0) + { + fprintf(output, "... repeated %u times\n", repeat_count); + } + } + + if (input_path) + fclose(input); + + if (in_place) + { + fclose(tmp_output); + /* Replace original with filtered version */ + if (rename(tmp_path, input_path) != 0) + { + fprintf(stderr, "Error: Failed to replace original file\n"); + free(tmp_path); + return 1; + } + free(tmp_path); + } + + return 0; +}