From 53183cbb04dff612f276909adb5cdd216d43e7d1 Mon Sep 17 00:00:00 2001 From: MrSoichi <21004580+MrSoichi@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:08:41 +0600 Subject: [PATCH 1/5] Add rate-limited traceback and heartbeat monitoring for Turing periodic tasks --- library/task_health.py | 51 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 library/task_health.py diff --git a/library/task_health.py b/library/task_health.py new file mode 100644 index 00000000..d615b5c6 --- /dev/null +++ b/library/task_health.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Keep periodic jobs observable without turning transient errors into dead workers. + +No sleeps, thread creation, or dependency on Windows here: scheduler.py retains +the original intervals and threads. Failures are rate-limited in the log. +""" +import time + + +class TaskHealth: + def __init__(self, retry_log_seconds=60, cpu_heartbeat_seconds=300, clock=None): + self.retry_log_seconds = retry_log_seconds + self.cpu_heartbeat_seconds = cpu_heartbeat_seconds + self.clock = clock or time.monotonic + self._state = {} + + def run(self, action, args, logger): + name = getattr(action, "__name__", type(action).__name__) + state = self._state.setdefault(name, { + "failures": 0, "last_failure_log": None, "last_heartbeat": None, + }) + try: + action(*args) + except Exception: + state["failures"] += 1 + now = self.clock() + if (state["last_failure_log"] is None or + now - state["last_failure_log"] >= self.retry_log_seconds): + # The first traceback and periodic reminders survive frozen GUI + # mode, where uncaught thread exceptions may miss log.log. + logger.exception( + "Periodic task %s failed %d time(s); will retry next tick", + name, state["failures"]) + state["last_failure_log"] = now + return False + + if state["failures"]: + logger.info("Periodic task %s recovered after %d failure(s)", + name, state["failures"]) + state["failures"] = 0 + state["last_failure_log"] = None + + # A very sparse CPU scheduler heartbeat proves the worker is still + # ticking. Its absence after resume is also useful diagnostic evidence. + if name == "CPUPercentage": + now = self.clock() + if (state["last_heartbeat"] is None or + now - state["last_heartbeat"] >= self.cpu_heartbeat_seconds): + logger.info("Periodic task CPUPercentage heartbeat: active") + state["last_heartbeat"] = now + return True From c94acecded3ad3524dd942381106f76ca5ff3be3 Mon Sep 17 00:00:00 2001 From: MrSoichi <21004580+MrSoichi@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:08:45 +0600 Subject: [PATCH 2/5] Test periodic task retry, recovery, heartbeat and nonfinite CPU% guard --- tests/library/test_task_health.py | 84 +++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/library/test_task_health.py diff --git a/tests/library/test_task_health.py b/tests/library/test_task_health.py new file mode 100644 index 00000000..fee7d5aa --- /dev/null +++ b/tests/library/test_task_health.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Regression tests for CPU% normalization and self-healing periodic workers.""" +import math +import unittest +from unittest.mock import Mock + +from library.sensors.cpu_sensor_health import cpu_percentage_value +from library.task_health import TaskHealth + + +class PercentageGuardTests(unittest.TestCase): + def test_valid_and_near_boundary(self): + for v in (0, 15.8, 100, "52.5"): + self.assertEqual(cpu_percentage_value(v), float(v)) + self.assertEqual(cpu_percentage_value(100.2), 100) + self.assertEqual(cpu_percentage_value(-0.1), 0) + + def test_invalid_never_reaches_int_or_draw(self): + for v in (None, math.nan, math.inf, -50, 200, "", "NaN", object()): + self.assertIsNone(cpu_percentage_value(v)) + + +class TaskHealthTests(unittest.TestCase): + def setUp(self): + self.t = [100.0] + self.logger = Mock() + self.guard = TaskHealth(clock=lambda: self.t[0]) + + def test_exception_does_not_end_task_and_recovery_is_logged(self): + side = [ValueError("CPU sensor momentarily unavailable"), None] + def CPUPercentage(): + maybe = side.pop(0) + if maybe: + raise maybe + self.assertFalse(self.guard.run(CPUPercentage, (), self.logger)) + self.assertEqual(self.logger.exception.call_count, 1) + self.assertTrue(self.guard.run(CPUPercentage, (), self.logger)) + self.logger.info.assert_any_call( + "Periodic task %s recovered after %d failure(s)", "CPUPercentage", 1) + self.logger.info.assert_any_call("Periodic task CPUPercentage heartbeat: active") + + def test_repeated_errors_have_bounded_logs(self): + def CPUPercentage(): + raise RuntimeError("bad reading") + for n in range(300): + self.t[0] = 100 + 0.1 * n + self.assertFalse(self.guard.run(CPUPercentage, (), self.logger)) + self.assertEqual(self.logger.exception.call_count, 1) + self.t[0] += 61 + self.guard.run(CPUPercentage, (), self.logger) + self.assertEqual(self.logger.exception.call_count, 2) + + def test_cpu_heartbeat_is_sparse_and_independent(self): + def CPUPercentage(): + return None + def CPUTemperature(): + return None + self.guard.run(CPUPercentage, (), self.logger) + self.t[0] += 299 + self.guard.run(CPUPercentage, (), self.logger) + self.guard.run(CPUTemperature, (), self.logger) + self.assertEqual(self.logger.info.call_count, 1) + self.t[0] += 1 + self.guard.run(CPUPercentage, (), self.logger) + self.assertEqual(self.logger.info.call_count, 2) + + def test_different_tasks_not_confused(self): + def CPUPercentage(): + raise IOError("broken") + def CPUFrequency(): + return None + self.assertFalse(self.guard.run(CPUPercentage, (), self.logger)) + self.assertTrue(self.guard.run(CPUFrequency, (), self.logger)) + self.assertEqual(self.logger.exception.call_count, 1) + + def test_keyboard_interrupt_not_swallowed(self): + def CPUPercentage(): + raise KeyboardInterrupt + with self.assertRaises(KeyboardInterrupt): + self.guard.run(CPUPercentage, (), self.logger) + + +if __name__ == "__main__": + unittest.main() From bed96ec8c79bb296df35cb1e9e255dee9d7a2433 Mon Sep 17 00:00:00 2001 From: MrSoichi <21004580+MrSoichi@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:09:12 +0600 Subject: [PATCH 3/5] Guard CPU percentage sampling and let periodic Turing tasks recover from exceptions --- library/sensors/cpu_sensor_health.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/library/sensors/cpu_sensor_health.py b/library/sensors/cpu_sensor_health.py index dc5a57d0..db885597 100644 --- a/library/sensors/cpu_sensor_health.py +++ b/library/sensors/cpu_sensor_health.py @@ -59,3 +59,12 @@ def cpu_sensor_inventory(cpu): except Exception as exc: return f"could not enumerate sensors ({type(exc).__name__})" return ", ".join(items) if items else "CPU has no exposed sensors" + +def cpu_percentage_value(value): + """Finite load percentage, with only tiny sampling overshoot clamped. + + A temporary None/NaN must not be converted with int() or sent to graphics. + It will be retried on the next scheduled read. + """ + number = finite_metric(value, -0.5, 100.5) + return min(100.0, max(0.0, number)) if number is not None else None From 1afa67dd7534459febad8a80485e033075b44a0d Mon Sep 17 00:00:00 2001 From: MrSoichi <21004580+MrSoichi@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:09:16 +0600 Subject: [PATCH 4/5] Guard CPU percentage sampling and let periodic Turing tasks recover from exceptions --- library/scheduler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/library/scheduler.py b/library/scheduler.py index d6a1edd5..b9f846e6 100644 --- a/library/scheduler.py +++ b/library/scheduler.py @@ -28,6 +28,10 @@ import library.config as config import library.stats as stats +from library.log import logger +from library.task_health import TaskHealth + +_TASK_HEALTH = TaskHealth() STOPPING = False @@ -62,7 +66,9 @@ def periodic(scheduler, periodic_interval, action, actionargs=()): # If the program is not stopping: re-schedule the task for future execution scheduler.enter(periodic_interval, 1, periodic, (scheduler, periodic_interval, action, actionargs)) - action(*actionargs) + # Preserve this scheduler thread even when a sensor or draw fails. + # Capture the traceback in log.log, then retry at the next tick. + _TASK_HEALTH.run(action, actionargs, logger) @wraps(func) def wrap( From bcf49c78fff07557cd57690e4af605e588733220 Mon Sep 17 00:00:00 2001 From: MrSoichi <21004580+MrSoichi@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:09:19 +0600 Subject: [PATCH 5/5] Guard CPU percentage sampling and let periodic Turing tasks recover from exceptions --- library/stats.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/library/stats.py b/library/stats.py index 70c5935d..067fd910 100644 --- a/library/stats.py +++ b/library/stats.py @@ -40,6 +40,7 @@ import library.config as config from library.display import display from library.log import logger +from library.sensors.cpu_sensor_health import cpu_percentage_value DEFAULT_HISTORY_SIZE = 10 @@ -264,16 +265,27 @@ class CPU: last_values_cpu_temperature = [] last_values_cpu_fan_speed = [] last_values_cpu_frequency = [] + _percentage_available = None @classmethod def percentage(cls): theme_data = config.THEME_DATA['STATS']['CPU']['PERCENTAGE'] - cpu_percentage = sensors.Cpu.percentage( + raw_percentage = sensors.Cpu.percentage( interval=theme_data.get("INTERVAL", None) ) + cpu_percentage = cpu_percentage_value(raw_percentage) + if cpu_percentage is None: + if cls._percentage_available is not False: + logger.warning("CPU percentage unavailable (%r); will retry next tick", + raw_percentage) + cls._percentage_available = False + return + + if cls._percentage_available is False: + logger.info("CPU percentage recovered: %.1f%%", cpu_percentage) + cls._percentage_available = True save_last_value(cpu_percentage, cls.last_values_cpu_percentage, theme_data['LINE_GRAPH'].get("HISTORY_SIZE", DEFAULT_HISTORY_SIZE)) - # logger.debug(f"CPU Percentage: {cpu_percentage}") display_themed_progress_bar(theme_data['GRAPH'], cpu_percentage) display_themed_percent_radial_bar(theme_data['RADIAL'], cpu_percentage)