Keep Turing CPU percentage alive after Windows resume: guarded scheduler and sensor diagnostics

library/scheduler.py

@@ -28,6 +28,10 @@
 
 import library.config as config
 import library.stats as stats
+from library.log import logger
+from library.task_health import TaskHealth
+
+_TASK_HEALTH = TaskHealth()
 
 STOPPING = False
 
@@ -62,7 +66,9 @@ def periodic(scheduler, periodic_interval, action, actionargs=()):
                 # If the program is not stopping: re-schedule the task for future execution
                 scheduler.enter(periodic_interval, 1, periodic,
                                 (scheduler, periodic_interval, action, actionargs))
-            action(*actionargs)
+            # Preserve this scheduler thread even when a sensor or draw fails.
+            # Capture the traceback in log.log, then retry at the next tick.
+            _TASK_HEALTH.run(action, actionargs, logger)
 
         @wraps(func)
         def wrap(

library/sensors/cpu_sensor_health.py

@@ -59,3 +59,12 @@ def cpu_sensor_inventory(cpu):
     except Exception as exc:
         return f"could not enumerate sensors ({type(exc).__name__})"
     return ", ".join(items) if items else "CPU has no exposed sensors"
+
+def cpu_percentage_value(value):
+    """Finite load percentage, with only tiny sampling overshoot clamped.
+
+    A temporary None/NaN must not be converted with int() or sent to graphics.
+    It will be retried on the next scheduled read.
+    """
+    number = finite_metric(value, -0.5, 100.5)
+    return min(100.0, max(0.0, number)) if number is not None else None

library/stats.py

@@ -40,6 +40,7 @@
 import library.config as config
 from library.display import display
 from library.log import logger
+from library.sensors.cpu_sensor_health import cpu_percentage_value
 
 DEFAULT_HISTORY_SIZE = 10
 
@@ -264,16 +265,27 @@ class CPU:
     last_values_cpu_temperature = []
     last_values_cpu_fan_speed = []
     last_values_cpu_frequency = []
+    _percentage_available = None
 
     @classmethod
     def percentage(cls):
         theme_data = config.THEME_DATA['STATS']['CPU']['PERCENTAGE']
-        cpu_percentage = sensors.Cpu.percentage(
+        raw_percentage = sensors.Cpu.percentage(
             interval=theme_data.get("INTERVAL", None)
         )
+        cpu_percentage = cpu_percentage_value(raw_percentage)
+        if cpu_percentage is None:
+            if cls._percentage_available is not False:
+                logger.warning("CPU percentage unavailable (%r); will retry next tick",
+                               raw_percentage)
+            cls._percentage_available = False
+            return
+
+        if cls._percentage_available is False:
+            logger.info("CPU percentage recovered: %.1f%%", cpu_percentage)
+        cls._percentage_available = True
         save_last_value(cpu_percentage, cls.last_values_cpu_percentage,
                         theme_data['LINE_GRAPH'].get("HISTORY_SIZE", DEFAULT_HISTORY_SIZE))
-        # logger.debug(f"CPU Percentage: {cpu_percentage}")
 
         display_themed_progress_bar(theme_data['GRAPH'], cpu_percentage)
         display_themed_percent_radial_bar(theme_data['RADIAL'], cpu_percentage)

library/task_health.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Keep periodic jobs observable without turning transient errors into dead workers.
+
+No sleeps, thread creation, or dependency on Windows here: scheduler.py retains
+the original intervals and threads. Failures are rate-limited in the log.
+"""
+import time
+
+
+class TaskHealth:
+    def __init__(self, retry_log_seconds=60, cpu_heartbeat_seconds=300, clock=None):
+        self.retry_log_seconds = retry_log_seconds
+        self.cpu_heartbeat_seconds = cpu_heartbeat_seconds
+        self.clock = clock or time.monotonic
+        self._state = {}
+
+    def run(self, action, args, logger):
+        name = getattr(action, "__name__", type(action).__name__)
+        state = self._state.setdefault(name, {
+            "failures": 0, "last_failure_log": None, "last_heartbeat": None,
+        })
+        try:
+            action(*args)
+        except Exception:
+            state["failures"] += 1
+            now = self.clock()
+            if (state["last_failure_log"] is None or
+                    now - state["last_failure_log"] >= self.retry_log_seconds):
+                # The first traceback and periodic reminders survive frozen GUI
+                # mode, where uncaught thread exceptions may miss log.log.
+                logger.exception(
+                    "Periodic task %s failed %d time(s); will retry next tick",
+                    name, state["failures"])
+                state["last_failure_log"] = now
+            return False
+
+        if state["failures"]:
+            logger.info("Periodic task %s recovered after %d failure(s)",
+                        name, state["failures"])
+            state["failures"] = 0
+            state["last_failure_log"] = None
+
+        # A very sparse CPU scheduler heartbeat proves the worker is still
+        # ticking. Its absence after resume is also useful diagnostic evidence.
+        if name == "CPUPercentage":
+            now = self.clock()
+            if (state["last_heartbeat"] is None or
+                    now - state["last_heartbeat"] >= self.cpu_heartbeat_seconds):
+                logger.info("Periodic task CPUPercentage heartbeat: active")
+                state["last_heartbeat"] = now
+        return True

tests/library/test_task_health.py

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Regression tests for CPU% normalization and self-healing periodic workers."""
+import math
+import unittest
+from unittest.mock import Mock
+
+from library.sensors.cpu_sensor_health import cpu_percentage_value
+from library.task_health import TaskHealth
+
+
+class PercentageGuardTests(unittest.TestCase):
+    def test_valid_and_near_boundary(self):
+        for v in (0, 15.8, 100, "52.5"):
+            self.assertEqual(cpu_percentage_value(v), float(v))
+        self.assertEqual(cpu_percentage_value(100.2), 100)
+        self.assertEqual(cpu_percentage_value(-0.1), 0)
+
+    def test_invalid_never_reaches_int_or_draw(self):
+        for v in (None, math.nan, math.inf, -50, 200, "", "NaN", object()):
+            self.assertIsNone(cpu_percentage_value(v))
+
+
+class TaskHealthTests(unittest.TestCase):
+    def setUp(self):
+        self.t = [100.0]
+        self.logger = Mock()
+        self.guard = TaskHealth(clock=lambda: self.t[0])
+
+    def test_exception_does_not_end_task_and_recovery_is_logged(self):
+        side = [ValueError("CPU sensor momentarily unavailable"), None]
+        def CPUPercentage():
+            maybe = side.pop(0)
+            if maybe:
+                raise maybe
+        self.assertFalse(self.guard.run(CPUPercentage, (), self.logger))
+        self.assertEqual(self.logger.exception.call_count, 1)
+        self.assertTrue(self.guard.run(CPUPercentage, (), self.logger))
+        self.logger.info.assert_any_call(
+            "Periodic task %s recovered after %d failure(s)", "CPUPercentage", 1)
+        self.logger.info.assert_any_call("Periodic task CPUPercentage heartbeat: active")
+
+    def test_repeated_errors_have_bounded_logs(self):
+        def CPUPercentage():
+            raise RuntimeError("bad reading")
+        for n in range(300):
+            self.t[0] = 100 + 0.1 * n
+            self.assertFalse(self.guard.run(CPUPercentage, (), self.logger))
+        self.assertEqual(self.logger.exception.call_count, 1)
+        self.t[0] += 61
+        self.guard.run(CPUPercentage, (), self.logger)
+        self.assertEqual(self.logger.exception.call_count, 2)
+
+    def test_cpu_heartbeat_is_sparse_and_independent(self):
+        def CPUPercentage():
+            return None
+        def CPUTemperature():
+            return None
+        self.guard.run(CPUPercentage, (), self.logger)
+        self.t[0] += 299
+        self.guard.run(CPUPercentage, (), self.logger)
+        self.guard.run(CPUTemperature, (), self.logger)
+        self.assertEqual(self.logger.info.call_count, 1)
+        self.t[0] += 1
+        self.guard.run(CPUPercentage, (), self.logger)
+        self.assertEqual(self.logger.info.call_count, 2)
+
+    def test_different_tasks_not_confused(self):
+        def CPUPercentage():
+            raise IOError("broken")
+        def CPUFrequency():
+            return None
+        self.assertFalse(self.guard.run(CPUPercentage, (), self.logger))
+        self.assertTrue(self.guard.run(CPUFrequency, (), self.logger))
+        self.assertEqual(self.logger.exception.call_count, 1)
+
+    def test_keyboard_interrupt_not_swallowed(self):
+        def CPUPercentage():
+            raise KeyboardInterrupt
+        with self.assertRaises(KeyboardInterrupt):
+            self.guard.run(CPUPercentage, (), self.logger)
+
+
+if __name__ == "__main__":
+    unittest.main()