From a7dea2c5d51c4c259b1da866b8f624b44acd657b Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 15:19:29 +0300
Subject: [PATCH 001/152] locking: move the locks into kernel/locking

The spinlock, the mutex and the seqlock were in the kernel root for
lack of a home, and the reader-writer locks to come would have joined
them there. The headers move to <locking/>.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/CMakeLists.txt                   |  3 +--
 kernel/arch/x86_64/ioapic.c             |  2 +-
 kernel/arch/x86_64/pci_type1.c          |  2 +-
 kernel/arch/x86_64/vector_alloc.c       |  2 +-
 kernel/include/{ => locking}/mutex.h    |  2 +-
 kernel/include/{ => locking}/seq.h      |  2 +-
 kernel/include/{ => locking}/spinlock.h |  0
 kernel/include/memory/alloc_cache.h     |  2 +-
 kernel/include/memory/page.h            |  2 +-
 kernel/include/private/irq.h            |  2 +-
 kernel/irq/request.c                    |  2 +-
 kernel/locking/CMakeLists.txt           |  4 ++++
 kernel/{ => locking}/mutex.c            |  2 +-
 kernel/{ => locking}/spinlock.c         |  2 +-
 kernel/memory/alloc.c                   |  2 +-
 kernel/memory/buddy.c                   |  2 +-
 kernel/memory/page_table.c              |  2 +-
 kernel/memory/valloc.c                  |  2 +-
 kernel/time/counter_device.c            |  2 +-
 kernel/time/keeper.c                    |  4 ++--
 tests/CMakeLists.txt                    | 16 +++++++++++-----
 tests/test-cases/test_mutex.c           |  2 +-
 22 files changed, 35 insertions(+), 26 deletions(-)
 rename kernel/include/{ => locking}/mutex.h (95%)
 rename kernel/include/{ => locking}/seq.h (98%)
 rename kernel/include/{ => locking}/spinlock.h (100%)
 create mode 100644 kernel/locking/CMakeLists.txt
 rename kernel/{ => locking}/mutex.c (95%)
 rename kernel/{ => locking}/spinlock.c (98%)

diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt
index 4e57b995..e37be5e9 100644
--- a/kernel/CMakeLists.txt
+++ b/kernel/CMakeLists.txt
@@ -103,8 +103,6 @@ ultra_sources(
     console.c
     log.c
     symbols.c
-    spinlock.c
-    mutex.c
     unwind.c
     param.c
     log_ring.c
@@ -167,6 +165,7 @@ ultra_link_options(
 
 add_subdirectory(arch/${ULTRA_ARCH})
 add_subdirectory(common)
+add_subdirectory(locking)
 add_subdirectory(irq)
 add_subdirectory(memory)
 add_subdirectory(time)
diff --git a/kernel/arch/x86_64/ioapic.c b/kernel/arch/x86_64/ioapic.c
index 65cb19cd..4cf7b1a0 100644
--- a/kernel/arch/x86_64/ioapic.c
+++ b/kernel/arch/x86_64/ioapic.c
@@ -12,7 +12,7 @@
 #include <arch_constants.h>
 #include <init_level.h>
 #include <free_after_init.h>
-#include <spinlock.h>
+#include <locking/spinlock.h>
 
 #include <memory/io.h>
 #include <memory/alloc.h>
diff --git a/kernel/arch/x86_64/pci_type1.c b/kernel/arch/x86_64/pci_type1.c
index b38bbd22..17d8d69e 100644
--- a/kernel/arch/x86_64/pci_type1.c
+++ b/kernel/arch/x86_64/pci_type1.c
@@ -3,7 +3,7 @@
 #include <memory/io.h>
 #include <init_level.h>
 #include <free_after_init.h>
-#include <spinlock.h>
+#include <locking/spinlock.h>
 
 #include <arch/private/pci_type1.h>
 
diff --git a/kernel/arch/x86_64/vector_alloc.c b/kernel/arch/x86_64/vector_alloc.c
index 78de1541..20f4b7ba 100644
--- a/kernel/arch/x86_64/vector_alloc.c
+++ b/kernel/arch/x86_64/vector_alloc.c
@@ -2,7 +2,7 @@
 #include <arch/private/vectors.h>
 
 #include <smp.h>
-#include <spinlock.h>
+#include <locking/spinlock.h>
 #include <per_cpu.h>
 #include <bug.h>
 
diff --git a/kernel/include/mutex.h b/kernel/include/locking/mutex.h
similarity index 95%
rename from kernel/include/mutex.h
rename to kernel/include/locking/mutex.h
index a6a54da2..be727b0a 100644
--- a/kernel/include/mutex.h
+++ b/kernel/include/locking/mutex.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <spinlock.h>
+#include <locking/spinlock.h>
 
 /*
  * A mutual exclusion lock for process context only. Holders are
diff --git a/kernel/include/seq.h b/kernel/include/locking/seq.h
similarity index 98%
rename from kernel/include/seq.h
rename to kernel/include/locking/seq.h
index 7c4ed5ed..b8221afb 100644
--- a/kernel/include/seq.h
+++ b/kernel/include/locking/seq.h
@@ -3,7 +3,7 @@
 #include <common/types.h>
 #include <common/atomic.h>
 
-#include <spinlock.h>
+#include <locking/spinlock.h>
 #include <arch/cpu_helpers.h>
 
 #include <irq_helpers.h>
diff --git a/kernel/include/spinlock.h b/kernel/include/locking/spinlock.h
similarity index 100%
rename from kernel/include/spinlock.h
rename to kernel/include/locking/spinlock.h
diff --git a/kernel/include/memory/alloc_cache.h b/kernel/include/memory/alloc_cache.h
index 8b837a4e..adaae0f7 100644
--- a/kernel/include/memory/alloc_cache.h
+++ b/kernel/include/memory/alloc_cache.h
@@ -5,7 +5,7 @@
 
 #include <memory/alloc.h>
 
-#include <spinlock.h>
+#include <locking/spinlock.h>
 
 struct alloc_cache {
     const char *name;
diff --git a/kernel/include/memory/page.h b/kernel/include/memory/page.h
index 5a744748..cffbf9c4 100644
--- a/kernel/include/memory/page.h
+++ b/kernel/include/memory/page.h
@@ -40,7 +40,7 @@
 #include <memory/io.h>
 #include <memory/bug.h>
 #include <arch_constants.h>
-#include <spinlock.h>
+#include <locking/spinlock.h>
 
 /*
  * Page type values are split into two main categories:
diff --git a/kernel/include/private/irq.h b/kernel/include/private/irq.h
index 5db30bf1..897f0179 100644
--- a/kernel/include/private/irq.h
+++ b/kernel/include/private/irq.h
@@ -6,7 +6,7 @@
 #include <common/error.h>
 #include <common/list.h>
 
-#include <spinlock.h>
+#include <locking/spinlock.h>
 #include <bug.h>
 
 struct irq_domain;
diff --git a/kernel/irq/request.c b/kernel/irq/request.c
index cc222e86..0ae393ae 100644
--- a/kernel/irq/request.c
+++ b/kernel/irq/request.c
@@ -3,7 +3,7 @@
 #include <private/irq.h>
 
 #include <bug.h>
-#include <mutex.h>
+#include <locking/mutex.h>
 #include <init_level.h>
 #include <memory/alloc.h>
 #include <common/atomic.h>
diff --git a/kernel/locking/CMakeLists.txt b/kernel/locking/CMakeLists.txt
new file mode 100644
index 00000000..cfb50c51
--- /dev/null
+++ b/kernel/locking/CMakeLists.txt
@@ -0,0 +1,4 @@
+ultra_sources(
+    spinlock.c
+    mutex.c
+)
diff --git a/kernel/mutex.c b/kernel/locking/mutex.c
similarity index 95%
rename from kernel/mutex.c
rename to kernel/locking/mutex.c
index c28c7d8d..24ddb1ef 100644
--- a/kernel/mutex.c
+++ b/kernel/locking/mutex.c
@@ -1,4 +1,4 @@
-#include <mutex.h>
+#include <locking/mutex.h>
 #include <irq_helpers.h>
 #include <bug.h>
 
diff --git a/kernel/spinlock.c b/kernel/locking/spinlock.c
similarity index 98%
rename from kernel/spinlock.c
rename to kernel/locking/spinlock.c
index 51083632..6e6a84de 100644
--- a/kernel/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -1,7 +1,7 @@
 #include <common/atomic.h>
 #include <arch/cpu_helpers.h>
 
-#include <spinlock.h>
+#include <locking/spinlock.h>
 #include <irq_helpers.h>
 
 #define CURRENT_TICKET_SHIFT 16
diff --git a/kernel/memory/alloc.c b/kernel/memory/alloc.c
index 7f88dd3a..002f20c9 100644
--- a/kernel/memory/alloc.c
+++ b/kernel/memory/alloc.c
@@ -15,7 +15,7 @@
 #include <private/buddy.h>
 
 #include <bug.h>
-#include <spinlock.h>
+#include <locking/spinlock.h>
 #include <free_after_init.h>
 #include <init_level.h>
 #include <log.h>
diff --git a/kernel/memory/buddy.c b/kernel/memory/buddy.c
index 71cf1b2a..5412d157 100644
--- a/kernel/memory/buddy.c
+++ b/kernel/memory/buddy.c
@@ -17,7 +17,7 @@
 
 #include <free_after_init.h>
 #include <bug.h>
-#include <spinlock.h>
+#include <locking/spinlock.h>
 #include <init_level.h>
 
 #define BUDDY_NUM_ORDERS (BUDDY_MAX_ORDER + 1)
diff --git a/kernel/memory/page_table.c b/kernel/memory/page_table.c
index 29e518eb..32fcc9d5 100644
--- a/kernel/memory/page_table.c
+++ b/kernel/memory/page_table.c
@@ -1,7 +1,7 @@
 #include <memory/page_table.h>
 #include <memory/page.h>
 
-#include <spinlock.h>
+#include <locking/spinlock.h>
 
 #define MAKE_GENERIC_PTN_POPULATE_IMPL(lvl, child_lvl)                 \
     bool pt##lvl##_populate(                                           \
diff --git a/kernel/memory/valloc.c b/kernel/memory/valloc.c
index 6d5f4268..038b598c 100644
--- a/kernel/memory/valloc.c
+++ b/kernel/memory/valloc.c
@@ -27,7 +27,7 @@
 
 #include <free_after_init.h>
 #include <init_level.h>
-#include <spinlock.h>
+#include <locking/spinlock.h>
 #include <irq_helpers.h>
 #include <per_cpu.h>
 
diff --git a/kernel/time/counter_device.c b/kernel/time/counter_device.c
index 818a5c8f..fdc167d6 100644
--- a/kernel/time/counter_device.c
+++ b/kernel/time/counter_device.c
@@ -8,7 +8,7 @@
 #include <private/time/keeper.h>
 
 #include <log.h>
-#include <spinlock.h>
+#include <locking/spinlock.h>
 
 static LIST_HEAD(s_counter_devices);
 static DEFINE_SPINLOCK(s_counter_lock);
diff --git a/kernel/time/keeper.c b/kernel/time/keeper.c
index 194b1a2e..46d0b28a 100644
--- a/kernel/time/keeper.c
+++ b/kernel/time/keeper.c
@@ -5,8 +5,8 @@
 
 #include <bug.h>
 #include <init_level.h>
-#include <spinlock.h>
-#include <seq.h>
+#include <locking/spinlock.h>
+#include <locking/seq.h>
 #include <time/time.h>
 
 #include <arch/cpu_helpers.h>
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index ed9d2b83..2d4d2e00 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -191,10 +191,16 @@ KERNEL_FILE(INCLUDE_FILE "bug.h")
 KERNEL_FILE(INCLUDE_PATH "private" INCLUDE_FILE "bug.h")
 KERNEL_FILE(INCLUDE_FILE "panic.h")
 KERNEL_FILE(INCLUDE_FILE "linker.h")
-KERNEL_FILE(INCLUDE_FILE "seq.h")
+KERNEL_FILE(INCLUDE_PATH "locking" INCLUDE_FILE "seq.h")
 KERNEL_FILE(INCLUDE_PATH "private" INCLUDE_FILE "init_level.h")
-KERNEL_FILE(INCLUDE_FILE "spinlock.h" SOURCE_FILE "spinlock.c")
-KERNEL_FILE(INCLUDE_FILE "mutex.h" SOURCE_FILE "mutex.c")
+KERNEL_FILE(
+    INCLUDE_PATH "locking" INCLUDE_FILE "spinlock.h"
+    SOURCE_PATH "locking" SOURCE_FILE "spinlock.c"
+)
+KERNEL_FILE(
+    INCLUDE_PATH "locking" INCLUDE_FILE "mutex.h"
+    SOURCE_PATH "locking" SOURCE_FILE "mutex.c"
+)
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "vm_flags.h")
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "page_table.h")
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "address_space.h")
@@ -216,8 +222,8 @@ include("test-cases/files.cmake")
 add_executable(
     run_tests
     run_tests.cpp kernel_helpers.cpp test_harness.cpp memory_map.c ${TEST_SOURCES}
-    kernel-source/spinlock.c kernel-source/memory/units.c
-    kernel-source/common/bit.c kernel-source/mutex.c
+    kernel-source/locking/spinlock.c kernel-source/memory/units.c
+    kernel-source/common/bit.c kernel-source/locking/mutex.c
     kernel-source/common/string_container.c kernel-source/common/conversions.c
 )
 set_property(
diff --git a/tests/test-cases/test_mutex.c b/tests/test-cases/test_mutex.c
index 4972a971..378203e2 100644
--- a/tests/test-cases/test_mutex.c
+++ b/tests/test-cases/test_mutex.c
@@ -1,4 +1,4 @@
-#include <mutex.h>
+#include <locking/mutex.h>
 
 #include <test_harness.h>
 

From 9257bb822f7c3dc41bf727ce29799f6290411f6b Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 02:08:18 +0300
Subject: [PATCH 002/152] everywhere: move implementation detail headers into
 detail/

The header split going forward is going to work like the following:
- include/ -> public API
- include/private -> private API (for neighbor subsystems)
- include/private/arch -> private API that is meant to be called or
                          implemented by the architecture's "half" of
                          a generic subsystem
- include/detail -> private API, but may be leaked via the public
                    header because it needs either fields or other
                    helpers to be available inline
- include/arch/ -> public, arch-specific API
- include/arch/private -> private arch-specific API
- include/arch/detail -> the arch-specific "half" of the implementation
                         of a generic subsystem

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .../include/arch/{private => detail}/unwind.h |  0
 kernel/arch/aarch64/unwind.S                  |  2 +-
 kernel/arch/aarch64/unwind.c                  |  2 +-
 .../x86_64/include/arch/{ => detail}/bug.h    |  0
 .../arch/{private => detail}/init_level.h     |  0
 .../arch/{private/io.h => detail/io_access.h} |  0
 .../arch/{private => detail}/irq_helpers.h    |  0
 .../include/arch/{ => detail}/page_table.h    |  0
 .../arch/{ => detail}/page_table_extras.h     |  0
 .../include/arch/{ => detail}/per_cpu.h       |  0
 .../include/arch/{private => detail}/string.h |  0
 .../include/arch/{private => detail}/unwind.h |  0
 kernel/arch/x86_64/include/arch/smp.h         |  2 +-
 kernel/arch/x86_64/unwind.S                   |  2 +-
 kernel/arch/x86_64/unwind.c                   |  2 +-
 kernel/common/string.c                        |  4 ++--
 kernel/include/bug.h                          |  6 ++---
 kernel/include/common/rb_tree_aggregated.h    |  2 +-
 kernel/include/detail/bug.h                   | 23 +++++++++++++++++++
 .../{private => detail/common}/rb_tree.h      |  0
 .../include/{private => detail}/init_level.h  |  4 ++--
 kernel/include/{private => detail}/log_ring.h |  0
 kernel/include/init_level.h                   |  2 +-
 kernel/include/irq_helpers.h                  |  2 +-
 kernel/include/linker_common.ld.h             |  2 +-
 kernel/include/log_ring.h                     |  2 +-
 kernel/include/memory/page_table.h            | 12 +++++-----
 kernel/include/per_cpu.h                      |  2 +-
 kernel/include/per_cpu_decls.h                |  2 +-
 kernel/include/private/bug.h                  | 22 ------------------
 kernel/include/private/irq.h                  |  6 ++---
 kernel/include/private/unwind.h               |  2 +-
 kernel/memory/io.c                            | 16 ++++++++++++-
 kernel/unwind.c                               |  1 -
 tests/CMakeLists.txt                          |  8 +++----
 .../synthetic-include/arch/{ => detail}/bug.h |  0
 .../arch/{ => detail}/page_table.h            |  0
 tests/synthetic-include/init_level.h          |  2 +-
 38 files changed, 72 insertions(+), 58 deletions(-)
 rename kernel/arch/aarch64/include/arch/{private => detail}/unwind.h (100%)
 rename kernel/arch/x86_64/include/arch/{ => detail}/bug.h (100%)
 rename kernel/arch/x86_64/include/arch/{private => detail}/init_level.h (100%)
 rename kernel/arch/x86_64/include/arch/{private/io.h => detail/io_access.h} (100%)
 rename kernel/arch/x86_64/include/arch/{private => detail}/irq_helpers.h (100%)
 rename kernel/arch/x86_64/include/arch/{ => detail}/page_table.h (100%)
 rename kernel/arch/x86_64/include/arch/{ => detail}/page_table_extras.h (100%)
 rename kernel/arch/x86_64/include/arch/{ => detail}/per_cpu.h (100%)
 rename kernel/arch/x86_64/include/arch/{private => detail}/string.h (100%)
 rename kernel/arch/x86_64/include/arch/{private => detail}/unwind.h (100%)
 create mode 100644 kernel/include/detail/bug.h
 rename kernel/include/{private => detail/common}/rb_tree.h (100%)
 rename kernel/include/{private => detail}/init_level.h (97%)
 rename kernel/include/{private => detail}/log_ring.h (100%)
 rename tests/synthetic-include/arch/{ => detail}/bug.h (100%)
 rename tests/synthetic-include/arch/{ => detail}/page_table.h (100%)

diff --git a/kernel/arch/aarch64/include/arch/private/unwind.h b/kernel/arch/aarch64/include/arch/detail/unwind.h
similarity index 100%
rename from kernel/arch/aarch64/include/arch/private/unwind.h
rename to kernel/arch/aarch64/include/arch/detail/unwind.h
diff --git a/kernel/arch/aarch64/unwind.S b/kernel/arch/aarch64/unwind.S
index 8c4cb572..3c219ea2 100644
--- a/kernel/arch/aarch64/unwind.S
+++ b/kernel/arch/aarch64/unwind.S
@@ -1,4 +1,4 @@
-#include <arch/private/unwind.h>
+#include <arch/detail/unwind.h>
 
 .section ".text", "ax", @progbits
 
diff --git a/kernel/arch/aarch64/unwind.c b/kernel/arch/aarch64/unwind.c
index 8e726051..ad823171 100644
--- a/kernel/arch/aarch64/unwind.c
+++ b/kernel/arch/aarch64/unwind.c
@@ -1,5 +1,5 @@
 #include <private/arch/unwind.h>
-#include <arch/private/unwind.h>
+#include <arch/detail/unwind.h>
 
 void arch_registers_to_dwarf_registers(
     struct registers *regs, ptr_t *dwarf_regs
diff --git a/kernel/arch/x86_64/include/arch/bug.h b/kernel/arch/x86_64/include/arch/detail/bug.h
similarity index 100%
rename from kernel/arch/x86_64/include/arch/bug.h
rename to kernel/arch/x86_64/include/arch/detail/bug.h
diff --git a/kernel/arch/x86_64/include/arch/private/init_level.h b/kernel/arch/x86_64/include/arch/detail/init_level.h
similarity index 100%
rename from kernel/arch/x86_64/include/arch/private/init_level.h
rename to kernel/arch/x86_64/include/arch/detail/init_level.h
diff --git a/kernel/arch/x86_64/include/arch/private/io.h b/kernel/arch/x86_64/include/arch/detail/io_access.h
similarity index 100%
rename from kernel/arch/x86_64/include/arch/private/io.h
rename to kernel/arch/x86_64/include/arch/detail/io_access.h
diff --git a/kernel/arch/x86_64/include/arch/private/irq_helpers.h b/kernel/arch/x86_64/include/arch/detail/irq_helpers.h
similarity index 100%
rename from kernel/arch/x86_64/include/arch/private/irq_helpers.h
rename to kernel/arch/x86_64/include/arch/detail/irq_helpers.h
diff --git a/kernel/arch/x86_64/include/arch/page_table.h b/kernel/arch/x86_64/include/arch/detail/page_table.h
similarity index 100%
rename from kernel/arch/x86_64/include/arch/page_table.h
rename to kernel/arch/x86_64/include/arch/detail/page_table.h
diff --git a/kernel/arch/x86_64/include/arch/page_table_extras.h b/kernel/arch/x86_64/include/arch/detail/page_table_extras.h
similarity index 100%
rename from kernel/arch/x86_64/include/arch/page_table_extras.h
rename to kernel/arch/x86_64/include/arch/detail/page_table_extras.h
diff --git a/kernel/arch/x86_64/include/arch/per_cpu.h b/kernel/arch/x86_64/include/arch/detail/per_cpu.h
similarity index 100%
rename from kernel/arch/x86_64/include/arch/per_cpu.h
rename to kernel/arch/x86_64/include/arch/detail/per_cpu.h
diff --git a/kernel/arch/x86_64/include/arch/private/string.h b/kernel/arch/x86_64/include/arch/detail/string.h
similarity index 100%
rename from kernel/arch/x86_64/include/arch/private/string.h
rename to kernel/arch/x86_64/include/arch/detail/string.h
diff --git a/kernel/arch/x86_64/include/arch/private/unwind.h b/kernel/arch/x86_64/include/arch/detail/unwind.h
similarity index 100%
rename from kernel/arch/x86_64/include/arch/private/unwind.h
rename to kernel/arch/x86_64/include/arch/detail/unwind.h
diff --git a/kernel/arch/x86_64/include/arch/smp.h b/kernel/arch/x86_64/include/arch/smp.h
index 913364e6..a4ff36af 100644
--- a/kernel/arch/x86_64/include/arch/smp.h
+++ b/kernel/arch/x86_64/include/arch/smp.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <per_cpu_decls.h>
-#include <arch/per_cpu.h>
+#include <arch/detail/per_cpu.h>
 
 DECLARE_PER_CPU(u32, g_this_cpu_id);
 
diff --git a/kernel/arch/x86_64/unwind.S b/kernel/arch/x86_64/unwind.S
index ce113809..d376ceda 100644
--- a/kernel/arch/x86_64/unwind.S
+++ b/kernel/arch/x86_64/unwind.S
@@ -1,4 +1,4 @@
-#include <arch/private/unwind.h>
+#include <arch/detail/unwind.h>
 #include <arch/private/asm_helpers.h>
 
 ASM_PRELUDE
diff --git a/kernel/arch/x86_64/unwind.c b/kernel/arch/x86_64/unwind.c
index fa72eb35..696f8cfb 100644
--- a/kernel/arch/x86_64/unwind.c
+++ b/kernel/arch/x86_64/unwind.c
@@ -1,5 +1,5 @@
 #include <private/arch/unwind.h>
-#include <arch/private/unwind.h>
+#include <arch/detail/unwind.h>
 
 #include <symbols.h>
 #include <unwind.h>
diff --git a/kernel/common/string.c b/kernel/common/string.c
index d28fb49c..6840e3aa 100644
--- a/kernel/common/string.c
+++ b/kernel/common/string.c
@@ -1,8 +1,8 @@
 #include <common/string.h>
 #include <common/attributes.h>
 
-#if HAS_INCLUDE(<arch/private/string.h>)
-#include <arch/private/string.h>
+#if HAS_INCLUDE(<arch/detail/string.h>)
+#include <arch/detail/string.h>
 #endif
 
 /*
diff --git a/kernel/include/bug.h b/kernel/include/bug.h
index 8c206e71..ce8a735e 100644
--- a/kernel/include/bug.h
+++ b/kernel/include/bug.h
@@ -5,10 +5,10 @@
 #include <common/types.h>
 #include <log.h>
 
-#include <private/bug.h>
+#include <detail/bug.h>
 
-#if HAS_INCLUDE(<arch/bug.h>)
-#include <arch/bug.h>
+#if HAS_INCLUDE(<arch/detail/bug.h>)
+#include <arch/detail/bug.h>
 #endif
 
 #ifdef ARCH_BUG_TRAP_INSTRUCTION
diff --git a/kernel/include/common/rb_tree_aggregated.h b/kernel/include/common/rb_tree_aggregated.h
index 1db83a37..3f59425e 100644
--- a/kernel/include/common/rb_tree_aggregated.h
+++ b/kernel/include/common/rb_tree_aggregated.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <private/rb_tree.h>
+#include <detail/common/rb_tree.h>
 
 #include <common/attributes.h>
 #include <common/minmax.h>
diff --git a/kernel/include/detail/bug.h b/kernel/include/detail/bug.h
new file mode 100644
index 00000000..03b157c0
--- /dev/null
+++ b/kernel/include/detail/bug.h
@@ -0,0 +1,23 @@
+#pragma once
+
+#include <common/helpers.h>
+#include <linker.h>
+
+#define BUG_FLAG_WARNING (1 << 0)
+#define BUG_FLAG_ONCE (1 << 1)
+#define BUG_FLAG_DONE (1 << 2)
+
+#define BUG_TRAP(trap_flags)                                    \
+    asm volatile(                                               \
+        "990: " ARCH_BUG_TRAP_INSTRUCTION "\n"                  \
+        ".pushsection .rodata.str1.1, \"aMS\", %progbits, 1\n"  \
+        "991: .string \"" __FILE__ "\"\n"                       \
+        ".popsection\n"                                         \
+        ".pushsection ." TO_STR(BUG_TABLE_SECTION) ", \"aw\"\n" \
+        ".balign 4\n"                                           \
+        ".long 990b - .\n"                                      \
+        ".long 991b - .\n"                                      \
+        ".short " TO_STR(__LINE__) "\n"                         \
+        ".short " TO_STR(trap_flags) "\n"                       \
+        ".popsection\n"                                         \
+    )
diff --git a/kernel/include/private/rb_tree.h b/kernel/include/detail/common/rb_tree.h
similarity index 100%
rename from kernel/include/private/rb_tree.h
rename to kernel/include/detail/common/rb_tree.h
diff --git a/kernel/include/private/init_level.h b/kernel/include/detail/init_level.h
similarity index 97%
rename from kernel/include/private/init_level.h
rename to kernel/include/detail/init_level.h
index 8c915a1b..9ccd108c 100644
--- a/kernel/include/private/init_level.h
+++ b/kernel/include/detail/init_level.h
@@ -53,8 +53,8 @@
     INIT_LEVEL(GENERIC_MODULES_INITIALIZED) \
     ARCH_INIT_LEVELS_AFTER_GENERIC_MODULES_INITIALIZED
 
-#if HAS_INCLUDE(<arch/private/init_level.h>)
-    #include <arch/private/init_level.h>
+#if HAS_INCLUDE(<arch/detail/init_level.h>)
+#include <arch/detail/init_level.h>
 #endif
 
 #ifndef ARCH_INIT_LEVELS_AFTER_NONE
diff --git a/kernel/include/private/log_ring.h b/kernel/include/detail/log_ring.h
similarity index 100%
rename from kernel/include/private/log_ring.h
rename to kernel/include/detail/log_ring.h
diff --git a/kernel/include/init_level.h b/kernel/include/init_level.h
index cfe655e7..91e3ebfa 100644
--- a/kernel/include/init_level.h
+++ b/kernel/include/init_level.h
@@ -4,7 +4,7 @@
 #include <common/types.h>
 #include <common/error.h>
 
-#include <private/init_level.h>
+#include <detail/init_level.h>
 
 #include <bug.h>
 
diff --git a/kernel/include/irq_helpers.h b/kernel/include/irq_helpers.h
index 9bfd388d..875e568e 100644
--- a/kernel/include/irq_helpers.h
+++ b/kernel/include/irq_helpers.h
@@ -3,7 +3,7 @@
 #include <common/types.h>
 #include <common/attributes.h>
 
-#include <arch/private/irq_helpers.h>
+#include <arch/detail/irq_helpers.h>
 
 static ALWAYS_INLINE bool irq_enabled(void)
 {
diff --git a/kernel/include/linker_common.ld.h b/kernel/include/linker_common.ld.h
index f20c2ac6..63a4e57a 100644
--- a/kernel/include/linker_common.ld.h
+++ b/kernel/include/linker_common.ld.h
@@ -3,7 +3,7 @@
 #include <linker.h>
 
 #include <arch_constants.h>
-#include <private/init_level.h>
+#include <detail/init_level.h>
 
 #define PHDR_READ  (1 << 2)
 #define PHDR_WRITE (1 << 1)
diff --git a/kernel/include/log_ring.h b/kernel/include/log_ring.h
index da74bed5..bde7e99b 100644
--- a/kernel/include/log_ring.h
+++ b/kernel/include/log_ring.h
@@ -11,7 +11,7 @@
  * only support statically defining the log ring, so the caller must be able to
  * "see" all the internal data structures.
  */
-#include <private/log_ring.h>
+#include <detail/log_ring.h>
 
 /*
  * Define a log ring 'name', that has 2^'size_shift' bytes of text space,
diff --git a/kernel/include/memory/page_table.h b/kernel/include/memory/page_table.h
index 3ab04c0a..a56ca3ed 100644
--- a/kernel/include/memory/page_table.h
+++ b/kernel/include/memory/page_table.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <arch/page_table.h>
+#include <arch/detail/page_table.h>
 
 #include <common/atomic.h>
 #include <common/helpers.h>
@@ -158,7 +158,7 @@
  * Porting to other architectures
  * ==============================
  *
- * <arch/page_table.h> is the only mandatory header, it is expected to
+ * <arch/detail/page_table.h> is the only mandatory header, it is expected to
  * provide:
  *
  *   pt_entry_word              one machine word, loadable and
@@ -181,10 +181,10 @@
  * values only, never a struct ptN pointer, which is what makes a plain
  * load or store unrepresentable outside this file.
  *
- * <arch/page_table_extras.h> is included (if provided) after the
+ * <arch/detail/page_table_extras.h> is included (if provided) after the
  * generic definitions and holds anything that needs a struct ptN
  * pointer or a generic helper defined before it. The matching guard
- * still belongs in <arch/page_table.h>, since the guards are tested
+ * still belongs in <arch/detail/page_table.h>, since the guards are tested
  * before this header is read.
  *
  * ptN_entry_none decides whether an entry is empty. It must ignore
@@ -624,8 +624,8 @@ pt_prot pt_prot_from_vm_prot(enum vm_prot);
 typedef struct pt5 pt_root;
 #define pt_root_from_address_space(as, va) pt5_from_pt5_base((as)->pt, va)
 
-#if HAS_INCLUDE(<arch/page_table_extras.h>)
-#include <arch/page_table_extras.h>
+#if HAS_INCLUDE(<arch/detail/page_table_extras.h>)
+#include <arch/detail/page_table_extras.h>
 #endif
 
 #undef MAKE_PT_TYPE
diff --git a/kernel/include/per_cpu.h b/kernel/include/per_cpu.h
index 04003359..ebb9e524 100644
--- a/kernel/include/per_cpu.h
+++ b/kernel/include/per_cpu.h
@@ -6,7 +6,7 @@
 #include <smp.h>
 #include <per_cpu_decls.h>
 
-#include <arch/per_cpu.h>
+#include <arch/detail/per_cpu.h>
 
 #define per_cpu_ptr(ptr, cpu_number) \
     PTR_ADD_HIDE_UB(ptr, g_per_cpu_offset[cpu_number])
diff --git a/kernel/include/per_cpu_decls.h b/kernel/include/per_cpu_decls.h
index 7b79d071..1f7851cf 100644
--- a/kernel/include/per_cpu_decls.h
+++ b/kernel/include/per_cpu_decls.h
@@ -4,7 +4,7 @@
 #include <common/types.h>
 #include <linker.h>
 
-#include <arch/per_cpu.h>
+#include <arch/detail/per_cpu.h>
 
 #ifndef ARCH_PER_CPU_ATTRIBUTES
     #define ARCH_PER_CPU_ATTRIBUTES
diff --git a/kernel/include/private/bug.h b/kernel/include/private/bug.h
index b06e6bcf..d5440a41 100644
--- a/kernel/include/private/bug.h
+++ b/kernel/include/private/bug.h
@@ -1,27 +1,5 @@
 #pragma once
 
-#include <common/helpers.h>
-#include <linker.h>
-
-#define BUG_FLAG_WARNING (1 << 0)
-#define BUG_FLAG_ONCE (1 << 1)
-#define BUG_FLAG_DONE (1 << 2)
-
-#define BUG_TRAP(trap_flags)                                    \
-    asm volatile(                                               \
-        "990: " ARCH_BUG_TRAP_INSTRUCTION "\n"                  \
-        ".pushsection .rodata.str1.1, \"aMS\", %progbits, 1\n"  \
-        "991: .string \"" __FILE__ "\"\n"                       \
-        ".popsection\n"                                         \
-        ".pushsection ." TO_STR(BUG_TABLE_SECTION) ", \"aw\"\n" \
-        ".balign 4\n"                                           \
-        ".long 990b - .\n"                                      \
-        ".long 991b - .\n"                                      \
-        ".short " TO_STR(__LINE__) "\n"                         \
-        ".short " TO_STR(trap_flags) "\n"                       \
-        ".popsection\n"                                         \
-    )
-
 struct registers;
 
 void finish_warn_report(struct registers *regs);
diff --git a/kernel/include/private/irq.h b/kernel/include/private/irq.h
index 897f0179..4a7bc77f 100644
--- a/kernel/include/private/irq.h
+++ b/kernel/include/private/irq.h
@@ -15,10 +15,10 @@ struct irq_chip;
 /*
  * Allocation facts only the architecture's own levels fill and read.
  * An architecture with such facts defines the struct in
- * <arch/irq_alloc_request.h>, the empty default covers the rest.
+ * <arch/detail/irq_alloc_request.h>, the empty default covers the rest.
  */
-#if HAS_INCLUDE(<arch/irq_alloc_request.h>)
-#include <arch/irq_alloc_request.h>
+#if HAS_INCLUDE(<arch/detail/irq_alloc_request.h>)
+#include <arch/detail/irq_alloc_request.h>
 #else
 struct arch_irq_alloc_request {
 };
diff --git a/kernel/include/private/unwind.h b/kernel/include/private/unwind.h
index 58c559e0..1b3290b6 100644
--- a/kernel/include/private/unwind.h
+++ b/kernel/include/private/unwind.h
@@ -5,7 +5,7 @@
 #include <common/attributes.h>
 
 #include <arch/registers.h>
-#include <arch/private/unwind.h>
+#include <arch/detail/unwind.h>
 
 struct eh_data {
     const u8 *cursor;
diff --git a/kernel/memory/io.c b/kernel/memory/io.c
index 853245e1..d7b64eda 100644
--- a/kernel/memory/io.c
+++ b/kernel/memory/io.c
@@ -20,7 +20,21 @@
 #include <log.h>
 
 #include <private/arch/io.h>
-#include <arch/private/io.h>
+/*
+ * API provided by <arch/detail/io_access.h>:
+ *
+ * [Mandatory (64-bit versions gated by ULTRA_ARCH_WIDTH >= 8)]
+ *   arch_mmio_{read,write}{8,16,32,64}{,_relaxed}()
+ *
+ * [Optional, gated by ARCH_HAS_CUSTOM_MMIO_RW_MANY_{STRICT,RELAXED}]
+ *   arch_mmio_{read,write}N{,_relaxed}_many()
+ *
+ * [Optional, gated by ARCH_HAS_CUSTOM_PIO from <arch/detail/io.h>]
+ *   arch_{map,unmap}_pio()
+ *   arch_pio_{read,write}{8,16,32}{,_relaxed,_slowdown}()
+ *   arch_pio_{read,write}{8,16,32}{,_relaxed}_many()
+ */
+#include <arch/detail/io_access.h>
 
 /*
  * Aligns the physical address down to the nearest PT1_SIZE and expands the
diff --git a/kernel/unwind.c b/kernel/unwind.c
index 30201c41..ac1e25ca 100644
--- a/kernel/unwind.c
+++ b/kernel/unwind.c
@@ -1,6 +1,5 @@
 #define MSG_FMT(msg) "unwind: " msg
 
-#include <arch/private/unwind.h>
 #include <private/arch/unwind.h>
 #include <private/unwind.h>
 #include <private/symbols.h>
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 2d4d2e00..29b66e44 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -97,7 +97,7 @@ KERNEL_FILE(
 KERNEL_FILE(
     SOURCE_FILE "drivers/smbios.c" INCLUDE_FILE "smbios.h"
 )
-KERNEL_FILE(INCLUDE_FILE "private/log_ring.h")
+KERNEL_FILE(INCLUDE_FILE "detail/log_ring.h")
 KERNEL_FILE(INCLUDE_FILE "smbios_structures.h")
 KERNEL_FILE(SOURCE_FILE "param.c"  INCLUDE_FILE "param.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "helpers.h")
@@ -149,7 +149,7 @@ KERNEL_FILE(
     SOURCE_PATH "common" SOURCE_FILE "rb_tree.c"
     INCLUDE_PATH "common" INCLUDE_FILE "rb_tree.h"
 )
-KERNEL_FILE(INCLUDE_PATH "private" INCLUDE_FILE "rb_tree.h")
+KERNEL_FILE(INCLUDE_PATH "detail/common" INCLUDE_FILE "rb_tree.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "rb_tree_aggregated.h")
 KERNEL_FILE(
     SOURCE_PATH "memory" SOURCE_FILE "alloc.c"
@@ -188,11 +188,11 @@ KERNEL_FILE(INCLUDE_PATH "boot" INCLUDE_FILE "ultra_protocol.h")
 KERNEL_FILE(INCLUDE_FILE "config_helpers.h")
 KERNEL_FILE(INCLUDE_FILE "log.h")
 KERNEL_FILE(INCLUDE_FILE "bug.h")
-KERNEL_FILE(INCLUDE_PATH "private" INCLUDE_FILE "bug.h")
+KERNEL_FILE(INCLUDE_PATH "detail" INCLUDE_FILE "bug.h")
 KERNEL_FILE(INCLUDE_FILE "panic.h")
 KERNEL_FILE(INCLUDE_FILE "linker.h")
 KERNEL_FILE(INCLUDE_PATH "locking" INCLUDE_FILE "seq.h")
-KERNEL_FILE(INCLUDE_PATH "private" INCLUDE_FILE "init_level.h")
+KERNEL_FILE(INCLUDE_PATH "detail" INCLUDE_FILE "init_level.h")
 KERNEL_FILE(
     INCLUDE_PATH "locking" INCLUDE_FILE "spinlock.h"
     SOURCE_PATH "locking" SOURCE_FILE "spinlock.c"
diff --git a/tests/synthetic-include/arch/bug.h b/tests/synthetic-include/arch/detail/bug.h
similarity index 100%
rename from tests/synthetic-include/arch/bug.h
rename to tests/synthetic-include/arch/detail/bug.h
diff --git a/tests/synthetic-include/arch/page_table.h b/tests/synthetic-include/arch/detail/page_table.h
similarity index 100%
rename from tests/synthetic-include/arch/page_table.h
rename to tests/synthetic-include/arch/detail/page_table.h
diff --git a/tests/synthetic-include/init_level.h b/tests/synthetic-include/init_level.h
index 88e34f69..9cfc1720 100644
--- a/tests/synthetic-include/init_level.h
+++ b/tests/synthetic-include/init_level.h
@@ -4,7 +4,7 @@
 #include <common/types.h>
 #include <common/error.h>
 
-#include <private/init_level.h>
+#include <detail/init_level.h>
 
 #include <bug.h>
 

From ab9871ae9f2e1dbc71d5bf91745f67487684e6be Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Mon, 26 Oct 2026 03:23:20 +0300
Subject: [PATCH 003/152] scripts: add a lint check for private/ includes in
 public headers

Leaking private headers via the public ones completely invalidates the
whole idea behind having private headers, so check that we don't do
that.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 scripts/lint.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/scripts/lint.py b/scripts/lint.py
index 042e455a..985b41eb 100755
--- a/scripts/lint.py
+++ b/scripts/lint.py
@@ -159,6 +159,32 @@ def check_includes(ctx: Context) -> List[Finding]:
     return findings
 
 
+def check_private_includes(ctx: Context) -> List[Finding]:
+    findings = []
+
+    for path in tracked_sources(ctx):
+        # Linker script headers never reach C code
+        if not path.endswith(".h") or path.endswith(".ld.h"):
+            continue
+
+        if "private" in path.split("/"):
+            continue
+
+        text = read_source(ctx, path)
+
+        for number, line in enumerate(text.split("\n"), 1):
+            match = INCLUDE.match(line)
+            if match is None:
+                continue
+
+            if "private" in match.group("path").split("/"):
+                findings.append(Finding(
+                    path, number, "private/ header included by a public one"
+                ))
+
+    return findings
+
+
 def check_tabs(ctx: Context) -> List[Finding]:
     findings = []
 
@@ -659,6 +685,7 @@ def check_configs(ctx: Context) -> List[Finding]:
     "formats": check_formats,
     "includes": check_includes,
     "null": check_null,
+    "private-includes": check_private_includes,
     "tabs": check_tabs,
     "variable-prefixes": check_variable_prefixes,
 }

From a3aced3f90d9f53d9d81c8c086c69347119b3af9 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 5 Nov 2026 22:31:15 +0300
Subject: [PATCH 004/152] scripts: add a lint check for statement expression
 macros

A statement expression macro must indent its body once and put the
closing }) in the first colum. Check it, and convert RUNTIME_COMPARE()
to follow this rule.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/common/minmax.h |  8 ++++----
 scripts/lint.py                | 37 ++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/kernel/include/common/minmax.h b/kernel/include/common/minmax.h
index 23d8797f..526164b0 100644
--- a/kernel/include/common/minmax.h
+++ b/kernel/include/common/minmax.h
@@ -5,10 +5,10 @@
 #define COMPARE(x, y, op) ((x) op (y) ? (x) : (y))
 
 #define RUNTIME_COMPARE(x, y, x_name, y_name, op) ({ \
-        typeof(x) x_name = x;                        \
-        typeof(y) y_name = y;                        \
-        COMPARE(x_name, y_name, op);                 \
-    })
+    typeof(x) x_name = x;                            \
+    typeof(y) y_name = y;                            \
+    COMPARE(x_name, y_name, op);                     \
+})
 
 #define DO_COMPARE(x, y, op)                                            \
     CHOOSE_EXPR(                                                        \
diff --git a/scripts/lint.py b/scripts/lint.py
index 985b41eb..ffb6bcbe 100755
--- a/scripts/lint.py
+++ b/scripts/lint.py
@@ -198,6 +198,42 @@ def check_tabs(ctx: Context) -> List[Finding]:
     return findings
 
 
+STATEMENT_EXPRESSION = re.compile(r"^#define\b.*\(\{\s*\\$")
+
+
+def indent_of(line: str) -> int:
+    return len(line) - len(line.lstrip(" "))
+
+
+def check_statement_expressions(ctx: Context) -> List[Finding]:
+    findings = []
+
+    for path in tracked_sources(ctx):
+        lines = read_source(ctx, path).split("\n")
+
+        for index, line in enumerate(lines):
+            if not STATEMENT_EXPRESSION.match(line):
+                continue
+
+            if indent_of(lines[index + 1]) != 4:
+                findings.append(Finding(
+                    path, index + 2,
+                    "indent the body of a statement expression once"
+                ))
+
+            end = index + 1
+            while lines[end].rstrip().endswith("\\"):
+                end += 1
+
+            if lines[end].strip().startswith("})") and indent_of(lines[end]):
+                findings.append(Finding(
+                    path, end + 1,
+                    "put the }) of a statement expression in the first column"
+                ))
+
+    return findings
+
+
 def check_file_end(ctx: Context) -> List[Finding]:
     findings = []
 
@@ -686,6 +722,7 @@ def check_configs(ctx: Context) -> List[Finding]:
     "includes": check_includes,
     "null": check_null,
     "private-includes": check_private_includes,
+    "statement-expressions": check_statement_expressions,
     "tabs": check_tabs,
     "variable-prefixes": check_variable_prefixes,
 }

From dd06d8cec0e26d7d54738596352a10eca716b891 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 12 Nov 2026 13:15:30 +0300
Subject: [PATCH 005/152] scripts: add a lint check for semicolons in
 definition macros

A DEFINE_*() or DECLARE_*() macro must leave the semicolon to its
users, so that a use reads like any other definition. Check it, and
convert DEFINE_MUTEX() and DEFINE_SPINLOCK() to follow this rule.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/locking/mutex.h    |  2 +-
 kernel/include/locking/spinlock.h |  2 +-
 scripts/lint.py                   | 27 +++++++++++++++++++++++++++
 tests/test-cases/test_mutex.c     |  2 +-
 4 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/kernel/include/locking/mutex.h b/kernel/include/locking/mutex.h
index be727b0a..5c89209d 100644
--- a/kernel/include/locking/mutex.h
+++ b/kernel/include/locking/mutex.h
@@ -16,7 +16,7 @@ struct mutex {
 };
 
 #define MUTEX_INIT(name) (struct mutex) { SPINLOCK_INIT(name.lock) }
-#define DEFINE_MUTEX(name) struct mutex name = MUTEX_INIT(name);
+#define DEFINE_MUTEX(name) struct mutex name = MUTEX_INIT(name)
 
 void mutex_init(struct mutex*);
 
diff --git a/kernel/include/locking/spinlock.h b/kernel/include/locking/spinlock.h
index 980e7660..03573161 100644
--- a/kernel/include/locking/spinlock.h
+++ b/kernel/include/locking/spinlock.h
@@ -20,7 +20,7 @@ struct spinlock {
 };
 
 #define SPINLOCK_INIT(name) (struct spinlock) { { .control = 0 } }
-#define DEFINE_SPINLOCK(name) struct spinlock name = SPINLOCK_INIT(name);
+#define DEFINE_SPINLOCK(name) struct spinlock name = SPINLOCK_INIT(name)
 
 static inline void spin_lock_init(struct spinlock *lock)
 {
diff --git a/scripts/lint.py b/scripts/lint.py
index ffb6bcbe..2ffc5a7e 100755
--- a/scripts/lint.py
+++ b/scripts/lint.py
@@ -234,6 +234,32 @@ def check_statement_expressions(ctx: Context) -> List[Finding]:
     return findings
 
 
+DEFINITION_MACRO = re.compile(r"^#define (?:DEFINE|DECLARE)_\w+")
+
+
+def check_definition_macros(ctx: Context) -> List[Finding]:
+    findings = []
+
+    for path in tracked_sources(ctx):
+        lines = read_source(ctx, path).split("\n")
+
+        for index, line in enumerate(lines):
+            if not DEFINITION_MACRO.match(line):
+                continue
+
+            end = index
+            while lines[end].rstrip().endswith("\\"):
+                end += 1
+
+            if lines[end].rstrip().endswith(";"):
+                findings.append(Finding(
+                    path, end + 1,
+                    "drop the ; at the end of a definition macro"
+                ))
+
+    return findings
+
+
 def check_file_end(ctx: Context) -> List[Finding]:
     findings = []
 
@@ -716,6 +742,7 @@ def check_configs(ctx: Context) -> List[Finding]:
 CHECKS: Dict[str, Check] = {
     "comments": check_comments,
     "configs": check_configs,
+    "definition-macros": check_definition_macros,
     "enum-types": check_enum_types,
     "file-end": check_file_end,
     "formats": check_formats,
diff --git a/tests/test-cases/test_mutex.c b/tests/test-cases/test_mutex.c
index 378203e2..b4d29ac6 100644
--- a/tests/test-cases/test_mutex.c
+++ b/tests/test-cases/test_mutex.c
@@ -32,7 +32,7 @@ TEST_CASE(mutex_try_lock_semantics)
 
 TEST_CASE(mutex_static_init)
 {
-    DEFINE_MUTEX(static_mutex)
+    DEFINE_MUTEX(static_mutex);
 
     ASSERT(mutex_try_lock(&static_mutex));
     ASSERT(!mutex_try_lock(&static_mutex));

From 9828b93e9eb5fd65e4fd21366c24ab8cd38345e6 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 02:22:44 +0300
Subject: [PATCH 006/152] io: drop the empty private/arch/io.h

Nothing is declared in it anymore and both of its includes are already
reached through memory/io.h.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/private/arch/io.h | 9 ---------
 kernel/memory/io.c               | 1 -
 2 files changed, 10 deletions(-)
 delete mode 100644 kernel/include/private/arch/io.h

diff --git a/kernel/include/private/arch/io.h b/kernel/include/private/arch/io.h
deleted file mode 100644
index 1f907910..00000000
--- a/kernel/include/private/arch/io.h
+++ /dev/null
@@ -1,9 +0,0 @@
-#pragma once
-
-#include <common/types.h>
-#include <common/error.h>
-#include <common/attributes.h>
-
-#include <memory/io.h>
-
-#include <arch/io.h>
diff --git a/kernel/memory/io.c b/kernel/memory/io.c
index d7b64eda..0fe0984c 100644
--- a/kernel/memory/io.c
+++ b/kernel/memory/io.c
@@ -19,7 +19,6 @@
 #include <init_level.h>
 #include <log.h>
 
-#include <private/arch/io.h>
 /*
  * API provided by <arch/detail/io_access.h>:
  *

From 0fa34f4dbf0761c0db66574b2713c7858e18182f Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 03:01:18 +0300
Subject: [PATCH 007/152] io: move struct io_window into detail/io.h

Callers only hold an io window and hand it back, its layout is an
implementation detail. Move it out of the public header together with
the pio address type and the io type enum, and the arch header that
defines them into arch/detail/.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .../x86_64/include/arch/{ => detail}/io.h     |  0
 .../x86_64/include/arch/detail/io_access.h    |  2 +-
 kernel/include/detail/io.h                    | 36 +++++++++++++++++++
 kernel/include/memory/io.h                    | 31 ++++++----------
 4 files changed, 47 insertions(+), 22 deletions(-)
 rename kernel/arch/x86_64/include/arch/{ => detail}/io.h (100%)
 create mode 100644 kernel/include/detail/io.h

diff --git a/kernel/arch/x86_64/include/arch/io.h b/kernel/arch/x86_64/include/arch/detail/io.h
similarity index 100%
rename from kernel/arch/x86_64/include/arch/io.h
rename to kernel/arch/x86_64/include/arch/detail/io.h
diff --git a/kernel/arch/x86_64/include/arch/detail/io_access.h b/kernel/arch/x86_64/include/arch/detail/io_access.h
index 7a0d0841..12a81cba 100644
--- a/kernel/arch/x86_64/include/arch/detail/io_access.h
+++ b/kernel/arch/x86_64/include/arch/detail/io_access.h
@@ -4,7 +4,7 @@
 #include <common/helpers.h>
 #include <common/error.h>
 
-#include <memory/io.h>
+#include <detail/io.h>
 
 #define X86_NUM_PORTS 0x10000
 
diff --git a/kernel/include/detail/io.h b/kernel/include/detail/io.h
new file mode 100644
index 00000000..a92ea130
--- /dev/null
+++ b/kernel/include/detail/io.h
@@ -0,0 +1,36 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/helpers.h>
+
+/*
+ * API provided by <arch/detail/io.h>:
+ *
+ * [Optional, gated by ARCH_HAS_CUSTOM_PIO]
+ *   A typedef for pio_addr_t, required if an architecture implements PIO
+ *   that is not emulated over MMIO.
+ */
+#if HAS_INCLUDE(<arch/detail/io.h>)
+#include <arch/detail/io.h>
+#endif
+
+enum io_type : u8 {
+    IO_TYPE_INVALID = 0,
+    IO_TYPE_PORT_IO,
+    IO_TYPE_MEM_IO,
+    IO_TYPE_UNMAPPED = 0xFF,
+};
+
+#ifndef ARCH_HAS_CUSTOM_PIO
+typedef void *pio_addr_t;
+#endif
+
+typedef struct io_window {
+    union {
+        void *mmio_address;
+        pio_addr_t port_address;
+    };
+
+    size_t length;
+    u8 type;
+} io_window;
diff --git a/kernel/include/memory/io.h b/kernel/include/memory/io.h
index 8c010624..9ae97850 100644
--- a/kernel/include/memory/io.h
+++ b/kernel/include/memory/io.h
@@ -5,7 +5,7 @@
 #include <common/bit.h>
 
 #include <arch/memory.h>
-#include <arch/io.h>
+#include <detail/io.h>
 
 #define MAX_PHYS_ADDR BIT_OF_TYPE(phys_addr_t, MAX_PHYS_BITS)
 
@@ -21,26 +21,15 @@ static inline void *phys_to_virt(phys_addr_t phys)
     return (void*)(phys + g_direct_map_base);
 }
 
-enum io_type : u8 {
-    IO_TYPE_INVALID = 0,
-    IO_TYPE_PORT_IO,
-    IO_TYPE_MEM_IO,
-    IO_TYPE_UNMAPPED = 0xFF,
-};
-
-#ifndef ARCH_HAS_CUSTOM_PIO
-typedef void *pio_addr_t;
-#endif
-
-typedef struct io_window {
-    union {
-        void *mmio_address;
-        pio_addr_t port_address;
-    };
-
-    size_t length;
-    u8 type;
-} io_window;
+/*
+ * An opaque MMIO/PIO handle, mapped with one of io_window_map* helpers and
+ * accessed via io{read/write}* or io_window_raw_ptr for memory mapped as
+ * cached. Use io_window_unmap() to unmap.
+ *
+ * NOTE: May be defined as a static/stack variable, but all fields are
+ *       internal.
+ */
+struct io_window;
 
 /*
  * Helpers for mapping MMIO into the CPU address space.

From e6d0bbb3c49c7c9dd1cca2811768afc02470c961 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 03:01:18 +0300
Subject: [PATCH 008/152] spinlock: move struct spinlock into detail/spinlock.h

Only the spin_*() helpers touch its fields, so spinlock.h keeps a
forward declaration and the definition moves out of the way.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/locking/spinlock.h | 19 +++++++++++++++++++
 kernel/include/locking/spinlock.h        | 23 +++++++----------------
 tests/CMakeLists.txt                     |  1 +
 3 files changed, 27 insertions(+), 16 deletions(-)
 create mode 100644 kernel/include/detail/locking/spinlock.h

diff --git a/kernel/include/detail/locking/spinlock.h b/kernel/include/detail/locking/spinlock.h
new file mode 100644
index 00000000..d1af68f1
--- /dev/null
+++ b/kernel/include/detail/locking/spinlock.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include <common/types.h>
+
+struct spinlock {
+    union {
+        u32 control;
+
+        struct {
+        #ifdef ULTRA_ARCH_LITTLE_ENDIAN
+            u16 last_freed_ticket;
+            u16 current_ticket;
+        #else
+            u16 current_ticket;
+            u16 last_freed_ticket;
+        #endif
+        };
+    };
+};
diff --git a/kernel/include/locking/spinlock.h b/kernel/include/locking/spinlock.h
index 03573161..2c4a0357 100644
--- a/kernel/include/locking/spinlock.h
+++ b/kernel/include/locking/spinlock.h
@@ -2,22 +2,13 @@
 
 #include <common/types.h>
 
-struct spinlock {
-    union {
-        u32 control;
-
-        struct {
-        #ifdef ULTRA_ARCH_LITTLE_ENDIAN
-            u16 last_freed_ticket;
-            u16 current_ticket;
-        #else
-            u16 current_ticket;
-            u16 last_freed_ticket;
-        #endif
-
-        };
-    };
-};
+#include <detail/locking/spinlock.h>
+
+/*
+ * An opaque spinlock, initialized with SPINLOCK_INIT() or spin_lock_init().
+ * Use the spin_* helpers, all fields are internal.
+ */
+struct spinlock;
 
 #define SPINLOCK_INIT(name) (struct spinlock) { { .control = 0 } }
 #define DEFINE_SPINLOCK(name) struct spinlock name = SPINLOCK_INIT(name)
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 29b66e44..93f27e30 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -197,6 +197,7 @@ KERNEL_FILE(
     INCLUDE_PATH "locking" INCLUDE_FILE "spinlock.h"
     SOURCE_PATH "locking" SOURCE_FILE "spinlock.c"
 )
+KERNEL_FILE(INCLUDE_PATH "detail/locking" INCLUDE_FILE "spinlock.h")
 KERNEL_FILE(
     INCLUDE_PATH "locking" INCLUDE_FILE "mutex.h"
     SOURCE_PATH "locking" SOURCE_FILE "mutex.c"

From 715bde9ed806c9abd91c92933519c57c532df3ab Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 03:01:18 +0300
Subject: [PATCH 009/152] mutex: move struct mutex into detail/mutex.h

Only the mutex_*() helpers touch its fields, so mutex.h keeps the
documented forward declaration and the definition moves out of the
way.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/locking/mutex.h | 7 +++++++
 kernel/include/locking/mutex.h        | 6 +++---
 tests/CMakeLists.txt                  | 1 +
 3 files changed, 11 insertions(+), 3 deletions(-)
 create mode 100644 kernel/include/detail/locking/mutex.h

diff --git a/kernel/include/detail/locking/mutex.h b/kernel/include/detail/locking/mutex.h
new file mode 100644
index 00000000..3620ae9f
--- /dev/null
+++ b/kernel/include/detail/locking/mutex.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include <locking/spinlock.h>
+
+struct mutex {
+    struct spinlock lock;
+};
diff --git a/kernel/include/locking/mutex.h b/kernel/include/locking/mutex.h
index 5c89209d..308ff3cd 100644
--- a/kernel/include/locking/mutex.h
+++ b/kernel/include/locking/mutex.h
@@ -2,6 +2,8 @@
 
 #include <locking/spinlock.h>
 
+#include <detail/locking/mutex.h>
+
 /*
  * A mutual exclusion lock for process context only. Holders are
  * allowed to block, so it must never be acquired from interrupt
@@ -11,9 +13,7 @@
  * spinlock that must be free when taken, but every caller must
  * already honor the contract as if it could sleep.
  */
-struct mutex {
-    struct spinlock lock;
-};
+struct mutex;
 
 #define MUTEX_INIT(name) (struct mutex) { SPINLOCK_INIT(name.lock) }
 #define DEFINE_MUTEX(name) struct mutex name = MUTEX_INIT(name)
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 93f27e30..7539c3ee 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -202,6 +202,7 @@ KERNEL_FILE(
     INCLUDE_PATH "locking" INCLUDE_FILE "mutex.h"
     SOURCE_PATH "locking" SOURCE_FILE "mutex.c"
 )
+KERNEL_FILE(INCLUDE_PATH "detail/locking" INCLUDE_FILE "mutex.h")
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "vm_flags.h")
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "page_table.h")
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "address_space.h")

From 29964d2bbaa01d6992fa7443223e5cba07010c6d Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 03:01:18 +0300
Subject: [PATCH 010/152] locking: define spin_lock_init() and mutex_init() via
 *_INIT()

The initial state of each lock was written twice, in its initializer
macro and in its init function, so a new field had to be added to
both. Going forward, an init function is defined through the macro next
to it, e.g. *lock = SPINLOCK_INIT(*lock).

The time keeper passed the address of its lock and sequence count
instead of the objects, which only built because neither macro uses its
argument. Pass the objects.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/locking/mutex.h    | 2 +-
 kernel/include/locking/spinlock.h | 2 +-
 kernel/locking/mutex.c            | 2 +-
 kernel/time/keeper.c              | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/kernel/include/locking/mutex.h b/kernel/include/locking/mutex.h
index 308ff3cd..e1d35016 100644
--- a/kernel/include/locking/mutex.h
+++ b/kernel/include/locking/mutex.h
@@ -15,7 +15,7 @@
  */
 struct mutex;
 
-#define MUTEX_INIT(name) (struct mutex) { SPINLOCK_INIT(name.lock) }
+#define MUTEX_INIT(name) (struct mutex) { SPINLOCK_INIT((name).lock) }
 #define DEFINE_MUTEX(name) struct mutex name = MUTEX_INIT(name)
 
 void mutex_init(struct mutex*);
diff --git a/kernel/include/locking/spinlock.h b/kernel/include/locking/spinlock.h
index 2c4a0357..fd50422f 100644
--- a/kernel/include/locking/spinlock.h
+++ b/kernel/include/locking/spinlock.h
@@ -15,7 +15,7 @@ struct spinlock;
 
 static inline void spin_lock_init(struct spinlock *lock)
 {
-    lock->control = 0;
+    *lock = SPINLOCK_INIT(*lock);
 }
 
 void spin_lock(struct spinlock*);
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 24ddb1ef..386f09ef 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -4,7 +4,7 @@
 
 void mutex_init(struct mutex *mutex)
 {
-    spin_lock_init(&mutex->lock);
+    *mutex = MUTEX_INIT(*mutex);
 }
 
 void mutex_lock(struct mutex *mutex)
diff --git a/kernel/time/keeper.c b/kernel/time/keeper.c
index 46d0b28a..52e19464 100644
--- a/kernel/time/keeper.c
+++ b/kernel/time/keeper.c
@@ -19,8 +19,8 @@ static struct keeper {
     u64 ns;
     u64 prev_cycles;
 } s_ctx = {
-    .lock = SPINLOCK_INIT(&s_ctx.lock),
-    .time_seq = SEQCOUNT_INIT(&s_ctx.time_seq),
+    .lock = SPINLOCK_INIT(s_ctx.lock),
+    .time_seq = SEQCOUNT_INIT(s_ctx.time_seq),
 };
 
 static void do_tick(void)

From 87849fce41047692e5c0656b4e581b7ab1b850a6 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 03:01:18 +0300
Subject: [PATCH 011/152] log_ring: add a forward declaration of struct
 log_ring

struct log_ring is defined in detail/log_ring.h, so log_ring.h never
said the type exists to its reader.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/log_ring.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/kernel/include/log_ring.h b/kernel/include/log_ring.h
index bde7e99b..c68265eb 100644
--- a/kernel/include/log_ring.h
+++ b/kernel/include/log_ring.h
@@ -13,6 +13,12 @@
  */
 #include <detail/log_ring.h>
 
+/*
+ * An opaque log ring, defined with MAKE_LOG_RING(). Use the
+ * log_ring_* helpers, all fields are internal.
+ */
+struct log_ring;
+
 /*
  * Define a log ring 'name', that has 2^'size_shift' bytes of text space,
  * with an expected average message length of 2^'avg_msg_length_shift' bytes.

From 2923280e6cd6fc3f1fc196b770b982beabe3d823 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 03:01:19 +0300
Subject: [PATCH 012/152] abortable_instructions: move the interface into
 private/arch

Only the subsystem and its arch half use the table entry, its flags and
the entry point of the exception handlers, so the public header has no
readers.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .../arch/private/abortable_instructions.h     |  2 +-
 kernel/include/abortable_instructions.h       | 31 -------------------
 .../private/arch/abortable_instructions.h     | 29 ++++++++++++++++-
 3 files changed, 29 insertions(+), 33 deletions(-)
 delete mode 100644 kernel/include/abortable_instructions.h

diff --git a/kernel/arch/x86_64/include/arch/private/abortable_instructions.h b/kernel/arch/x86_64/include/arch/private/abortable_instructions.h
index 92f4134a..451c39cd 100644
--- a/kernel/arch/x86_64/include/arch/private/abortable_instructions.h
+++ b/kernel/arch/x86_64/include/arch/private/abortable_instructions.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <abortable_instructions.h>
+#include <private/arch/abortable_instructions.h>
 #include <linker.h>
 #include <common/attributes.h>
 
diff --git a/kernel/include/abortable_instructions.h b/kernel/include/abortable_instructions.h
deleted file mode 100644
index 62f69518..00000000
--- a/kernel/include/abortable_instructions.h
+++ /dev/null
@@ -1,31 +0,0 @@
-#pragma once
-
-/*
- * This instruction should be handled by arch specific code via
- * arch_handle_abortable_instruction()
- */
-#define ABORTABLE_INSTRUCTION_HAS_ARCH_HANDLER (1 << 0)
-
-/*
- * Set the respective error code if an exception occurs while executing this
- * instruction.
- */
-#define ABORTABLE_INSTRUCTION_ENOSYS_ON_ERROR (1 << 1)
-#define ABORTABLE_INSTRUCTION_EFAULT_ON_ERROR (1 << 2)
-
-#ifndef __ASSEMBLER__
-#include <common/types.h>
-#include <arch/registers.h>
-
-struct abortable_instruction {
-    i32 try_pc_disp;
-    i32 abort_pc_disp;
-
-    u32 flags;
-    u32 arch_flags;
-};
-
-// Called by the arch-specific exception handling code
-bool handle_abortable_instruction(struct registers*);
-
-#endif
diff --git a/kernel/include/private/arch/abortable_instructions.h b/kernel/include/private/arch/abortable_instructions.h
index 5d14dee4..5ef410d1 100644
--- a/kernel/include/private/arch/abortable_instructions.h
+++ b/kernel/include/private/arch/abortable_instructions.h
@@ -1,8 +1,33 @@
 #pragma once
 
-#include <abortable_instructions.h>
+/*
+ * This instruction should be handled by arch specific code via
+ * arch_handle_abortable_instruction()
+ */
+#define ABORTABLE_INSTRUCTION_HAS_ARCH_HANDLER (1 << 0)
+
+/*
+ * Set the respective error code if an exception occurs while executing this
+ * instruction.
+ */
+#define ABORTABLE_INSTRUCTION_ENOSYS_ON_ERROR (1 << 1)
+#define ABORTABLE_INSTRUCTION_EFAULT_ON_ERROR (1 << 2)
+
+#ifndef __ASSEMBLER__
+#include <common/types.h>
 #include <arch/registers.h>
 
+struct abortable_instruction {
+    i32 try_pc_disp;
+    i32 abort_pc_disp;
+
+    u32 flags;
+    u32 arch_flags;
+};
+
+// Called by the arch-specific exception handling code
+bool handle_abortable_instruction(struct registers*);
+
 /*
  * Called by generic code if ABORTABLE_INSTRUCTION_HAS_ARCH_HANDLER is set.
  * Returns true if the instruction was handled entirely by arch-specific code
@@ -11,3 +36,5 @@
 bool arch_handle_abortable_instruction(
     struct registers *regs, const struct abortable_instruction *ai
 );
+
+#endif

From 10bf0710c85f62d7217fb8ec8ff11caf7a0edb0b Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 18 Oct 2026 23:38:32 +0300
Subject: [PATCH 013/152] unwind: stop eh_consume() shadowing the caller's ret

The statement expression declares its own ret, which shadows the ret
every caller assigns the result to. Rename it to consume_ret so that
-Wshadow can be enabled.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/unwind.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/kernel/unwind.c b/kernel/unwind.c
index ac1e25ca..04944192 100644
--- a/kernel/unwind.c
+++ b/kernel/unwind.c
@@ -66,14 +66,14 @@ static u64 s_num_fdes;
 } while (0)
 
 #define eh_consume(eh, dst) ({                             \
-    error_t ret = ERANGE;                                  \
+    error_t consume_ret = ERANGE;                          \
                                                            \
     if (likely(sizeof(typeof(dst)) <= (eh)->bytes_left)) { \
         EH_CONSUME_UNSAFE(eh, dst);                        \
-        ret = EOK;                                         \
+        consume_ret = EOK;                                 \
     }                                                      \
                                                            \
-    ret;                                                   \
+    consume_ret;                                           \
 })
 
 static void eh_data_init(struct eh_data *eh, const u8 *cursor)

From 88a7fbc56e696d81d8e9586917f10b998fe1f0f8 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 18 Oct 2026 23:38:33 +0300
Subject: [PATCH 014/152] time: stop shadowing cycles in
 time_keeper_set_counter_device()

The old device's final reading is declared in an inner block under the
same name as the new device's. Rename it to old_cycles.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/time/keeper.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/kernel/time/keeper.c b/kernel/time/keeper.c
index 52e19464..318ad8dc 100644
--- a/kernel/time/keeper.c
+++ b/kernel/time/keeper.c
@@ -65,11 +65,11 @@ void time_keeper_set_counter_device(struct counter_device *cd)
 
     // If we already had a device, accumulate its final time before dumping it
     if (s_ctx.cd) {
-        u64 cycles, delta_cycles;
+        u64 old_cycles, delta_cycles;
 
-        cycles = counter_device_read(s_ctx.cd);
+        old_cycles = counter_device_read(s_ctx.cd);
         delta_cycles = counter_device_delta(
-            s_ctx.cd, s_ctx.prev_cycles, cycles
+            s_ctx.cd, s_ctx.prev_cycles, old_cycles
         );
         atomic_store_relaxed(
             &s_ctx.ns,

From 9e7255f98178ffa73ef2157df52f35cb251b0ada Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 18 Oct 2026 23:38:33 +0300
Subject: [PATCH 015/152] seq: make SEQLOCK_INIT() compile

SEQLOCK_INIT() builds a struct seqlock, but seqlock is a typedef of an
anonymous struct, so the macro fails to compile on its first use. It
also hands the whole seqlock to SEQCOUNT_INIT() and SPINLOCK_INIT()
instead of its members. Nothing uses it yet.

Name the struct like struct seqcount, drop the typedef and initialize
each member from its own field.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/locking/seq.h | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/kernel/include/locking/seq.h b/kernel/include/locking/seq.h
index b8221afb..ecacef45 100644
--- a/kernel/include/locking/seq.h
+++ b/kernel/include/locking/seq.h
@@ -69,27 +69,29 @@ static inline bool seqcount_read_retry(const struct seqcount *s, u32 start_seq)
     return atomic_load_relaxed(&s->sequence) != start_seq;
 }
 
-typedef struct {
+struct seqlock {
     struct seqcount seqcount;
     struct spinlock lock;
-} seqlock;
+};
 
-#define SEQLOCK_INIT(name) (struct seqlock) \
-    { .seqcount = SEQCOUNT_INIT(name), .lock = SPINLOCK_INIT(name) }
+#define SEQLOCK_INIT(name) (struct seqlock) {   \
+    .seqcount = SEQCOUNT_INIT((name).seqcount), \
+    .lock = SPINLOCK_INIT((name).lock),         \
+}
 
-static inline void seqlock_write_lock(seqlock *sl)
+static inline void seqlock_write_lock(struct seqlock *sl)
 {
     spin_lock(&sl->lock);
     seqcount_write_begin(&sl->seqcount);
 }
 
-static inline void seqlock_write_unlock(seqlock *sl)
+static inline void seqlock_write_unlock(struct seqlock *sl)
 {
     seqcount_write_end(&sl->seqcount);
     spin_unlock(&sl->lock);
 }
 
-static inline irq_state_t seqlock_write_lock_irq_save(seqlock *sl)
+static inline irq_state_t seqlock_write_lock_irq_save(struct seqlock *sl)
 {
     irq_state_t state;
 
@@ -100,7 +102,7 @@ static inline irq_state_t seqlock_write_lock_irq_save(seqlock *sl)
 }
 
 static inline void seqlock_write_unlock_irq_restore(
-    seqlock *sl, irq_state_t state
+    struct seqlock *sl, irq_state_t state
 )
 {
     seqcount_write_end(&sl->seqcount);

From d32ecf626539697c7dde2bf175a5b57e1fbcb0c3 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 18 Oct 2026 23:38:33 +0300
Subject: [PATCH 016/152] time: don't drop the nanosecond fraction on every
 keeper tick

do_tick() converts the cycles since the previous tick to nanoseconds,
rounds down and throws the remainder away. The keeper falls behind the
counter by up to a nanosecond per tick, and ns_since_boot() can go back
by one: a reader that sampled the counter after the tick did, but still
saw the previous base, gets the time without the rounding.

Keep the remainder in units of the device's shift and add it to the
next conversion. A device switch rounds it off once, since the next
device has a different shift.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/private/time/counter_device.h | 40 ++++++++++++++++++--
 kernel/time/keeper.c                         | 16 +++++---
 tests/test-cases/test_time.c                 | 36 ++++++++++++++++++
 3 files changed, 83 insertions(+), 9 deletions(-)

diff --git a/kernel/include/private/time/counter_device.h b/kernel/include/private/time/counter_device.h
index c07be03e..c1773de7 100644
--- a/kernel/include/private/time/counter_device.h
+++ b/kernel/include/private/time/counter_device.h
@@ -2,14 +2,48 @@
 
 #include <time/counter_device.h>
 
+#include <common/bit.h>
+
+/*
+ * Convert cycles to nanoseconds, adding *in_out_frac, the fraction of a
+ * nanosecond left over by a previous conversion with this device, and
+ * storing the new one back. The fraction is in units of 2^-shift.
+ */
+static inline u64 counter_device_cycles_to_ns_frac(
+    struct counter_device *dev, u64 cycles, u64 *in_out_frac
+)
+{
+    u64 ns, frac, frac_mask;
+
+    frac_mask = BIT_U64(dev->shift) - 1;
+
+    if (likely(cycles < dev->max_cycles_before_u64_overflow)) {
+        u64 shifted_ns;
+
+        shifted_ns = cycles * dev->mult;
+        ns = shifted_ns >> dev->shift;
+        frac = shifted_ns & frac_mask;
+    } else {
+        u128 shifted_ns;
+
+        shifted_ns = (u128)cycles * dev->mult;
+        ns = (u64)(shifted_ns >> dev->shift);
+        frac = (u64)shifted_ns & frac_mask;
+    }
+
+    frac += *in_out_frac;
+    *in_out_frac = frac & frac_mask;
+
+    return ns + (frac >> dev->shift);
+}
+
 static inline u64 counter_device_cycles_to_ns(
     struct counter_device *dev, u64 cycles
 )
 {
-    if (likely(cycles < dev->max_cycles_before_u64_overflow))
-        return (cycles * dev->mult) >> dev->shift;
+    u64 frac = 0;
 
-    return (u64)(((u128)cycles * dev->mult) >> dev->shift);
+    return counter_device_cycles_to_ns_frac(dev, cycles, &frac);
 }
 
 static inline u64 counter_device_delta(
diff --git a/kernel/time/keeper.c b/kernel/time/keeper.c
index 318ad8dc..87c9a2e0 100644
--- a/kernel/time/keeper.c
+++ b/kernel/time/keeper.c
@@ -17,6 +17,7 @@ static struct keeper {
     struct counter_device *cd;
 
     u64 ns;
+    u64 ns_frac;
     u64 prev_cycles;
 } s_ctx = {
     .lock = SPINLOCK_INIT(s_ctx.lock),
@@ -25,7 +26,7 @@ static struct keeper {
 
 static void do_tick(void)
 {
-    u64 cycles, delta_cycles, delta_ns;
+    u64 cycles, delta_cycles, delta_ns, ns_frac = s_ctx.ns_frac;
     struct counter_device *cd = s_ctx.cd;
 
     if (unlikely(cd == nullptr))
@@ -33,7 +34,7 @@ static void do_tick(void)
 
     cycles = counter_device_read(cd);
     delta_cycles = counter_device_delta(cd, s_ctx.prev_cycles, cycles);
-    delta_ns = counter_device_cycles_to_ns(cd, delta_cycles);
+    delta_ns = counter_device_cycles_to_ns_frac(cd, delta_cycles, &ns_frac);
 
     seqcount_write_begin(&s_ctx.time_seq);
 
@@ -41,6 +42,7 @@ static void do_tick(void)
         &s_ctx.ns,
         atomic_load_relaxed(&s_ctx.ns) + delta_ns
     );
+    atomic_store_relaxed(&s_ctx.ns_frac, ns_frac);
     atomic_store_relaxed(&s_ctx.prev_cycles, cycles);
 
     seqcount_write_end(&s_ctx.time_seq);
@@ -65,7 +67,7 @@ void time_keeper_set_counter_device(struct counter_device *cd)
 
     // If we already had a device, accumulate its final time before dumping it
     if (s_ctx.cd) {
-        u64 old_cycles, delta_cycles;
+        u64 old_cycles, delta_cycles, ns_frac = s_ctx.ns_frac;
 
         old_cycles = counter_device_read(s_ctx.cd);
         delta_cycles = counter_device_delta(
@@ -74,8 +76,9 @@ void time_keeper_set_counter_device(struct counter_device *cd)
         atomic_store_relaxed(
             &s_ctx.ns,
             atomic_load_relaxed(&s_ctx.ns) +
-            counter_device_cycles_to_ns(s_ctx.cd, delta_cycles)
+            counter_device_cycles_to_ns_frac(s_ctx.cd, delta_cycles, &ns_frac)
         );
+        atomic_store_relaxed(&s_ctx.ns_frac, 0);
     }
 
     if (cd)
@@ -91,7 +94,7 @@ void time_keeper_set_counter_device(struct counter_device *cd)
 u64 ns_since_boot(void)
 {
     u32 seq;
-    u64 base, prev_cycles, now, delta_cycles, delta_ns;
+    u64 base, ns_frac, prev_cycles, now, delta_cycles, delta_ns;
     struct counter_device *cd;
 
     do {
@@ -104,6 +107,7 @@ u64 ns_since_boot(void)
         if (cd == nullptr)
             return base;
 
+        ns_frac = atomic_load_relaxed(&s_ctx.ns_frac);
         prev_cycles = atomic_load_relaxed(&s_ctx.prev_cycles);
         barrier_acquire();
 
@@ -111,7 +115,7 @@ u64 ns_since_boot(void)
     } while (seqcount_read_retry(&s_ctx.time_seq, seq));
 
     delta_cycles = counter_device_delta(cd, prev_cycles, now);
-    delta_ns = counter_device_cycles_to_ns(cd, delta_cycles);
+    delta_ns = counter_device_cycles_to_ns_frac(cd, delta_cycles, &ns_frac);
 
     return base + delta_ns;
 }
diff --git a/tests/test-cases/test_time.c b/tests/test-cases/test_time.c
index 4b3e3236..88932086 100644
--- a/tests/test-cases/test_time.c
+++ b/tests/test-cases/test_time.c
@@ -39,6 +39,7 @@ static void time_test_reset(void)
     seqcount_init(&s_ctx.time_seq);
     s_ctx.cd = nullptr;
     s_ctx.ns = 0;
+    s_ctx.ns_frac = 0;
     s_ctx.prev_cycles = 0;
 
     g_counter_value = 0;
@@ -298,6 +299,41 @@ TEST_CASE(keeper_reports_live_delta)
     ASSERT_EQ(ns_since_boot(), 1500);
 }
 
+TEST_CASE(keeper_carries_the_fraction_across_ticks)
+{
+    struct counter_device dev = TEST_CD("tsc", COUNTER_MASK(64), IDEAL);
+    u64 i;
+
+    time_test_reset();
+    counter_device_register(&dev, 400 * MHZ);
+
+    for (i = 1; i <= 1000; i++) {
+        g_counter_value = i;
+        time_keeper_tick();
+    }
+
+    ASSERT_EQ(ns_since_boot(), reference_cycles_to_ns(&dev, 1000));
+}
+
+TEST_CASE(keeper_time_does_not_step_back_across_a_tick)
+{
+    struct counter_device dev = TEST_CD("tsc", COUNTER_MASK(64), IDEAL);
+    u64 before;
+
+    time_test_reset();
+    counter_device_register(&dev, 400 * MHZ);
+
+    g_counter_value = 2;
+    before = ns_since_boot();
+
+    // The tick read the counter before the reader did, but published after it
+    g_counter_value = 1;
+    time_keeper_tick();
+
+    g_counter_value = 2;
+    ASSERT_EQ(ns_since_boot(), before);
+}
+
 /*
  * Swapping the active device must bank the elapsed time from the outgoing
  * device before continuing on the new one, so the timeline stays monotonic.

From 68d363fe6ff5a9a0028fb43f626e4aec64526715 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 18 Oct 2026 23:38:33 +0300
Subject: [PATCH 017/152] build: enable -Wshadow

A shadowed variable compiles silently and is easy to mistake for the
outer one. Nothing in the tree shadows anymore, so keep it that way.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt
index e37be5e9..85446467 100644
--- a/kernel/CMakeLists.txt
+++ b/kernel/CMakeLists.txt
@@ -129,6 +129,7 @@ ultra_compile_options(
     -ggdb3
     -Wall
     -Wextra
+    -Wshadow
     -Werror
     -Wstrict-prototypes
     -Wmissing-prototypes

From 180148f0d8260e8773041a12a91101e00f21dfb2 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 16 Oct 2026 23:08:44 +0300
Subject: [PATCH 018/152] everywhere: rename the IRQ flag helpers to
 local_irq_*()

irq_* prefixed APIs are going to be reserved for actual interaction with
IRQ lines and hardware, not CPU flags. Rename to reflect that and to
avoid confusion about what this API does.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .../x86_64/include/arch/detail/irq_helpers.h   |  8 ++++----
 kernel/arch/x86_64/xapic.c                     |  4 ++--
 kernel/include/irq_helpers.h                   | 18 +++++++++---------
 kernel/include/locking/seq.h                   |  4 ++--
 kernel/locking/spinlock.c                      |  8 ++++----
 kernel/log_ring.c                              | 10 +++++-----
 tests/synthetic-include/irq_helpers.h          |  6 +++---
 7 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/kernel/arch/x86_64/include/arch/detail/irq_helpers.h b/kernel/arch/x86_64/include/arch/detail/irq_helpers.h
index 5a2d95d2..c92abe05 100644
--- a/kernel/arch/x86_64/include/arch/detail/irq_helpers.h
+++ b/kernel/arch/x86_64/include/arch/detail/irq_helpers.h
@@ -5,22 +5,22 @@
 
 #define X86_RFLAGS_IF (1 << 9)
 
-static ALWAYS_INLINE void arch_irq_enable(void)
+static ALWAYS_INLINE void arch_local_irq_enable(void)
 {
     asm volatile("sti" ::: "memory");
 }
 
-static ALWAYS_INLINE void arch_irq_disable(void)
+static ALWAYS_INLINE void arch_local_irq_disable(void)
 {
     asm volatile("cli" ::: "memory");
 }
 
-static ALWAYS_INLINE bool arch_irq_is_enabled_state(irq_state_t flags)
+static ALWAYS_INLINE bool arch_local_irq_is_enabled_state(irq_state_t flags)
 {
     return flags & X86_RFLAGS_IF;
 }
 
-static ALWAYS_INLINE irq_state_t arch_irq_state(void)
+static ALWAYS_INLINE irq_state_t arch_local_irq_state(void)
 {
     irq_state_t state;
 
diff --git a/kernel/arch/x86_64/xapic.c b/kernel/arch/x86_64/xapic.c
index 3aacb912..92fb2432 100644
--- a/kernel/arch/x86_64/xapic.c
+++ b/kernel/arch/x86_64/xapic.c
@@ -43,7 +43,7 @@ static void xapic_icr_write(u32 value, u32 dest_apic_id)
      * Both halves must reach the same APIC back to back, neither a
      * migration nor an IPI sent from an interrupt may come between.
      */
-    irq_state = irq_state_save();
+    irq_state = local_irq_state_save();
 
     xapic_wait_icr_idle();
 
@@ -51,7 +51,7 @@ static void xapic_icr_write(u32 value, u32 dest_apic_id)
     xapic_write(APIC_REG_ICR2, dest_apic_id << 24);
     xapic_write(APIC_REG_ICR, value);
 
-    irq_state_restore(irq_state);
+    local_irq_state_restore(irq_state);
 }
 
 static void xapic_send_ipi_self(u8 vector)
diff --git a/kernel/include/irq_helpers.h b/kernel/include/irq_helpers.h
index 875e568e..fa3e0ee7 100644
--- a/kernel/include/irq_helpers.h
+++ b/kernel/include/irq_helpers.h
@@ -5,9 +5,9 @@
 
 #include <arch/detail/irq_helpers.h>
 
-static ALWAYS_INLINE bool irq_enabled(void)
+static ALWAYS_INLINE bool local_irq_enabled(void)
 {
-    return arch_irq_is_enabled_state(arch_irq_state());
+    return arch_local_irq_is_enabled_state(arch_local_irq_state());
 }
 
 // True while executing in hardware interrupt context on this CPU
@@ -17,22 +17,22 @@ bool in_hard_irq(void);
  * Save the arch-specific IRQ state and disable IRQs if they were previously
  * enabled.
  */
-static ALWAYS_INLINE irq_state_t irq_state_save(void)
+static ALWAYS_INLINE irq_state_t local_irq_state_save(void)
 {
     irq_state_t state;
 
-    state = arch_irq_state();
-    if (arch_irq_is_enabled_state(state))
-        arch_irq_disable();
+    state = arch_local_irq_state();
+    if (arch_local_irq_is_enabled_state(state))
+        arch_local_irq_disable();
 
     return state;
 }
 
 // Restore the IRQ state from the arch-specific state saved earlier
-static ALWAYS_INLINE void irq_state_restore(irq_state_t state)
+static ALWAYS_INLINE void local_irq_state_restore(irq_state_t state)
 {
-    if (!arch_irq_is_enabled_state(state))
+    if (!arch_local_irq_is_enabled_state(state))
         return;
 
-    arch_irq_enable();
+    arch_local_irq_enable();
 }
diff --git a/kernel/include/locking/seq.h b/kernel/include/locking/seq.h
index ecacef45..95dc4ffe 100644
--- a/kernel/include/locking/seq.h
+++ b/kernel/include/locking/seq.h
@@ -33,7 +33,7 @@ static inline irq_state_t seqcount_write_begin_irq_save(struct seqcount *s)
 {
     irq_state_t state;
 
-    state = irq_state_save();
+    state = local_irq_state_save();
     seqcount_write_begin(s);
 
     return state;
@@ -44,7 +44,7 @@ static inline void seqcount_write_end_irq_restore(
 )
 {
     seqcount_write_end(s);
-    irq_state_restore(state);
+    local_irq_state_restore(state);
 }
 
 static inline u32 seqcount_read_begin(const struct seqcount *s)
diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index 6e6a84de..fa34f7a9 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -37,7 +37,7 @@ irq_state_t spin_lock_irq_save(struct spinlock *lock)
 {
     irq_state_t irq_state;
 
-    irq_state = irq_state_save();
+    irq_state = local_irq_state_save();
     spin_lock(lock);
 
     return irq_state;
@@ -60,13 +60,13 @@ bool spin_try_lock_irq_save(struct spinlock *lock, irq_state_t *out_irq_state)
 {
     irq_state_t irq_state;
 
-    irq_state = irq_state_save();
+    irq_state = local_irq_state_save();
     if (spin_try_lock(lock)) {
         *out_irq_state = irq_state;
         return true;
     }
 
-    irq_state_restore(irq_state);
+    local_irq_state_restore(irq_state);
     return false;
 }
 
@@ -81,7 +81,7 @@ void spin_unlock(struct spinlock *lock)
 void spin_unlock_irq_restore(struct spinlock *lock, irq_state_t irq_state)
 {
     spin_unlock(lock);
-    irq_state_restore(irq_state);
+    local_irq_state_restore(irq_state);
 }
 
 bool spin_is_locked(struct spinlock *lock)
diff --git a/kernel/log_ring.c b/kernel/log_ring.c
index 07a5dbcb..014b4b86 100644
--- a/kernel/log_ring.c
+++ b/kernel/log_ring.c
@@ -985,12 +985,12 @@ error_t log_ring_reserve(
         return ENOSPC;
 
     timestamp = ns_since_boot();
-    res->details.irq_state = irq_state_save();
+    res->details.irq_state = local_irq_state_save();
     res->details.ring = ring;
 
     ret = log_descriptor_alloc(ring, &res->details.id);
     if (is_error(ret)) {
-        irq_state_restore(res->details.irq_state);
+        local_irq_state_restore(res->details.irq_state);
         return ret;
     }
 
@@ -1046,7 +1046,7 @@ error_t log_ring_reserve_extend(
     struct log_descriptor_ring *desc_ring = &ring->descriptor_ring;
     bool did_reopen;
 
-    res->details.irq_state = irq_state_save();
+    res->details.irq_state = local_irq_state_save();
     res->details.id = atomic_load_acquire(&desc_ring->head_id);
     res->details.ring = ring;
 
@@ -1055,7 +1055,7 @@ error_t log_ring_reserve_extend(
         DESCRIPTOR_STATE_COMMITTED, DESCRIPTOR_STATE_RESERVED
     );
     if (!did_reopen) {
-        irq_state_restore(res->details.irq_state);
+        local_irq_state_restore(res->details.irq_state);
         return EBUSY;
     }
 
@@ -1134,7 +1134,7 @@ static bool update_reservation_state(
     );
     WARN_ON_WITH_MSG(!success, "likely state corruption, reserved record gone");
 
-    irq_state_restore(res->details.irq_state);
+    local_irq_state_restore(res->details.irq_state);
     return success;
 }
 
diff --git a/tests/synthetic-include/irq_helpers.h b/tests/synthetic-include/irq_helpers.h
index 8f0f1f64..fa290479 100644
--- a/tests/synthetic-include/irq_helpers.h
+++ b/tests/synthetic-include/irq_helpers.h
@@ -2,12 +2,12 @@
 
 static __thread bool g_enabled = true;
 
-static inline bool irq_enabled(void)
+static inline bool local_irq_enabled(void)
 {
     return g_enabled;
 }
 
-static inline irq_state_t irq_state_save(void)
+static inline irq_state_t local_irq_state_save(void)
 {
     bool was_enabled = g_enabled;
 
@@ -15,7 +15,7 @@ static inline irq_state_t irq_state_save(void)
     return was_enabled;
 }
 
-static inline void irq_state_restore(irq_state_t state)
+static inline void local_irq_state_restore(irq_state_t state)
 {
     g_enabled = state;
 }

From 2fa34dd77bd88da5c18373badca2b7244f27d49d Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 12:26:54 +0300
Subject: [PATCH 019/152] tests: share the emulated interrupt state between
 files

The flag was a static in the stub header, so every translation unit had
its own copy, and a lock taken with interrupts disabled in spinlock.c
left them enabled as far as the test could tell. Define it once in
kernel_helpers.cpp.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 tests/kernel_helpers.cpp              |  6 ++++++
 tests/synthetic-include/irq_helpers.h | 12 +++++++-----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/tests/kernel_helpers.cpp b/tests/kernel_helpers.cpp
index b439f67f..8f31015c 100644
--- a/tests/kernel_helpers.cpp
+++ b/tests/kernel_helpers.cpp
@@ -14,6 +14,12 @@ extern "C" {
 
 struct boot_context g_boot_ctx;
 
+extern "C" {
+
+__thread bool g_test_irqs_enabled = true;
+
+}
+
 /*
  * There is no loader-provided memory map in userspace, so boot_alloc_init()
  * has nothing to walk. Tests seed the boot allocator by writing its range
diff --git a/tests/synthetic-include/irq_helpers.h b/tests/synthetic-include/irq_helpers.h
index fa290479..a9566b19 100644
--- a/tests/synthetic-include/irq_helpers.h
+++ b/tests/synthetic-include/irq_helpers.h
@@ -1,23 +1,25 @@
 #pragma once
 
-static __thread bool g_enabled = true;
+#include <common/types.h>
+
+extern __thread bool g_test_irqs_enabled;
 
 static inline bool local_irq_enabled(void)
 {
-    return g_enabled;
+    return g_test_irqs_enabled;
 }
 
 static inline irq_state_t local_irq_state_save(void)
 {
-    bool was_enabled = g_enabled;
+    bool was_enabled = g_test_irqs_enabled;
 
-    g_enabled = false;
+    g_test_irqs_enabled = false;
     return was_enabled;
 }
 
 static inline void local_irq_state_restore(irq_state_t state)
 {
-    g_enabled = state;
+    g_test_irqs_enabled = state;
 }
 
 static inline bool in_hard_irq(void)

From 013d6de8a637eb0b5a1b8e54e48e4d2ea8c05a23 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 16 Oct 2026 22:46:33 +0300
Subject: [PATCH 020/152] kernel: add local_irq_{enable,disable}()

So that generic code doesn't have to use the arch_* variant directly.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/irq_helpers.h          | 10 ++++++++++
 tests/synthetic-include/irq_helpers.h | 10 ++++++++++
 2 files changed, 20 insertions(+)

diff --git a/kernel/include/irq_helpers.h b/kernel/include/irq_helpers.h
index fa3e0ee7..9279c7f9 100644
--- a/kernel/include/irq_helpers.h
+++ b/kernel/include/irq_helpers.h
@@ -36,3 +36,13 @@ static ALWAYS_INLINE void local_irq_state_restore(irq_state_t state)
 
     arch_local_irq_enable();
 }
+
+static ALWAYS_INLINE void local_irq_disable(void)
+{
+    arch_local_irq_disable();
+}
+
+static ALWAYS_INLINE void local_irq_enable(void)
+{
+    arch_local_irq_enable();
+}
diff --git a/tests/synthetic-include/irq_helpers.h b/tests/synthetic-include/irq_helpers.h
index a9566b19..148cfe97 100644
--- a/tests/synthetic-include/irq_helpers.h
+++ b/tests/synthetic-include/irq_helpers.h
@@ -17,6 +17,16 @@ static inline irq_state_t local_irq_state_save(void)
     return was_enabled;
 }
 
+static inline void local_irq_enable(void)
+{
+    g_test_irqs_enabled = true;
+}
+
+static inline void local_irq_disable(void)
+{
+    g_test_irqs_enabled = false;
+}
+
 static inline void local_irq_state_restore(irq_state_t state)
 {
     g_test_irqs_enabled = state;

From 7dc6583da624dd1a92bf6bc5600ef8f7af9c82df Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sat, 3 Oct 2026 21:49:49 +0300
Subject: [PATCH 021/152] panic: disable interrupts when panicking

A handler interrupting the panic could log into the ring being dumped or
block on a lock the panic path holds.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/panic.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/kernel/panic.c b/kernel/panic.c
index 315f12cc..0a94589f 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -6,6 +6,7 @@
 #include <arch/cpu_helpers.h>
 
 #include <panic.h>
+#include <irq_helpers.h>
 #include <log.h>
 
 static bool s_in_panic;
@@ -21,6 +22,8 @@ void panic(const char *fmt, ...)
     va_list vlist;
     struct nested_printf npf;
 
+    local_irq_disable();
+
     if (atomic_xchg(&s_in_panic, true, MO_ACQ_REL))
         goto hang;
 

From 6f1cb749b76d4127242a683f9d5a9ed59c987793 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 22 Oct 2026 00:08:46 +0300
Subject: [PATCH 022/152] arch/x86_64: move the MSR numbers into
 msr_definitions.h

Assembly is about to need MSR numbers, but msr.h is C and its digit
separators do not assemble. Keep the numbers as plain defines in a
header of their own, which msr.h includes.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/include/arch/private/msr.h            | 9 ++-------
 .../arch/x86_64/include/arch/private/msr_definitions.h   | 8 ++++++++
 2 files changed, 10 insertions(+), 7 deletions(-)
 create mode 100644 kernel/arch/x86_64/include/arch/private/msr_definitions.h

diff --git a/kernel/arch/x86_64/include/arch/private/msr.h b/kernel/arch/x86_64/include/arch/private/msr.h
index 443d0cb8..10ad214b 100644
--- a/kernel/arch/x86_64/include/arch/private/msr.h
+++ b/kernel/arch/x86_64/include/arch/private/msr.h
@@ -1,16 +1,11 @@
 #pragma once
 
+#include <arch/private/msr_definitions.h>
+
 #include <common/types.h>
 #include <common/error.h>
 #include <common/bit.h>
 
-#define MSR_IA32_EFER 0xC0000080 // please use efer_feature_enable() instead
-#define MSR_FS_BASE 0xC000'0100
-#define MSR_GS_BASE 0xC000'0101
-#define MSR_KERNEL_GS_BASE 0xC000'0102
-#define MSR_IA32_APIC_BASE  0x0000'001B
-#define MSR_IA32_PAT 0x00000277
-
 // MSR_IA32_APIC_BASE
 #define IA32_APIC_BASE_ADDRESS_MASK MAKE_BIT_MASK(51, 12)
 #define IA32_APIC_BASE_IS_ENABLED BIT(11)
diff --git a/kernel/arch/x86_64/include/arch/private/msr_definitions.h b/kernel/arch/x86_64/include/arch/private/msr_definitions.h
new file mode 100644
index 00000000..be506e52
--- /dev/null
+++ b/kernel/arch/x86_64/include/arch/private/msr_definitions.h
@@ -0,0 +1,8 @@
+#pragma once
+
+#define MSR_IA32_EFER 0xC0000080 // please use efer_feature_enable() instead
+#define MSR_FS_BASE 0xC0000100
+#define MSR_GS_BASE 0xC0000101
+#define MSR_KERNEL_GS_BASE 0xC0000102
+#define MSR_IA32_APIC_BASE  0x0000001B
+#define MSR_IA32_PAT 0x00000277

From 38ae03a242e14cbbfd8603d864e49271da5550f8 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sat, 3 Oct 2026 21:49:49 +0300
Subject: [PATCH 023/152] arch/x86_64: make per-cpu usable as early as possible

Going forward, the contract with the arch-specific code is going to be
that upon calling the generic entry() per-cpu variables are already
online and usable on the BSP.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/entry.S | 7 +++++++
 kernel/arch/x86_64/entry.c | 4 ----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/kernel/arch/x86_64/entry.S b/kernel/arch/x86_64/entry.S
index c4559ef2..2e90c41c 100644
--- a/kernel/arch/x86_64/entry.S
+++ b/kernel/arch/x86_64/entry.S
@@ -1,4 +1,5 @@
 #include <arch/private/asm_helpers.h>
+#include <arch/private/msr_definitions.h>
 
 #include <unwind_hints.h>
 #include <linker.h>
@@ -11,6 +12,12 @@ ASM_PRELUDE
 
 ASM_GLOBAL_FUNCTION x86_entry_asm
     UNWIND_HINT_UNDEFINED(rip)
+
+    mov ecx, MSR_GS_BASE
+    xor eax, eax
+    xor edx, edx
+    wrmsr
+
     call x86_entry
 ASM_FUNCTION_END x86_entry_asm
 
diff --git a/kernel/arch/x86_64/entry.c b/kernel/arch/x86_64/entry.c
index 8c3fc8f5..8d2bb512 100644
--- a/kernel/arch/x86_64/entry.c
+++ b/kernel/arch/x86_64/entry.c
@@ -6,7 +6,6 @@
 #include <arch/private/idt.h>
 #include <arch/private/cpu.h>
 #include <arch/private/cr.h>
-#include <arch/private/msr.h>
 #include <arch/private/smp.h>
 #include <arch/private/apic.h>
 #include <arch/private/hypervisor.h>
@@ -35,9 +34,6 @@ static error_t INIT_CODE x86_early_init(void)
 
     cpu_info_setup(&g_cpu_info);
 
-    // Enable early per-cpu variables for the BSP
-    wrmsr_or_die(MSR_GS_BASE, 0);
-
     cr0_setup();
 
     pr_info(

From 556f4563f69a3daa2cc05323542903d5479ae360 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 12:26:54 +0300
Subject: [PATCH 024/152] tests: add spinlock tests

Cover locking, static initialization, try-lock and the interrupt state
around the irq-saving variants, which the tests can observe now that the
emulated interrupt state is shared.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 tests/test-cases/files.cmake     |  1 +
 tests/test-cases/test_spinlock.c | 86 ++++++++++++++++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 tests/test-cases/test_spinlock.c

diff --git a/tests/test-cases/files.cmake b/tests/test-cases/files.cmake
index fb49bd54..2808ec23 100644
--- a/tests/test-cases/files.cmake
+++ b/tests/test-cases/files.cmake
@@ -12,6 +12,7 @@ add_test_cases(
     test_rbtree.c
     test_cpu_mask.c
     test_cpu_mask_out_of_line.c
+    test_spinlock.c
     test_mutex.c
     test_vector_alloc.c
     test_irq.c
diff --git a/tests/test-cases/test_spinlock.c b/tests/test-cases/test_spinlock.c
new file mode 100644
index 00000000..5efc00f0
--- /dev/null
+++ b/tests/test-cases/test_spinlock.c
@@ -0,0 +1,86 @@
+#include <locking/spinlock.h>
+#include <irq_helpers.h>
+
+#include <test_harness.h>
+
+TEST_CASE(spinlock_lock_unlock)
+{
+    struct spinlock lock;
+
+    spin_lock_init(&lock);
+    ASSERT(!spin_is_locked(&lock));
+
+    spin_lock(&lock);
+    ASSERT(spin_is_locked(&lock));
+    ASSERT(!spin_try_lock(&lock));
+    spin_unlock(&lock);
+
+    ASSERT(!spin_is_locked(&lock));
+    ASSERT(spin_try_lock(&lock));
+    spin_unlock(&lock);
+}
+
+TEST_CASE(spinlock_static_init)
+{
+    DEFINE_SPINLOCK(static_lock);
+
+    ASSERT(spin_try_lock(&static_lock));
+    ASSERT(!spin_try_lock(&static_lock));
+    spin_unlock(&static_lock);
+}
+
+TEST_CASE(spinlock_irq_save_restores_enabled)
+{
+    struct spinlock lock;
+    irq_state_t state;
+
+    spin_lock_init(&lock);
+    ASSERT(local_irq_enabled());
+
+    state = spin_lock_irq_save(&lock);
+    ASSERT(!local_irq_enabled());
+    spin_unlock_irq_restore(&lock, state);
+
+    ASSERT(local_irq_enabled());
+}
+
+TEST_CASE(spinlock_irq_save_keeps_disabled)
+{
+    struct spinlock lock;
+    irq_state_t outer, inner;
+
+    spin_lock_init(&lock);
+
+    outer = local_irq_state_save();
+    ASSERT(!local_irq_enabled());
+
+    inner = spin_lock_irq_save(&lock);
+    ASSERT(!local_irq_enabled());
+    spin_unlock_irq_restore(&lock, inner);
+    ASSERT(!local_irq_enabled());
+
+    local_irq_state_restore(outer);
+    ASSERT(local_irq_enabled());
+}
+
+TEST_CASE(spinlock_try_lock_irq_save)
+{
+    struct spinlock lock;
+    irq_state_t held, state;
+
+    spin_lock_init(&lock);
+
+    ASSERT(spin_try_lock_irq_save(&lock, &state));
+    ASSERT(!local_irq_enabled());
+    spin_unlock_irq_restore(&lock, state);
+    ASSERT(local_irq_enabled());
+
+    held = spin_lock_irq_save(&lock);
+    spin_unlock(&lock);
+    local_irq_state_restore(held);
+
+    spin_lock(&lock);
+    ASSERT(!spin_try_lock_irq_save(&lock, &state));
+    ASSERT(local_irq_enabled());
+    spin_unlock(&lock);
+}

From bcaeae75e9c869829b091374e5b01c0313923c07 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sat, 3 Oct 2026 22:10:20 +0300
Subject: [PATCH 025/152] arch/x86_64: make this_cpu_add() and this_cpu_sub()
 interrupt safe

They were compound assignments on a gs-qualified lvalue, which only
becomes a single instruction with optimizations on. A debug build loads,
modifies and stores, so an interrupt updating the same variable in
between loses its update.

Spell them out as one instruction. The gs prefix is written explicitly
since clang drops it for a gs-qualified memory operand.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .../arch/x86_64/include/arch/detail/per_cpu.h | 29 ++++++++++++++-----
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/kernel/arch/x86_64/include/arch/detail/per_cpu.h b/kernel/arch/x86_64/include/arch/detail/per_cpu.h
index f1f8fb5d..fddf5cd9 100644
--- a/kernel/arch/x86_64/include/arch/detail/per_cpu.h
+++ b/kernel/arch/x86_64/include/arch/detail/per_cpu.h
@@ -19,12 +19,25 @@
 #define this_cpu_write_32(val, x) (X86_APPLY_SEG_GS(val) = (x))
 #define this_cpu_write_64(val, x) (X86_APPLY_SEG_GS(val) = (x))
 
-#define this_cpu_add_8(val, x) (X86_APPLY_SEG_GS(val) += (x))
-#define this_cpu_add_16(val, x) (X86_APPLY_SEG_GS(val) += (x))
-#define this_cpu_add_32(val, x) (X86_APPLY_SEG_GS(val) += (x))
-#define this_cpu_add_64(val, x) (X86_APPLY_SEG_GS(val) += (x))
+/*
+ * This uses inline assembly instead of raw C with __seg_gs because debug
+ * builds at O0 make some of these operations RMW and thus not compatible
+ * with this_cpu_* semantics, which are assumed to be atomic.
+ */
+#define X86_THIS_CPU_MODIFY(insn, constraint, type, val, x) \
+    asm volatile(                                           \
+        insn " %[value], %%gs:%[var]"                       \
+        : [var] "+m" (val)                                  \
+        : [value] constraint ((type)(x))                    \
+        : "cc"                                              \
+    )
+
+#define this_cpu_add_8(val, x) X86_THIS_CPU_MODIFY("addb", "qi", u8, val, x)
+#define this_cpu_add_16(val, x) X86_THIS_CPU_MODIFY("addw", "ri", u16, val, x)
+#define this_cpu_add_32(val, x) X86_THIS_CPU_MODIFY("addl", "ri", u32, val, x)
+#define this_cpu_add_64(val, x) X86_THIS_CPU_MODIFY("addq", "re", u64, val, x)
 
-#define this_cpu_sub_8(val, x) (X86_APPLY_SEG_GS(val) -= (x))
-#define this_cpu_sub_16(val, x) (X86_APPLY_SEG_GS(val) -= (x))
-#define this_cpu_sub_32(val, x) (X86_APPLY_SEG_GS(val) -= (x))
-#define this_cpu_sub_64(val, x) (X86_APPLY_SEG_GS(val) -= (x))
+#define this_cpu_sub_8(val, x) X86_THIS_CPU_MODIFY("subb", "qi", u8, val, x)
+#define this_cpu_sub_16(val, x) X86_THIS_CPU_MODIFY("subw", "ri", u16, val, x)
+#define this_cpu_sub_32(val, x) X86_THIS_CPU_MODIFY("subl", "ri", u32, val, x)
+#define this_cpu_sub_64(val, x) X86_THIS_CPU_MODIFY("subq", "re", u64, val, x)

From 2bbf9e7fdb96fd8f8c253fc62af4274edf24a661 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 13:39:21 +0300
Subject: [PATCH 026/152] smp: add for_each_present_cpu()

A convenience macro to walk all CPUs.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/smp.c      | 5 +++--
 kernel/include/smp.h          | 3 +++
 kernel/per_cpu.c              | 3 ++-
 tests/synthetic-include/smp.h | 3 +++
 4 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/kernel/arch/x86_64/smp.c b/kernel/arch/x86_64/smp.c
index 89f039e7..2d4c784d 100644
--- a/kernel/arch/x86_64/smp.c
+++ b/kernel/arch/x86_64/smp.c
@@ -14,6 +14,7 @@
 #include <bug.h>
 #include <panic.h>
 #include <per_cpu.h>
+#include <smp.h>
 #include <free_after_init.h>
 #include <init_level.h>
 
@@ -78,7 +79,7 @@ static INIT_CODE void register_apic_id(u32 apic_id)
                  apic_id == g_bsp_apic_id))
         return;
 
-    for (i = 0; i < g_num_present_cpus; ++i) {
+    for_each_present_cpu(i) {
         // APIC has been registered before
         if (s_early_cpu_to_apic_id[i] == apic_id)
             return;
@@ -375,7 +376,7 @@ static error_t INIT_CODE x86_per_cpu_setup(void)
     // Do the boot CPU setup here, AP CPUs call it elsewhere
     this_cpu_enable_per_cpu(0);
 
-    for (i = 0; i < g_num_present_cpus; ++i) {
+    for_each_present_cpu(i) {
         per_cpu(g_this_cpu_id, i) = i;
         per_cpu(g_this_cpu_apic_id, i) = s_early_cpu_to_apic_id[i];
     }
diff --git a/kernel/include/smp.h b/kernel/include/smp.h
index e2ee979c..5b04932b 100644
--- a/kernel/include/smp.h
+++ b/kernel/include/smp.h
@@ -10,3 +10,6 @@ extern u32 g_num_online_cpus;
 extern struct cpu_mask g_online_cpus;
 
 #define for_each_online_cpu(cpu) for_each_cpu_in(cpu, &g_online_cpus)
+
+#define for_each_present_cpu(cpu)                        \
+    for ((cpu) = 0; (cpu) < g_num_present_cpus; (cpu)++)
diff --git a/kernel/per_cpu.c b/kernel/per_cpu.c
index cdb3dfde..1dc84a48 100644
--- a/kernel/per_cpu.c
+++ b/kernel/per_cpu.c
@@ -4,6 +4,7 @@
 
 #include <linker.h>
 #include <per_cpu.h>
+#include <smp.h>
 #include <log.h>
 #include <bug.h>
 #include <panic.h>
@@ -57,7 +58,7 @@ static error_t INIT_CODE per_cpu_setup(void)
     this_cpu_offset = g_per_cpu_base;
     this_cpu_offset -= (ptr_t)SECTION_MARKER_BEGIN(PER_CPU_SECTION);
 
-    for (i = 0; i < g_num_present_cpus; i++) {
+    for_each_present_cpu(i) {
         g_per_cpu_offset[i] = this_cpu_offset;
 
         // Give every CPU its own copy of the static per-cpu section
diff --git a/tests/synthetic-include/smp.h b/tests/synthetic-include/smp.h
index 8c221960..4ce2a258 100644
--- a/tests/synthetic-include/smp.h
+++ b/tests/synthetic-include/smp.h
@@ -6,3 +6,6 @@ extern u32 g_num_present_cpus;
 extern struct cpu_mask g_online_cpus;
 
 #define for_each_online_cpu(cpu) for_each_cpu_in(cpu, &g_online_cpus)
+
+#define for_each_present_cpu(cpu)                        \
+    for ((cpu) = 0; (cpu) < g_num_present_cpus; (cpu)++)

From 3375763f5a10a07b085303c8c6eeecaf0b0b2ce8 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 13:39:21 +0300
Subject: [PATCH 027/152] common/bit: add BIT_FIND_FIRST_SET()

BIT_UNIT_FIND_FIRST_SET() takes a long, so BIT_FIELD_SHIFT() would
truncate a u64 mask where long is 32-bit. Replace it with a helper for a
value of any integer type, which picks the builtin by the size of the
value.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/common/bit.c         |  2 +-
 kernel/include/common/bit.h | 10 ++++++++--
 tests/test-cases/test_bit.c | 13 +++++++++++++
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/kernel/common/bit.c b/kernel/common/bit.c
index 521bb5cb..5482ea4e 100644
--- a/kernel/common/bit.c
+++ b/kernel/common/bit.c
@@ -38,7 +38,7 @@ reg_t find_next_bit_base(
 
         // If there are any valid 1s left, find the lowest one
         if (unit != 0) {
-            bit = BIT_UNIT_FIND_FIRST_SET(unit) - 1;
+            bit = BIT_FIND_FIRST_SET(unit) - 1;
             return (i * BITS_PER_UNIT) + bit;
         }
     }
diff --git a/kernel/include/common/bit.h b/kernel/include/common/bit.h
index 2eaa82f0..2b7587bd 100644
--- a/kernel/include/common/bit.h
+++ b/kernel/include/common/bit.h
@@ -43,8 +43,14 @@
 #define MAKE_BIT_MASK(end_bit, start_bit) \
     MAKE_BIT_MASK_OF_TYPE(end_bit, start_bit, reg_t)
 
-#define BIT_UNIT_FIND_FIRST_SET(unit) __builtin_ffsl(unit)
-#define BIT_FIELD_SHIFT(mask) (BIT_UNIT_FIND_FIRST_SET(mask) - 1)
+// 1-based index of the lowest set bit of 'x', 0 if no bit is set
+#define BIT_FIND_FIRST_SET(x) (         \
+    sizeof(x) <= sizeof(unsigned int) ? \
+        __builtin_ffs((int)(x)) :       \
+        __builtin_ffsll((long long)(x)) \
+)
+
+#define BIT_FIELD_SHIFT(mask) (BIT_FIND_FIRST_SET(mask) - 1)
 
 #define BIT_FIELD_VALIDATE_INPUT(mask, value)                                    \
     EMBED_STATIC_ASSERT(IS_UNSIGNED_TYPE(mask), "field mask must be unsigned") + \
diff --git a/tests/test-cases/test_bit.c b/tests/test-cases/test_bit.c
index 2b5047e4..e4ebe050 100644
--- a/tests/test-cases/test_bit.c
+++ b/tests/test-cases/test_bit.c
@@ -34,6 +34,19 @@ TEST_CASE(bit_mask_helpers)
     ASSERT_EQ(MAKE_BIT_MASK_U64(63, 32), 0xFFFFFFFF00000000ull);
 }
 
+TEST_CASE(bit_find_first_set_of_any_width)
+{
+    ASSERT_EQ(BIT_FIND_FIRST_SET((u8)0), 0);
+    ASSERT_EQ(BIT_FIND_FIRST_SET(BIT_U8(7)), 8);
+    ASSERT_EQ(BIT_FIND_FIRST_SET(BIT_U16(9) | BIT_U16(15)), 10);
+    ASSERT_EQ(BIT_FIND_FIRST_SET(BIT_U32(31)), 32);
+    ASSERT_EQ(BIT_FIND_FIRST_SET(MAKE_BIT_MASK_U32(31, 4)), 5);
+    ASSERT_EQ(BIT_FIND_FIRST_SET((u64)0), 0);
+    ASSERT_EQ(BIT_FIND_FIRST_SET(BIT_U64(40) | BIT_U64(63)), 41);
+    ASSERT_EQ(BIT_FIND_FIRST_SET(BIT_U64(63)), 64);
+    ASSERT_EQ(BIT_FIND_FIRST_SET(-8), 4);
+}
+
 TEST_CASE(bit_field_shift_make_read_write_max_fits)
 {
     u32 data;

From f5c3ad7f18f87dc74755a72cb50ee8e1673dfdc2 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 14:59:13 +0300
Subject: [PATCH 028/152] types: add cpu_t

CPU numbers were plain u32 next to APIC IDs of the same type, and loops
over the CPUs used u32 and size_t interchangeably. Give logical CPU
numbers and counts of CPUs a type of their own, printed with %u.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .../x86_64/include/arch/private/vector_alloc.h     |  4 ++--
 kernel/arch/x86_64/include/arch/smp.h              |  4 +++-
 kernel/arch/x86_64/lapic_domain.c                  |  4 ++--
 kernel/arch/x86_64/smp.c                           |  8 ++++----
 kernel/arch/x86_64/vector_alloc.c                  |  6 +++---
 kernel/include/common/types.h                      |  3 +++
 kernel/include/cpu_mask.h                          | 14 +++++++-------
 kernel/include/smp.h                               |  4 ++--
 kernel/per_cpu.c                                   |  3 ++-
 kernel/smp.c                                       |  4 ++--
 scripts/lint.py                                    |  2 ++
 tests/synthetic-include/smp.h                      |  2 +-
 tests/test-cases/test_cpu_mask.c                   |  3 ++-
 tests/test-cases/test_cpu_mask_out_of_line.c       |  5 +++--
 tests/test-cases/test_vector_alloc.c               | 12 +++++++-----
 15 files changed, 45 insertions(+), 33 deletions(-)

diff --git a/kernel/arch/x86_64/include/arch/private/vector_alloc.h b/kernel/arch/x86_64/include/arch/private/vector_alloc.h
index 7add9876..b116dc6e 100644
--- a/kernel/arch/x86_64/include/arch/private/vector_alloc.h
+++ b/kernel/arch/x86_64/include/arch/private/vector_alloc.h
@@ -8,7 +8,7 @@
  * by 'allowed' (NULL means any online CPU).
  */
 error_t vector_alloc(
-    const struct cpu_mask *allowed, u32 *out_cpu, u8 *out_vector
+    const struct cpu_mask *allowed, cpu_t *out_cpu, u8 *out_vector
 );
 
-void vector_free(u32 cpu, u8 vector);
+void vector_free(cpu_t cpu, u8 vector);
diff --git a/kernel/arch/x86_64/include/arch/smp.h b/kernel/arch/x86_64/include/arch/smp.h
index a4ff36af..76a6509a 100644
--- a/kernel/arch/x86_64/include/arch/smp.h
+++ b/kernel/arch/x86_64/include/arch/smp.h
@@ -1,8 +1,10 @@
 #pragma once
 
+#include <common/helpers.h>
 #include <per_cpu_decls.h>
 #include <arch/detail/per_cpu.h>
 
-DECLARE_PER_CPU(u32, g_this_cpu_id);
+DECLARE_PER_CPU(cpu_t, g_this_cpu_id);
 
+BUILD_BUG_ON(sizeof(cpu_t) != sizeof(u32));
 #define unstable_cpu_id() this_cpu_read_32(g_this_cpu_id)
diff --git a/kernel/arch/x86_64/lapic_domain.c b/kernel/arch/x86_64/lapic_domain.c
index 4901389c..a949873c 100644
--- a/kernel/arch/x86_64/lapic_domain.c
+++ b/kernel/arch/x86_64/lapic_domain.c
@@ -21,7 +21,7 @@
 static DEFINE_PER_CPU(struct irq*, s_dynamic_vectors[NUM_DYNAMIC_VECTORS]);
 static DEFINE_PER_CPU(u64, s_num_unexpected);
 
-static struct irq **dynamic_vector_slot(u32 cpu, u8 vector)
+static struct irq **dynamic_vector_slot(cpu_t cpu, u8 vector)
 {
     BUG_ON(vector < VECTOR_DYNAMIC_FIRST || vector > VECTOR_DYNAMIC_LAST);
     return &per_cpu(s_dynamic_vectors[vector - VECTOR_DYNAMIC_FIRST], cpu);
@@ -59,7 +59,7 @@ IRQ_HANDLER {
 
 // The domain's private bookkeeping for one allocated interrupt
 struct lapic_route {
-    u32 cpu;
+    cpu_t cpu;
     u32 dest_apic_id;
     u8 vector;
 };
diff --git a/kernel/arch/x86_64/smp.c b/kernel/arch/x86_64/smp.c
index 2d4c784d..f1cf8d6b 100644
--- a/kernel/arch/x86_64/smp.c
+++ b/kernel/arch/x86_64/smp.c
@@ -24,7 +24,7 @@
 #define LAPIC_ID_NONE 0xFF
 #define X2APIC_ID_NONE 0xFFFF'FFFF
 
-DEFINE_PER_CPU(u32, g_this_cpu_id);
+DEFINE_PER_CPU(cpu_t, g_this_cpu_id);
 DEFINE_PER_CPU(u32, g_this_cpu_apic_id);
 
 static INIT_DATA u32 s_early_cpu_to_apic_id[ULTRA_MAX_CPUS] = {
@@ -72,7 +72,7 @@ static INIT_CODE uacpi_iteration_decision check_if_madt_has_lapic(
 
 static INIT_CODE void register_apic_id(u32 apic_id)
 {
-    u32 i;
+    cpu_t i;
 
     // Don't allow registering the BSP if it's not also the boot CPU
     if (unlikely(g_bsp_apic_id != g_boot_cpu_apic_id &&
@@ -358,7 +358,7 @@ void INIT_CODE setup_smp_topology(void)
     uacpi_table_unref(&tbl);
 }
 
-static void this_cpu_enable_per_cpu(u32 my_id)
+static void this_cpu_enable_per_cpu(cpu_t my_id)
 {
     struct descriptor_ptr gdt_ptr = {
         .limit = sizeof(g_this_cpu_gdt) - 1,
@@ -371,7 +371,7 @@ static void this_cpu_enable_per_cpu(u32 my_id)
 
 static error_t INIT_CODE x86_per_cpu_setup(void)
 {
-    size_t i;
+    cpu_t i;
 
     // Do the boot CPU setup here, AP CPUs call it elsewhere
     this_cpu_enable_per_cpu(0);
diff --git a/kernel/arch/x86_64/vector_alloc.c b/kernel/arch/x86_64/vector_alloc.c
index 20f4b7ba..4c1def57 100644
--- a/kernel/arch/x86_64/vector_alloc.c
+++ b/kernel/arch/x86_64/vector_alloc.c
@@ -15,11 +15,11 @@ static DEFINE_PER_CPU(struct vector_pool, s_pool);
 static struct spinlock s_lock;
 
 error_t vector_alloc(
-    const struct cpu_mask *allowed, u32 *out_cpu, u8 *out_vector
+    const struct cpu_mask *allowed, cpu_t *out_cpu, u8 *out_vector
 )
 {
     struct vector_pool *pool, *best_pool = nullptr;
-    u32 cpu, best_cpu = 0;
+    cpu_t cpu, best_cpu = 0;
     reg_t bit;
 
     spin_lock(&s_lock);
@@ -59,7 +59,7 @@ error_t vector_alloc(
     return EOK;
 }
 
-void vector_free(u32 cpu, u8 vector)
+void vector_free(cpu_t cpu, u8 vector)
 {
     struct vector_pool *pool;
     reg_t bit;
diff --git a/kernel/include/common/types.h b/kernel/include/common/types.h
index a232399a..acb9d63b 100644
--- a/kernel/include/common/types.h
+++ b/kernel/include/common/types.h
@@ -48,6 +48,9 @@ typedef u64 phys_addr_t;
 typedef ptr_t virt_addr_t;
 typedef size_t pfn_t;
 
+// A logical CPU number, or a number of CPUs
+typedef u32 cpu_t;
+
 #define LITERAL_SUFFIX_i32
 #define LITERAL_SUFFIX_i64 ll
 #define LITERAL_SUFFIX_u32 u
diff --git a/kernel/include/cpu_mask.h b/kernel/include/cpu_mask.h
index 4c212885..3d775e57 100644
--- a/kernel/include/cpu_mask.h
+++ b/kernel/include/cpu_mask.h
@@ -15,7 +15,7 @@ struct cpu_mask {
  * operation may ever touch bits past it.
  */
 #if IS_ENABLED(CPU_MASK_OUT_OF_LINE)
-extern u32 g_num_present_cpus;
+extern cpu_t g_num_present_cpus;
 #define CPU_MASK_NUM_BITS g_num_present_cpus
 #else
 #define CPU_MASK_NUM_BITS ULTRA_MAX_CPUS
@@ -24,17 +24,17 @@ extern u32 g_num_present_cpus;
 #define CPU_MASK_NUM_BYTES \
     (NUM_UNITS_FOR_BITS(CPU_MASK_NUM_BITS) * sizeof(reg_t))
 
-static inline void cpu_mask_set(struct cpu_mask *mask, u32 cpu)
+static inline void cpu_mask_set(struct cpu_mask *mask, cpu_t cpu)
 {
     bit_set(mask->bits, cpu);
 }
 
-static inline void cpu_mask_clear(struct cpu_mask *mask, u32 cpu)
+static inline void cpu_mask_clear(struct cpu_mask *mask, cpu_t cpu)
 {
     bit_clear(mask->bits, cpu);
 }
 
-static inline bool cpu_mask_test(const struct cpu_mask *mask, u32 cpu)
+static inline bool cpu_mask_test(const struct cpu_mask *mask, cpu_t cpu)
 {
     return bit_test(mask->bits, cpu);
 }
@@ -59,19 +59,19 @@ static inline void cpu_mask_and(
     bit_array_and(dst->bits, lhs->bits, rhs->bits, CPU_MASK_NUM_BITS);
 }
 
-static inline u32 cpu_mask_weight(const struct cpu_mask *mask)
+static inline cpu_t cpu_mask_weight(const struct cpu_mask *mask)
 {
     return bit_array_count_set(mask->bits, CPU_MASK_NUM_BITS);
 }
 
 // Returns CPU_MASK_NUM_BITS if no set bits are present
-static inline u32 cpu_mask_first(const struct cpu_mask *mask)
+static inline cpu_t cpu_mask_first(const struct cpu_mask *mask)
 {
     return find_first_set_bit(mask->bits, CPU_MASK_NUM_BITS);
 }
 
 // Returns CPU_MASK_NUM_BITS if no set bits are present after 'prev'
-static inline u32 cpu_mask_next(const struct cpu_mask *mask, u32 prev)
+static inline cpu_t cpu_mask_next(const struct cpu_mask *mask, cpu_t prev)
 {
     return find_next_set_bit(mask->bits, CPU_MASK_NUM_BITS, prev + 1);
 }
diff --git a/kernel/include/smp.h b/kernel/include/smp.h
index 5b04932b..c4c8b873 100644
--- a/kernel/include/smp.h
+++ b/kernel/include/smp.h
@@ -4,8 +4,8 @@
 
 #include <cpu_mask.h>
 
-extern u32 g_num_present_cpus;
-extern u32 g_num_online_cpus;
+extern cpu_t g_num_present_cpus;
+extern cpu_t g_num_online_cpus;
 
 extern struct cpu_mask g_online_cpus;
 
diff --git a/kernel/per_cpu.c b/kernel/per_cpu.c
index 1dc84a48..a203a6d8 100644
--- a/kernel/per_cpu.c
+++ b/kernel/per_cpu.c
@@ -27,7 +27,8 @@ extern u8 SECTION_MARKER_END(PER_CPU_SECTION)[];
 
 static error_t INIT_CODE per_cpu_setup(void)
 {
-    size_t static_size, per_cpu_size, alloc_size, i;
+    size_t static_size, per_cpu_size, alloc_size;
+    cpu_t i;
     ptr_t this_cpu_offset;
     void *this_cpu_ptr;
     phys_addr_t addr;
diff --git a/kernel/smp.c b/kernel/smp.c
index ff43ccd7..3c9b330d 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -1,6 +1,6 @@
 #include <smp.h>
 
-u32 g_num_present_cpus = 0;
-u32 g_num_online_cpus = 0;
+cpu_t g_num_present_cpus = 0;
+cpu_t g_num_online_cpus = 0;
 
 struct cpu_mask g_online_cpus;
diff --git a/scripts/lint.py b/scripts/lint.py
index 2ffc5a7e..61282d6a 100755
--- a/scripts/lint.py
+++ b/scripts/lint.py
@@ -345,6 +345,7 @@ class Rule(NamedTuple):
 
 INT_RULE = Rule(SHORT_LENGTHS, SIGNED + "c", "%d")
 UINT_RULE = Rule(SHORT_LENGTHS, UNSIGNED, "%u or %X")
+CPU_RULE = Rule(SHORT_LENGTHS, "u", "%u")
 CHAR_RULE = Rule(("",), "cd", "%c or %d")
 U64_RULE = Rule(("ll",), UNSIGNED, "%llu or %llX")
 I64_RULE = Rule(("ll",), SIGNED, "%lld")
@@ -361,6 +362,7 @@ class Rule(NamedTuple):
     "ptr_t": SIZE_RULE,
     "reg_t": SIZE_RULE,
     "virt_addr_t": SIZE_RULE,
+    "cpu_t": CPU_RULE,
     "ssize_t": SSIZE_RULE,
     "u64": U64_RULE,
     "i64": I64_RULE,
diff --git a/tests/synthetic-include/smp.h b/tests/synthetic-include/smp.h
index 4ce2a258..ff1f7607 100644
--- a/tests/synthetic-include/smp.h
+++ b/tests/synthetic-include/smp.h
@@ -2,7 +2,7 @@
 
 #include <cpu_mask.h>
 
-extern u32 g_num_present_cpus;
+extern cpu_t g_num_present_cpus;
 extern struct cpu_mask g_online_cpus;
 
 #define for_each_online_cpu(cpu) for_each_cpu_in(cpu, &g_online_cpus)
diff --git a/tests/test-cases/test_cpu_mask.c b/tests/test-cases/test_cpu_mask.c
index a7a2002c..a762d39f 100644
--- a/tests/test-cases/test_cpu_mask.c
+++ b/tests/test-cases/test_cpu_mask.c
@@ -34,7 +34,8 @@ TEST_CASE(cpu_mask_iteration_order)
 {
     struct cpu_mask mask = { };
     u32 expected[] = { 2, 63, 64, 100, ULTRA_MAX_CPUS - 1 };
-    u32 cpu, i = 0;
+    cpu_t cpu;
+    u32 i = 0;
 
     ASSERT_EQ(cpu_mask_first(&mask), ULTRA_MAX_CPUS);
 
diff --git a/tests/test-cases/test_cpu_mask_out_of_line.c b/tests/test-cases/test_cpu_mask_out_of_line.c
index acb30641..fd5505ac 100644
--- a/tests/test-cases/test_cpu_mask_out_of_line.c
+++ b/tests/test-cases/test_cpu_mask_out_of_line.c
@@ -17,7 +17,7 @@
 
 #include <test_harness.h>
 
-u32 g_num_present_cpus;
+cpu_t g_num_present_cpus;
 
 static size_t s_last_alloc_size;
 
@@ -62,7 +62,8 @@ TEST_CASE(cpu_mask_out_of_line_ops_are_bounded)
 {
     struct cpu_mask full = { };
     cpu_mask_handle handle;
-    u32 cpu, count = 0;
+    cpu_t cpu;
+    u32 count = 0;
 
     g_num_present_cpus = 70;
 
diff --git a/tests/test-cases/test_vector_alloc.c b/tests/test-cases/test_vector_alloc.c
index f16606f1..6deee68b 100644
--- a/tests/test-cases/test_vector_alloc.c
+++ b/tests/test-cases/test_vector_alloc.c
@@ -12,7 +12,7 @@ static struct vector_pool s_fake_pools[ULTRA_MAX_CPUS];
 
 static void reset_state(void)
 {
-    u32 i;
+    cpu_t i;
 
     memset(s_fake_pools, 0, sizeof(s_fake_pools));
     for (i = 0; i < ULTRA_MAX_CPUS; i++)
@@ -24,7 +24,8 @@ static void reset_state(void)
 
 TEST_CASE(vector_alloc_sequential_and_exhaustion)
 {
-    u32 cpu, i;
+    cpu_t cpu;
+    u32 i;
     u8 vector, first_vector;
 
     reset_state();
@@ -51,7 +52,7 @@ TEST_CASE(vector_alloc_sequential_and_exhaustion)
 
 TEST_CASE(vector_alloc_least_busy)
 {
-    u32 cpu;
+    cpu_t cpu;
     u8 vector;
 
     reset_state();
@@ -80,7 +81,8 @@ TEST_CASE(vector_alloc_least_busy)
 TEST_CASE(vector_alloc_mask_filter_is_strict)
 {
     struct cpu_mask allowed = { };
-    u32 cpu, i;
+    cpu_t cpu;
+    u32 i;
     u8 vector;
 
     reset_state();
@@ -107,7 +109,7 @@ TEST_CASE(vector_alloc_mask_filter_is_strict)
 TEST_CASE(vector_alloc_offline_cpus_skipped)
 {
     struct cpu_mask allowed = { };
-    u32 cpu;
+    cpu_t cpu;
     u8 vector;
 
     reset_state();

From dc9538d67d3ee1a56bc21e673df6f8dc5e3be529 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 14:59:14 +0300
Subject: [PATCH 029/152] arch/x86_64: add apic_id_t

An APIC ID addresses a local APIC and is not a CPU number, yet both
were u32 and sit side by side where one is mapped to the other. Give
APIC IDs a type of their own.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/apic.c                      | 10 +++++-----
 kernel/arch/x86_64/include/arch/private/apic.h | 15 +++++++++------
 kernel/arch/x86_64/include/arch/private/smp.h  |  3 ++-
 kernel/arch/x86_64/lapic_domain.c              |  2 +-
 kernel/arch/x86_64/smp.c                       | 10 +++++-----
 kernel/arch/x86_64/x2apic.c                    |  2 +-
 kernel/arch/x86_64/xapic.c                     |  2 +-
 7 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/kernel/arch/x86_64/apic.c b/kernel/arch/x86_64/apic.c
index cc508a6d..239d0dd4 100644
--- a/kernel/arch/x86_64/apic.c
+++ b/kernel/arch/x86_64/apic.c
@@ -19,9 +19,9 @@ static UNUSED_DECL u64 s_freq_hz;
 struct apic *g_apic = nullptr;
 enum apic_mode g_apic_mode = APIC_MODE_NONE;
 
-u32 g_bsp_apic_id = APIC_ID_NONE;
-u32 g_boot_cpu_apic_id = APIC_ID_NONE;
-u32 g_max_apic_id = 0xFF - 1;
+apic_id_t g_bsp_apic_id = APIC_ID_NONE;
+apic_id_t g_boot_cpu_apic_id = APIC_ID_NONE;
+apic_id_t g_max_apic_id = 0xFF - 1;
 
 extern struct apic g_xapic;
 extern struct apic g_x2apic;
@@ -78,12 +78,12 @@ void apic_eoi(void)
     g_apic->write(APIC_REG_EOI, 0);
 }
 
-void apic_icr_write(u32 value, u32 dest_apic_id)
+void apic_icr_write(u32 value, apic_id_t dest_apic_id)
 {
     g_apic->icr_write(value, dest_apic_id);
 }
 
-void apic_send_fixed_ipi(u32 dest_apic_id, u8 vector)
+void apic_send_fixed_ipi(apic_id_t dest_apic_id, u8 vector)
 {
     apic_icr_write(APIC_ICR_DELIVERY_FIXED | vector, dest_apic_id);
 }
diff --git a/kernel/arch/x86_64/include/arch/private/apic.h b/kernel/arch/x86_64/include/arch/private/apic.h
index 5c84096d..78cb5a57 100644
--- a/kernel/arch/x86_64/include/arch/private/apic.h
+++ b/kernel/arch/x86_64/include/arch/private/apic.h
@@ -3,6 +3,9 @@
 #include <common/types.h>
 #include <common/bit.h>
 
+// The hardware ID of a local APIC
+typedef u32 apic_id_t;
+
 enum apic_reg : u32 {
     APIC_REG_ID = 0x20,
     APIC_REG_VERSION = 0x30,
@@ -70,16 +73,16 @@ enum apic_mode : u32 {
 extern struct apic *g_apic;
 extern enum apic_mode g_apic_mode;
 
-extern u32 g_bsp_apic_id;
-extern u32 g_boot_cpu_apic_id;
-extern u32 g_max_apic_id;
+extern apic_id_t g_bsp_apic_id;
+extern apic_id_t g_boot_cpu_apic_id;
+extern apic_id_t g_max_apic_id;
 
 struct apic {
     void (*setup)(void);
     u32 (*read)(enum apic_reg);
     void (*write)(enum apic_reg, u32 value);
 
-    void (*icr_write)(u32 value, u32 dest_apic_id);
+    void (*icr_write)(u32 value, apic_id_t dest_apic_id);
     void (*send_ipi_self)(u8 vector);
 };
 
@@ -92,8 +95,8 @@ void apic_eoi(void);
 bool apic_vector_in_isr(u8 vector);
 bool apic_vector_in_tmr(u8 vector);
 
-void apic_icr_write(u32 value, u32 dest_apic_id);
-void apic_send_fixed_ipi(u32 dest_apic_id, u8 vector);
+void apic_icr_write(u32 value, apic_id_t dest_apic_id);
+void apic_send_fixed_ipi(apic_id_t dest_apic_id, u8 vector);
 void apic_send_ipi_self(u8 vector);
 
 void apic_set_known_frequency(u64 hz);
diff --git a/kernel/arch/x86_64/include/arch/private/smp.h b/kernel/arch/x86_64/include/arch/private/smp.h
index ad2c1483..7b2f4099 100644
--- a/kernel/arch/x86_64/include/arch/private/smp.h
+++ b/kernel/arch/x86_64/include/arch/private/smp.h
@@ -1,7 +1,8 @@
 #pragma once
 
+#include <arch/private/apic.h>
 #include <per_cpu.h>
 
-DECLARE_PER_CPU(u32, g_this_cpu_apic_id);
+DECLARE_PER_CPU(apic_id_t, g_this_cpu_apic_id);
 
 void setup_smp_topology(void);
diff --git a/kernel/arch/x86_64/lapic_domain.c b/kernel/arch/x86_64/lapic_domain.c
index a949873c..932d629a 100644
--- a/kernel/arch/x86_64/lapic_domain.c
+++ b/kernel/arch/x86_64/lapic_domain.c
@@ -60,7 +60,7 @@ IRQ_HANDLER {
 // The domain's private bookkeeping for one allocated interrupt
 struct lapic_route {
     cpu_t cpu;
-    u32 dest_apic_id;
+    apic_id_t dest_apic_id;
     u8 vector;
 };
 
diff --git a/kernel/arch/x86_64/smp.c b/kernel/arch/x86_64/smp.c
index f1cf8d6b..1dfbdcbc 100644
--- a/kernel/arch/x86_64/smp.c
+++ b/kernel/arch/x86_64/smp.c
@@ -25,9 +25,9 @@
 #define X2APIC_ID_NONE 0xFFFF'FFFF
 
 DEFINE_PER_CPU(cpu_t, g_this_cpu_id);
-DEFINE_PER_CPU(u32, g_this_cpu_apic_id);
+DEFINE_PER_CPU(apic_id_t, g_this_cpu_apic_id);
 
-static INIT_DATA u32 s_early_cpu_to_apic_id[ULTRA_MAX_CPUS] = {
+static INIT_DATA apic_id_t s_early_cpu_to_apic_id[ULTRA_MAX_CPUS] = {
     [0 ... (ULTRA_MAX_CPUS - 1)] = APIC_ID_NONE,
 };
 
@@ -70,7 +70,7 @@ static INIT_CODE uacpi_iteration_decision check_if_madt_has_lapic(
     return UACPI_ITERATION_DECISION_CONTINUE;
 }
 
-static INIT_CODE void register_apic_id(u32 apic_id)
+static INIT_CODE void register_apic_id(apic_id_t apic_id)
 {
     cpu_t i;
 
@@ -111,7 +111,7 @@ static INIT_CODE void register_apic_id(u32 apic_id)
     g_num_present_cpus++;
 }
 
-static INIT_CODE error_t register_cpu(u32 apic_id)
+static INIT_CODE error_t register_cpu(apic_id_t apic_id)
 {
     /*
      * This is the first CPU to be registered, we expect that it's the BSP since
@@ -193,7 +193,7 @@ static INIT_CODE uacpi_iteration_decision detect_lapics(
 )
 {
     struct madt_iter_ctx *ctx = user;
-    u32 apic_id;
+    apic_id_t apic_id;
     u32 flags;
 
     switch (hdr->type) {
diff --git a/kernel/arch/x86_64/x2apic.c b/kernel/arch/x86_64/x2apic.c
index 2278640a..e25ba775 100644
--- a/kernel/arch/x86_64/x2apic.c
+++ b/kernel/arch/x86_64/x2apic.c
@@ -29,7 +29,7 @@ static void x2apic_write(enum apic_reg reg, u32 data)
 }
 
 // A single 64-bit write, no busy bit or separate destination half
-static void x2apic_icr_write(u32 value, u32 dest_apic_id)
+static void x2apic_icr_write(u32 value, apic_id_t dest_apic_id)
 {
     u64 icr;
 
diff --git a/kernel/arch/x86_64/xapic.c b/kernel/arch/x86_64/xapic.c
index 92fb2432..911a6b39 100644
--- a/kernel/arch/x86_64/xapic.c
+++ b/kernel/arch/x86_64/xapic.c
@@ -35,7 +35,7 @@ static void xapic_wait_icr_idle(void)
         arch_cpu_relax();
 }
 
-static void xapic_icr_write(u32 value, u32 dest_apic_id)
+static void xapic_icr_write(u32 value, apic_id_t dest_apic_id)
 {
     irq_state_t irq_state;
 

From 6fdd212f0062c69bdcc55a3e9f65c9a04b4601ce Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 15:28:11 +0300
Subject: [PATCH 030/152] arch/x86_64: initialize the x2APIC ops in declaration
 order

g_xapic follows the layout of struct apic, g_x2apic put setup after the
register accessors. Give setup its own group in the declaration, the
way g_xapic already spaces it, and follow it in both.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/include/arch/private/apic.h | 1 +
 kernel/arch/x86_64/x2apic.c                    | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/kernel/arch/x86_64/include/arch/private/apic.h b/kernel/arch/x86_64/include/arch/private/apic.h
index 78cb5a57..8532e891 100644
--- a/kernel/arch/x86_64/include/arch/private/apic.h
+++ b/kernel/arch/x86_64/include/arch/private/apic.h
@@ -79,6 +79,7 @@ extern apic_id_t g_max_apic_id;
 
 struct apic {
     void (*setup)(void);
+
     u32 (*read)(enum apic_reg);
     void (*write)(enum apic_reg, u32 value);
 
diff --git a/kernel/arch/x86_64/x2apic.c b/kernel/arch/x86_64/x2apic.c
index e25ba775..ec48bb47 100644
--- a/kernel/arch/x86_64/x2apic.c
+++ b/kernel/arch/x86_64/x2apic.c
@@ -59,9 +59,10 @@ static void INIT_CODE x2apic_setup(void)
 }
 
 struct apic DATA_REFERENCES_INIT_DATA g_x2apic = {
+    .setup = x2apic_setup,
+
     .read = x2apic_read,
     .write = x2apic_write,
-    .setup = x2apic_setup,
 
     .icr_write = x2apic_icr_write,
     .send_ipi_self = x2apic_send_ipi_self,

From 50cd0adf9bc0c2f042b2675b3b6818f231361404 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 15:17:27 +0300
Subject: [PATCH 031/152] arch/x86_64: fix the boot CPU's APIC ID in xAPIC mode

In xAPIC mode the ID is in bits 31:24 of the register, but it was stored
raw, so a boot CPU with a non-zero ID never matched its MADT entry. This
went unnoticed since the boot CPU usually has ID 0, and x2APIC capable
machines, KVM included, run in x2APIC mode where the register is the ID.

Give each mode a read_id() op that returns the decoded ID.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/apic.c                      | 7 ++++++-
 kernel/arch/x86_64/include/arch/private/apic.h | 5 +++++
 kernel/arch/x86_64/x2apic.c                    | 6 ++++++
 kernel/arch/x86_64/xapic.c                     | 6 ++++++
 4 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/kernel/arch/x86_64/apic.c b/kernel/arch/x86_64/apic.c
index 239d0dd4..b45262d2 100644
--- a/kernel/arch/x86_64/apic.c
+++ b/kernel/arch/x86_64/apic.c
@@ -59,7 +59,7 @@ void INIT_CODE apic_detect(void)
     pr_info("using %s mode\n", s_apic_mode_strings[g_apic_mode]);
     g_apic->setup();
 
-    g_boot_cpu_apic_id = g_apic->read(APIC_REG_ID);
+    g_boot_cpu_apic_id = apic_read_id();
     if (unlikely(!(apic_base_msr & IA32_APIC_BASE_IS_BSP))) {
         pr_warn("boot CPU (0x%08X) is not the BSP!\n", g_boot_cpu_apic_id);
         return;
@@ -78,6 +78,11 @@ void apic_eoi(void)
     g_apic->write(APIC_REG_EOI, 0);
 }
 
+apic_id_t apic_read_id(void)
+{
+    return g_apic->read_id();
+}
+
 void apic_icr_write(u32 value, apic_id_t dest_apic_id)
 {
     g_apic->icr_write(value, dest_apic_id);
diff --git a/kernel/arch/x86_64/include/arch/private/apic.h b/kernel/arch/x86_64/include/arch/private/apic.h
index 8532e891..bea33bd9 100644
--- a/kernel/arch/x86_64/include/arch/private/apic.h
+++ b/kernel/arch/x86_64/include/arch/private/apic.h
@@ -8,6 +8,9 @@ typedef u32 apic_id_t;
 
 enum apic_reg : u32 {
     APIC_REG_ID = 0x20,
+        // x2APIC mode uses the whole register as the ID
+        #define APIC_ID_XAPIC MAKE_BIT_MASK_U32(31, 24)
+
     APIC_REG_VERSION = 0x30,
     APIC_REG_TPR = 0x80,
     APIC_REG_APR = 0x90,
@@ -83,6 +86,7 @@ struct apic {
     u32 (*read)(enum apic_reg);
     void (*write)(enum apic_reg, u32 value);
 
+    apic_id_t (*read_id)(void);
     void (*icr_write)(u32 value, apic_id_t dest_apic_id);
     void (*send_ipi_self)(u8 vector);
 };
@@ -96,6 +100,7 @@ void apic_eoi(void);
 bool apic_vector_in_isr(u8 vector);
 bool apic_vector_in_tmr(u8 vector);
 
+apic_id_t apic_read_id(void);
 void apic_icr_write(u32 value, apic_id_t dest_apic_id);
 void apic_send_fixed_ipi(apic_id_t dest_apic_id, u8 vector);
 void apic_send_ipi_self(u8 vector);
diff --git a/kernel/arch/x86_64/x2apic.c b/kernel/arch/x86_64/x2apic.c
index ec48bb47..a5c7329b 100644
--- a/kernel/arch/x86_64/x2apic.c
+++ b/kernel/arch/x86_64/x2apic.c
@@ -28,6 +28,11 @@ static void x2apic_write(enum apic_reg reg, u32 data)
     wrmsr_or_die(x2apic_reg_to_offset(reg), data);
 }
 
+static apic_id_t x2apic_read_id(void)
+{
+    return x2apic_read(APIC_REG_ID);
+}
+
 // A single 64-bit write, no busy bit or separate destination half
 static void x2apic_icr_write(u32 value, apic_id_t dest_apic_id)
 {
@@ -64,6 +69,7 @@ struct apic DATA_REFERENCES_INIT_DATA g_x2apic = {
     .read = x2apic_read,
     .write = x2apic_write,
 
+    .read_id = x2apic_read_id,
     .icr_write = x2apic_icr_write,
     .send_ipi_self = x2apic_send_ipi_self,
 };
diff --git a/kernel/arch/x86_64/xapic.c b/kernel/arch/x86_64/xapic.c
index 911a6b39..33336760 100644
--- a/kernel/arch/x86_64/xapic.c
+++ b/kernel/arch/x86_64/xapic.c
@@ -28,6 +28,11 @@ static void xapic_write(enum apic_reg reg, u32 data)
     iowrite32(&s_xapic_io, (u32)reg, data);
 }
 
+static apic_id_t xapic_read_id(void)
+{
+    return BIT_FIELD_READ(xapic_read(APIC_REG_ID), APIC_ID_XAPIC);
+}
+
 // A new command written while one is still being sent is dropped
 static void xapic_wait_icr_idle(void)
 {
@@ -83,6 +88,7 @@ struct apic DATA_REFERENCES_INIT_DATA g_xapic = {
     .read = xapic_read,
     .write = xapic_write,
 
+    .read_id = xapic_read_id,
     .icr_write = xapic_icr_write,
     .send_ipi_self = xapic_send_ipi_self,
 };

From 47745904736a32151a7843dfaf4105397d4533dc Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 21 Oct 2026 15:23:26 +0300
Subject: [PATCH 032/152] arch/x86_64: use BIT_FIELD_MAKE() for the xAPIC ICR
 destination

Every other APIC register field is a mask under its register, the ICR2
destination was a bare shift.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/include/arch/private/apic.h | 2 ++
 kernel/arch/x86_64/xapic.c                     | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/kernel/arch/x86_64/include/arch/private/apic.h b/kernel/arch/x86_64/include/arch/private/apic.h
index bea33bd9..6e348422 100644
--- a/kernel/arch/x86_64/include/arch/private/apic.h
+++ b/kernel/arch/x86_64/include/arch/private/apic.h
@@ -36,6 +36,8 @@ enum apic_reg : u32 {
         #define APIC_ICR_DEST_SELF BIT_U32(18)
 
     APIC_REG_ICR2 = 0x310,
+        #define APIC_ICR2_XAPIC_DEST MAKE_BIT_MASK_U32(31, 24)
+
     APIC_REG_LVT_TIMER = 0x320,
     APIC_REG_LVT_THERMAL = 0x330,
     APIC_REG_LVT_PERF = 0x340,
diff --git a/kernel/arch/x86_64/xapic.c b/kernel/arch/x86_64/xapic.c
index 33336760..dceff586 100644
--- a/kernel/arch/x86_64/xapic.c
+++ b/kernel/arch/x86_64/xapic.c
@@ -53,7 +53,9 @@ static void xapic_icr_write(u32 value, apic_id_t dest_apic_id)
     xapic_wait_icr_idle();
 
     // The destination half must be programmed first, ICR sends
-    xapic_write(APIC_REG_ICR2, dest_apic_id << 24);
+    xapic_write(
+        APIC_REG_ICR2, BIT_FIELD_MAKE(APIC_ICR2_XAPIC_DEST, dest_apic_id)
+    );
     xapic_write(APIC_REG_ICR, value);
 
     local_irq_state_restore(irq_state);

From 21f766a3ea49db9db274ef7af41f067e12b22f6c Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 14:32:52 +0300
Subject: [PATCH 033/152] sched: introduce struct task and current

The scheduler needs an object to switch between and a way to find the
running one. Every CPU runs its idle task from the first instruction of
entry(), and the arch decides where current lives, x86_64 keeps it
per-CPU.

Stack dumps now name the task they were taken in.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/CMakeLists.txt                         |  1 +
 .../arch/x86_64/include/arch/detail/current.h |  9 ++++++
 kernel/arch/x86_64/smp.c                      |  9 ++++--
 kernel/include/detail/sched/task.h            | 11 +++++++
 kernel/include/private/sched/task.h           |  6 ++++
 kernel/include/sched/task.h                   |  9 ++++++
 kernel/log.c                                  |  2 ++
 kernel/sched/CMakeLists.txt                   |  3 ++
 kernel/sched/task.c                           | 30 +++++++++++++++++++
 tests/CMakeLists.txt                          |  7 +++++
 tests/synthetic-include/arch/detail/current.h |  7 +++++
 tests/task_state.c                            |  5 ++++
 12 files changed, 96 insertions(+), 3 deletions(-)
 create mode 100644 kernel/arch/x86_64/include/arch/detail/current.h
 create mode 100644 kernel/include/detail/sched/task.h
 create mode 100644 kernel/include/private/sched/task.h
 create mode 100644 kernel/include/sched/task.h
 create mode 100644 kernel/sched/CMakeLists.txt
 create mode 100644 kernel/sched/task.c
 create mode 100644 tests/synthetic-include/arch/detail/current.h
 create mode 100644 tests/task_state.c

diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt
index 85446467..761fe75b 100644
--- a/kernel/CMakeLists.txt
+++ b/kernel/CMakeLists.txt
@@ -169,6 +169,7 @@ add_subdirectory(common)
 add_subdirectory(locking)
 add_subdirectory(irq)
 add_subdirectory(memory)
+add_subdirectory(sched)
 add_subdirectory(time)
 add_subdirectory(drivers)
 
diff --git a/kernel/arch/x86_64/include/arch/detail/current.h b/kernel/arch/x86_64/include/arch/detail/current.h
new file mode 100644
index 00000000..2e045b67
--- /dev/null
+++ b/kernel/arch/x86_64/include/arch/detail/current.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <per_cpu.h>
+
+struct task;
+
+DECLARE_PER_CPU(struct task*, g_this_cpu_task);
+
+#define current this_cpu_read(g_this_cpu_task)
diff --git a/kernel/arch/x86_64/smp.c b/kernel/arch/x86_64/smp.c
index 1dfbdcbc..73bfb826 100644
--- a/kernel/arch/x86_64/smp.c
+++ b/kernel/arch/x86_64/smp.c
@@ -6,6 +6,7 @@
 #include <arch/private/msr.h>
 #include <arch/private/descriptors.h>
 #include <arch/private/cpu.h>
+#include <private/sched/task.h>
 
 #include <uacpi/tables.h>
 #include <uacpi/acpi.h>
@@ -26,6 +27,7 @@
 
 DEFINE_PER_CPU(cpu_t, g_this_cpu_id);
 DEFINE_PER_CPU(apic_id_t, g_this_cpu_apic_id);
+DEFINE_PER_CPU(struct task*, g_this_cpu_task) = &g_this_cpu_idle_task;
 
 static INIT_DATA apic_id_t s_early_cpu_to_apic_id[ULTRA_MAX_CPUS] = {
     [0 ... (ULTRA_MAX_CPUS - 1)] = APIC_ID_NONE,
@@ -373,14 +375,15 @@ static error_t INIT_CODE x86_per_cpu_setup(void)
 {
     cpu_t i;
 
-    // Do the boot CPU setup here, AP CPUs call it elsewhere
-    this_cpu_enable_per_cpu(0);
-
     for_each_present_cpu(i) {
         per_cpu(g_this_cpu_id, i) = i;
         per_cpu(g_this_cpu_apic_id, i) = s_early_cpu_to_apic_id[i];
+        per_cpu(g_this_cpu_task, i) = per_cpu_ptr(&g_this_cpu_idle_task, i);
     }
 
+    // Do the boot CPU setup here, AP CPUs call it elsewhere
+    this_cpu_enable_per_cpu(0);
+
     /*
      * Duplicate the early boot CPU feature detection data into its own per-cpu
      * copy. From this point on g_cpu_info contains an AND mask of features of
diff --git a/kernel/include/detail/sched/task.h b/kernel/include/detail/sched/task.h
new file mode 100644
index 00000000..ea17c70d
--- /dev/null
+++ b/kernel/include/detail/sched/task.h
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <common/types.h>
+
+#include <sched/task.h>
+
+#define TASK_NAME_MAX 32
+
+struct task {
+    char name[TASK_NAME_MAX];
+};
diff --git a/kernel/include/private/sched/task.h b/kernel/include/private/sched/task.h
new file mode 100644
index 00000000..7b9aa4d1
--- /dev/null
+++ b/kernel/include/private/sched/task.h
@@ -0,0 +1,6 @@
+#pragma once
+
+#include <per_cpu.h>
+#include <detail/sched/task.h>
+
+DECLARE_PER_CPU(struct task, g_this_cpu_idle_task);
diff --git a/kernel/include/sched/task.h b/kernel/include/sched/task.h
new file mode 100644
index 00000000..feeafe37
--- /dev/null
+++ b/kernel/include/sched/task.h
@@ -0,0 +1,9 @@
+#pragma once
+
+// Arch-specific way to fetch 'current'
+#include <arch/detail/current.h>
+
+// A thread of execution, 'current' is the one running on this CPU
+struct task;
+
+const char *task_name(const struct task *task);
diff --git a/kernel/log.c b/kernel/log.c
index 89ac1c17..040d434a 100644
--- a/kernel/log.c
+++ b/kernel/log.c
@@ -14,6 +14,7 @@
 #include <log_ring.h>
 #include <param.h>
 #include <panic.h>
+#include <sched/task.h>
 #include <arch_constants.h>
 
 #include <time/units.h>
@@ -459,6 +460,7 @@ void dump_stack(enum log_level level, struct registers *regs)
     };
 
     pr_lvl(level, "Hardware: %s\n", s_hw_id);
+    pr_lvl(level, "Task: %s\n", task_name(current));
     pr_lvl(level, "Call trace (most recent call first):\n");
     unwind_walk(regs, do_dump_frame, &state);
 }
diff --git a/kernel/sched/CMakeLists.txt b/kernel/sched/CMakeLists.txt
new file mode 100644
index 00000000..7965cf08
--- /dev/null
+++ b/kernel/sched/CMakeLists.txt
@@ -0,0 +1,3 @@
+ultra_sources(
+    task.c
+)
diff --git a/kernel/sched/task.c b/kernel/sched/task.c
new file mode 100644
index 00000000..c78791cd
--- /dev/null
+++ b/kernel/sched/task.c
@@ -0,0 +1,30 @@
+#include <private/sched/task.h>
+
+#include <common/format.h>
+
+#include <smp.h>
+#include <init_level.h>
+#include <free_after_init.h>
+
+DEFINE_PER_CPU(struct task, g_this_cpu_idle_task) = {
+    .name = "idle/0",
+};
+
+static error_t INIT_CODE idle_tasks_set_names(void)
+{
+    struct task *task;
+    cpu_t cpu;
+
+    for_each_present_cpu(cpu) {
+        task = per_cpu_ptr(&g_this_cpu_idle_task, cpu);
+        snprintf(task->name, sizeof(task->name), "idle/%u", cpu);
+    }
+
+    return EOK;
+}
+INIT_CALL_POST(PER_CPU_AVAILABLE, idle_tasks_set_names);
+
+const char *task_name(const struct task *task)
+{
+    return task->name;
+}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 7539c3ee..879fcf68 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -193,6 +193,12 @@ KERNEL_FILE(INCLUDE_FILE "panic.h")
 KERNEL_FILE(INCLUDE_FILE "linker.h")
 KERNEL_FILE(INCLUDE_PATH "locking" INCLUDE_FILE "seq.h")
 KERNEL_FILE(INCLUDE_PATH "detail" INCLUDE_FILE "init_level.h")
+KERNEL_FILE(
+    INCLUDE_PATH "sched" INCLUDE_FILE "task.h"
+    SOURCE_PATH "sched" SOURCE_FILE "task.c"
+)
+KERNEL_FILE(INCLUDE_PATH "detail/sched" INCLUDE_FILE "task.h")
+KERNEL_FILE(INCLUDE_PATH "private/sched" INCLUDE_FILE "task.h")
 KERNEL_FILE(
     INCLUDE_PATH "locking" INCLUDE_FILE "spinlock.h"
     SOURCE_PATH "locking" SOURCE_FILE "spinlock.c"
@@ -224,6 +230,7 @@ include("test-cases/files.cmake")
 add_executable(
     run_tests
     run_tests.cpp kernel_helpers.cpp test_harness.cpp memory_map.c ${TEST_SOURCES}
+    task_state.c
     kernel-source/locking/spinlock.c kernel-source/memory/units.c
     kernel-source/common/bit.c kernel-source/locking/mutex.c
     kernel-source/common/string_container.c kernel-source/common/conversions.c
diff --git a/tests/synthetic-include/arch/detail/current.h b/tests/synthetic-include/arch/detail/current.h
new file mode 100644
index 00000000..d064873d
--- /dev/null
+++ b/tests/synthetic-include/arch/detail/current.h
@@ -0,0 +1,7 @@
+#pragma once
+
+struct task;
+
+extern __thread struct task *g_test_current;
+
+#define current g_test_current
diff --git a/tests/task_state.c b/tests/task_state.c
new file mode 100644
index 00000000..5d55306c
--- /dev/null
+++ b/tests/task_state.c
@@ -0,0 +1,5 @@
+#include <detail/sched/task.h>
+
+static struct task s_test_task;
+
+__thread struct task *g_test_current = &s_test_task;

From a7380da77f0227ee5edd618bc1dd75a45ed629ce Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sat, 3 Oct 2026 21:56:41 +0300
Subject: [PATCH 034/152] kernel: introduce the preempt count

The scheduler has to know when the running context must not be switched
away from: preemption is disabled or a hard interrupt handler is
running. Track both in one count, which also makes hard interrupt
context tracking generic. Every CPU starts with preemption disabled.

The generic count lives in struct task and needs no atomics. x86_64
keeps it per-CPU instead, where a single instruction updates it.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .../arch/x86_64/include/arch/detail/preempt.h | 34 +++++++++++
 kernel/arch/x86_64/include/arch/private/idt.h |  3 -
 kernel/arch/x86_64/irq.c                      | 30 +++-------
 kernel/arch/x86_64/lapic_domain.c             |  2 +
 kernel/include/detail/preempt.h               | 45 ++++++++++++++
 kernel/include/detail/sched/task.h            | 10 ++++
 kernel/include/irq_helpers.h                  |  3 -
 kernel/include/preempt.h                      | 24 ++++++++
 kernel/include/private/arch/preempt.h         | 23 ++++++++
 kernel/irq/request.c                          |  1 +
 kernel/locking/mutex.c                        |  2 +-
 kernel/sched/task.c                           |  4 ++
 tests/CMakeLists.txt                          |  3 +
 tests/run_tests.cpp                           |  1 +
 tests/synthetic-include/irq_helpers.h         |  5 --
 tests/task_state.c                            |  8 +++
 tests/test-cases/files.cmake                  |  1 +
 tests/test-cases/test_preempt.c               | 58 +++++++++++++++++++
 tests/test_harness.h                          |  1 +
 19 files changed, 223 insertions(+), 35 deletions(-)
 create mode 100644 kernel/arch/x86_64/include/arch/detail/preempt.h
 create mode 100644 kernel/include/detail/preempt.h
 create mode 100644 kernel/include/preempt.h
 create mode 100644 kernel/include/private/arch/preempt.h
 create mode 100644 tests/test-cases/test_preempt.c

diff --git a/kernel/arch/x86_64/include/arch/detail/preempt.h b/kernel/arch/x86_64/include/arch/detail/preempt.h
new file mode 100644
index 00000000..8cd2f8cd
--- /dev/null
+++ b/kernel/arch/x86_64/include/arch/detail/preempt.h
@@ -0,0 +1,34 @@
+#pragma once
+
+#define ARCH_HAS_CUSTOM_PREEMPT_COUNT
+
+#include <common/types.h>
+#include <common/attributes.h>
+#include <common/bit.h>
+
+#include <per_cpu.h>
+
+/*
+ * Stored inverted, so that a count of zero means both that preemption
+ * is enabled and that a reschedule is wanted, which lets the enable
+ * path decide with one decrement.
+ */
+#define X86_PREEMPT_RESCHED_NOT_NEEDED BIT_U32(31)
+
+DECLARE_PER_CPU(u32, g_this_cpu_preempt_count);
+
+static ALWAYS_INLINE u32 preempt_count(void)
+{
+    return this_cpu_read(g_this_cpu_preempt_count) &
+           ~X86_PREEMPT_RESCHED_NOT_NEEDED;
+}
+
+static ALWAYS_INLINE void preempt_count_add(u32 value)
+{
+    this_cpu_add(g_this_cpu_preempt_count, value);
+}
+
+static ALWAYS_INLINE void preempt_count_sub(u32 value)
+{
+    this_cpu_sub(g_this_cpu_preempt_count, value);
+}
diff --git a/kernel/arch/x86_64/include/arch/private/idt.h b/kernel/arch/x86_64/include/arch/private/idt.h
index 4a0710fb..3e68b2df 100644
--- a/kernel/arch/x86_64/include/arch/private/idt.h
+++ b/kernel/arch/x86_64/include/arch/private/idt.h
@@ -63,9 +63,6 @@
     void X86_IRQ_DISPATCH(struct registers *regs); \
     void X86_IRQ_DISPATCH(struct registers *regs)
 
-void hard_irq_enter(void);
-void hard_irq_exit(void);
-
 void fixed_vector_enter(u8 vector);
 void fixed_vector_exit(void);
 
diff --git a/kernel/arch/x86_64/irq.c b/kernel/arch/x86_64/irq.c
index a09a6ccc..3d6776f2 100644
--- a/kernel/arch/x86_64/irq.c
+++ b/kernel/arch/x86_64/irq.c
@@ -4,34 +4,18 @@
 #include <arch/private/vectors.h>
 #include <arch/registers.h>
 
+#include <private/arch/preempt.h>
+
 #include <per_cpu.h>
-#include <irq_helpers.h>
+#include <preempt.h>
 #include <log.h>
 #include <bug.h>
 
-static DEFINE_PER_CPU(bool, s_in_hard_irq);
-static DEFINE_PER_CPU(u64, s_num_fixed_invocations[NUM_FIXED_VECTORS]);
-
-bool in_hard_irq(void)
-{
-    return this_cpu_read(s_in_hard_irq);
-}
+// Every CPU starts out with preemption disabled
+DEFINE_PER_CPU(u32, g_this_cpu_preempt_count) =
+    X86_PREEMPT_RESCHED_NOT_NEEDED | PREEMPT_DISABLE_DEPTH(1);
 
-/*
- * Handlers run with interrupts disabled for their entire duration,
- * so hard interrupt context can never nest. NMIs and exceptions take
- * a different path and are not part of this context.
- */
-void hard_irq_enter(void)
-{
-    BUG_ON(this_cpu_read(s_in_hard_irq));
-    this_cpu_write(s_in_hard_irq, true);
-}
-
-void hard_irq_exit(void)
-{
-    this_cpu_write(s_in_hard_irq, false);
-}
+static DEFINE_PER_CPU(u64, s_num_fixed_invocations[NUM_FIXED_VECTORS]);
 
 void fixed_vector_enter(u8 vector)
 {
diff --git a/kernel/arch/x86_64/lapic_domain.c b/kernel/arch/x86_64/lapic_domain.c
index 932d629a..03900097 100644
--- a/kernel/arch/x86_64/lapic_domain.c
+++ b/kernel/arch/x86_64/lapic_domain.c
@@ -10,8 +10,10 @@
 #include <arch/smp.h>
 
 #include <private/irq.h>
+#include <private/arch/preempt.h>
 
 #include <per_cpu.h>
+#include <preempt.h>
 #include <init_level.h>
 #include <free_after_init.h>
 #include <memory/alloc.h>
diff --git a/kernel/include/detail/preempt.h b/kernel/include/detail/preempt.h
new file mode 100644
index 00000000..1ebe040e
--- /dev/null
+++ b/kernel/include/detail/preempt.h
@@ -0,0 +1,45 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/atomic.h>
+#include <common/attributes.h>
+#include <common/helpers.h>
+#include <common/bit.h>
+
+#define PREEMPT_DISABLE_UNIT BIT_U32(0)
+#define PREEMPT_DISABLE_MASK MAKE_BIT_MASK_U32(7, 0)
+#define PREEMPT_DISABLE_DEPTH(x) ((x) * PREEMPT_DISABLE_UNIT)
+
+#define PREEMPT_HARD_IRQ_UNIT BIT_U32(8)
+#define PREEMPT_HARD_IRQ_MASK MAKE_BIT_MASK_U32(11, 8)
+#define PREEMPT_HARD_IRQ_DEPTH(x) ((x) * PREEMPT_HARD_IRQ_UNIT)
+
+#if HAS_INCLUDE(<arch/detail/preempt.h>)
+#include <arch/detail/preempt.h>
+#endif
+
+#ifndef ARCH_HAS_CUSTOM_PREEMPT_COUNT
+#include <sched/task.h>
+#include <detail/sched/task.h>
+
+static ALWAYS_INLINE u32 preempt_count(void)
+{
+    return atomic_load_relaxed(&current->preempt_count);
+}
+
+static ALWAYS_INLINE void preempt_count_add(u32 value)
+{
+    u32 count;
+
+    count = atomic_load_relaxed(&current->preempt_count);
+    atomic_store_relaxed(&current->preempt_count, count + value);
+}
+
+static ALWAYS_INLINE void preempt_count_sub(u32 value)
+{
+    u32 count;
+
+    count = atomic_load_relaxed(&current->preempt_count);
+    atomic_store_relaxed(&current->preempt_count, count - value);
+}
+#endif
diff --git a/kernel/include/detail/sched/task.h b/kernel/include/detail/sched/task.h
index ea17c70d..50ac1a3b 100644
--- a/kernel/include/detail/sched/task.h
+++ b/kernel/include/detail/sched/task.h
@@ -1,11 +1,21 @@
 #pragma once
 
 #include <common/types.h>
+#include <common/helpers.h>
 
 #include <sched/task.h>
 
+// We need to know whether this arch keeps preempt_count in struct task
+#if HAS_INCLUDE(<arch/detail/preempt.h>)
+#include <arch/detail/preempt.h>
+#endif
+
 #define TASK_NAME_MAX 32
 
 struct task {
+#ifndef ARCH_HAS_CUSTOM_PREEMPT_COUNT
+    u32 preempt_count;
+#endif
+
     char name[TASK_NAME_MAX];
 };
diff --git a/kernel/include/irq_helpers.h b/kernel/include/irq_helpers.h
index 9279c7f9..7780731f 100644
--- a/kernel/include/irq_helpers.h
+++ b/kernel/include/irq_helpers.h
@@ -10,9 +10,6 @@ static ALWAYS_INLINE bool local_irq_enabled(void)
     return arch_local_irq_is_enabled_state(arch_local_irq_state());
 }
 
-// True while executing in hardware interrupt context on this CPU
-bool in_hard_irq(void);
-
 /*
  * Save the arch-specific IRQ state and disable IRQs if they were previously
  * enabled.
diff --git a/kernel/include/preempt.h b/kernel/include/preempt.h
new file mode 100644
index 00000000..163f88f4
--- /dev/null
+++ b/kernel/include/preempt.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include <common/attributes.h>
+#include <common/atomic.h>
+
+#include <detail/preempt.h>
+
+static ALWAYS_INLINE void preempt_disable(void)
+{
+    preempt_count_add(PREEMPT_DISABLE_UNIT);
+    compiler_barrier();
+}
+
+static ALWAYS_INLINE void preempt_enable(void)
+{
+    compiler_barrier();
+    preempt_count_sub(PREEMPT_DISABLE_UNIT);
+}
+
+// True while executing in hardware interrupt context on this CPU
+static ALWAYS_INLINE bool in_hard_irq(void)
+{
+    return preempt_count() & PREEMPT_HARD_IRQ_MASK;
+}
diff --git a/kernel/include/private/arch/preempt.h b/kernel/include/private/arch/preempt.h
new file mode 100644
index 00000000..34b2a43e
--- /dev/null
+++ b/kernel/include/private/arch/preempt.h
@@ -0,0 +1,23 @@
+#pragma once
+
+#include <common/attributes.h>
+
+#include <bug.h>
+#include <preempt.h>
+
+/*
+ * The hook arch-specific interrupt handling code calls upon entry into an
+ * IRQ handler (NMI and exceptions never call this).
+ *
+ * This does basic hard irq bookkeeping & makes sure IRQ contexts never nest.
+ */
+static ALWAYS_INLINE void hard_irq_enter(void)
+{
+    BUG_ON(in_hard_irq());
+    preempt_count_add(PREEMPT_HARD_IRQ_UNIT);
+}
+
+static ALWAYS_INLINE void hard_irq_exit(void)
+{
+    preempt_count_sub(PREEMPT_HARD_IRQ_UNIT);
+}
diff --git a/kernel/irq/request.c b/kernel/irq/request.c
index 0ae393ae..e6db72be 100644
--- a/kernel/irq/request.c
+++ b/kernel/irq/request.c
@@ -4,6 +4,7 @@
 
 #include <bug.h>
 #include <locking/mutex.h>
+#include <preempt.h>
 #include <init_level.h>
 #include <memory/alloc.h>
 #include <common/atomic.h>
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 386f09ef..7d09a1f2 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -1,5 +1,5 @@
 #include <locking/mutex.h>
-#include <irq_helpers.h>
+#include <preempt.h>
 #include <bug.h>
 
 void mutex_init(struct mutex *mutex)
diff --git a/kernel/sched/task.c b/kernel/sched/task.c
index c78791cd..cee3f6dd 100644
--- a/kernel/sched/task.c
+++ b/kernel/sched/task.c
@@ -3,11 +3,15 @@
 #include <common/format.h>
 
 #include <smp.h>
+#include <preempt.h>
 #include <init_level.h>
 #include <free_after_init.h>
 
 DEFINE_PER_CPU(struct task, g_this_cpu_idle_task) = {
     .name = "idle/0",
+#ifndef ARCH_HAS_CUSTOM_PREEMPT_COUNT
+    .preempt_count = PREEMPT_DISABLE_DEPTH(1),
+#endif
 };
 
 static error_t INIT_CODE idle_tasks_set_names(void)
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 879fcf68..0026314f 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -193,6 +193,9 @@ KERNEL_FILE(INCLUDE_FILE "panic.h")
 KERNEL_FILE(INCLUDE_FILE "linker.h")
 KERNEL_FILE(INCLUDE_PATH "locking" INCLUDE_FILE "seq.h")
 KERNEL_FILE(INCLUDE_PATH "detail" INCLUDE_FILE "init_level.h")
+KERNEL_FILE(INCLUDE_FILE "preempt.h")
+KERNEL_FILE(INCLUDE_PATH "detail" INCLUDE_FILE "preempt.h")
+KERNEL_FILE(INCLUDE_PATH "private/arch" INCLUDE_FILE "preempt.h")
 KERNEL_FILE(
     INCLUDE_PATH "sched" INCLUDE_FILE "task.h"
     SOURCE_PATH "sched" SOURCE_FILE "task.c"
diff --git a/tests/run_tests.cpp b/tests/run_tests.cpp
index d273f229..662f6968 100644
--- a/tests/run_tests.cpp
+++ b/tests/run_tests.cpp
@@ -98,6 +98,7 @@ int main(int argc, char **argv)
 
             std::printf("Running test %s...", test->name);
             std::fflush(stdout);
+            test_task_reset();
             test->run();
             std::puts("OK");
         } catch (const std::exception& ex) {
diff --git a/tests/synthetic-include/irq_helpers.h b/tests/synthetic-include/irq_helpers.h
index 148cfe97..3620b3ac 100644
--- a/tests/synthetic-include/irq_helpers.h
+++ b/tests/synthetic-include/irq_helpers.h
@@ -31,8 +31,3 @@ static inline void local_irq_state_restore(irq_state_t state)
 {
     g_test_irqs_enabled = state;
 }
-
-static inline bool in_hard_irq(void)
-{
-    return false;
-}
diff --git a/tests/task_state.c b/tests/task_state.c
index 5d55306c..b414492e 100644
--- a/tests/task_state.c
+++ b/tests/task_state.c
@@ -1,5 +1,13 @@
 #include <detail/sched/task.h>
 
+#include <test_harness.h>
+
 static struct task s_test_task;
 
 __thread struct task *g_test_current = &s_test_task;
+
+void test_task_reset(void)
+{
+    g_test_current = &s_test_task;
+    s_test_task.preempt_count = 0;
+}
diff --git a/tests/test-cases/files.cmake b/tests/test-cases/files.cmake
index 2808ec23..5a0bccab 100644
--- a/tests/test-cases/files.cmake
+++ b/tests/test-cases/files.cmake
@@ -12,6 +12,7 @@ add_test_cases(
     test_rbtree.c
     test_cpu_mask.c
     test_cpu_mask_out_of_line.c
+    test_preempt.c
     test_spinlock.c
     test_mutex.c
     test_vector_alloc.c
diff --git a/tests/test-cases/test_preempt.c b/tests/test-cases/test_preempt.c
new file mode 100644
index 00000000..4d3d55c2
--- /dev/null
+++ b/tests/test-cases/test_preempt.c
@@ -0,0 +1,58 @@
+#include <preempt.h>
+#include <private/arch/preempt.h>
+
+#include <test_harness.h>
+
+TEST_CASE(preempt_disable_nests)
+{
+    u32 base = preempt_count();
+
+    preempt_disable();
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(1));
+
+    preempt_disable();
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(2));
+
+    preempt_enable();
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(1));
+
+    preempt_enable();
+    ASSERT_EQ(preempt_count(), base);
+}
+
+TEST_CASE(preempt_hard_irq_context)
+{
+    u32 base = preempt_count();
+
+    ASSERT(!in_hard_irq());
+
+    hard_irq_enter();
+    ASSERT(in_hard_irq());
+    ASSERT_EQ(preempt_count(), base + PREEMPT_HARD_IRQ_DEPTH(1));
+
+    hard_irq_exit();
+    ASSERT(!in_hard_irq());
+    ASSERT_EQ(preempt_count(), base);
+}
+
+TEST_CASE(preempt_hard_irq_independent_of_depth)
+{
+    u32 base = preempt_count();
+
+    preempt_disable();
+    ASSERT(!in_hard_irq());
+
+    hard_irq_enter();
+    preempt_disable();
+    ASSERT(in_hard_irq());
+    ASSERT_EQ(
+        preempt_count(),
+        base + PREEMPT_DISABLE_DEPTH(2) + PREEMPT_HARD_IRQ_DEPTH(1)
+    );
+    preempt_enable();
+    hard_irq_exit();
+
+    ASSERT(!in_hard_irq());
+    preempt_enable();
+    ASSERT_EQ(preempt_count(), base);
+}
diff --git a/tests/test_harness.h b/tests/test_harness.h
index c0ee21b8..be22248a 100644
--- a/tests/test_harness.h
+++ b/tests/test_harness.h
@@ -39,6 +39,7 @@ void malloc_phys_range(uint64_t start, uint64_t size);
 
 // Done automatically after each test case
 void reset_phys_ranges(void);
+void test_task_reset(void);
 
 uint64_t translate_virt_to_phys(void* virt);
 void *translate_phys_to_virt(uint64_t phys);

From 9b7e1fcc7e9f9e84112cd37d78d7ea37690edcaa Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sat, 3 Oct 2026 21:58:21 +0300
Subject: [PATCH 035/152] spinlock: disable preemption while a lock is held

This is critical for locks that don't disable IRQs on acquire, since
it will be possible for them to get preempted while holding a lock
otherwise.

For locks that do disable local IRQs this is less important, but there's
still the issue of preemption latency if we don't check whether
something wanted us to reschedule while we were in the critical section
but couldn't.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/locking/spinlock.c        | 32 ++++++++++++++--
 tests/test-cases/test_spinlock.c | 64 ++++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+), 4 deletions(-)

diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index fa34f7a9..b4e6388e 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -3,6 +3,7 @@
 
 #include <locking/spinlock.h>
 #include <irq_helpers.h>
+#include <preempt.h>
 
 #define CURRENT_TICKET_SHIFT 16
 #define LAST_FREED_TICKET_SHIFT 0
@@ -18,6 +19,8 @@ void spin_lock(struct spinlock *lock)
     u32 ctrl;
     u16 my_ticket;
 
+    preempt_disable();
+
     ctrl = atomic_fetch_add(
         &lock->control, 1u << CURRENT_TICKET_SHIFT, MO_ACQ_REL
     );
@@ -43,7 +46,7 @@ irq_state_t spin_lock_irq_save(struct spinlock *lock)
     return irq_state;
 }
 
-bool spin_try_lock(struct spinlock *lock)
+static bool do_try_lock(struct spinlock *lock)
 {
     u32 ctrl;
 
@@ -56,21 +59,35 @@ bool spin_try_lock(struct spinlock *lock)
     );
 }
 
+bool spin_try_lock(struct spinlock *lock)
+{
+    preempt_disable();
+
+    if (do_try_lock(lock))
+        return true;
+
+    preempt_enable();
+    return false;
+}
+
 bool spin_try_lock_irq_save(struct spinlock *lock, irq_state_t *out_irq_state)
 {
     irq_state_t irq_state;
 
     irq_state = local_irq_state_save();
-    if (spin_try_lock(lock)) {
+    preempt_disable();
+
+    if (do_try_lock(lock)) {
         *out_irq_state = irq_state;
         return true;
     }
 
     local_irq_state_restore(irq_state);
+    preempt_enable();
     return false;
 }
 
-void spin_unlock(struct spinlock *lock)
+static void do_unlock(struct spinlock *lock)
 {
     u32 ctrl;
 
@@ -78,10 +95,17 @@ void spin_unlock(struct spinlock *lock)
     atomic_store_release(&lock->last_freed_ticket, LAST_FREED_TICKET(ctrl) + 1);
 }
 
+void spin_unlock(struct spinlock *lock)
+{
+    do_unlock(lock);
+    preempt_enable();
+}
+
 void spin_unlock_irq_restore(struct spinlock *lock, irq_state_t irq_state)
 {
-    spin_unlock(lock);
+    do_unlock(lock);
     local_irq_state_restore(irq_state);
+    preempt_enable();
 }
 
 bool spin_is_locked(struct spinlock *lock)
diff --git a/tests/test-cases/test_spinlock.c b/tests/test-cases/test_spinlock.c
index 5efc00f0..e96e7aa3 100644
--- a/tests/test-cases/test_spinlock.c
+++ b/tests/test-cases/test_spinlock.c
@@ -1,5 +1,6 @@
 #include <locking/spinlock.h>
 #include <irq_helpers.h>
+#include <preempt.h>
 
 #include <test_harness.h>
 
@@ -84,3 +85,66 @@ TEST_CASE(spinlock_try_lock_irq_save)
     ASSERT(local_irq_enabled());
     spin_unlock(&lock);
 }
+
+TEST_CASE(spinlock_disables_preemption)
+{
+    struct spinlock outer, inner;
+    u32 base = preempt_count();
+
+    spin_lock_init(&outer);
+    spin_lock_init(&inner);
+
+    spin_lock(&outer);
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(1));
+
+    spin_lock(&inner);
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(2));
+
+    spin_unlock(&inner);
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(1));
+
+    spin_unlock(&outer);
+    ASSERT_EQ(preempt_count(), base);
+}
+
+TEST_CASE(spinlock_irq_save_disables_preemption)
+{
+    struct spinlock lock;
+    irq_state_t state;
+    u32 base = preempt_count();
+
+    spin_lock_init(&lock);
+
+    state = spin_lock_irq_save(&lock);
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(1));
+
+    spin_unlock_irq_restore(&lock, state);
+    ASSERT_EQ(preempt_count(), base);
+}
+
+TEST_CASE(spinlock_try_lock_preemption)
+{
+    struct spinlock lock;
+    irq_state_t state;
+    u32 base = preempt_count();
+
+    spin_lock_init(&lock);
+
+    ASSERT(spin_try_lock(&lock));
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(1));
+
+    ASSERT(!spin_try_lock(&lock));
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(1));
+
+    ASSERT(!spin_try_lock_irq_save(&lock, &state));
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(1));
+
+    spin_unlock(&lock);
+    ASSERT_EQ(preempt_count(), base);
+
+    ASSERT(spin_try_lock_irq_save(&lock, &state));
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(1));
+
+    spin_unlock_irq_restore(&lock, state);
+    ASSERT_EQ(preempt_count(), base);
+}

From ce8be7201ec2620a735d7d9412ffb3e671907162 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sat, 3 Oct 2026 22:00:41 +0300
Subject: [PATCH 036/152] log: flush the console one record at a time

Don't try to flush the entire thing in one go, instead lock it per
record and disable IRQs for each record so we don't get preempted in the
middle of flushing it, but also don't hang forever doing the flush.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/log.c | 107 ++++++++++++++++++++++++++-------------------------
 1 file changed, 54 insertions(+), 53 deletions(-)

diff --git a/kernel/log.c b/kernel/log.c
index 040d434a..d8da650f 100644
--- a/kernel/log.c
+++ b/kernel/log.c
@@ -14,6 +14,7 @@
 #include <log_ring.h>
 #include <param.h>
 #include <panic.h>
+#include <irq_helpers.h>
 #include <sched/task.h>
 #include <arch_constants.h>
 
@@ -210,7 +211,7 @@ static void format_record(
         out_buf_append(out, "\n", 1);
 }
 
-static void log_flush_console(struct console *con)
+static bool log_flush_console(struct console *con)
 {
     static char s_msg_buf[512], s_out_data[512 + 128];
 
@@ -229,36 +230,35 @@ static void log_flush_console(struct console *con)
     if (panic_in_progress())
         read_flags |= LOG_RING_READ_SKIP_UNPUBLISHED;
 
-    for (;;) {
-        ret = log_ring_read_with_flags(
-            &s_log_ring, con->log_seq_num, s_msg_buf, sizeof(s_msg_buf),
-            read_flags, &rec
-        );
-        if (ret != EOK)
-            break;
+    ret = log_ring_read_with_flags(
+        &s_log_ring, con->log_seq_num, s_msg_buf, sizeof(s_msg_buf),
+        read_flags, &rec
+    );
+    if (ret != EOK)
+        return false;
 
-        con->log_seq_num = rec.seq_num + 1;
+    con->log_seq_num = rec.seq_num + 1;
 
-        if (rec.level > s_log_level)
-            continue;
+    if (rec.level > s_log_level)
+        return true;
 
-        sec = rec.timestamp_ns / NS_PER_SEC;
-        usec = (rec.timestamp_ns % NS_PER_SEC) / 1000;
-        stamp_len = snprintf(
-            stamp, sizeof(stamp), "[%5llu.%06llu] ", sec, usec
-        );
-
-        out = (struct out_buf) {
-            .data = s_out_data,
-            .capacity = sizeof(s_out_data),
-        };
-        format_record(
-            &out, stamp, stamp_len, s_msg_buf, rec.length, rec.level,
-            con->flags
-        );
-
-        con->write(con, out.data, out.size);
-    }
+    sec = rec.timestamp_ns / NS_PER_SEC;
+    usec = (rec.timestamp_ns % NS_PER_SEC) / 1000;
+    stamp_len = snprintf(
+        stamp, sizeof(stamp), "[%5llu.%06llu] ", sec, usec
+    );
+
+    out = (struct out_buf) {
+        .data = s_out_data,
+        .capacity = sizeof(s_out_data),
+    };
+    format_record(
+        &out, stamp, stamp_len, s_msg_buf, rec.length, rec.level,
+        con->flags
+    );
+
+    con->write(con, out.data, out.size);
+    return true;
 }
 
 /*
@@ -296,40 +296,41 @@ static bool flush_try_acquire(void)
 
 static bool flush_release(void)
 {
-    u32 state = FLUSH_OWNED;
+    u32 state;
 
-    /*
-     * Fails when a record was left behind by another writer, which noticed
-     * there was already a flush in-progress that we owned.
-     */
-    if (atomic_cmpxchg_acq_rel(&s_flush_state, &state, 0))
-        return true;
-
-    /*
-     * Go back and take another pass so we flush the record(s) left behind by
-     * concurrent writers. Clear the pending bit so we notice if this happens
-     * again.
-     */
-    atomic_xchg(&s_flush_state, FLUSH_OWNED, MO_ACQ_REL);
-    return false;
+    state = atomic_xchg(&s_flush_state, 0, MO_ACQ_REL);
+    return !(state & FLUSH_PENDING);
 }
 
 void log_flush_consoles(void)
 {
     struct console *con;
+    irq_state_t irq_state;
+    bool advanced, clean;
 
-    /*
-     * Always allow panic to take over flushing, we have no other way to make
-     * sure the interrupted context that was doing the flushing ever gets to
-     * run again.
-     */
-    if (!flush_try_acquire() && !panic_in_progress())
-        return;
+    for (;;) {
+        irq_state = local_irq_state_save();
+
+        /*
+         * Always allow panic to take over flushing, we have no other way to
+         * make sure the interrupted context that was doing the flushing ever
+         * gets to run again.
+         */
+        if (!flush_try_acquire() && !panic_in_progress()) {
+            local_irq_state_restore(irq_state);
+            return;
+        }
 
-    do {
+        advanced = false;
         for (con = g_consoles; con; con = con->next)
-            log_flush_console(con);
-    } while (!flush_release());
+            advanced |= log_flush_console(con);
+
+        clean = flush_release();
+        local_irq_state_restore(irq_state);
+
+        if (!advanced && clean)
+            return;
+    }
 }
 
 static size_t extract_msg_level(const char *msg, enum log_level *out_level)

From e8c46814b8fbeed4de10f0b52994ed9feb726dd2 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sat, 3 Oct 2026 22:15:29 +0300
Subject: [PATCH 037/152] per_cpu: add this_cpu_xchg() and this_cpu_cmpxchg()

Read-modify-write of a per-CPU variable has to be safe against
interrupts and preemption without disabling either. Both are an unlocked
cmpxchg on x86_64. this_cpu_cmpxchg() follows the atomic helpers: it
returns whether the exchange happened and writes back the value it found
otherwise.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .../arch/x86_64/include/arch/detail/per_cpu.h | 60 +++++++++++++++++++
 kernel/include/per_cpu.h                      | 46 ++++++++++++++
 tests/synthetic-include/per_cpu.h             | 20 +++++++
 3 files changed, 126 insertions(+)

diff --git a/kernel/arch/x86_64/include/arch/detail/per_cpu.h b/kernel/arch/x86_64/include/arch/detail/per_cpu.h
index fddf5cd9..fbf40b67 100644
--- a/kernel/arch/x86_64/include/arch/detail/per_cpu.h
+++ b/kernel/arch/x86_64/include/arch/detail/per_cpu.h
@@ -41,3 +41,63 @@
 #define this_cpu_sub_16(val, x) X86_THIS_CPU_MODIFY("subw", "ri", u16, val, x)
 #define this_cpu_sub_32(val, x) X86_THIS_CPU_MODIFY("subl", "ri", u32, val, x)
 #define this_cpu_sub_64(val, x) X86_THIS_CPU_MODIFY("subq", "re", u64, val, x)
+
+/*
+ * Don't use xchg here, which implies the lock prefix when dealing with a
+ * memory operand. We don't need such semantics for per-cpu private variables,
+ * so a cmpxchg loop suffices.
+ */
+#define X86_THIS_CPU_XCHG(suffix, constraint, type, val, x) ({ \
+    type xchg_old, xchg_new;                                   \
+                                                               \
+    xchg_new = (type)(ptr_t)(typeof(val))(x);                  \
+    asm volatile(                                              \
+        "mov" suffix " %%gs:%[var], %[old]\n"                  \
+        "1:\n"                                                 \
+        "cmpxchg" suffix " %[new], %%gs:%[var]\n"              \
+        "jnz 1b"                                               \
+        : [old] "=&a" (xchg_old), [var] "+m" (val)             \
+        : [new] constraint (xchg_new)                          \
+        : "cc"                                                 \
+    );                                                         \
+                                                               \
+    (typeof(val))(ptr_t)xchg_old;                              \
+})
+
+#define this_cpu_xchg_8(val, x)             \
+    X86_THIS_CPU_XCHG("b", "q", u8, val, x)
+#define this_cpu_xchg_16(val, x)             \
+    X86_THIS_CPU_XCHG("w", "r", u16, val, x)
+#define this_cpu_xchg_32(val, x)             \
+    X86_THIS_CPU_XCHG("l", "r", u32, val, x)
+#define this_cpu_xchg_64(val, x)             \
+    X86_THIS_CPU_XCHG("q", "r", u64, val, x)
+
+#define X86_THIS_CPU_CMPXCHG(                                                \
+    suffix, constraint, type, val, expected, desired                         \
+) ({                                                                         \
+    type cmpxchg_old, cmpxchg_new;                                           \
+    bool cmpxchg_done;                                                       \
+                                                                             \
+    cmpxchg_old = (type)(ptr_t)(*(expected));                                \
+    cmpxchg_new = (type)(ptr_t)(typeof(val))(desired);                       \
+    asm volatile(                                                            \
+        "cmpxchg" suffix " %[new], %%gs:%[var]"                              \
+        : "=@ccz" (cmpxchg_done), [old] "+a" (cmpxchg_old), [var] "+m" (val) \
+        : [new] constraint (cmpxchg_new)                                     \
+    );                                                                       \
+                                                                             \
+    if (!cmpxchg_done)                                                       \
+        *(expected) = (typeof(val))(ptr_t)cmpxchg_old;                       \
+                                                                             \
+    cmpxchg_done;                                                            \
+})
+
+#define this_cpu_cmpxchg_8(val, expected, desired)             \
+    X86_THIS_CPU_CMPXCHG("b", "q", u8, val, expected, desired)
+#define this_cpu_cmpxchg_16(val, expected, desired)             \
+    X86_THIS_CPU_CMPXCHG("w", "r", u16, val, expected, desired)
+#define this_cpu_cmpxchg_32(val, expected, desired)             \
+    X86_THIS_CPU_CMPXCHG("l", "r", u32, val, expected, desired)
+#define this_cpu_cmpxchg_64(val, expected, desired)             \
+    X86_THIS_CPU_CMPXCHG("q", "r", u64, val, expected, desired)
diff --git a/kernel/include/per_cpu.h b/kernel/include/per_cpu.h
index ebb9e524..09c2ce5f 100644
--- a/kernel/include/per_cpu.h
+++ b/kernel/include/per_cpu.h
@@ -65,6 +65,48 @@
     }                                        \
 })
 
+#define DISPATCH_CPU_XCHG_OP(var, x) ({       \
+    typeof(var) cxchg_ret;                    \
+    BUILD_BUG_ON_BAD_CPU_OP(var, xchg);       \
+                                              \
+    switch (sizeof(var)) {                    \
+    case 1:                                   \
+        cxchg_ret = this_cpu_xchg_8(var, x);  \
+        break;                                \
+    case 2:                                   \
+        cxchg_ret = this_cpu_xchg_16(var, x); \
+        break;                                \
+    case 4:                                   \
+        cxchg_ret = this_cpu_xchg_32(var, x); \
+        break;                                \
+    case 8:                                   \
+        cxchg_ret = this_cpu_xchg_64(var, x); \
+        break;                                \
+    }                                         \
+    cxchg_ret;                                \
+})
+
+#define DISPATCH_CPU_CMPXCHG_OP(var, expected, desired) ({          \
+    bool ccmpxchg_ret;                                              \
+    BUILD_BUG_ON_BAD_CPU_OP(var, cmpxchg);                          \
+                                                                    \
+    switch (sizeof(var)) {                                          \
+    case 1:                                                         \
+        ccmpxchg_ret = this_cpu_cmpxchg_8(var, expected, desired);  \
+        break;                                                      \
+    case 2:                                                         \
+        ccmpxchg_ret = this_cpu_cmpxchg_16(var, expected, desired); \
+        break;                                                      \
+    case 4:                                                         \
+        ccmpxchg_ret = this_cpu_cmpxchg_32(var, expected, desired); \
+        break;                                                      \
+    case 8:                                                         \
+        ccmpxchg_ret = this_cpu_cmpxchg_64(var, expected, desired); \
+        break;                                                      \
+    }                                                               \
+    ccmpxchg_ret;                                                   \
+})
+
 #define this_cpu_read(var) DISPATCH_CPU_READ_OP(var, read)
 #define this_cpu_write(var, x) DISPATCH_CPU_WRITE_OP(var, write, x)
 
@@ -73,3 +115,7 @@
 
 #define this_cpu_inc(var) this_cpu_add(var, 1)
 #define this_cpu_dec(var) this_cpu_sub(var, 1)
+
+#define this_cpu_xchg(var, x) DISPATCH_CPU_XCHG_OP(var, x)
+#define this_cpu_cmpxchg(var, expected, desired) \
+    DISPATCH_CPU_CMPXCHG_OP(var, expected, desired)
diff --git a/tests/synthetic-include/per_cpu.h b/tests/synthetic-include/per_cpu.h
index 5fca2e98..239d3b40 100644
--- a/tests/synthetic-include/per_cpu.h
+++ b/tests/synthetic-include/per_cpu.h
@@ -26,5 +26,25 @@ extern ptr_t g_per_cpu_offset[];
 #define this_cpu_inc(var) this_cpu_add(var, 1)
 #define this_cpu_dec(var) this_cpu_sub(var, 1)
 
+#define this_cpu_xchg(var, x) ({ \
+    typeof(var) cxchg_ret;       \
+                                 \
+    cxchg_ret = (var);           \
+    (var) = (x);                 \
+    cxchg_ret;                   \
+})
+
+#define this_cpu_cmpxchg(var, expected, desired) ({ \
+    bool ccmpxchg_ret;                              \
+                                                    \
+    ccmpxchg_ret = (var) == *(expected);            \
+    if (ccmpxchg_ret)                               \
+        (var) = (desired);                          \
+    else                                            \
+        *(expected) = (var);                        \
+                                                    \
+    ccmpxchg_ret;                                   \
+})
+
 #define per_cpu_ptr(ptr, cpu_number) \
     PTR_ADD_HIDE_UB(ptr, g_per_cpu_offset[cpu_number])

From 87335bef9e944efd76822f12d533719d6f49e93b Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 22 Oct 2026 15:28:52 +0300
Subject: [PATCH 038/152] build: generate asm offsets from C

Assembly that reaches into C structs relies on hand-written offsets
checked by BUILD_BUG_ON. They must be updated on every layout change
and cannot follow a field that depends on the config. Compute them at
build time from the arch's asm_offsets.c instead.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/CMakeLists.txt                     |  1 +
 kernel/asm_offsets.cmake                  | 42 +++++++++++++++++++++++
 kernel/include/private/arch/asm_offsets.h | 22 ++++++++++++
 scripts/generate_asm_offsets.py           | 38 ++++++++++++++++++++
 4 files changed, 103 insertions(+)
 create mode 100644 kernel/asm_offsets.cmake
 create mode 100644 kernel/include/private/arch/asm_offsets.h
 create mode 100755 scripts/generate_asm_offsets.py

diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt
index 761fe75b..9aeb15ba 100644
--- a/kernel/CMakeLists.txt
+++ b/kernel/CMakeLists.txt
@@ -6,6 +6,7 @@ include(target_helpers.cmake)
 include(module.cmake)
 include(ld_template.cmake)
 include(ubsan.cmake)
+include(asm_offsets.cmake)
 
 # The base kernel executable that contains an empty stub symbol table
 set(
diff --git a/kernel/asm_offsets.cmake b/kernel/asm_offsets.cmake
new file mode 100644
index 00000000..357e8c06
--- /dev/null
+++ b/kernel/asm_offsets.cmake
@@ -0,0 +1,42 @@
+function(setup_asm_offsets SOURCE)
+    set(OFFSETS_FILE "${ULTRA_GENERATED_DIR}/asm_offsets.h")
+    set(OFFSETS_SCRIPT "${ULTRA_SCRIPTS_DIR}/generate_asm_offsets.py")
+    set(OFFSETS_OBJECTS "kernel-asm-offsets")
+
+    add_library(${OFFSETS_OBJECTS} OBJECT ${SOURCE})
+    target_link_libraries(${OFFSETS_OBJECTS} PRIVATE ${ULTRA_KERNEL_IFACE})
+    set_property(
+        SOURCE ${SOURCE}
+        TARGET_DIRECTORY ${OFFSETS_OBJECTS}
+        APPEND
+        PROPERTY COMPILE_OPTIONS
+        -S -fno-lto -g0
+    )
+
+    # Clang warns about -c being unused since we override it with -S here
+    if (CONFIG_TOOLCHAIN_CLANG)
+        set_property(
+            SOURCE ${SOURCE}
+            TARGET_DIRECTORY ${OFFSETS_OBJECTS}
+            APPEND
+            PROPERTY COMPILE_OPTIONS
+            -Wno-unused-command-line-argument
+        )
+    endif ()
+
+    add_custom_command(
+        OUTPUT ${OFFSETS_FILE}
+        COMMAND
+        Python3::Interpreter ${OFFSETS_SCRIPT}
+        $<TARGET_OBJECTS:${OFFSETS_OBJECTS}> ${OFFSETS_FILE}
+        DEPENDS
+        ${OFFSETS_OBJECTS} $<TARGET_OBJECTS:${OFFSETS_OBJECTS}>
+        ${OFFSETS_SCRIPT}
+        COMMENT "Generating asm offsets..."
+        VERBATIM
+        COMMAND_EXPAND_LISTS
+    )
+
+    add_custom_target(asm-offsets DEPENDS ${OFFSETS_FILE})
+    add_dependencies(${ULTRA_KERNEL_OBJECTS} asm-offsets)
+endfunction()
diff --git a/kernel/include/private/arch/asm_offsets.h b/kernel/include/private/arch/asm_offsets.h
new file mode 100644
index 00000000..da913d6e
--- /dev/null
+++ b/kernel/include/private/arch/asm_offsets.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include <common/types.h>
+
+/*
+ * Make 'name' a macro for the compile-time constant 'value' in
+ * <generated/asm_offsets.h>.
+ */
+#define ASM_CONSTANT(name, value)                               \
+    asm volatile("\n.ascii \"->" #name " %c0\"" :: "i"(value))
+
+// ASM_CONSTANT() of the offset of 'member' within 'type'
+#define ASM_OFFSET(name, type, member) \
+    ASM_CONSTANT(name, offsetof(type, member))
+
+/*
+ * To be defined by the arch in the translation unit that is passed to
+ * setup_asm_offsets() (usually asm_offsets.c). This function is never
+ * called, it's only used as a container for all ASM_{CONSTANT,OFFSET}()
+ * constants an arch wants defined in <generated/asm_offsets.h>.
+ */
+void asm_offsets(void);
diff --git a/scripts/generate_asm_offsets.py b/scripts/generate_asm_offsets.py
new file mode 100755
index 00000000..9d3c3d73
--- /dev/null
+++ b/scripts/generate_asm_offsets.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import re
+
+
+def write_if_changed(path: str, content: str) -> None:
+    try:
+        with open(path, "r") as f:
+            if f.read() == content:
+                return
+    except FileNotFoundError:
+        pass
+
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w") as f:
+        f.write(content)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser("Generate the asm offsets header")
+    parser.add_argument("asm_file")
+    parser.add_argument("out_file")
+    args = parser.parse_args()
+
+    with open(args.asm_file, "r") as f:
+        constants = re.findall(r'"->(\w+) (\S+)"', f.read())
+
+    content = "#pragma once\n"
+    if constants:
+        content += "\n"
+        content += "".join(f"#define {k} {v}\n" for k, v in constants)
+
+    write_if_changed(args.out_file, content)
+
+
+if __name__ == "__main__":
+    main()

From 4e165353e1687c31a093ac0750e8f93ec4a3c35d Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 22 Oct 2026 15:28:52 +0300
Subject: [PATCH 039/152] arch/x86_64: generate the register offsets

The offsets of struct registers used by the interrupt entry code were
written by hand and checked against the struct by BUILD_BUG_ON. Compute
them from the struct instead.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/CMakeLists.txt             |  2 ++
 kernel/arch/x86_64/asm_offsets.c              | 33 +++++++++++++++++
 kernel/arch/x86_64/exceptions.c               | 24 -------------
 .../include/arch/private/asm_registers.h      | 36 +++----------------
 4 files changed, 39 insertions(+), 56 deletions(-)
 create mode 100644 kernel/arch/x86_64/asm_offsets.c

diff --git a/kernel/arch/x86_64/CMakeLists.txt b/kernel/arch/x86_64/CMakeLists.txt
index 88060f93..25a81652 100644
--- a/kernel/arch/x86_64/CMakeLists.txt
+++ b/kernel/arch/x86_64/CMakeLists.txt
@@ -25,6 +25,8 @@ ultra_sources(
     pci_type1.c
     tsc.c
 )
+setup_asm_offsets(asm_offsets.c)
+
 ultra_include_directories(include)
 
 check_c_compiler_flag(
diff --git a/kernel/arch/x86_64/asm_offsets.c b/kernel/arch/x86_64/asm_offsets.c
new file mode 100644
index 00000000..c8569432
--- /dev/null
+++ b/kernel/arch/x86_64/asm_offsets.c
@@ -0,0 +1,33 @@
+#include <private/arch/asm_offsets.h>
+
+#include <arch/registers.h>
+
+void asm_offsets(void)
+{
+    ASM_OFFSET(R15_OFFSET, struct registers, r15);
+    ASM_OFFSET(R14_OFFSET, struct registers, r14);
+    ASM_OFFSET(R13_OFFSET, struct registers, r13);
+    ASM_OFFSET(R12_OFFSET, struct registers, r12);
+    ASM_OFFSET(RBP_OFFSET, struct registers, rbp);
+    ASM_OFFSET(RBX_OFFSET, struct registers, rbx);
+
+    ASM_OFFSET(R11_OFFSET, struct registers, r11);
+    ASM_OFFSET(R10_OFFSET, struct registers, r10);
+    ASM_OFFSET(R9_OFFSET, struct registers, r9);
+    ASM_OFFSET(R8_OFFSET, struct registers, r8);
+    ASM_OFFSET(RAX_OFFSET, struct registers, rax);
+    ASM_OFFSET(RCX_OFFSET, struct registers, rcx);
+    ASM_OFFSET(RDX_OFFSET, struct registers, rdx);
+    ASM_OFFSET(RSI_OFFSET, struct registers, rsi);
+    ASM_OFFSET(RDI_OFFSET, struct registers, rdi);
+
+    ASM_OFFSET(AUX_OFFSET, struct registers, error_code);
+
+    ASM_OFFSET(RIP_OFFSET, struct registers, rip);
+    ASM_OFFSET(CS_OFFSET, struct registers, cs);
+    ASM_OFFSET(FLAGS_OFFSET, struct registers, flags);
+    ASM_OFFSET(RSP_OFFSET, struct registers, rsp);
+    ASM_OFFSET(SS_OFFSET, struct registers, ss);
+
+    ASM_CONSTANT(REGISTERS_SIZE, sizeof(struct registers));
+}
diff --git a/kernel/arch/x86_64/exceptions.c b/kernel/arch/x86_64/exceptions.c
index fa061b88..2a39aeee 100644
--- a/kernel/arch/x86_64/exceptions.c
+++ b/kernel/arch/x86_64/exceptions.c
@@ -7,35 +7,11 @@
 
 #include <arch/private/idt.h>
 #include <arch/registers.h>
-#include <arch/private/asm_registers.h>
 #include <arch/private/abortable_instructions.h>
 
 #include <private/arch/abortable_instructions.h>
 #include <private/arch/bug.h>
 
-// Ensure both ASM and C code have the same idea about register layout
-BUILD_BUG_ON(R15_OFFSET != offsetof(struct registers, r15));
-BUILD_BUG_ON(R14_OFFSET != offsetof(struct registers, r14));
-BUILD_BUG_ON(R13_OFFSET != offsetof(struct registers, r13));
-BUILD_BUG_ON(R12_OFFSET != offsetof(struct registers, r12));
-BUILD_BUG_ON(RBP_OFFSET != offsetof(struct registers, rbp));
-BUILD_BUG_ON(RBX_OFFSET != offsetof(struct registers, rbx));
-BUILD_BUG_ON(R11_OFFSET != offsetof(struct registers, r11));
-BUILD_BUG_ON(R10_OFFSET != offsetof(struct registers, r10));
-BUILD_BUG_ON(R9_OFFSET  != offsetof(struct registers, r9));
-BUILD_BUG_ON(R8_OFFSET  != offsetof(struct registers, r8));
-BUILD_BUG_ON(RAX_OFFSET != offsetof(struct registers, rax));
-BUILD_BUG_ON(RCX_OFFSET != offsetof(struct registers, rcx));
-BUILD_BUG_ON(RDX_OFFSET != offsetof(struct registers, rdx));
-BUILD_BUG_ON(RSI_OFFSET != offsetof(struct registers, rsi));
-BUILD_BUG_ON(RDI_OFFSET != offsetof(struct registers, rdi));
-BUILD_BUG_ON(AUX_OFFSET != offsetof(struct registers, error_code));
-BUILD_BUG_ON(RIP_OFFSET != offsetof(struct registers, rip));
-BUILD_BUG_ON(CS_OFFSET  != offsetof(struct registers, cs));
-BUILD_BUG_ON(FLAGS_OFFSET != offsetof(struct registers, flags));
-BUILD_BUG_ON(RSP_OFFSET != offsetof(struct registers, rsp));
-BUILD_BUG_ON(SS_OFFSET != offsetof(struct registers, ss));
-
 #define STUB_EXCEPTION(x)                             \
     EXCEPTION_HANDLER(x)                              \
     {                                                 \
diff --git a/kernel/arch/x86_64/include/arch/private/asm_registers.h b/kernel/arch/x86_64/include/arch/private/asm_registers.h
index 4b6a8bd2..4df7db97 100644
--- a/kernel/arch/x86_64/include/arch/private/asm_registers.h
+++ b/kernel/arch/x86_64/include/arch/private/asm_registers.h
@@ -1,37 +1,9 @@
 #pragma once
 
-#define REG_AFTER(x) ((x) + ULTRA_ARCH_WIDTH)
+#include <generated/asm_offsets.h>
 
-#define R15_OFFSET 0
-#define R14_OFFSET REG_AFTER(R15_OFFSET)
-#define R13_OFFSET REG_AFTER(R14_OFFSET)
-#define R12_OFFSET REG_AFTER(R13_OFFSET)
-#define RBP_OFFSET REG_AFTER(R12_OFFSET)
-#define RBX_OFFSET REG_AFTER(RBP_OFFSET)
-
-#define R11_OFFSET REG_AFTER(RBX_OFFSET)
-#define R10_OFFSET REG_AFTER(R11_OFFSET)
-#define R9_OFFSET  REG_AFTER(R10_OFFSET)
-#define R8_OFFSET  REG_AFTER(R9_OFFSET)
-#define RAX_OFFSET REG_AFTER(R8_OFFSET)
-#define RCX_OFFSET REG_AFTER(RAX_OFFSET)
-#define RDX_OFFSET REG_AFTER(RCX_OFFSET)
-#define RSI_OFFSET REG_AFTER(RDX_OFFSET)
-#define RDI_OFFSET REG_AFTER(RSI_OFFSET)
-
-#define GP_REGS_END_OFFSET REG_AFTER(RDI_OFFSET)
-
-#define AUX_OFFSET GP_REGS_END_OFFSET
-
-#define IRETQ_FRAME_OFFSET REG_AFTER(AUX_OFFSET)
-
-#define RIP_OFFSET   IRETQ_FRAME_OFFSET
-#define CS_OFFSET    REG_AFTER(RIP_OFFSET)
-#define FLAGS_OFFSET REG_AFTER(CS_OFFSET)
-#define RSP_OFFSET   REG_AFTER(FLAGS_OFFSET)
-#define SS_OFFSET    REG_AFTER(RSP_OFFSET)
-
-#define IRETQ_FRAME_END_OFFSET REG_AFTER(SS_OFFSET)
+#define GP_REGS_END_OFFSET AUX_OFFSET
+#define IRETQ_FRAME_OFFSET RIP_OFFSET
 
 #define INTERRUPT_FRAME_OFFSET(x) ((x) - IRETQ_FRAME_OFFSET)
-#define INTERRUPT_FRAME_SIZE (IRETQ_FRAME_END_OFFSET - IRETQ_FRAME_OFFSET)
+#define INTERRUPT_FRAME_SIZE (REGISTERS_SIZE - IRETQ_FRAME_OFFSET)

From 495fa983afd22218063db9b6567ca46abd7f8bd1 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 14:32:54 +0300
Subject: [PATCH 040/152] ubsan: move the recursion guard into struct task

A global guard made a report in one task silence the reports of every
other task for as long as it took to print.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/sched/task.h  | 10 ++++++++++
 kernel/include/private/sched/task.h | 14 ++++++++++++++
 kernel/ubsan.c                      | 21 +++++++++++++++------
 3 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/kernel/include/detail/sched/task.h b/kernel/include/detail/sched/task.h
index 50ac1a3b..4f6f869d 100644
--- a/kernel/include/detail/sched/task.h
+++ b/kernel/include/detail/sched/task.h
@@ -2,7 +2,9 @@
 
 #include <common/types.h>
 #include <common/helpers.h>
+#include <common/bit.h>
 
+#include <config.h>
 #include <sched/task.h>
 
 // We need to know whether this arch keeps preempt_count in struct task
@@ -12,7 +14,15 @@
 
 #define TASK_NAME_MAX 32
 
+enum task_flags : u32 {
+    TASK_FLAG_NONE = 0,
+#if IS_ENABLED(UBSAN)
+    TASK_FLAG_IN_UBSAN_REPORT = BIT_U32(31),
+#endif
+};
+
 struct task {
+    enum task_flags flags;
 #ifndef ARCH_HAS_CUSTOM_PREEMPT_COUNT
     u32 preempt_count;
 #endif
diff --git a/kernel/include/private/sched/task.h b/kernel/include/private/sched/task.h
index 7b9aa4d1..b8c1f587 100644
--- a/kernel/include/private/sched/task.h
+++ b/kernel/include/private/sched/task.h
@@ -1,6 +1,20 @@
 #pragma once
 
+#include <common/atomic.h>
+
 #include <per_cpu.h>
 #include <detail/sched/task.h>
 
 DECLARE_PER_CPU(struct task, g_this_cpu_idle_task);
+
+static inline bool task_test_and_set_flag(
+    struct task *task, enum task_flags flag
+)
+{
+    return atomic_fetch_or(&task->flags, flag, MO_ACQ_REL) & flag;
+}
+
+static inline void task_clear_flag(struct task *task, enum task_flags flag)
+{
+    atomic_fetch_and(&task->flags, ~flag, MO_RELEASE);
+}
diff --git a/kernel/ubsan.c b/kernel/ubsan.c
index 43fa85bd..c139da92 100644
--- a/kernel/ubsan.c
+++ b/kernel/ubsan.c
@@ -7,17 +7,16 @@
 
 #include <log.h>
 #include <panic.h>
+#include <sched/task.h>
 
 #include <private/bug.h>
+#include <private/sched/task.h>
 #include <private/ubsan.h>
 
 #define UBSAN_MAX_INT_WIDTH BITS_PER_TYPE(u128)
 #define UBSAN_COLUMN_REPORTED UINT32_MAX
 #define UBSAN_VALUE_STR_SIZE 48
 
-// TODO: move this into struct task once we have it
-static bool s_in_report;
-
 static u32 type_bit_width(const struct ubsan_type *type)
 {
     return BIT_U32(BIT_FIELD_READ(type->info, UBSAN_TYPE_INFO_LOG2_WIDTH));
@@ -100,11 +99,21 @@ static void format_value(
     );
 }
 
+static bool ubsan_enter(void)
+{
+    return !task_test_and_set_flag(current, TASK_FLAG_IN_UBSAN_REPORT);
+}
+
+static void ubsan_leave(void)
+{
+    task_clear_flag(current, TASK_FLAG_IN_UBSAN_REPORT);
+}
+
 static bool report_begin(struct ubsan_source_location *loc, const char *check)
 {
     u32 column;
 
-    if (atomic_xchg(&s_in_report, true, MO_ACQ_REL))
+    if (!ubsan_enter())
         return false;
 
     /*
@@ -113,7 +122,7 @@ static bool report_begin(struct ubsan_source_location *loc, const char *check)
      */
     column = atomic_xchg(&loc->column, UBSAN_COLUMN_REPORTED, MO_RELAXED);
     if (column == UBSAN_COLUMN_REPORTED) {
-        atomic_store_release(&s_in_report, false);
+        ubsan_leave();
         return false;
     }
 
@@ -124,7 +133,7 @@ static bool report_begin(struct ubsan_source_location *loc, const char *check)
 static void report_end(void)
 {
     finish_warn_report(nullptr);
-    atomic_store_release(&s_in_report, false);
+    ubsan_leave();
 }
 
 void __ubsan_handle_out_of_bounds(

From a078735d6cf415be623e554b979b6b584f53dece Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 14:33:04 +0300
Subject: [PATCH 041/152] kernel: introduce arch_switch_to()

Basic context switching primitives.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/CMakeLists.txt             |  1 +
 kernel/arch/x86_64/asm_offsets.c              |  3 ++
 kernel/arch/x86_64/include/arch/detail/task.h |  7 +++++
 kernel/arch/x86_64/task.S                     | 31 +++++++++++++++++++
 kernel/include/detail/sched/task.h            |  6 ++++
 kernel/include/private/arch/task.h            | 10 ++++++
 tests/synthetic-include/arch/detail/task.h    |  9 ++++++
 7 files changed, 67 insertions(+)
 create mode 100644 kernel/arch/x86_64/include/arch/detail/task.h
 create mode 100644 kernel/arch/x86_64/task.S
 create mode 100644 kernel/include/private/arch/task.h
 create mode 100644 tests/synthetic-include/arch/detail/task.h

diff --git a/kernel/arch/x86_64/CMakeLists.txt b/kernel/arch/x86_64/CMakeLists.txt
index 25a81652..2a1ac382 100644
--- a/kernel/arch/x86_64/CMakeLists.txt
+++ b/kernel/arch/x86_64/CMakeLists.txt
@@ -24,6 +24,7 @@ ultra_sources(
     earlycon.c
     pci_type1.c
     tsc.c
+    task.S
 )
 setup_asm_offsets(asm_offsets.c)
 
diff --git a/kernel/arch/x86_64/asm_offsets.c b/kernel/arch/x86_64/asm_offsets.c
index c8569432..6dc9b6eb 100644
--- a/kernel/arch/x86_64/asm_offsets.c
+++ b/kernel/arch/x86_64/asm_offsets.c
@@ -1,6 +1,7 @@
 #include <private/arch/asm_offsets.h>
 
 #include <arch/registers.h>
+#include <detail/sched/task.h>
 
 void asm_offsets(void)
 {
@@ -30,4 +31,6 @@ void asm_offsets(void)
     ASM_OFFSET(SS_OFFSET, struct registers, ss);
 
     ASM_CONSTANT(REGISTERS_SIZE, sizeof(struct registers));
+
+    ASM_OFFSET(TASK_ARCH_SP_OFFSET, struct task, arch.sp);
 }
diff --git a/kernel/arch/x86_64/include/arch/detail/task.h b/kernel/arch/x86_64/include/arch/detail/task.h
new file mode 100644
index 00000000..81c9826f
--- /dev/null
+++ b/kernel/arch/x86_64/include/arch/detail/task.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include <common/types.h>
+
+struct arch_task {
+    virt_addr_t sp;
+};
diff --git a/kernel/arch/x86_64/task.S b/kernel/arch/x86_64/task.S
new file mode 100644
index 00000000..3f7ad731
--- /dev/null
+++ b/kernel/arch/x86_64/task.S
@@ -0,0 +1,31 @@
+#include <arch/private/unwind_hints.h>
+#include <arch/private/asm_helpers.h>
+
+#include <generated/asm_offsets.h>
+
+ASM_PRELUDE
+
+.extern g_this_cpu_task
+
+ASM_GLOBAL_FUNCTION arch_switch_to
+    PUSH_REG_WITH_UNWIND_HINT(rbp)
+    PUSH_REG_WITH_UNWIND_HINT(rbx)
+    PUSH_REG_WITH_UNWIND_HINT(r12)
+    PUSH_REG_WITH_UNWIND_HINT(r13)
+    PUSH_REG_WITH_UNWIND_HINT(r14)
+    PUSH_REG_WITH_UNWIND_HINT(r15)
+
+    mov [rdi + TASK_ARCH_SP_OFFSET], rsp
+    mov rsp, [rsi + TASK_ARCH_SP_OFFSET]
+    mov gs:[rip + g_this_cpu_task], rsi
+
+    POP_REG_WITH_UNWIND_HINT(r15)
+    POP_REG_WITH_UNWIND_HINT(r14)
+    POP_REG_WITH_UNWIND_HINT(r13)
+    POP_REG_WITH_UNWIND_HINT(r12)
+    POP_REG_WITH_UNWIND_HINT(rbx)
+    POP_REG_WITH_UNWIND_HINT(rbp)
+
+    mov rax, rdi
+    ret
+ASM_FUNCTION_END arch_switch_to
diff --git a/kernel/include/detail/sched/task.h b/kernel/include/detail/sched/task.h
index 4f6f869d..5abc9c77 100644
--- a/kernel/include/detail/sched/task.h
+++ b/kernel/include/detail/sched/task.h
@@ -7,6 +7,8 @@
 #include <config.h>
 #include <sched/task.h>
 
+#include <arch/detail/task.h>
+
 // We need to know whether this arch keeps preempt_count in struct task
 #if HAS_INCLUDE(<arch/detail/preempt.h>)
 #include <arch/detail/preempt.h>
@@ -28,4 +30,8 @@ struct task {
 #endif
 
     char name[TASK_NAME_MAX];
+    void *stack;
+    size_t stack_size;
+
+    struct arch_task arch;
 };
diff --git a/kernel/include/private/arch/task.h b/kernel/include/private/arch/task.h
new file mode 100644
index 00000000..32770974
--- /dev/null
+++ b/kernel/include/private/arch/task.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <sched/task.h>
+
+/*
+ * Perform a context switch from 'prev' to 'next'. Once the CPU is
+ * yielded back to 'prev', the return value contains the task that
+ * performed the switch.
+ */
+struct task *arch_switch_to(struct task *prev, struct task *next);
diff --git a/tests/synthetic-include/arch/detail/task.h b/tests/synthetic-include/arch/detail/task.h
new file mode 100644
index 00000000..145485b4
--- /dev/null
+++ b/tests/synthetic-include/arch/detail/task.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <ucontext.h>
+
+struct arch_task {
+    ucontext_t context;
+    void (*fn)(void *user);
+    void *user;
+};

From b7a0b57f9dd1a943c6a6cbbb88d5d8c9adc807a5 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 14:46:35 +0300
Subject: [PATCH 042/152] linker: add helpers to declare linker symbols and
 take their addresses

Every linker symbol was declared as a hand-written extern array and
cast to ptr_t wherever its address was needed. Add
DECLARE_LINKER_SYMBOL(), DECLARE_SECTION_MARKERS() and
DECLARE_TYPED_SECTION_MARKERS() for the declarations,
LINKER_SYMBOL_ADDR() and SECTION_MARKER_{BEGIN,END}_ADDR() for the
addresses, and convert the existing users.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/abortable_instructions.c            |  6 +++---
 kernel/arch/x86_64/hypervisor/hypervisor.c |  3 +--
 kernel/bug.c                               |  4 +---
 kernel/include/linker.h                    | 11 +++++++++++
 kernel/include/private/param.h             |  7 ++-----
 kernel/include/symbols.h                   | 16 ++++++++--------
 kernel/memory/init.c                       | 12 ++++++------
 kernel/per_cpu.c                           |  5 ++---
 kernel/unwind.c                            |  6 +++---
 9 files changed, 37 insertions(+), 33 deletions(-)

diff --git a/kernel/abortable_instructions.c b/kernel/abortable_instructions.c
index 5d0207ba..e97ce52b 100644
--- a/kernel/abortable_instructions.c
+++ b/kernel/abortable_instructions.c
@@ -3,9 +3,9 @@
 
 #include <private/arch/abortable_instructions.h>
 
-extern const struct abortable_instruction
-SECTION_ARRAY_BEGIN(ABORTABLE_INSTRUCTIONS_SECTION)[],
-SECTION_ARRAY_END(ABORTABLE_INSTRUCTIONS_SECTION)[];
+DECLARE_TYPED_SECTION_MARKERS(
+    const struct abortable_instruction, ABORTABLE_INSTRUCTIONS_SECTION
+);
 
 static const struct abortable_instruction *find_abortable_instruction(reg_t ip)
 {
diff --git a/kernel/arch/x86_64/hypervisor/hypervisor.c b/kernel/arch/x86_64/hypervisor/hypervisor.c
index c681abc9..8e7ffe1a 100644
--- a/kernel/arch/x86_64/hypervisor/hypervisor.c
+++ b/kernel/arch/x86_64/hypervisor/hypervisor.c
@@ -13,8 +13,7 @@
 static struct active_hypervisor s_hypervisor;
 static char s_hypervisor_signature[13];
 
-extern const struct hypervisor SECTION_ARRAY_BEGIN(HYPERVISORS_SECTION)[];
-extern const struct hypervisor SECTION_ARRAY_END(HYPERVISORS_SECTION)[];
+DECLARE_TYPED_SECTION_MARKERS(const struct hypervisor, HYPERVISORS_SECTION);
 
 /*
  * A placeholder hypervisor that is used in case we detect the presence of a
diff --git a/kernel/bug.c b/kernel/bug.c
index 4e100f90..8f47f6d3 100644
--- a/kernel/bug.c
+++ b/kernel/bug.c
@@ -52,9 +52,7 @@ BUILD_BUG_ON(offsetof(struct bug_entry, file_disp) != 4);
 BUILD_BUG_ON(offsetof(struct bug_entry, line) != 8);
 BUILD_BUG_ON(offsetof(struct bug_entry, flags) != 10);
 
-extern struct bug_entry
-SECTION_ARRAY_BEGIN(BUG_TABLE_SECTION)[],
-SECTION_ARRAY_END(BUG_TABLE_SECTION)[];
+DECLARE_TYPED_SECTION_MARKERS(struct bug_entry, BUG_TABLE_SECTION);
 
 static struct bug_entry *find_bug_entry(reg_t pc)
 {
diff --git a/kernel/include/linker.h b/kernel/include/linker.h
index 8ab72dd4..06546ee0 100644
--- a/kernel/include/linker.h
+++ b/kernel/include/linker.h
@@ -3,12 +3,23 @@
 #include <common/helpers.h>
 
 #define LINKER_SYMBOL(x) CONCAT(g_linker_symbol_, x)
+#define DECLARE_LINKER_SYMBOL(x) extern u8 LINKER_SYMBOL(x)[]
+#define LINKER_SYMBOL_ADDR(x) ((ptr_t)LINKER_SYMBOL(x))
+
 #define SECTION_NAME(sec) TO_STR(.##sec)
 
 #define SECTION_MARKER_BEGIN(x) CONCAT(LINKER_SYMBOL(x), _begin)
 #define SECTION_MARKER_END(x) CONCAT(LINKER_SYMBOL(x), _end)
+#define SECTION_MARKER_BEGIN_ADDR(x) ((ptr_t)SECTION_MARKER_BEGIN(x))
+#define SECTION_MARKER_END_ADDR(x) ((ptr_t)SECTION_MARKER_END(x))
 #define SECTION_SIZE(x) (SECTION_MARKER_END(x) - SECTION_MARKER_BEGIN(x))
 
+#define DECLARE_TYPED_SECTION_MARKERS(type, x) \
+    extern type SECTION_MARKER_BEGIN(x)[];     \
+    extern type SECTION_MARKER_END(x)[]
+
+#define DECLARE_SECTION_MARKERS(x) DECLARE_TYPED_SECTION_MARKERS(u8, x)
+
 #define SECTION_ARRAY_BEGIN(x) SECTION_MARKER_BEGIN(x)
 #define SECTION_ARRAY_END(x) SECTION_MARKER_END(x)
 #define SECTION_ARRAY_SIZE(x) SECTION_SIZE(x)
diff --git a/kernel/include/private/param.h b/kernel/include/private/param.h
index 4f652677..b24538e2 100644
--- a/kernel/include/private/param.h
+++ b/kernel/include/private/param.h
@@ -3,8 +3,5 @@
 #include <param.h>
 #include <linker.h>
 
-extern struct param SECTION_ARRAY_BEGIN(PARAMETERS_SECTION)[];
-extern struct param SECTION_ARRAY_END(PARAMETERS_SECTION)[];
-
-extern struct param SECTION_ARRAY_BEGIN(FREE_AFTER_INIT_PARAMETERS_SECTION)[];
-extern struct param SECTION_ARRAY_END(FREE_AFTER_INIT_PARAMETERS_SECTION)[];
+DECLARE_TYPED_SECTION_MARKERS(struct param, PARAMETERS_SECTION);
+DECLARE_TYPED_SECTION_MARKERS(struct param, FREE_AFTER_INIT_PARAMETERS_SECTION);
diff --git a/kernel/include/symbols.h b/kernel/include/symbols.h
index d8f16c30..110d6101 100644
--- a/kernel/include/symbols.h
+++ b/kernel/include/symbols.h
@@ -3,22 +3,22 @@
 
 #include <linker.h>
 
-extern u8 LINKER_SYMBOL(text_begin)[];
-extern u8 LINKER_SYMBOL(text_end)[];
+DECLARE_LINKER_SYMBOL(text_begin);
+DECLARE_LINKER_SYMBOL(text_end);
 
-extern u8 LINKER_SYMBOL(eh_frame_hdr_begin)[];
-extern u8 LINKER_SYMBOL(eh_frame_hdr_end)[];
+DECLARE_LINKER_SYMBOL(eh_frame_hdr_begin);
+DECLARE_LINKER_SYMBOL(eh_frame_hdr_end);
 
-extern u8 LINKER_SYMBOL(eh_frame_begin)[];
-extern u8 LINKER_SYMBOL(eh_frame_end)[];
+DECLARE_LINKER_SYMBOL(eh_frame_begin);
+DECLARE_LINKER_SYMBOL(eh_frame_end);
 
 // 127 chars & a NULL terminator
 #define MAX_SYMBOL_LENGTH (127 + 1)
 
 static inline bool address_is_kernel_code(ptr_t address)
 {
-    return address >= (ptr_t)LINKER_SYMBOL(text_begin) &&
-           address <= (ptr_t)LINKER_SYMBOL(text_end);
+    return address >= LINKER_SYMBOL_ADDR(text_begin) &&
+           address <= LINKER_SYMBOL_ADDR(text_end);
 }
 
 error_t symbol_lookup_by_address(
diff --git a/kernel/memory/init.c b/kernel/memory/init.c
index e5452441..20816beb 100644
--- a/kernel/memory/init.c
+++ b/kernel/memory/init.c
@@ -496,14 +496,14 @@ static void INIT_CODE direct_map_one(
     );
 }
 
-extern u8 LINKER_SYMBOL(executable_data_begin)[];
-extern u8 LINKER_SYMBOL(executable_data_end)[];
+DECLARE_LINKER_SYMBOL(executable_data_begin);
+DECLARE_LINKER_SYMBOL(executable_data_end);
 
-extern u8 LINKER_SYMBOL(readonly_data_begin)[];
-extern u8 LINKER_SYMBOL(readonly_data_end)[];
+DECLARE_LINKER_SYMBOL(readonly_data_begin);
+DECLARE_LINKER_SYMBOL(readonly_data_end);
 
-extern u8 LINKER_SYMBOL(readwrite_data_begin)[];
-extern u8 LINKER_SYMBOL(readwrite_data_end)[];
+DECLARE_LINKER_SYMBOL(readwrite_data_begin);
+DECLARE_LINKER_SYMBOL(readwrite_data_end);
 
 static void INIT_CODE map_kernel_segment(
     struct direct_mapping_ctx *ctx, struct ultra_kernel_info_attribute *ki,
diff --git a/kernel/per_cpu.c b/kernel/per_cpu.c
index a203a6d8..612da683 100644
--- a/kernel/per_cpu.c
+++ b/kernel/per_cpu.c
@@ -22,8 +22,7 @@
 ptr_t g_per_cpu_offset[ULTRA_MAX_CPUS];
 virt_addr_t g_per_cpu_base;
 
-extern u8 SECTION_MARKER_BEGIN(PER_CPU_SECTION)[];
-extern u8 SECTION_MARKER_END(PER_CPU_SECTION)[];
+DECLARE_SECTION_MARKERS(PER_CPU_SECTION);
 
 static error_t INIT_CODE per_cpu_setup(void)
 {
@@ -57,7 +56,7 @@ static error_t INIT_CODE per_cpu_setup(void)
      * and the new allocated per-cpu section, which is duplicated for each CPU.
      */
     this_cpu_offset = g_per_cpu_base;
-    this_cpu_offset -= (ptr_t)SECTION_MARKER_BEGIN(PER_CPU_SECTION);
+    this_cpu_offset -= SECTION_MARKER_BEGIN_ADDR(PER_CPU_SECTION);
 
     for_each_present_cpu(i) {
         g_per_cpu_offset[i] = this_cpu_offset;
diff --git a/kernel/unwind.c b/kernel/unwind.c
index 04944192..7c66e8cc 100644
--- a/kernel/unwind.c
+++ b/kernel/unwind.c
@@ -169,10 +169,10 @@ static error_t decode_value(
         *out_value = (ptr_t)data->cursor;
         break;
     case DW_EH_PE_textrel:
-        *out_value = (ptr_t)LINKER_SYMBOL(text_begin);
+        *out_value = LINKER_SYMBOL_ADDR(text_begin);
         break;
     case DW_EH_PE_datarel:
-        *out_value = (ptr_t)LINKER_SYMBOL(eh_frame_hdr_begin);
+        *out_value = LINKER_SYMBOL_ADDR(eh_frame_hdr_begin);
         break;
     case DW_EH_PE_funcrel:
     case DW_EH_PE_aligned:
@@ -255,7 +255,7 @@ static error_t INIT_CODE unwind_init(void)
     if (is_error(ret))
         return ret;
 
-    if (unlikely((ptr_t)LINKER_SYMBOL(eh_frame_begin) != value))
+    if (unlikely(LINKER_SYMBOL_ADDR(eh_frame_begin) != value))
         return EINVAL;
 
     ret = decode_value(&data, fde_count_encoding, &s_num_fdes);

From a97c9155f2d0a131ce0f3dc9066ec361cb2bfc09 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 8 Nov 2026 15:50:51 +0300
Subject: [PATCH 043/152] linker: add for_each_section_array_item()

A walk over a section array was either a for loop over its begin and
end markers, split across two lines, or an index loop that took the
address of each item. Add for_each_section_array_item() and convert the
existing loops.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/abortable_instructions.c            | 5 +----
 kernel/arch/x86_64/hypervisor/hypervisor.c | 3 +--
 kernel/bug.c                               | 5 +----
 kernel/include/linker.h                    | 5 +++++
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/kernel/abortable_instructions.c b/kernel/abortable_instructions.c
index e97ce52b..3bc2d81b 100644
--- a/kernel/abortable_instructions.c
+++ b/kernel/abortable_instructions.c
@@ -9,12 +9,9 @@ DECLARE_TYPED_SECTION_MARKERS(
 
 static const struct abortable_instruction *find_abortable_instruction(reg_t ip)
 {
-    ssize_t i;
     const struct abortable_instruction *ai;
 
-    for (i = 0; i < SECTION_ARRAY_SIZE(ABORTABLE_INSTRUCTIONS_SECTION); i++) {
-        ai = &SECTION_ARRAY_BEGIN(ABORTABLE_INSTRUCTIONS_SECTION)[i];
-
+    for_each_section_array_item(ai, ABORTABLE_INSTRUCTIONS_SECTION) {
         if (SELF_RELATIVE_TARGET(ai->try_pc_disp) == ip)
             return ai;
     }
diff --git a/kernel/arch/x86_64/hypervisor/hypervisor.c b/kernel/arch/x86_64/hypervisor/hypervisor.c
index 8e7ffe1a..032f95cc 100644
--- a/kernel/arch/x86_64/hypervisor/hypervisor.c
+++ b/kernel/arch/x86_64/hypervisor/hypervisor.c
@@ -74,8 +74,7 @@ void INIT_CODE x86_detect_hypervisor(void)
 
     cpuid_has_hypervisor = all_cpus_have(X86_FEATURE_HYPERVISOR);
 
-    for (hv = SECTION_ARRAY_BEGIN(HYPERVISORS_SECTION);
-         hv < SECTION_ARRAY_END(HYPERVISORS_SECTION); ++hv) {
+    for_each_section_array_item(hv, HYPERVISORS_SECTION) {
         u32 this_priority = 0;
 
         if (cpuid_has_hypervisor && hv->max_leaf >= BASE_HYPERVISOR_LEAF)
diff --git a/kernel/bug.c b/kernel/bug.c
index 8f47f6d3..04e32c2f 100644
--- a/kernel/bug.c
+++ b/kernel/bug.c
@@ -56,12 +56,9 @@ DECLARE_TYPED_SECTION_MARKERS(struct bug_entry, BUG_TABLE_SECTION);
 
 static struct bug_entry *find_bug_entry(reg_t pc)
 {
-    ssize_t i;
     struct bug_entry *entry;
 
-    for (i = 0; i < SECTION_ARRAY_SIZE(BUG_TABLE_SECTION); i++) {
-        entry = &SECTION_ARRAY_BEGIN(BUG_TABLE_SECTION)[i];
-
+    for_each_section_array_item(entry, BUG_TABLE_SECTION) {
         if (SELF_RELATIVE_TARGET(entry->pc_disp) == pc)
             return entry;
     }
diff --git a/kernel/include/linker.h b/kernel/include/linker.h
index 06546ee0..4ef2e624 100644
--- a/kernel/include/linker.h
+++ b/kernel/include/linker.h
@@ -27,6 +27,11 @@
 #define SECTION_ARRAY_ARGS(x) \
     SECTION_ARRAY_BEGIN(x), SECTION_ARRAY_SIZE(x)
 
+#define for_each_section_array_item(item, x) \
+    for ((item) = SECTION_ARRAY_BEGIN(x);    \
+         (item) < SECTION_ARRAY_END(x);      \
+         (item)++)
+
 #define ABORTABLE_INSTRUCTIONS_SECTION abortable_instructions
 #define BUG_TABLE_SECTION bug_table
 #define PARAMETERS_SECTION parameters

From 81d5d8f7d603e016aefa109ebf7afa2df6e10523 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 10 Nov 2026 23:17:37 +0300
Subject: [PATCH 044/152] linker: make SECTION_ARRAY_SIZE() a size_t

SECTION_ARRAY_SIZE() is the difference of two pointers, which is a
signed integer (ptrdiff_t). The users of this macro expect the number
of items in a section, which cannot be negative and makes the UX worse
for no reason, so make this a size_t instead.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/linker.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel/include/linker.h b/kernel/include/linker.h
index 4ef2e624..8ebc0992 100644
--- a/kernel/include/linker.h
+++ b/kernel/include/linker.h
@@ -22,7 +22,7 @@
 
 #define SECTION_ARRAY_BEGIN(x) SECTION_MARKER_BEGIN(x)
 #define SECTION_ARRAY_END(x) SECTION_MARKER_END(x)
-#define SECTION_ARRAY_SIZE(x) SECTION_SIZE(x)
+#define SECTION_ARRAY_SIZE(x) ((size_t)SECTION_SIZE(x))
 
 #define SECTION_ARRAY_ARGS(x) \
     SECTION_ARRAY_BEGIN(x), SECTION_ARRAY_SIZE(x)

From a3c0090d1d55fe3d99829f69e68d4b5452ff9f7d Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 14:46:35 +0300
Subject: [PATCH 045/152] tests: drop macOS unit-testing support

The unit tests are about to link with an ELF linker script fragment,
which Mach-O and the macOS linker cannot do. Drop the mach timer and
the override of a host PAGE_SIZE, make --unit-tests refuse to run
anywhere but Linux, and stop running them on macOS in CI.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .github/workflows/main.yml               |  7 +------
 build.py                                 |  4 ++++
 tests/synthetic-include/arch_constants.h |  8 --------
 tests/test_harness.cpp                   | 22 ----------------------
 4 files changed, 5 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3418745c..323a8cd3 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -96,12 +96,7 @@ jobs:
         run: "${{ github.workspace }}/build.py --arch ${{ matrix.arch }} --config ${{ matrix.preset }} --toolchain ${{ matrix.toolchain }} --no-tune-native"
 
   unit-tests:
-    runs-on: ${{ matrix.os }}
-
-    strategy:
-      fail-fast: true
-      matrix:
-        os: [ubuntu-latest, macos-latest]
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
diff --git a/build.py b/build.py
index 1028a309..69b5cc7a 100755
--- a/build.py
+++ b/build.py
@@ -584,6 +584,10 @@ def main() -> None:
         sys.exit(1)
 
     if args.unit_tests:
+        if this_os != "Linux":
+            print("Unit tests only support Linux hosts")
+            sys.exit(1)
+
         sys.exit(run_unit_tests(args, this_os.lower()))
 
     rc = bmc.early_action(args, hyper_pxe_loaders)
diff --git a/tests/synthetic-include/arch_constants.h b/tests/synthetic-include/arch_constants.h
index 53c467c3..daf7a604 100644
--- a/tests/synthetic-include/arch_constants.h
+++ b/tests/synthetic-include/arch_constants.h
@@ -2,13 +2,5 @@
 
 #include <common/types.h>
 
-/*
- * Undef first: host headers may leak their own definitions into harness
- * translation units, e.g. the mach headers on arm64 macOS, where the host
- * page size is 16K.
- */
-#undef PAGE_SIZE
-#undef PAGE_SHIFT
-
 #define PAGE_SHIFT 12
 #define PAGE_SIZE (LITERAL_OF_TYPE(reg_t, 1) << PAGE_SHIFT)
diff --git a/tests/test_harness.cpp b/tests/test_harness.cpp
index 59103a3e..11cd409c 100644
--- a/tests/test_harness.cpp
+++ b/tests/test_harness.cpp
@@ -3,20 +3,10 @@
 #include <set>
 #include <filesystem>
 
-#ifdef __APPLE__
-#include <mach/mach_time.h>
-#else
 #include <ctime>
-#endif
 
-/*
- * Included after any host headers so that the synthetic <arch_constants.h>
- * (pulled in here) overrides a possibly leaked host PAGE_SIZE.
- */
 #include <common/align.h>
 
-static_assert(PAGE_SIZE == 4096, "host PAGE_SIZE leaked into the harness");
-
 /*
  * struct page pulls in <common/bit.h>'s meta bit-field macros, which are not
  * valid C++, so the memory map is owned by memory_map.c and driven here through
@@ -226,20 +216,8 @@ void teardown_callback_register(void (*callback)(void), const char *file)
 
 uint64_t ns_timer()
 {
-#if defined(__APPLE__)
-    static struct mach_timebase_info tb;
-    static bool initialized;
-
-    if (!initialized) {
-        ASSERT_EQ(mach_timebase_info(&tb), KERN_SUCCESS);
-        initialized = true;
-    }
-
-    return (mach_absolute_time() * tb.numer) / tb.denom;
-#else
     struct timespec ts;
 
     ASSERT_EQ(clock_gettime(CLOCK_MONOTONIC, &ts), 0);
     return ts.tv_sec * NANOSECONDS_PER_SECOND + ts.tv_nsec;
-#endif
 }

From 5bec144360134e187c59266c494f8b0da0fe70f2 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 14:46:35 +0300
Subject: [PATCH 046/152] sched: introduce runqueues and scheduling classes

Keep policy out of the scheduler core. A per-CPU runqueue asks an
ordered table of classes for the next task: round-robin with a FIFO per
priority, then an idle class that always returns the CPU's idle task, so
a pick never fails.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/sched/rr.h       |   8 ++
 kernel/include/detail/sched/task.h     |  12 ++
 kernel/include/linker_common.ld.h      |  10 ++
 kernel/include/private/sched/classes.h |  21 ++++
 kernel/include/private/sched/core.h    |  71 ++++++++++++
 kernel/include/private/sched/idle.h    |   7 ++
 kernel/include/private/sched/rr.h      |  19 ++++
 kernel/include/private/sched/task.h    |   6 +
 kernel/sched/CMakeLists.txt            |   3 +
 kernel/sched/core.c                    |  75 ++++++++++++
 kernel/sched/idle.c                    |  33 ++++++
 kernel/sched/rr.c                      |  50 ++++++++
 tests/CMakeLists.txt                   |  38 +++++++
 tests/link.ld.template                 |   7 ++
 tests/synthetic-include/per_cpu.h      |   2 +
 tests/test-cases/files.cmake           |   1 +
 tests/test-cases/test_sched.c          | 151 +++++++++++++++++++++++++
 17 files changed, 514 insertions(+)
 create mode 100644 kernel/include/detail/sched/rr.h
 create mode 100644 kernel/include/private/sched/classes.h
 create mode 100644 kernel/include/private/sched/core.h
 create mode 100644 kernel/include/private/sched/idle.h
 create mode 100644 kernel/include/private/sched/rr.h
 create mode 100644 kernel/sched/core.c
 create mode 100644 kernel/sched/idle.c
 create mode 100644 kernel/sched/rr.c
 create mode 100644 tests/link.ld.template
 create mode 100644 tests/test-cases/test_sched.c

diff --git a/kernel/include/detail/sched/rr.h b/kernel/include/detail/sched/rr.h
new file mode 100644
index 00000000..9906dd86
--- /dev/null
+++ b/kernel/include/detail/sched/rr.h
@@ -0,0 +1,8 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/list.h>
+
+struct rr_entity {
+    struct list_link link;
+};
diff --git a/kernel/include/detail/sched/task.h b/kernel/include/detail/sched/task.h
index 5abc9c77..6d3e853e 100644
--- a/kernel/include/detail/sched/task.h
+++ b/kernel/include/detail/sched/task.h
@@ -7,6 +7,7 @@
 #include <config.h>
 #include <sched/task.h>
 
+#include <detail/sched/rr.h>
 #include <arch/detail/task.h>
 
 // We need to know whether this arch keeps preempt_count in struct task
@@ -14,6 +15,8 @@
 #include <arch/detail/preempt.h>
 #endif
 
+struct sched_class;
+
 #define TASK_NAME_MAX 32
 
 enum task_flags : u32 {
@@ -29,6 +32,15 @@ struct task {
     u32 preempt_count;
 #endif
 
+    cpu_t cpu;
+    u8 prio;
+    bool on_rq;
+    const struct sched_class *sched_class;
+
+    union {
+        struct rr_entity rr;
+    };
+
     char name[TASK_NAME_MAX];
     void *stack;
     size_t stack_size;
diff --git a/kernel/include/linker_common.ld.h b/kernel/include/linker_common.ld.h
index 63a4e57a..23a6e4ad 100644
--- a/kernel/include/linker_common.ld.h
+++ b/kernel/include/linker_common.ld.h
@@ -4,6 +4,7 @@
 
 #include <arch_constants.h>
 #include <detail/init_level.h>
+#include <private/sched/classes.h>
 
 #define PHDR_READ  (1 << 2)
 #define PHDR_WRITE (1 << 1)
@@ -72,9 +73,18 @@
 
 #define INIT_LEVEL_CB_SECTIONS INIT_LEVELS
 
+#define SCHED_CLASS(x) SPECIAL_SECTION(SCHED_CLASS_SECTION(x))
+
+#define SCHED_CLASS_SECTIONS                         \
+    . = ALIGN(ULTRA_ARCH_WIDTH);                     \
+    SECTION_MARKER_BEGIN(SCHED_CLASSES_SECTION) = .; \
+    SCHED_CLASSES                                    \
+    SECTION_MARKER_END(SCHED_CLASSES_SECTION) = .;
+
 #define RODATA                                           \
     *(.rodata .rodata.*)                                 \
     *(.INIT_DATA_REFERENCE_RODATA_SECTION)               \
+    SCHED_CLASS_SECTIONS                                 \
     NATURALLY_ALIGNED_MARKED_SECTION(PARAMETERS_SECTION)
 
 #define RODATA_FREE_AFTER_INIT_BEGIN \
diff --git a/kernel/include/private/sched/classes.h b/kernel/include/private/sched/classes.h
new file mode 100644
index 00000000..b2c861f1
--- /dev/null
+++ b/kernel/include/private/sched/classes.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include <common/helpers.h>
+
+#define SCHED_CLASSES_SECTION sched_classes
+#define SCHED_CLASS_SECTION(name) CONCAT(sched_class_, name)
+
+#define SCHED_CLASS_INSTANCE(name) g_##name##_sched_class
+#define DECLARE_SCHED_CLASS(name)                              \
+    extern const struct sched_class SCHED_CLASS_INSTANCE(name)
+
+/*
+ * All scheduling classes ordered by rank, higher first.
+ *
+ * NOTE: This ordering is further enforced by the linker, a higher rank class is
+ *       always placed at a lower address. We use this fact for O(1) comparisons
+ *       between the classes of different tasks.
+ */
+#define SCHED_CLASSES \
+    SCHED_CLASS(rr)   \
+    SCHED_CLASS(idle)
diff --git a/kernel/include/private/sched/core.h b/kernel/include/private/sched/core.h
new file mode 100644
index 00000000..82e06c61
--- /dev/null
+++ b/kernel/include/private/sched/core.h
@@ -0,0 +1,71 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/attributes.h>
+
+#include <per_cpu.h>
+#include <linker.h>
+#include <locking/spinlock.h>
+
+#include <private/sched/rr.h>
+#include <private/sched/classes.h>
+#include <private/sched/task.h>
+
+struct rq;
+
+/*
+ * A set of operations of one scheduling algorithm. Every task belongs to
+ * exactly one of these classes. Each class keeps its own state in the runqueue
+ * and is responsible for ordering all tasks that belong to it.
+ *
+ * The set of classes is fixed at build time and the scheduling core orders them
+ * internally by rank. A class only gets to pick when every class above it has
+ * nothing runnable
+ *
+ * Every operation is called with the runqueue locked and interrupts disabled.
+ */
+struct sched_class {
+    /*
+     * Add a task that became runnable, or remove one that stopped being
+     * runnable. A task stays queued while it runs.
+     */
+    void (*enqueue)(struct rq *rq, struct task *task);
+    void (*dequeue)(struct rq *rq, struct task *task);
+
+    /*
+     * Return the task this class wants to run next, leaving it queued, or
+     * nullptr if the class has nothing runnable.
+     */
+    struct task *(*pick)(struct rq *rq);
+};
+
+#define DEFINE_SCHED_CLASS(name)                                      \
+    SECTION_VAR(SCHED_CLASS_SECTION(name), const, struct sched_class) \
+    SCHED_CLASS_INSTANCE(name)
+
+DECLARE_TYPED_SECTION_MARKERS(const struct sched_class, SCHED_CLASSES_SECTION);
+
+struct rq {
+    struct spinlock lock;
+
+    struct task *curr;
+    struct task *idle;
+
+    /*
+     * Number of runnable tasks on this runqueue, including the currently
+     * running task. The idle task is not accounted here.
+     */
+    u32 nr_running;
+
+    struct rr_rq rr;
+};
+
+DECLARE_PER_CPU(struct rq, g_this_cpu_rq);
+
+#define this_rq() this_cpu_ptr(&g_this_cpu_rq)
+#define cpu_rq(cpu) per_cpu_ptr(&g_this_cpu_rq, cpu)
+#define task_rq(task) cpu_rq((task)->cpu)
+
+void rq_enqueue(struct rq *rq, struct task *task);
+void rq_dequeue(struct rq *rq, struct task *task);
+struct task *rq_pick_next(struct rq *rq);
diff --git a/kernel/include/private/sched/idle.h b/kernel/include/private/sched/idle.h
new file mode 100644
index 00000000..2aa1f396
--- /dev/null
+++ b/kernel/include/private/sched/idle.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include <private/sched/classes.h>
+
+struct sched_class;
+
+DECLARE_SCHED_CLASS(idle);
diff --git a/kernel/include/private/sched/rr.h b/kernel/include/private/sched/rr.h
new file mode 100644
index 00000000..ffef2746
--- /dev/null
+++ b/kernel/include/private/sched/rr.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/list.h>
+
+#include <sched/task.h>
+#include <private/sched/task.h>
+#include <private/sched/classes.h>
+
+struct sched_class;
+
+struct rr_rq {
+    struct list_link queues[TASK_NUM_PRIOS];
+    u32 non_empty_queues_mask;
+};
+
+DECLARE_SCHED_CLASS(rr);
+
+void rr_rq_init(struct rr_rq *rr);
diff --git a/kernel/include/private/sched/task.h b/kernel/include/private/sched/task.h
index b8c1f587..43967bf1 100644
--- a/kernel/include/private/sched/task.h
+++ b/kernel/include/private/sched/task.h
@@ -5,6 +5,12 @@
 #include <per_cpu.h>
 #include <detail/sched/task.h>
 
+// Lower priority value runs first
+#define TASK_PRIO_HIGHEST 0
+#define TASK_PRIO_DEFAULT 16
+#define TASK_PRIO_LOWEST 31
+#define TASK_NUM_PRIOS (TASK_PRIO_LOWEST + 1)
+
 DECLARE_PER_CPU(struct task, g_this_cpu_idle_task);
 
 static inline bool task_test_and_set_flag(
diff --git a/kernel/sched/CMakeLists.txt b/kernel/sched/CMakeLists.txt
index 7965cf08..815dd636 100644
--- a/kernel/sched/CMakeLists.txt
+++ b/kernel/sched/CMakeLists.txt
@@ -1,3 +1,6 @@
 ultra_sources(
     task.c
+    core.c
+    rr.c
+    idle.c
 )
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
new file mode 100644
index 00000000..6ad8a222
--- /dev/null
+++ b/kernel/sched/core.c
@@ -0,0 +1,75 @@
+#include <private/sched/core.h>
+#include <private/sched/rr.h>
+#include <private/sched/idle.h>
+
+#include <common/helpers.h>
+
+#include <bug.h>
+#include <linker.h>
+#include <smp.h>
+#include <init_level.h>
+#include <free_after_init.h>
+
+DEFINE_PER_CPU(struct rq, g_this_cpu_rq);
+
+void rq_enqueue(struct rq *rq, struct task *task)
+{
+    BUG_ON(!spin_is_locked(&rq->lock));
+    BUG_ON(task->on_rq);
+
+    task->sched_class->enqueue(rq, task);
+    task->on_rq = true;
+    rq->nr_running++;
+}
+
+void rq_dequeue(struct rq *rq, struct task *task)
+{
+    BUG_ON(!spin_is_locked(&rq->lock));
+    BUG_ON(!task->on_rq);
+
+    task->sched_class->dequeue(rq, task);
+    task->on_rq = false;
+    rq->nr_running--;
+}
+
+struct task *rq_pick_next(struct rq *rq)
+{
+    const struct sched_class *class;
+    struct task *task;
+
+    BUG_ON(!spin_is_locked(&rq->lock));
+
+    for_each_section_array_item(class, SCHED_CLASSES_SECTION) {
+        task = class->pick(rq);
+        if (task != nullptr)
+            return task;
+    }
+
+    BUG();
+}
+
+static void rq_init(struct rq *rq, cpu_t cpu)
+{
+    struct task *idle;
+
+    idle = per_cpu_ptr(&g_this_cpu_idle_task, cpu);
+    idle->cpu = cpu;
+    idle->sched_class = &SCHED_CLASS_INSTANCE(idle);
+
+    spin_lock_init(&rq->lock);
+    rr_rq_init(&rq->rr);
+
+    rq->idle = idle;
+    rq->curr = idle;
+}
+
+static error_t INIT_CODE sched_init(void)
+{
+    cpu_t cpu;
+
+    for_each_present_cpu(cpu)
+        rq_init(cpu_rq(cpu), cpu);
+
+    return EOK;
+}
+INIT_CALL_POST(PER_CPU_AVAILABLE, sched_init);
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
new file mode 100644
index 00000000..592a749b
--- /dev/null
+++ b/kernel/sched/idle.c
@@ -0,0 +1,33 @@
+#include <private/sched/core.h>
+#include <private/sched/idle.h>
+
+#include <common/helpers.h>
+
+#include <bug.h>
+
+static void idle_enqueue(struct rq *rq, struct task *task)
+{
+    UNREFERENCED_PARAMETER(rq);
+    UNREFERENCED_PARAMETER(task);
+
+    BUG();
+}
+
+static void idle_dequeue(struct rq *rq, struct task *task)
+{
+    UNREFERENCED_PARAMETER(rq);
+    UNREFERENCED_PARAMETER(task);
+
+    BUG();
+}
+
+static struct task *idle_pick(struct rq *rq)
+{
+    return rq->idle;
+}
+
+DEFINE_SCHED_CLASS(idle) = {
+    .enqueue = idle_enqueue,
+    .dequeue = idle_dequeue,
+    .pick = idle_pick,
+};
diff --git a/kernel/sched/rr.c b/kernel/sched/rr.c
new file mode 100644
index 00000000..6fcd38b8
--- /dev/null
+++ b/kernel/sched/rr.c
@@ -0,0 +1,50 @@
+#include <private/sched/core.h>
+#include <private/sched/rr.h>
+
+#include <common/bit.h>
+
+void rr_rq_init(struct rr_rq *rr)
+{
+    size_t i;
+
+    for (i = 0; i < TASK_NUM_PRIOS; i++)
+        list_init(&rr->queues[i]);
+
+    rr->non_empty_queues_mask = 0;
+}
+
+static void rr_enqueue(struct rq *rq, struct task *task)
+{
+    struct rr_rq *rr = &rq->rr;
+
+    list_insert_prev(&rr->queues[task->prio], &task->rr.link);
+    rr->non_empty_queues_mask |= BIT_U32(task->prio);
+}
+
+static void rr_dequeue(struct rq *rq, struct task *task)
+{
+    struct rr_rq *rr = &rq->rr;
+
+    list_remove(&task->rr.link);
+
+    if (list_is_empty(&rr->queues[task->prio]))
+        rr->non_empty_queues_mask &= ~BIT_U32(task->prio);
+}
+
+static struct task *rr_pick(struct rq *rq)
+{
+    struct rr_rq *rr = &rq->rr;
+    u32 prio;
+
+    if (rr->non_empty_queues_mask == 0)
+        return nullptr;
+
+    prio = BIT_FIND_FIRST_SET(rr->non_empty_queues_mask) - 1;
+    return list_first_entry(&rr->queues[prio], struct task, rr.link);
+}
+
+DEFINE_SCHED_CLASS(rr) = {
+    .enqueue = rr_enqueue,
+    .dequeue = rr_dequeue,
+    .pick = rr_pick,
+};
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 0026314f..d9507ae5 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -191,17 +191,32 @@ KERNEL_FILE(INCLUDE_FILE "bug.h")
 KERNEL_FILE(INCLUDE_PATH "detail" INCLUDE_FILE "bug.h")
 KERNEL_FILE(INCLUDE_FILE "panic.h")
 KERNEL_FILE(INCLUDE_FILE "linker.h")
+KERNEL_FILE(INCLUDE_FILE "linker_common.ld.h")
 KERNEL_FILE(INCLUDE_PATH "locking" INCLUDE_FILE "seq.h")
 KERNEL_FILE(INCLUDE_PATH "detail" INCLUDE_FILE "init_level.h")
 KERNEL_FILE(INCLUDE_FILE "preempt.h")
 KERNEL_FILE(INCLUDE_PATH "detail" INCLUDE_FILE "preempt.h")
 KERNEL_FILE(INCLUDE_PATH "private/arch" INCLUDE_FILE "preempt.h")
+KERNEL_FILE(INCLUDE_PATH "detail/sched" INCLUDE_FILE "rr.h")
 KERNEL_FILE(
     INCLUDE_PATH "sched" INCLUDE_FILE "task.h"
     SOURCE_PATH "sched" SOURCE_FILE "task.c"
 )
 KERNEL_FILE(INCLUDE_PATH "detail/sched" INCLUDE_FILE "task.h")
 KERNEL_FILE(INCLUDE_PATH "private/sched" INCLUDE_FILE "task.h")
+KERNEL_FILE(
+    INCLUDE_PATH "private/sched" INCLUDE_FILE "core.h"
+    SOURCE_PATH "sched" SOURCE_FILE "core.c"
+)
+KERNEL_FILE(INCLUDE_PATH "private/sched" INCLUDE_FILE "classes.h")
+KERNEL_FILE(
+    INCLUDE_PATH "private/sched" INCLUDE_FILE "rr.h"
+    SOURCE_PATH "sched" SOURCE_FILE "rr.c"
+)
+KERNEL_FILE(
+    INCLUDE_PATH "private/sched" INCLUDE_FILE "idle.h"
+    SOURCE_PATH "sched" SOURCE_FILE "idle.c"
+)
 KERNEL_FILE(
     INCLUDE_PATH "locking" INCLUDE_FILE "spinlock.h"
     SOURCE_PATH "locking" SOURCE_FILE "spinlock.c"
@@ -254,5 +269,28 @@ target_compile_definitions(
     ULTRA_ARCH_WIDTH=8
 )
 
+set(TEST_LD_TEMPLATE "${CMAKE_CURRENT_SOURCE_DIR}/link.ld.template")
+set(TEST_LD_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/link.ld")
+
+add_custom_command(
+    OUTPUT ${TEST_LD_SCRIPT}
+    COMMAND
+        ${CMAKE_C_COMPILER} -E -xc -P -D__ASSEMBLER__
+        "-I$<JOIN:$<TARGET_PROPERTY:run_tests,INCLUDE_DIRECTORIES>,;-I>"
+        "-D$<JOIN:$<TARGET_PROPERTY:run_tests,COMPILE_DEFINITIONS>,;-D>"
+        ${TEST_LD_TEMPLATE} -o ${TEST_LD_SCRIPT}
+    DEPENDS
+        ${TEST_LD_TEMPLATE}
+        ${EXTERNAL_KERNEL_FILES_LOCAL}
+    COMMAND_EXPAND_LISTS
+)
+
+add_custom_target(link_script DEPENDS ${TEST_LD_SCRIPT})
+add_dependencies(link_script external_files)
+add_dependencies(run_tests link_script)
+
+target_link_options(run_tests PRIVATE "-Wl,-T,${TEST_LD_SCRIPT}")
+set_property(TARGET run_tests APPEND PROPERTY LINK_DEPENDS ${TEST_LD_SCRIPT})
+
 set_property(TARGET run_tests PROPERTY C_STANDARD 23)
 set_property(TARGET run_tests PROPERTY CXX_STANDARD 20)
diff --git a/tests/link.ld.template b/tests/link.ld.template
new file mode 100644
index 00000000..27592e81
--- /dev/null
+++ b/tests/link.ld.template
@@ -0,0 +1,7 @@
+#include <linker_common.ld.h>
+
+SECTIONS
+{
+    .sched_classes : { SCHED_CLASS_SECTIONS }
+}
+INSERT AFTER .data;
diff --git a/tests/synthetic-include/per_cpu.h b/tests/synthetic-include/per_cpu.h
index 239d3b40..b477f436 100644
--- a/tests/synthetic-include/per_cpu.h
+++ b/tests/synthetic-include/per_cpu.h
@@ -48,3 +48,5 @@ extern ptr_t g_per_cpu_offset[];
 
 #define per_cpu_ptr(ptr, cpu_number) \
     PTR_ADD_HIDE_UB(ptr, g_per_cpu_offset[cpu_number])
+
+#define this_cpu_ptr(ptr) (ptr)
diff --git a/tests/test-cases/files.cmake b/tests/test-cases/files.cmake
index 5a0bccab..b89ce519 100644
--- a/tests/test-cases/files.cmake
+++ b/tests/test-cases/files.cmake
@@ -13,6 +13,7 @@ add_test_cases(
     test_cpu_mask.c
     test_cpu_mask_out_of_line.c
     test_preempt.c
+    test_sched.c
     test_spinlock.c
     test_mutex.c
     test_vector_alloc.c
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
new file mode 100644
index 00000000..5adc6df1
--- /dev/null
+++ b/tests/test-cases/test_sched.c
@@ -0,0 +1,151 @@
+#include <kernel-source/sched/task.c>
+#include <kernel-source/sched/core.c>
+#include <kernel-source/sched/rr.c>
+#include <kernel-source/sched/idle.c>
+
+#include <test_harness.h>
+
+#define NUM_TEST_TASKS 8
+
+static struct task s_tasks[NUM_TEST_TASKS];
+
+static struct rq *reset_state(void)
+{
+    struct rq *rq;
+    size_t i;
+
+    g_num_present_cpus = 1;
+    g_per_cpu_offset[0] = 0;
+
+    memset(&g_this_cpu_rq, 0, sizeof(g_this_cpu_rq));
+    sched_init();
+
+    for (i = 0; i < NUM_TEST_TASKS; i++) {
+        s_tasks[i] = (struct task) {
+            .prio = TASK_PRIO_DEFAULT,
+            .sched_class = &SCHED_CLASS_INSTANCE(rr),
+        };
+    }
+
+    rq = cpu_rq(0);
+    spin_lock(&rq->lock);
+
+    return rq;
+}
+
+static void finish(struct rq *rq)
+{
+    spin_unlock(&rq->lock);
+}
+
+TEST_CASE(sched_init_installs_idle)
+{
+    struct rq *rq;
+
+    rq = reset_state();
+
+    ASSERT_EQ(rq->idle, &g_this_cpu_idle_task);
+    ASSERT_EQ(rq->curr, &g_this_cpu_idle_task);
+    ASSERT_EQ(rq->nr_running, 0);
+    ASSERT_EQ(g_this_cpu_idle_task.cpu, 0);
+    ASSERT_EQ(g_this_cpu_idle_task.sched_class, &SCHED_CLASS_INSTANCE(idle));
+
+    finish(rq);
+}
+
+TEST_CASE(sched_pick_idle_when_empty)
+{
+    struct rq *rq;
+
+    rq = reset_state();
+
+    ASSERT_EQ(rq_pick_next(rq), rq->idle);
+
+    rq_enqueue(rq, &s_tasks[0]);
+    ASSERT_EQ(rq_pick_next(rq), &s_tasks[0]);
+
+    rq_dequeue(rq, &s_tasks[0]);
+    ASSERT_EQ(rq_pick_next(rq), rq->idle);
+
+    finish(rq);
+}
+
+TEST_CASE(sched_rr_fifo_within_priority)
+{
+    struct rq *rq;
+    size_t i;
+
+    rq = reset_state();
+
+    for (i = 0; i < NUM_TEST_TASKS; i++)
+        rq_enqueue(rq, &s_tasks[i]);
+    ASSERT_EQ(rq->nr_running, NUM_TEST_TASKS);
+
+    for (i = 0; i < NUM_TEST_TASKS; i++) {
+        ASSERT_EQ(rq_pick_next(rq), &s_tasks[i]);
+        ASSERT(s_tasks[i].on_rq);
+
+        rq_dequeue(rq, &s_tasks[i]);
+        ASSERT(!s_tasks[i].on_rq);
+    }
+
+    ASSERT_EQ(rq->nr_running, 0);
+    ASSERT_EQ(rq_pick_next(rq), rq->idle);
+
+    finish(rq);
+}
+
+TEST_CASE(sched_rr_priority_order)
+{
+    struct rq *rq;
+
+    rq = reset_state();
+
+    s_tasks[0].prio = TASK_PRIO_LOWEST;
+    s_tasks[1].prio = TASK_PRIO_DEFAULT;
+    s_tasks[2].prio = TASK_PRIO_HIGHEST;
+    s_tasks[3].prio = TASK_PRIO_DEFAULT;
+
+    rq_enqueue(rq, &s_tasks[0]);
+    ASSERT_EQ(rq_pick_next(rq), &s_tasks[0]);
+
+    rq_enqueue(rq, &s_tasks[1]);
+    ASSERT_EQ(rq_pick_next(rq), &s_tasks[1]);
+
+    rq_enqueue(rq, &s_tasks[2]);
+    ASSERT_EQ(rq_pick_next(rq), &s_tasks[2]);
+
+    rq_enqueue(rq, &s_tasks[3]);
+    ASSERT_EQ(rq_pick_next(rq), &s_tasks[2]);
+
+    rq_dequeue(rq, &s_tasks[2]);
+    ASSERT_EQ(rq_pick_next(rq), &s_tasks[1]);
+
+    rq_dequeue(rq, &s_tasks[1]);
+    ASSERT_EQ(rq_pick_next(rq), &s_tasks[3]);
+
+    rq_dequeue(rq, &s_tasks[3]);
+    ASSERT_EQ(rq_pick_next(rq), &s_tasks[0]);
+
+    rq_dequeue(rq, &s_tasks[0]);
+    ASSERT_EQ(rq_pick_next(rq), rq->idle);
+
+    finish(rq);
+}
+
+TEST_CASE(sched_rr_requeue_goes_to_the_tail)
+{
+    struct rq *rq;
+
+    rq = reset_state();
+
+    rq_enqueue(rq, &s_tasks[0]);
+    rq_enqueue(rq, &s_tasks[1]);
+    ASSERT_EQ(rq_pick_next(rq), &s_tasks[0]);
+
+    rq_dequeue(rq, &s_tasks[0]);
+    rq_enqueue(rq, &s_tasks[0]);
+    ASSERT_EQ(rq_pick_next(rq), &s_tasks[1]);
+
+    finish(rq);
+}

From 317ca42fedd319104fa311fd3d34fcd82c823c46 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 23 Oct 2026 15:00:51 +0300
Subject: [PATCH 047/152] common/minmax: add within_bounds() and
 within_bounds_inclusive()

Range checks were open-coded with two comparisons. Add helpers for a
half-open range and for one that includes its last value. Each takes a
single unsigned comparison, so the inclusive one also works for a range
that ends at the top of the type.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/common/minmax.h | 35 +++++++++++++++++++
 tests/test-cases/files.cmake   |  1 +
 tests/test-cases/test_minmax.c | 61 ++++++++++++++++++++++++++++++++++
 3 files changed, 97 insertions(+)
 create mode 100644 tests/test-cases/test_minmax.c

diff --git a/kernel/include/common/minmax.h b/kernel/include/common/minmax.h
index 526164b0..a11eb118 100644
--- a/kernel/include/common/minmax.h
+++ b/kernel/include/common/minmax.h
@@ -20,3 +20,38 @@
 
 #define MIN(x, y) DO_COMPARE(x, y, <)
 #define MAX(x, y) DO_COMPARE(x, y, >)
+
+// Add 0u to keep u8 or a u16 from being promoted to a signed int
+#define BOUNDS_TYPE(value, begin, end) typeof((value) + (begin) + (end) + 0u)
+
+#define RUNTIME_WITHIN_BOUNDS(value, begin, end, op, uv, ub, ue) ({ \
+    BOUNDS_TYPE(value, begin, end) uv = (value);                    \
+    BOUNDS_TYPE(value, begin, end) ub = (begin);                    \
+    BOUNDS_TYPE(value, begin, end) ue = (end);                      \
+                                                                    \
+    STATIC_ASSERT(                                                  \
+        IS_UNSIGNED_TYPE(value) && IS_UNSIGNED_TYPE(begin) &&       \
+        IS_UNSIGNED_TYPE(end), "bounds must be unsigned"            \
+    );                                                              \
+    uv - ub op ue - ub;                                             \
+})
+
+#define DO_WITHIN_BOUNDS(value, begin, end, op)          \
+    RUNTIME_WITHIN_BOUNDS(                               \
+        value, begin, end, op, CONCAT(uv, __COUNTER__),  \
+        CONCAT(ub, __COUNTER__), CONCAT(ue, __COUNTER__) \
+    )
+
+/*
+ * True if 'value' is within [begin, end). Every argument must be unsigned and
+ * is evaluated once.
+ */
+#define within_bounds(value, begin, end) \
+    DO_WITHIN_BOUNDS(value, begin, end, <)
+
+/*
+ * True if 'value' is within [begin, end]. Every argument must be unsigned and
+ * is evaluated once.
+ */
+#define within_bounds_inclusive(value, first, last) \
+    DO_WITHIN_BOUNDS(value, first, last, <=)
diff --git a/tests/test-cases/files.cmake b/tests/test-cases/files.cmake
index b89ce519..c51f8103 100644
--- a/tests/test-cases/files.cmake
+++ b/tests/test-cases/files.cmake
@@ -5,6 +5,7 @@ add_test_cases(
     test_bit.c
     test_console.c
     test_string.c
+    test_minmax.c
     test_smbios.c
     test_list.c
     test_slist.c
diff --git a/tests/test-cases/test_minmax.c b/tests/test-cases/test_minmax.c
new file mode 100644
index 00000000..7c6618ed
--- /dev/null
+++ b/tests/test-cases/test_minmax.c
@@ -0,0 +1,61 @@
+#include <common/minmax.h>
+#include <common/types.h>
+#include <common/bit.h>
+
+#include <test_harness.h>
+
+TEST_CASE(within_bounds_excludes_the_end)
+{
+    ASSERT_FALSE(within_bounds(9u, 10u, 20u));
+    ASSERT_TRUE(within_bounds(10u, 10u, 20u));
+    ASSERT_TRUE(within_bounds(19u, 10u, 20u));
+    ASSERT_FALSE(within_bounds(20u, 10u, 20u));
+    ASSERT_FALSE(within_bounds(10u, 10u, 10u));
+}
+
+TEST_CASE(within_bounds_inclusive_includes_the_last_value)
+{
+    ASSERT_FALSE(within_bounds_inclusive(9u, 10u, 20u));
+    ASSERT_TRUE(within_bounds_inclusive(10u, 10u, 20u));
+    ASSERT_TRUE(within_bounds_inclusive(20u, 10u, 20u));
+    ASSERT_FALSE(within_bounds_inclusive(21u, 10u, 20u));
+    ASSERT_TRUE(within_bounds_inclusive(10u, 10u, 10u));
+}
+
+TEST_CASE(within_bounds_reaches_the_top_of_the_type)
+{
+    u64 max;
+
+    max = UNSIGNED_MAX(u64);
+
+    ASSERT_TRUE(within_bounds(max - 1, max - 1, max));
+    ASSERT_FALSE(within_bounds(max, max - 1, max));
+    ASSERT_TRUE(within_bounds_inclusive(max, max - 1, max));
+    ASSERT_FALSE(within_bounds_inclusive((u64)0, max - 1, max));
+}
+
+TEST_CASE(within_bounds_of_narrow_types)
+{
+    u8 value;
+
+    value = 200;
+
+    ASSERT_TRUE(within_bounds(value, (u8)100, (u8)255));
+    ASSERT_FALSE(within_bounds((u8)50, (u8)100, (u8)255));
+    ASSERT_FALSE(within_bounds((u8)255, (u8)100, (u8)255));
+    ASSERT_TRUE(within_bounds_inclusive((u8)255, (u8)100, (u8)255));
+}
+
+TEST_CASE(within_bounds_evaluates_each_argument_once)
+{
+    u32 value, begin, end;
+
+    value = 5;
+    begin = 0;
+    end = 10;
+
+    ASSERT_TRUE(within_bounds(value++, begin++, end++));
+    ASSERT_EQ(value, 6);
+    ASSERT_EQ(begin, 1);
+    ASSERT_EQ(end, 11);
+}

From 49c0031d24214ff2282c48086ca8262f5c56a1ef Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 23 Oct 2026 15:02:26 +0300
Subject: [PATCH 048/152] everywhere: use within_bounds() for open-coded range
 checks

All no-op conversions, except a slight difference for the boot
allocator, which compared the start exclusively, but an exact match on
it has already returned by then, so the half-open check behaves the
same.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/ioapic.c | 8 ++++++--
 kernel/include/symbols.h    | 7 +++++--
 kernel/memory/boot_alloc.c  | 2 +-
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/kernel/arch/x86_64/ioapic.c b/kernel/arch/x86_64/ioapic.c
index 4cf7b1a0..ac96478e 100644
--- a/kernel/arch/x86_64/ioapic.c
+++ b/kernel/arch/x86_64/ioapic.c
@@ -23,6 +23,7 @@
 #include <common/string.h>
 #include <common/format.h>
 #include <common/bit.h>
+#include <common/minmax.h>
 
 #define IOAPIC_IOREGSEL 0x00
 #define IOAPIC_IOWIN 0x10
@@ -83,11 +84,14 @@ static u32 ioapic_num_pins(struct ioapic *ioapic)
 
 static struct ioapic *find_ioapic_for_gsi(u32 gsi)
 {
+    struct ioapic *ioapic;
     size_t i;
 
     for (i = 0; i < s_num_ioapics; i++) {
-        if (s_ioapics[i].gsi_base <= gsi && gsi <= s_ioapics[i].gsi_last)
-            return &s_ioapics[i];
+        ioapic = &s_ioapics[i];
+
+        if (within_bounds_inclusive(gsi, ioapic->gsi_base, ioapic->gsi_last))
+            return ioapic;
     }
 
     return nullptr;
diff --git a/kernel/include/symbols.h b/kernel/include/symbols.h
index 110d6101..fed30d57 100644
--- a/kernel/include/symbols.h
+++ b/kernel/include/symbols.h
@@ -1,5 +1,6 @@
 #include <common/types.h>
 #include <common/error.h>
+#include <common/minmax.h>
 
 #include <linker.h>
 
@@ -17,8 +18,10 @@ DECLARE_LINKER_SYMBOL(eh_frame_end);
 
 static inline bool address_is_kernel_code(ptr_t address)
 {
-    return address >= LINKER_SYMBOL_ADDR(text_begin) &&
-           address <= LINKER_SYMBOL_ADDR(text_end);
+    return within_bounds_inclusive(
+        address, LINKER_SYMBOL_ADDR(text_begin),
+                 LINKER_SYMBOL_ADDR(text_end)
+    );
 }
 
 error_t symbol_lookup_by_address(
diff --git a/kernel/memory/boot_alloc.c b/kernel/memory/boot_alloc.c
index f12279a4..97f4b563 100644
--- a/kernel/memory/boot_alloc.c
+++ b/kernel/memory/boot_alloc.c
@@ -97,7 +97,7 @@ static ssize_t INIT_CODE find_range(
     if (right >= 0) {
         struct memory_range *mr = &s_buffer[right];
 
-        if (mr->physical_address < value && value < mr_end(mr))
+        if (within_bounds(value, mr->physical_address, mr_end(mr)))
             return right;
     }
 

From 9b5dd64298080d373e16d19cf391783ed2d03101 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 25 Oct 2026 01:10:00 +0300
Subject: [PATCH 049/152] symbols: regenerate the symbol tables when the
 generator changes

The generated tables only depended on the kernel binary, so a change to
the generator left them stale until a clean build.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/symbols.cmake | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/kernel/symbols.cmake b/kernel/symbols.cmake
index 70e9a63a..bf0ba10e 100644
--- a/kernel/symbols.cmake
+++ b/kernel/symbols.cmake
@@ -10,14 +10,18 @@ function(ultra_symbol_file)
         set(BINARY_FLAGS "--binary;${ARG_BINARY}")
     endif ()
 
+    # Regenerate the tables when the generator changes
+    set(GENERATOR ${ULTRA_SCRIPTS_DIR}/generate_symbol_tables.py)
+    set(DEPENDENCY_FLAGS "DEPENDS;${GENERATOR}")
+
     if (ARG_DEPENDENCY)
-        set(DEPENDENCY_FLAGS "DEPENDS;${ARG_BINARY}")
+        list(APPEND DEPENDENCY_FLAGS ${ARG_BINARY})
     endif ()
 
     add_custom_command(
         OUTPUT ${ARG_OUTPUT_PATH}
         COMMAND
-        Python3::Interpreter ${ULTRA_SCRIPTS_DIR}/generate_symbol_tables.py
+        Python3::Interpreter ${GENERATOR}
         ${ARG_OUTPUT_PATH} ${BINARY_FLAGS}
         ${DEPENDENCY_FLAGS}
         COMMAND_EXPAND_LISTS

From de93983613b9f81ac11e8c834e96a4244ba75e2c Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 25 Oct 2026 14:12:44 +0300
Subject: [PATCH 050/152] scripts: drop the forwarding constructors of the
 symbol classes

LinuxSymbol and UltraSymbol only passed their arguments on to Symbol,
which they inherit the constructor from anyway.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 scripts/generate_symbol_tables.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/scripts/generate_symbol_tables.py b/scripts/generate_symbol_tables.py
index dcc6e058..ad4b1572 100755
--- a/scripts/generate_symbol_tables.py
+++ b/scripts/generate_symbol_tables.py
@@ -123,9 +123,6 @@ def sort_by_name_comparator(lhs: 'Symbol', rhs: 'Symbol') -> int:
 
 
 class LinuxSymbol(Symbol):
-    def __init__(self, name: str, type: str, index: int, address: int) -> None:
-        super().__init__(name, type, index, address)
-
     def canonical_name(self) -> str:
         # Linux stores the type of the symbol as the first byte of the name
         return self._type + self._name
@@ -148,9 +145,6 @@ def is_linker(self) -> bool:
 
 
 class UltraSymbol(Symbol):
-    def __init__(self, name: str, type: str, index: int, address: int) -> None:
-        super().__init__(name, type, index, address)
-
     def canonical_name(self) -> str:
         return self.name
 

From 7bedb1dffd9e0d577a6acdfea97ac949258eddf6 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 23 Oct 2026 15:02:26 +0300
Subject: [PATCH 051/152] symbols: rename the code lookup to
 symbol_lookup_code_by_address()

symbol_lookup_by_address() only looks up functions, and the symbol table
is about to gain data with a lookup of its own.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/common/format.c   | 2 +-
 kernel/include/symbols.h | 2 +-
 kernel/symbols.c         | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/kernel/common/format.c b/kernel/common/format.c
index 35975bf5..30ba58b7 100644
--- a/kernel/common/format.c
+++ b/kernel/common/format.c
@@ -185,7 +185,7 @@ static void write_symbol(
     error_t ret;
     int len;
 
-    ret = symbol_lookup_by_address(pc - is_return_address, sym, &offset);
+    ret = symbol_lookup_code_by_address(pc - is_return_address, sym, &offset);
     if (is_error(ret)) {
         len = snprintf(sym, sizeof(sym), "unknown/garbage <0x%016zX>", pc);
         write_many(fb_state, sym, len);
diff --git a/kernel/include/symbols.h b/kernel/include/symbols.h
index fed30d57..a53726ee 100644
--- a/kernel/include/symbols.h
+++ b/kernel/include/symbols.h
@@ -24,7 +24,7 @@ static inline bool address_is_kernel_code(ptr_t address)
     );
 }
 
-error_t symbol_lookup_by_address(
+error_t symbol_lookup_code_by_address(
     ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
     size_t *out_offset_within
 );
diff --git a/kernel/symbols.c b/kernel/symbols.c
index 7c094e50..3fad2df0 100644
--- a/kernel/symbols.c
+++ b/kernel/symbols.c
@@ -53,7 +53,7 @@ static const u8 *get_compressed_symbol_name_cursor(u32 index)
     return cursor;
 }
 
-error_t symbol_lookup_by_address(
+error_t symbol_lookup_code_by_address(
     ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
     size_t *out_offset_within
 )

From 8503ae2e5c27f039ddb1b1e48b6a0368a7e137c3 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 25 Oct 2026 01:13:24 +0300
Subject: [PATCH 052/152] symbols: anchor symbol addresses to the start of text

Every symbol sits at or above the start of text, so measure the
addresses from there and drop g_symbol_base, which only pointed at the
first symbol a few bytes in.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/private/symbols.h  |  5 ++---
 kernel/symbols.c                  |  2 +-
 scripts/generate_symbol_tables.py | 29 ++++++++++++++---------------
 3 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/kernel/include/private/symbols.h b/kernel/include/private/symbols.h
index 3ba76e9a..b585ed8c 100644
--- a/kernel/include/private/symbols.h
+++ b/kernel/include/private/symbols.h
@@ -35,12 +35,11 @@ extern const char g_symbol_token_table[];
 extern const u16 g_symbol_token_offsets[];
 
 /*
- * An array of 'g_symbol_count' symbol addresses relative to 'g_symbol_base'
- * sorted in ascending order. Note that both this table and
+ * An array of 'g_symbol_count' symbol addresses relative to the start of
+ * text sorted in ascending order. Note that both this table and
  * 'g_symbol_compressed_names' are sorted in the same order.
  */
 extern const u32 g_symbol_relative_addresses[];
-extern const ptr_t g_symbol_base;
 
 /*
  * An array of 'g_symbol_count' 3-byte indices of names sorted in
diff --git a/kernel/symbols.c b/kernel/symbols.c
index 3fad2df0..8059523c 100644
--- a/kernel/symbols.c
+++ b/kernel/symbols.c
@@ -66,7 +66,7 @@ error_t symbol_lookup_code_by_address(
     if (unlikely(g_symbol_count == 0))
         return ENOSYS;
 
-    rel_address = address - g_symbol_base;
+    rel_address = address - LINKER_SYMBOL_ADDR(text_begin);
 
     while (end - begin > 1) {
         i = begin + ((end - begin) / 2);
diff --git a/scripts/generate_symbol_tables.py b/scripts/generate_symbol_tables.py
index ad4b1572..250d09f4 100755
--- a/scripts/generate_symbol_tables.py
+++ b/scripts/generate_symbol_tables.py
@@ -165,6 +165,7 @@ def contains(self, addr: int) -> bool:
 
 class Context(ABC):
     MAX_SYMBOL_LENGTH: int
+    HAS_RELATIVE_BASE: bool = False
 
     def __init__(
         self, text_start_name: str, text_end_name: str,
@@ -199,6 +200,9 @@ def symbols(self) -> List[Symbol]:
     def start_of_text(self) -> int:
         return self._text.begin
 
+    def address_base(self) -> int:
+        return self.start_of_text()
+
     def _should_drop(self, symbol: Symbol) -> bool:
         return not self._text.contains(symbol.address)
 
@@ -255,11 +259,15 @@ def sort_by_name(self) -> None:
 
 class LinuxContext(Context):
     MAX_SYMBOL_LENGTH = 511
+    HAS_RELATIVE_BASE = True
 
     def __init__(self, binary_path: Optional[str]):
         self.__init_text = Section("_sinittext", "_einittext", 0, 0)
         super().__init__("_stext", "_etext", binary_path)
 
+    def address_base(self) -> int:
+        return self.symbols[0].address if self.symbols else 0
+
     @staticmethod
     def make_symbol(name: str, type: str, index: int, address: int) -> Symbol:
         return LinuxSymbol(name, type, index, address)
@@ -572,7 +580,6 @@ def __table_type_to_string(table_type: TableType) -> str:
             TableType.SYMBOL_NAMES: "compressed_names",
             TableType.SYMBOL_MARKERS: "name_offsets",
             TableType.SYMBOL_ADDRESSES: "relative_addresses",
-            TableType.SYMBOL_BASE: "base",
             TableType.SYMBOL_INDICES: "name_index_to_address_index",
             TableType.TOKEN_TABLE: "token_table",
             TableType.TOKEN_OFFSETS: "token_offsets",
@@ -610,18 +617,9 @@ def _emit_value(self, value: Value) -> None:
     def emit(self, table_type: TableType, value: Value) -> None:
         assert value.type not in (ValueType.U8_ARRAY, ValueType.U16_ARRAY)
 
-        if table_type == TableType.SYMBOL_BASE:
-            self._write("const ptr_t ")
-            self._write(CGenerator.__table_type_to_string(table_type))
-        else:
-            self._emit_label(table_type, value.type)
-
+        self._emit_label(table_type, value.type)
         self._write(" = ")
         self._emit_value(value)
-
-        if table_type == TableType.SYMBOL_BASE:
-            self._write(" + (ptr_t)g_linker_symbol_text_begin")
-
         self._write(";\n\n")
 
     class ArrayEmitter(TableGenerator.ArrayEmitter):
@@ -856,17 +854,18 @@ def main() -> None:
             for marker in offset_markers:
                 arr.emit(Value.u16(marker))
 
-        first_symbol_address = ctx.symbols[0].address if ctx.symbols else 0
+        base = ctx.address_base()
 
         with gen.array(TableType.SYMBOL_ADDRESSES, ValueType.U32) as arr:
             for symbol in ctx.symbols:
                 arr.emit(
-                    Value.u32(symbol.address - first_symbol_address),
+                    Value.u32(symbol.address - base),
                     comment=symbol.canonical_name()
                 )
 
-        first_symbol_offset = first_symbol_address - ctx.start_of_text()
-        gen.emit(TableType.SYMBOL_BASE, Value.u32(first_symbol_offset))
+        if ctx.HAS_RELATIVE_BASE:
+            offset = base - ctx.start_of_text()
+            gen.emit(TableType.SYMBOL_BASE, Value.u32(offset))
 
         ctx.sort_by_name()
 

From 336ea221ed1e734eb62d8da5d4f65f82a2a71a17 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 25 Oct 2026 01:15:11 +0300
Subject: [PATCH 053/152] scripts: match the current kallsyms format in the
 Linux mode

kallsyms dropped the relative base: the offsets are now either
PC-relative references to _text or absolute addresses. Labels are 4-byte
aligned and the tables carry their sizes. Follow it, with --pc-relative
for the former.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 scripts/generate_symbol_tables.py | 154 +++++++++++++++++++-----------
 1 file changed, 97 insertions(+), 57 deletions(-)

diff --git a/scripts/generate_symbol_tables.py b/scripts/generate_symbol_tables.py
index 250d09f4..7062ef51 100755
--- a/scripts/generate_symbol_tables.py
+++ b/scripts/generate_symbol_tables.py
@@ -165,7 +165,6 @@ def contains(self, addr: int) -> bool:
 
 class Context(ABC):
     MAX_SYMBOL_LENGTH: int
-    HAS_RELATIVE_BASE: bool = False
 
     def __init__(
         self, text_start_name: str, text_end_name: str,
@@ -200,7 +199,7 @@ def symbols(self) -> List[Symbol]:
     def start_of_text(self) -> int:
         return self._text.begin
 
-    def address_base(self) -> int:
+    def anchor(self) -> int:
         return self.start_of_text()
 
     def _should_drop(self, symbol: Symbol) -> bool:
@@ -213,7 +212,9 @@ def make_symbol(name: str, type: str, index: int, address: int) -> Symbol:
 
     @staticmethod
     @abstractmethod
-    def make_table_generator(out_path: str) -> 'TableGenerator':
+    def make_table_generator(
+        out_path: str, pc_relative: bool
+    ) -> 'TableGenerator':
         pass
 
     def _on_symbol_added(self, symbol: Symbol) -> None:
@@ -259,26 +260,32 @@ def sort_by_name(self) -> None:
 
 class LinuxContext(Context):
     MAX_SYMBOL_LENGTH = 511
-    HAS_RELATIVE_BASE = True
 
     def __init__(self, binary_path: Optional[str]):
         self.__init_text = Section("_sinittext", "_einittext", 0, 0)
+        self.__text_anchor = 0
         super().__init__("_stext", "_etext", binary_path)
 
-    def address_base(self) -> int:
-        return self.symbols[0].address if self.symbols else 0
+    def anchor(self) -> int:
+        return self.__text_anchor
 
     @staticmethod
     def make_symbol(name: str, type: str, index: int, address: int) -> Symbol:
         return LinuxSymbol(name, type, index, address)
 
     @staticmethod
-    def make_table_generator(out_path: str) -> 'TableGenerator':
-        return GASGenerator(out_path)
+    def make_table_generator(
+        out_path: str, pc_relative: bool
+    ) -> 'TableGenerator':
+        return GASGenerator(out_path, pc_relative)
 
     def _on_symbol_added(self, symbol: Symbol) -> None:
         super()._on_symbol_added(symbol)
 
+        if symbol.name == "_text":
+            self.__text_anchor = symbol.address
+            return
+
         if symbol.name == self.__init_text.start_name:
             self.__init_text.begin = symbol.address
             return
@@ -322,7 +329,10 @@ def make_symbol(name: str, type: str, index: int, address: int) -> Symbol:
         return UltraSymbol(name, type, index, address)
 
     @staticmethod
-    def make_table_generator(out_path: str) -> 'TableGenerator':
+    def make_table_generator(
+        out_path: str, pc_relative: bool
+    ) -> 'TableGenerator':
+        assert not pc_relative
         return CGenerator(out_path)
 
     def _should_drop(self, symbol):
@@ -496,7 +506,6 @@ class TableType(Enum):
     SYMBOL_NAMES = 1
     SYMBOL_MARKERS = 2
     SYMBOL_ADDRESSES = 3
-    SYMBOL_BASE = 4
     SYMBOL_INDICES = 5
     TOKEN_TABLE = 6
     TOKEN_OFFSETS = 7
@@ -553,6 +562,10 @@ def array(self, table_type: TableType, type: ValueType) -> ArrayEmitter:
     def emit(self, table_type: TableType, value: Value) -> None:
         pass
 
+    @abstractmethod
+    def emit_addresses(self, symbols: List[Symbol], anchor: int) -> None:
+        pass
+
 
 class CGenerator(TableGenerator):
     def __init__(self, path: str) -> None:
@@ -657,37 +670,56 @@ def emit(
     def array(self, table_type: TableType, type: ValueType) -> ArrayEmitter:
         return CGenerator.ArrayEmitter(self, table_type, type)
 
+    def emit_addresses(self, symbols: List[Symbol], anchor: int) -> None:
+        with self.array(TableType.SYMBOL_ADDRESSES, ValueType.U32) as arr:
+            for symbol in symbols:
+                arr.emit(
+                    Value.u32(symbol.address - anchor),
+                    comment=symbol.canonical_name()
+                )
+
 
 class GASGenerator(TableGenerator):
-    def __init__(self, path: str) -> None:
+    SIZED_TABLES = (
+        TableType.SYMBOL_NAMES, TableType.SYMBOL_MARKERS,
+        TableType.TOKEN_TABLE, TableType.SYMBOL_ADDRESSES,
+    )
+
+    def __init__(self, path: str, pc_relative: bool) -> None:
         super().__init__(path)
 
-        self._write("#include <asm/bitsperlong.h>\n")
-        self._write("#if BITS_PER_LONG == 64\n")
-        self._write("#define PTR .quad\n")
-        self._write("#define ALGN .balign 8\n")
-        self._write("#else\n")
-        self._write("#define PTR .long\n")
-        self._write("#define ALGN .balign 4\n")
-        self._write("#endif\n")
+        self._address_entry = (
+            GASGenerator._pc_relative_entry if pc_relative
+            else GASGenerator._absolute_entry
+        )
         self._write("\t.section .rodata, \"a\"\n")
 
-    def _emit_label(self, table_type: TableType) -> None:
-        name = "kallsyms_" + {
+    @staticmethod
+    def _label(table_type: TableType) -> str:
+        return "kallsyms_" + {
             TableType.SYMBOL_COUNT: "num_syms",
             TableType.SYMBOL_NAMES: "names",
             TableType.SYMBOL_MARKERS: "markers",
             TableType.SYMBOL_ADDRESSES: "offsets",
-            TableType.SYMBOL_BASE: "relative_base",
             TableType.SYMBOL_INDICES: "seqs_of_names",
             TableType.TOKEN_TABLE: "token_table",
             TableType.TOKEN_OFFSETS: "token_index",
         }[table_type]
 
+    def _emit_label(self, table_type: TableType) -> None:
+        name = GASGenerator._label(table_type)
+
         self._write(f".globl {name}\n")
-        self._write("\tALGN\n")
+        self._write("\t.balign 4\n")
         self._write(f"{name}:\n")
 
+    def _emit_end(self, table_type: TableType) -> None:
+        if table_type in GASGenerator.SIZED_TABLES:
+            name = GASGenerator._label(table_type)
+            self._write(f".size {name}, . - {name}\n")
+
+        self._write("\n")
+
     def _emit_comment_or_newline(self, comment: Optional[str]) -> None:
         if comment:
             self._write(f"\t/* {comment} */\n")
@@ -712,31 +744,51 @@ def _make_gas_prologue(table_type: TableType, value: Value) -> str:
         return f".{string_type}{prefix}"
 
     @staticmethod
-    def _alternate_form_hex_fmt(value: Value) -> str:
+    def _alternate_form_hex_fmt(value: int) -> str:
         # Python doesn't handle the alternate form ("#") for hex numbers
         # which are 0 correctly (it unconditionally prefixes them with 0x)
         # so handle the zero case manually
-        if not value.data:
+        if not value:
             return "0"
 
-        return f"0x{value.data:x}"
+        return f"0x{value:x}"
 
     def emit(self, table_type: TableType, value: Value) -> None:
         self._emit_label(table_type)
 
-        if table_type == TableType.SYMBOL_BASE:
-            assert value.type == ValueType.U32
-
-            offset = GASGenerator._alternate_form_hex_fmt(value)
-            self._write(f"\tPTR\t_text + {offset}\n\n")
-            return
-
         prologue = GASGenerator._make_gas_prologue(table_type, value)
         self._write(f"\t{prologue}{value.data}\n\n")
 
+    @staticmethod
+    def _pc_relative_entry(symbol: Symbol, anchor: int) -> str:
+        offset = symbol.address - anchor
+        if not -(1 << 31) <= offset < (1 << 31):
+            raise RuntimeError(
+                f"Symbol {symbol.name} is out of range of _text"
+            )
+
+        return f"_text - . + ({offset})"
+
+    @staticmethod
+    def _absolute_entry(symbol: Symbol, anchor: int) -> str:
+        return GASGenerator._alternate_form_hex_fmt(
+            symbol.address & 0xFFFFFFFF
+        )
+
+    def emit_addresses(self, symbols: List[Symbol], anchor: int) -> None:
+        self._emit_label(TableType.SYMBOL_ADDRESSES)
+
+        for symbol in symbols:
+            self._write(f"\t.long\t{self._address_entry(symbol, anchor)}")
+            self._emit_comment_or_newline(symbol.canonical_name())
+
+        self._emit_end(TableType.SYMBOL_ADDRESSES)
+
     class ArrayEmitter(TableGenerator.ArrayEmitter):
+        _parent: 'GASGenerator'
+
         def __init__(
-            self, parent: 'TableGenerator', table_type: TableType,
+            self, parent: 'GASGenerator', table_type: TableType,
             type: ValueType
         ) -> None:
             super().__init__(parent, table_type, type)
@@ -749,12 +801,9 @@ def __exit__(
             self, exc_type: Optional[type[BaseException]],
             ex: Optional[BaseException], traceback: Optional[TracebackType]
         ) -> Optional[bool]:
-            self._parent._write("\n")
+            self._parent._emit_end(self._table_type)
             return None
 
-        def __hex_format(self):
-            return self._table_type == TableType.SYMBOL_ADDRESSES
-
         def emit(
             self, value: Value, comment: Optional[str] = None
         ) -> None:
@@ -762,12 +811,7 @@ def emit(
             self._parent._write(f"\t{prologue}")
 
             if value.type in (ValueType.U8, ValueType.U16, ValueType.U32):
-                if self.__hex_format():
-                    data = GASGenerator._alternate_form_hex_fmt(value)
-                else:
-                    data = str(value.data)
-
-                self._parent._write(data)
+                self._parent._write(str(value.data))
             elif value.type in (ValueType.U8_ARRAY, ValueType.U16_ARRAY):
                 assert isinstance(value.data, list)
                 hex_width = 2 if value.type == ValueType.U8_ARRAY else 4
@@ -795,8 +839,15 @@ def main() -> None:
     parser.add_argument("--linux-mode", action="store_true",
                         help="Enables GAS backend & linux-specific symbols"
                              " (fully compatible with kallsyms)")
+    parser.add_argument("--pc-relative", action="store_true",
+                        help="Emit the offsets as PC-relative references "
+                             "to _text instead of absolute addresses "
+                             "(--linux-mode only)")
     args = parser.parse_args()
 
+    if args.pc_relative and not args.linux_mode:
+        parser.error("--pc-relative requires --linux-mode")
+
     ctx = make_context(
         Mode.LINUX if args.linux_mode else Mode.ULTRA,
         args.binary
@@ -806,7 +857,7 @@ def main() -> None:
     byte_offset = 0
     offset_markers: List[int] = []
 
-    with ctx.make_table_generator(args.out_file) as gen:
+    with ctx.make_table_generator(args.out_file, args.pc_relative) as gen:
         gen.emit(TableType.SYMBOL_COUNT, Value.u32(len(ctx.symbols)))
 
         with gen.array(TableType.SYMBOL_NAMES, ValueType.U8_ARRAY) as arr:
@@ -854,18 +905,7 @@ def main() -> None:
             for marker in offset_markers:
                 arr.emit(Value.u16(marker))
 
-        base = ctx.address_base()
-
-        with gen.array(TableType.SYMBOL_ADDRESSES, ValueType.U32) as arr:
-            for symbol in ctx.symbols:
-                arr.emit(
-                    Value.u32(symbol.address - base),
-                    comment=symbol.canonical_name()
-                )
-
-        if ctx.HAS_RELATIVE_BASE:
-            offset = base - ctx.start_of_text()
-            gen.emit(TableType.SYMBOL_BASE, Value.u32(offset))
+        gen.emit_addresses(ctx.symbols, ctx.anchor())
 
         ctx.sort_by_name()
 

From 62543646151ca07496f21c4139ea6351dc8ddf53 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 25 Oct 2026 01:15:54 +0300
Subject: [PATCH 054/152] symbols: add symbol sizes to the symbol table

Read the ELF size of every symbol and emit it next to the address, so a
lookup can tell how far a symbol extends.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/private/symbols.h  |  6 ++++
 scripts/generate_symbol_tables.py | 49 ++++++++++++++++++++++++-------
 2 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/kernel/include/private/symbols.h b/kernel/include/private/symbols.h
index b585ed8c..43a5dd0f 100644
--- a/kernel/include/private/symbols.h
+++ b/kernel/include/private/symbols.h
@@ -41,6 +41,12 @@ extern const u16 g_symbol_token_offsets[];
  */
 extern const u32 g_symbol_relative_addresses[];
 
+/*
+ * An array of 'g_symbol_count' symbol sizes in bytes, in the same order as
+ * 'g_symbol_relative_addresses'.
+ */
+extern const u32 g_symbol_sizes[];
+
 /*
  * An array of 'g_symbol_count' 3-byte indices of names sorted in
  * lexicographical order that map names to their respective index in
diff --git a/scripts/generate_symbol_tables.py b/scripts/generate_symbol_tables.py
index 7062ef51..8f12752f 100755
--- a/scripts/generate_symbol_tables.py
+++ b/scripts/generate_symbol_tables.py
@@ -39,11 +39,14 @@ def to_uleb128(number: int, byte_idx: int) -> int:
 
 
 class Symbol(ABC):
-    def __init__(self, name: str, type: str, index: int, address: int) -> None:
+    def __init__(
+        self, name: str, type: str, index: int, address: int, size: int
+    ) -> None:
         self._name = name
         self._type = type
         self._index = index
         self._address = address
+        self._size = size
 
     @property
     def name(self) -> str:
@@ -65,6 +68,10 @@ def index(self, value: int) -> None:
     def address(self) -> int:
         return self._address
 
+    @property
+    def size(self) -> int:
+        return self._size
+
     def is_weak(self) -> bool:
         # w is not considered as it's filtered out earlier
         return self._type == "W"
@@ -165,6 +172,7 @@ def contains(self, addr: int) -> bool:
 
 class Context(ABC):
     MAX_SYMBOL_LENGTH: int
+    HAS_SIZES: bool = False
 
     def __init__(
         self, text_start_name: str, text_end_name: str,
@@ -175,14 +183,16 @@ def __init__(
 
         if binary_path:
             nm_output = subprocess.check_output(
-                ["nm", "-n", binary_path], universal_newlines=True
+                ["nm", "-n", "-S", binary_path], universal_newlines=True
             ).splitlines()
 
             raw_symbols = [e.split(" ") for e in nm_output]
             for sym in raw_symbols:
-                assert len(sym) == 3
+                assert len(sym) in (3, 4)
+                size = int(sym[1], 16) if len(sym) == 4 else 0
                 self.__add_symbol(
-                    sym[2], sym[1], len(self.__symbols), int(sym[0], 16)
+                    sym[-1], sym[-2], len(self.__symbols), int(sym[0], 16),
+                    size
                 )
 
             self.__symbols = list(filter(
@@ -207,7 +217,9 @@ def _should_drop(self, symbol: Symbol) -> bool:
 
     @staticmethod
     @abstractmethod
-    def make_symbol(name: str, type: str, index: int, address: int) -> Symbol:
+    def make_symbol(
+        name: str, type: str, index: int, address: int, size: int
+    ) -> Symbol:
         pass
 
     @staticmethod
@@ -227,7 +239,7 @@ def _on_symbol_added(self, symbol: Symbol) -> None:
             return
 
     def __add_symbol(
-        self, name: str, type: str, index: int, address: int
+        self, name: str, type: str, index: int, address: int, size: int
     ) -> None:
         # Explicitly skip the following:
         # - Undefined symbol: U
@@ -247,7 +259,7 @@ def __add_symbol(
                 f'Symbol "{name}" is too long, adjust MAX_SYMBOL_LENGTH'
             )
 
-        symbol = self.make_symbol(name, type, index, address)
+        symbol = self.make_symbol(name, type, index, address, size)
 
         self.__symbols.append(symbol)
         self._on_symbol_added(symbol)
@@ -270,8 +282,10 @@ def anchor(self) -> int:
         return self.__text_anchor
 
     @staticmethod
-    def make_symbol(name: str, type: str, index: int, address: int) -> Symbol:
-        return LinuxSymbol(name, type, index, address)
+    def make_symbol(
+        name: str, type: str, index: int, address: int, size: int
+    ) -> Symbol:
+        return LinuxSymbol(name, type, index, address, size)
 
     @staticmethod
     def make_table_generator(
@@ -314,6 +328,7 @@ def _should_drop(self, symbol):
 
 class UltraContext(Context):
     MAX_SYMBOL_LENGTH = 127
+    HAS_SIZES = True
     LINKER_SYMBOL_PREFIX: str = "g_linker_symbol_"
 
     @staticmethod
@@ -325,8 +340,10 @@ def text_end_symbol() -> str:
         return UltraContext.LINKER_SYMBOL_PREFIX + "text_end"
 
     @staticmethod
-    def make_symbol(name: str, type: str, index: int, address: int) -> Symbol:
-        return UltraSymbol(name, type, index, address)
+    def make_symbol(
+        name: str, type: str, index: int, address: int, size: int
+    ) -> Symbol:
+        return UltraSymbol(name, type, index, address, size)
 
     @staticmethod
     def make_table_generator(
@@ -509,6 +526,7 @@ class TableType(Enum):
     SYMBOL_INDICES = 5
     TOKEN_TABLE = 6
     TOKEN_OFFSETS = 7
+    SYMBOL_SIZES = 8
 
 
 class TableGenerator(ABC):
@@ -596,6 +614,7 @@ def __table_type_to_string(table_type: TableType) -> str:
             TableType.SYMBOL_INDICES: "name_index_to_address_index",
             TableType.TOKEN_TABLE: "token_table",
             TableType.TOKEN_OFFSETS: "token_offsets",
+            TableType.SYMBOL_SIZES: "sizes",
         }[table_type]
 
     def _emit_label(
@@ -907,6 +926,14 @@ def main() -> None:
 
         gen.emit_addresses(ctx.symbols, ctx.anchor())
 
+        if ctx.HAS_SIZES:
+            with gen.array(TableType.SYMBOL_SIZES, ValueType.U32) as arr:
+                for symbol in ctx.symbols:
+                    arr.emit(
+                        Value.u32(symbol.size),
+                        comment=symbol.canonical_name()
+                    )
+
         ctx.sort_by_name()
 
         with gen.array(TableType.SYMBOL_INDICES, ValueType.U8_ARRAY) as arr:

From 75c376cae49804742b802a67df6f807985e29d8a Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 25 Oct 2026 01:16:17 +0300
Subject: [PATCH 055/152] symbols: return the symbol size from
 symbol_lookup_code_by_address()

Let a caller ask for the size of the symbol it found, now that the table
has it.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/common/format.c   | 4 +++-
 kernel/include/symbols.h | 2 +-
 kernel/symbols.c         | 4 +++-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/kernel/common/format.c b/kernel/common/format.c
index 30ba58b7..e1dc73f9 100644
--- a/kernel/common/format.c
+++ b/kernel/common/format.c
@@ -185,7 +185,9 @@ static void write_symbol(
     error_t ret;
     int len;
 
-    ret = symbol_lookup_code_by_address(pc - is_return_address, sym, &offset);
+    ret = symbol_lookup_code_by_address(
+        pc - is_return_address, sym, &offset, nullptr
+    );
     if (is_error(ret)) {
         len = snprintf(sym, sizeof(sym), "unknown/garbage <0x%016zX>", pc);
         write_many(fb_state, sym, len);
diff --git a/kernel/include/symbols.h b/kernel/include/symbols.h
index a53726ee..84bd309e 100644
--- a/kernel/include/symbols.h
+++ b/kernel/include/symbols.h
@@ -26,5 +26,5 @@ static inline bool address_is_kernel_code(ptr_t address)
 
 error_t symbol_lookup_code_by_address(
     ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
-    size_t *out_offset_within
+    size_t *out_offset_within, size_t *out_size
 );
diff --git a/kernel/symbols.c b/kernel/symbols.c
index 8059523c..60a5dcd7 100644
--- a/kernel/symbols.c
+++ b/kernel/symbols.c
@@ -55,7 +55,7 @@ static const u8 *get_compressed_symbol_name_cursor(u32 index)
 
 error_t symbol_lookup_code_by_address(
     ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
-    size_t *out_offset_within
+    size_t *out_offset_within, size_t *out_size
 )
 {
     u32 rel_address, symbol_base;
@@ -80,6 +80,8 @@ error_t symbol_lookup_code_by_address(
     symbol_base = g_symbol_relative_addresses[begin];
     if (out_offset_within != nullptr)
         *out_offset_within = rel_address - symbol_base;
+    if (out_size != nullptr)
+        *out_size = g_symbol_sizes[begin];
 
     uncompress_symbol(
         get_compressed_symbol_name_cursor(begin),

From bb0e19c7a09edfcaf5d992ce9b71b8569189cac9 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 23 Oct 2026 14:32:13 +0300
Subject: [PATCH 056/152] common/format: print symbol sizes in %pSM and %pRSM

Print the size of the symbol after the offset, so a frame outside of its
function stands out in a backtrace.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/common/format.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/kernel/common/format.c b/kernel/common/format.c
index e1dc73f9..a47b4572 100644
--- a/kernel/common/format.c
+++ b/kernel/common/format.c
@@ -181,12 +181,12 @@ static void write_symbol(
 )
 {
     char sym[MAX_SYMBOL_LENGTH];
-    size_t offset;
+    size_t offset, size;
     error_t ret;
     int len;
 
     ret = symbol_lookup_code_by_address(
-        pc - is_return_address, sym, &offset, nullptr
+        pc - is_return_address, sym, &offset, &size
     );
     if (is_error(ret)) {
         len = snprintf(sym, sizeof(sym), "unknown/garbage <0x%016zX>", pc);
@@ -198,7 +198,7 @@ static void write_symbol(
 
     write_cstr(fb_state, sym);
     if (offset) {
-        len = snprintf(sym, sizeof(sym), "+%zu", offset);
+        len = snprintf(sym, sizeof(sym), "+%zu/%zu", offset, size);
         write_many(fb_state, sym, len);
     }
 }

From 874e7701a5b45aed114ff56695d5b8f626842623 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 23 Oct 2026 13:40:02 +0300
Subject: [PATCH 057/152] symbols: add data symbols to the symbol table

Only functions could be named by address. With --all-symbols the
generator keeps the data symbols that have a size as well, matching
kallsyms --all-symbols in the Linux mode. The table moves the data after
it, so the build takes one more relink to settle, and the table's own
symbols stay out since their sizes change between links.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/CMakeLists.txt             |  4 +-
 kernel/symbols.cmake              |  2 +-
 scripts/generate_symbol_tables.py | 62 ++++++++++++++++++++++++++-----
 3 files changed, 56 insertions(+), 12 deletions(-)

diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt
index 9aeb15ba..5a9b6f71 100644
--- a/kernel/CMakeLists.txt
+++ b/kernel/CMakeLists.txt
@@ -84,9 +84,9 @@ ultra_compile_definitions(
 set_property(GLOBAL PROPERTY ULTRA_OBJECT_TARGETS "${ULTRA_KERNEL_OBJECTS}")
 
 # Number of times to re-link the kernel to stabilize symbols
-set(ULTRA_NUM_RELINKS 1)
+set(ULTRA_NUM_RELINKS 2)
 if (CONFIG_LTO)
-    set(ULTRA_NUM_RELINKS 2)
+    set(ULTRA_NUM_RELINKS 3)
 endif ()
 
 setup_symbol_table_link_steps(${ULTRA_NUM_RELINKS})
diff --git a/kernel/symbols.cmake b/kernel/symbols.cmake
index bf0ba10e..ab5f5d02 100644
--- a/kernel/symbols.cmake
+++ b/kernel/symbols.cmake
@@ -22,7 +22,7 @@ function(ultra_symbol_file)
         OUTPUT ${ARG_OUTPUT_PATH}
         COMMAND
         Python3::Interpreter ${GENERATOR}
-        ${ARG_OUTPUT_PATH} ${BINARY_FLAGS}
+        ${ARG_OUTPUT_PATH} ${BINARY_FLAGS} --all-symbols
         ${DEPENDENCY_FLAGS}
         COMMAND_EXPAND_LISTS
     )
diff --git a/scripts/generate_symbol_tables.py b/scripts/generate_symbol_tables.py
index 8f12752f..b4b3d4fe 100755
--- a/scripts/generate_symbol_tables.py
+++ b/scripts/generate_symbol_tables.py
@@ -176,9 +176,10 @@ class Context(ABC):
 
     def __init__(
         self, text_start_name: str, text_end_name: str,
-        binary_path: Optional[str]
+        binary_path: Optional[str], all_symbols: bool
     ) -> None:
         self._text = Section(text_start_name, text_end_name, 0, 0)
+        self._all_symbols = all_symbols
         self.__symbols: List[Symbol] = []
 
         if binary_path:
@@ -273,10 +274,10 @@ def sort_by_name(self) -> None:
 class LinuxContext(Context):
     MAX_SYMBOL_LENGTH = 511
 
-    def __init__(self, binary_path: Optional[str]):
+    def __init__(self, binary_path: Optional[str], all_symbols: bool):
         self.__init_text = Section("_sinittext", "_einittext", 0, 0)
         self.__text_anchor = 0
-        super().__init__("_stext", "_etext", binary_path)
+        super().__init__("_stext", "_etext", binary_path, all_symbols)
 
     def anchor(self) -> int:
         return self.__text_anchor
@@ -309,6 +310,9 @@ def _on_symbol_added(self, symbol: Symbol) -> None:
             return
 
     def _should_drop(self, symbol):
+        if self._all_symbols:
+            return False
+
         if symbol.name.startswith(("__start_", "__stop_")):
             return False
 
@@ -352,24 +356,60 @@ def make_table_generator(
         assert not pc_relative
         return CGenerator(out_path)
 
+    def _on_symbol_added(self, symbol: Symbol) -> None:
+        super()._on_symbol_added(symbol)
+
+        for section in self.__data_sections:
+            if symbol.name == section.start_name:
+                section.begin = symbol.address
+            elif symbol.name == section.end_name:
+                section.end = symbol.address
+
+    def __is_data(self, symbol: Symbol) -> bool:
+        return any(
+            section.begin <= symbol.address < section.end
+            for section in self.__data_sections
+        )
+
     def _should_drop(self, symbol):
         # We don't need linker symbols since they can't appear in traces
         if symbol.name.startswith(UltraContext.LINKER_SYMBOL_PREFIX):
             return True
+
+        if self.__is_data(symbol):
+            # Keep data if asked, but only if it has an associated size
+            if not self._all_symbols or symbol.size == 0:
+                return True
+
+            # The table changes size between links, keep its own symbols out
+            if symbol.name.startswith(CGenerator.SYMBOL_PREFIX):
+                return True
+
+            return False
+
         return super()._should_drop(symbol)
 
-    def __init__(self, binary_path: Optional[str]):
+    def __init__(self, binary_path: Optional[str], all_symbols: bool):
+        self.__data_sections = [
+            Section(
+                UltraContext.LINKER_SYMBOL_PREFIX + f"{kind}_data_begin",
+                UltraContext.LINKER_SYMBOL_PREFIX + f"{kind}_data_end", 0, 0
+            )
+            for kind in ("readonly", "readwrite")
+        ]
         super().__init__(
             UltraContext.text_begin_symbol(), UltraContext.text_end_symbol(),
-            binary_path
+            binary_path, all_symbols
         )
 
 
-def make_context(mode: Mode, binary_path: Optional[str]) -> Context:
+def make_context(
+    mode: Mode, binary_path: Optional[str], all_symbols: bool
+) -> Context:
     return {
         Mode.LINUX: LinuxContext,
         Mode.ULTRA: UltraContext,
-    }[mode](binary_path)
+    }[mode](binary_path, all_symbols)
 
 
 class TokenTable:
@@ -586,6 +626,8 @@ def emit_addresses(self, symbols: List[Symbol], anchor: int) -> None:
 
 
 class CGenerator(TableGenerator):
+    SYMBOL_PREFIX = "g_symbol_"
+
     def __init__(self, path: str) -> None:
         super().__init__(path)
 
@@ -606,7 +648,7 @@ def __type_to_string(type: ValueType) -> str:
 
     @staticmethod
     def __table_type_to_string(table_type: TableType) -> str:
-        return "g_symbol_" + {
+        return CGenerator.SYMBOL_PREFIX + {
             TableType.SYMBOL_COUNT: "count",
             TableType.SYMBOL_NAMES: "compressed_names",
             TableType.SYMBOL_MARKERS: "name_offsets",
@@ -858,6 +900,8 @@ def main() -> None:
     parser.add_argument("--linux-mode", action="store_true",
                         help="Enables GAS backend & linux-specific symbols"
                              " (fully compatible with kallsyms)")
+    parser.add_argument("--all-symbols", action="store_true",
+                        help="Include data symbols as well")
     parser.add_argument("--pc-relative", action="store_true",
                         help="Emit the offsets as PC-relative references "
                              "to _text instead of absolute addresses "
@@ -869,7 +913,7 @@ def main() -> None:
 
     ctx = make_context(
         Mode.LINUX if args.linux_mode else Mode.ULTRA,
-        args.binary
+        args.binary, args.all_symbols
     )
     token_table = TokenTable(ctx.symbols)
 

From d3d5184c5676bddaf41dd09e14fc84d920cd1117 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 25 Oct 2026 01:25:38 +0300
Subject: [PATCH 058/152] symbols: add symbol_lookup_data_by_address()

A helper for looking up a data (variable) symbol by address, now that
we have this available. Note that the bounds are respected unlike code
symbol lookups.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/symbols.h | 25 ++++++++++++++++++++++++
 kernel/symbols.c         | 42 ++++++++++++++++++++++++++++++++--------
 2 files changed, 59 insertions(+), 8 deletions(-)

diff --git a/kernel/include/symbols.h b/kernel/include/symbols.h
index 84bd309e..1a12186d 100644
--- a/kernel/include/symbols.h
+++ b/kernel/include/symbols.h
@@ -13,6 +13,12 @@ DECLARE_LINKER_SYMBOL(eh_frame_hdr_end);
 DECLARE_LINKER_SYMBOL(eh_frame_begin);
 DECLARE_LINKER_SYMBOL(eh_frame_end);
 
+DECLARE_LINKER_SYMBOL(readonly_data_begin);
+DECLARE_LINKER_SYMBOL(readonly_data_end);
+
+DECLARE_LINKER_SYMBOL(readwrite_data_begin);
+DECLARE_LINKER_SYMBOL(readwrite_data_end);
+
 // 127 chars & a NULL terminator
 #define MAX_SYMBOL_LENGTH (127 + 1)
 
@@ -24,7 +30,26 @@ static inline bool address_is_kernel_code(ptr_t address)
     );
 }
 
+static inline bool address_is_kernel_data(ptr_t address)
+{
+    return within_bounds(address, LINKER_SYMBOL_ADDR(readonly_data_begin),
+                                  LINKER_SYMBOL_ADDR(readonly_data_end)) ||
+           within_bounds(address, LINKER_SYMBOL_ADDR(readwrite_data_begin),
+                                  LINKER_SYMBOL_ADDR(readwrite_data_end));
+}
+
 error_t symbol_lookup_code_by_address(
     ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
     size_t *out_offset_within, size_t *out_size
 );
+
+/*
+ * Same as symbol_lookup_code_by_address(), but for data (variables).
+ *
+ * NOTE: unlike code symbol lookups, this honors the 'size' of the symbol and
+ *       returns ENOENT if an address happens to be out-of-bounds.
+ */
+error_t symbol_lookup_data_by_address(
+    ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
+    size_t *out_offset_within, size_t *out_size
+);
diff --git a/kernel/symbols.c b/kernel/symbols.c
index 60a5dcd7..5c2b5a74 100644
--- a/kernel/symbols.c
+++ b/kernel/symbols.c
@@ -53,16 +53,13 @@ static const u8 *get_compressed_symbol_name_cursor(u32 index)
     return cursor;
 }
 
-error_t symbol_lookup_code_by_address(
+static error_t do_lookup_symbol(
     ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
-    size_t *out_offset_within, size_t *out_size
+    size_t *out_offset_within, size_t *out_size, bool ensure_within_bounds
 )
 {
-    u32 rel_address, symbol_base;
-    u32 i, begin = 0, end = g_symbol_count;
+    u32 rel_address, offset, i, begin = 0, end = g_symbol_count;
 
-    if (unlikely(!address_is_kernel_code(address)))
-        return EINVAL;
     if (unlikely(g_symbol_count == 0))
         return ENOSYS;
 
@@ -77,9 +74,12 @@ error_t symbol_lookup_code_by_address(
             end = i;
     }
 
-    symbol_base = g_symbol_relative_addresses[begin];
+    offset = rel_address - g_symbol_relative_addresses[begin];
+    if (ensure_within_bounds && offset >= g_symbol_sizes[begin])
+        return ENOENT;
+
     if (out_offset_within != nullptr)
-        *out_offset_within = rel_address - symbol_base;
+        *out_offset_within = offset;
     if (out_size != nullptr)
         *out_size = g_symbol_sizes[begin];
 
@@ -89,3 +89,29 @@ error_t symbol_lookup_code_by_address(
     );
     return EOK;
 }
+
+error_t symbol_lookup_code_by_address(
+    ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
+    size_t *out_offset_within, size_t *out_size
+)
+{
+    if (unlikely(!address_is_kernel_code(address)))
+        return EINVAL;
+
+    return do_lookup_symbol(
+        address, out_name_buf, out_offset_within, out_size, false
+    );
+}
+
+error_t symbol_lookup_data_by_address(
+    ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
+    size_t *out_offset_within, size_t *out_size
+)
+{
+    if (unlikely(!address_is_kernel_data(address)))
+        return EINVAL;
+
+    return do_lookup_symbol(
+        address, out_name_buf, out_offset_within, out_size, true
+    );
+}

From 10d54948e48e40b2e564c85f880c2d6872546054 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 8 Nov 2026 16:45:47 +0300
Subject: [PATCH 059/152] symbols: return the name from the lookups as a struct
 string

The lookups returned an error code and wrote the name into the caller's
buffer, which only held a name on EOK. No caller tells EINVAL, ENOSYS
and ENOENT apart, so return the name as a view of that buffer instead.
Going forward, a lookup that finds nothing returns NULL_STR(), which
the caller checks with str_is_null().

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/abortable_instructions.c |  1 +
 kernel/common/format.c          | 13 ++++++++----
 kernel/include/symbols.h        | 16 +++++++++------
 kernel/symbols.c                | 36 ++++++++++++++++-----------------
 4 files changed, 38 insertions(+), 28 deletions(-)

diff --git a/kernel/abortable_instructions.c b/kernel/abortable_instructions.c
index 3bc2d81b..db17f1a2 100644
--- a/kernel/abortable_instructions.c
+++ b/kernel/abortable_instructions.c
@@ -1,4 +1,5 @@
 #include <common/attributes.h>
+#include <common/error.h>
 #include <symbols.h>
 
 #include <private/arch/abortable_instructions.h>
diff --git a/kernel/common/format.c b/kernel/common/format.c
index a47b4572..f3a5ce6a 100644
--- a/kernel/common/format.c
+++ b/kernel/common/format.c
@@ -81,6 +81,11 @@ static void write_many(
     fb_state->bytes_written += count;
 }
 
+static void write_str(struct fmt_buf_state *fb_state, struct string str)
+{
+    write_many(fb_state, str.text, str.size);
+}
+
 static char hex_char(bool upper, u64 value)
 {
     static const char s_upper_hex[] = "0123456789ABCDEF";
@@ -181,14 +186,14 @@ static void write_symbol(
 )
 {
     char sym[MAX_SYMBOL_LENGTH];
+    struct string name;
     size_t offset, size;
-    error_t ret;
     int len;
 
-    ret = symbol_lookup_code_by_address(
+    name = symbol_lookup_code_by_address(
         pc - is_return_address, sym, &offset, &size
     );
-    if (is_error(ret)) {
+    if (str_is_null(name)) {
         len = snprintf(sym, sizeof(sym), "unknown/garbage <0x%016zX>", pc);
         write_many(fb_state, sym, len);
         return;
@@ -196,7 +201,7 @@ static void write_symbol(
 
     offset += is_return_address;
 
-    write_cstr(fb_state, sym);
+    write_str(fb_state, name);
     if (offset) {
         len = snprintf(sym, sizeof(sym), "+%zu/%zu", offset, size);
         write_many(fb_state, sym, len);
diff --git a/kernel/include/symbols.h b/kernel/include/symbols.h
index 1a12186d..756e54d8 100644
--- a/kernel/include/symbols.h
+++ b/kernel/include/symbols.h
@@ -1,5 +1,5 @@
 #include <common/types.h>
-#include <common/error.h>
+#include <common/string_container.h>
 #include <common/minmax.h>
 
 #include <linker.h>
@@ -38,8 +38,12 @@ static inline bool address_is_kernel_data(ptr_t address)
                                   LINKER_SYMBOL_ADDR(readwrite_data_end));
 }
 
-error_t symbol_lookup_code_by_address(
-    ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
+/*
+ * Look up the code symbol that 'address' falls into. Returns its name as a
+ * view of 'out_name_buf', or NULL_STR() if nothing matched.
+ */
+struct string symbol_lookup_code_by_address(
+    ptr_t address, char out_name_buf[static MAX_SYMBOL_LENGTH],
     size_t *out_offset_within, size_t *out_size
 );
 
@@ -47,9 +51,9 @@ error_t symbol_lookup_code_by_address(
  * Same as symbol_lookup_code_by_address(), but for data (variables).
  *
  * NOTE: unlike code symbol lookups, this honors the 'size' of the symbol and
- *       returns ENOENT if an address happens to be out-of-bounds.
+ *       returns NULL_STR() if an address happens to be out-of-bounds.
  */
-error_t symbol_lookup_data_by_address(
-    ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
+struct string symbol_lookup_data_by_address(
+    ptr_t address, char out_name_buf[static MAX_SYMBOL_LENGTH],
     size_t *out_offset_within, size_t *out_size
 );
diff --git a/kernel/symbols.c b/kernel/symbols.c
index 5c2b5a74..6648ab9f 100644
--- a/kernel/symbols.c
+++ b/kernel/symbols.c
@@ -17,26 +17,28 @@ static u16 get_compressed_symbol_length(const u8 **cursor)
     return len;
 }
 
-static void uncompress_symbol(
-    const u8 *compressed_cursor, char out_buf[MAX_SYMBOL_LENGTH]
+static struct string uncompress_symbol(
+    const u8 *compressed_cursor, char out_buf[static MAX_SYMBOL_LENGTH]
 )
 {
     u16 len;
     const char *token;
-    char *out_cursor = out_buf;
+    char *out_cursor = out_buf, *out_last;
 
     len = get_compressed_symbol_length(&compressed_cursor);
+    out_last = &out_buf[MAX_SYMBOL_LENGTH - 1];
 
     while (len--) {
         token = &g_symbol_token_table[
             g_symbol_token_offsets[*compressed_cursor++]
         ];
 
-        while (*token)
+        while (*token && out_cursor != out_last)
             *out_cursor++ = *token++;
     }
 
     *out_cursor = '\0';
+    return MAKE_STR(out_buf, out_cursor - out_buf);
 }
 
 static const u8 *get_compressed_symbol_name_cursor(u32 index)
@@ -53,15 +55,15 @@ static const u8 *get_compressed_symbol_name_cursor(u32 index)
     return cursor;
 }
 
-static error_t do_lookup_symbol(
-    ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
+static struct string do_lookup_symbol(
+    ptr_t address, char out_name_buf[static MAX_SYMBOL_LENGTH],
     size_t *out_offset_within, size_t *out_size, bool ensure_within_bounds
 )
 {
     u32 rel_address, offset, i, begin = 0, end = g_symbol_count;
 
     if (unlikely(g_symbol_count == 0))
-        return ENOSYS;
+        return NULL_STR();
 
     rel_address = address - LINKER_SYMBOL_ADDR(text_begin);
 
@@ -76,40 +78,38 @@ static error_t do_lookup_symbol(
 
     offset = rel_address - g_symbol_relative_addresses[begin];
     if (ensure_within_bounds && offset >= g_symbol_sizes[begin])
-        return ENOENT;
+        return NULL_STR();
 
     if (out_offset_within != nullptr)
         *out_offset_within = offset;
     if (out_size != nullptr)
         *out_size = g_symbol_sizes[begin];
 
-    uncompress_symbol(
-        get_compressed_symbol_name_cursor(begin),
-        out_name_buf
+    return uncompress_symbol(
+        get_compressed_symbol_name_cursor(begin), out_name_buf
     );
-    return EOK;
 }
 
-error_t symbol_lookup_code_by_address(
-    ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
+struct string symbol_lookup_code_by_address(
+    ptr_t address, char out_name_buf[static MAX_SYMBOL_LENGTH],
     size_t *out_offset_within, size_t *out_size
 )
 {
     if (unlikely(!address_is_kernel_code(address)))
-        return EINVAL;
+        return NULL_STR();
 
     return do_lookup_symbol(
         address, out_name_buf, out_offset_within, out_size, false
     );
 }
 
-error_t symbol_lookup_data_by_address(
-    ptr_t address, char out_name_buf[MAX_SYMBOL_LENGTH],
+struct string symbol_lookup_data_by_address(
+    ptr_t address, char out_name_buf[static MAX_SYMBOL_LENGTH],
     size_t *out_offset_within, size_t *out_size
 )
 {
     if (unlikely(!address_is_kernel_data(address)))
-        return EINVAL;
+        return NULL_STR();
 
     return do_lookup_symbol(
         address, out_name_buf, out_offset_within, out_size, true

From 82cf198f94205b84f1cbde94fc94b4203b3650b9 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 8 Nov 2026 17:04:05 +0300
Subject: [PATCH 060/152] common/attributes: support restrict for C++

C++ has no restrict, so a header that uses it can't be included by the
C++ files of the unit-test harness. Map it to __restrict, which g++ and
clang++ accept, next to the NORETURN shim.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/common/attributes.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/kernel/include/common/attributes.h b/kernel/include/common/attributes.h
index 445750fd..9b012ccb 100644
--- a/kernel/include/common/attributes.h
+++ b/kernel/include/common/attributes.h
@@ -8,6 +8,10 @@
 #define NORETURN _Noreturn
 #endif
 
+#ifdef __cplusplus
+#define restrict __restrict
+#endif
+
 #define PACKED __attribute__((packed))
 
 #ifdef __clang__

From 6f23ba8fb8e509d57ab55b9dd8c8542436c092bb Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 8 Nov 2026 16:45:47 +0300
Subject: [PATCH 061/152] common/string_container: add str_printf()

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/common/format.c                   | 11 +++++------
 kernel/common/string_container.c         | 18 ++++++++++++++++++
 kernel/include/common/string_container.h | 10 ++++++++++
 3 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/kernel/common/format.c b/kernel/common/format.c
index f3a5ce6a..d61e95f6 100644
--- a/kernel/common/format.c
+++ b/kernel/common/format.c
@@ -186,16 +186,15 @@ static void write_symbol(
 )
 {
     char sym[MAX_SYMBOL_LENGTH];
-    struct string name;
+    struct string name, text;
     size_t offset, size;
-    int len;
 
     name = symbol_lookup_code_by_address(
         pc - is_return_address, sym, &offset, &size
     );
     if (str_is_null(name)) {
-        len = snprintf(sym, sizeof(sym), "unknown/garbage <0x%016zX>", pc);
-        write_many(fb_state, sym, len);
+        text = str_printf(sym, sizeof(sym), "unknown/garbage <0x%016zX>", pc);
+        write_str(fb_state, text);
         return;
     }
 
@@ -203,8 +202,8 @@ static void write_symbol(
 
     write_str(fb_state, name);
     if (offset) {
-        len = snprintf(sym, sizeof(sym), "+%zu/%zu", offset, size);
-        write_many(fb_state, sym, len);
+        text = str_printf(sym, sizeof(sym), "+%zu/%zu", offset, size);
+        write_str(fb_state, text);
     }
 }
 
diff --git a/kernel/common/string_container.c b/kernel/common/string_container.c
index d3a3dae9..8dd974eb 100644
--- a/kernel/common/string_container.c
+++ b/kernel/common/string_container.c
@@ -1,5 +1,6 @@
 #include <common/string_container.h>
 #include <common/ctype.h>
+#include <common/format.h>
 
 bool str_equals(struct string lhs, struct string rhs)
 {
@@ -118,3 +119,20 @@ ssize_t str_find(struct string str, struct string needle, size_t starting_at)
 
     return -1;
 }
+
+struct string str_printf(
+    char *restrict buffer, size_t capacity, const char *fmt, ...
+)
+{
+    va_list list;
+    int written;
+
+    va_start(list, fmt);
+    written = vscnprintf(buffer, capacity, fmt, list);
+    va_end(list);
+
+    if (is_nerror(written))
+        return NULL_STR();
+
+    return MAKE_STR(buffer, written);
+}
diff --git a/kernel/include/common/string_container.h b/kernel/include/common/string_container.h
index 52143cfa..b03e3a08 100644
--- a/kernel/include/common/string_container.h
+++ b/kernel/include/common/string_container.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <common/types.h>
+#include <common/attributes.h>
 #include <common/string.h>
 #include <common/ctype.h>
 
@@ -168,3 +169,12 @@ static inline void str_terminated_copy(char *dst, struct string str)
     memcpy(dst, str.text, str.size);
     dst[str.size] = '\0';
 }
+
+/*
+ * Same as scnprintf(), but returns what was written as a view of 'buffer'.
+ * Returns NULL_STR() if 'fmt' is invalid.
+ */
+PRINTF_DECL(3, 4)
+struct string str_printf(
+    char *restrict buffer, size_t capacity, const char *fmt, ...
+);

From 2e598e1b1f2e13ed9f2c463c2f4cedf586402290 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 25 Oct 2026 01:25:38 +0300
Subject: [PATCH 062/152] io: let virt_to_phys() take a const pointer

It only turns the pointer into a number, so there is no reason to make a
caller with a const pointer cast it away.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/memory/io.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel/include/memory/io.h b/kernel/include/memory/io.h
index 9ae97850..45fdb284 100644
--- a/kernel/include/memory/io.h
+++ b/kernel/include/memory/io.h
@@ -11,7 +11,7 @@
 
 extern ptr_t g_direct_map_base;
 
-static inline phys_addr_t virt_to_phys(void *virt)
+static inline phys_addr_t virt_to_phys(const void *virt)
 {
     return (ptr_t)virt - g_direct_map_base;
 }

From e94f273177ed6a201adde2c52971f56205f6ad76 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 23 Oct 2026 13:43:12 +0300
Subject: [PATCH 063/152] memory: add mem_describe()

Reports could only print a bare address. Add mem_describe(), which asks
each part of the kernel that owns memory to describe an address, starting
with the page allocator: for a struct page or a direct map address it
gives the pfn, type and order of the block. Everything is read with
try_memcpy(), so a bad address is safe to describe from any context.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/memory/describe.h |  16 ++++
 kernel/include/memory/io.h       |  12 +++
 kernel/include/memory/page.h     |   1 +
 kernel/memory/CMakeLists.txt     |   1 +
 kernel/memory/describe.c         | 160 +++++++++++++++++++++++++++++++
 5 files changed, 190 insertions(+)
 create mode 100644 kernel/include/memory/describe.h
 create mode 100644 kernel/memory/describe.c

diff --git a/kernel/include/memory/describe.h b/kernel/include/memory/describe.h
new file mode 100644
index 00000000..9bdb4406
--- /dev/null
+++ b/kernel/include/memory/describe.h
@@ -0,0 +1,16 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/string_container.h>
+
+#define MAX_MEM_DESCRIPTION_LENGTH 128
+
+/*
+ * Describe the virtual address pointed to by 'addr' to the best of the kernel's
+ * knowledge. This queries every possible subsystem to see if it's able to
+ * recognize the address as something it is responsible for and produce a short
+ * description of what it points at.
+ */
+struct string mem_describe(
+    const void *addr, char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
+);
diff --git a/kernel/include/memory/io.h b/kernel/include/memory/io.h
index 45fdb284..85b93708 100644
--- a/kernel/include/memory/io.h
+++ b/kernel/include/memory/io.h
@@ -3,6 +3,7 @@
 #include <common/types.h>
 #include <common/error.h>
 #include <common/bit.h>
+#include <common/minmax.h>
 
 #include <arch/memory.h>
 #include <detail/io.h>
@@ -21,6 +22,17 @@ static inline void *phys_to_virt(phys_addr_t phys)
     return (void*)(phys + g_direct_map_base);
 }
 
+static inline bool address_is_direct_map(virt_addr_t address)
+{
+    return within_bounds(address, g_direct_map_base,
+                                  g_direct_map_base + MAX_PHYS_ADDR);
+}
+
+static inline bool pointer_is_direct_map(const void *ptr)
+{
+    return address_is_direct_map((virt_addr_t)ptr);
+}
+
 /*
  * An opaque MMIO/PIO handle, mapped with one of io_window_map* helpers and
  * accessed via io{read/write}* or io_window_raw_ptr for memory mapped as
diff --git a/kernel/include/memory/page.h b/kernel/include/memory/page.h
index cffbf9c4..ac3c0dba 100644
--- a/kernel/include/memory/page.h
+++ b/kernel/include/memory/page.h
@@ -136,6 +136,7 @@ static inline bool page_type_is_buddy_managed(enum page_type type)
 #define META_TO_ORDER(meta) ((u8)BIT_FIELD_READ(meta, META_ORDER_MASK))
 
 #define META_TO_BLOCK_PTR(meta) ((meta) & ~META_TAIL_BIT)
+#define META_TO_BLOCK(meta) ((struct page_block*)META_TO_BLOCK_PTR(meta))
 
 #define META_MAKE_BLOCK(type, order)          \
     (BIT_FIELD_MAKE(META_TYPE_MASK, (type)) | \
diff --git a/kernel/memory/CMakeLists.txt b/kernel/memory/CMakeLists.txt
index aa0f7a53..2748f15d 100644
--- a/kernel/memory/CMakeLists.txt
+++ b/kernel/memory/CMakeLists.txt
@@ -7,4 +7,5 @@ ultra_sources(
     page_table.c
     buddy.c
     valloc.c
+    describe.c
 )
diff --git a/kernel/memory/describe.c b/kernel/memory/describe.c
new file mode 100644
index 00000000..4be32df4
--- /dev/null
+++ b/kernel/memory/describe.c
@@ -0,0 +1,160 @@
+#include <common/format.h>
+#include <common/helpers.h>
+#include <common/minmax.h>
+
+#include <memory/describe.h>
+#include <memory/io.h>
+#include <memory/page.h>
+
+#include <init_level.h>
+#include <unsafe_access.h>
+
+static const char *const s_page_type_names[] = {
+    [PAGE_TYPE_RESERVED] = "reserved",
+    [PAGE_TYPE_BUDDY] = "free",
+    [PAGE_TYPE_GENERIC_FROZEN] = "frozen",
+    [PAGE_TYPE_KHEAP] = "kheap",
+    [PAGE_TYPE_KHEAP_LARGE] = "kheap large",
+    [PAGE_TYPE_PTDESC] = "page table",
+    [PAGE_TYPE_VALLOC_MANAGED] = "valloc",
+    [PAGE_TYPE_GENERIC] = "generic",
+    [PAGE_TYPE_ANON] = "anon",
+    [PAGE_TYPE_PAGE_CACHE] = "page cache",
+};
+
+static const char *page_type_name(enum page_type type)
+{
+    if (unlikely(type >= ARRAY_SIZE(s_page_type_names) ||
+                 s_page_type_names[type] == nullptr))
+        return "unknown";
+
+    return s_page_type_names[type];
+}
+
+// A pfn or a tail link can point anywhere, check before reading
+static bool address_is_struct_page(ptr_t address)
+{
+    return within_bounds(address, MEMORY_MAP_BASE, MEMORY_MAP_END) &&
+           (address - MEMORY_MAP_BASE) % sizeof(struct page) == 0;
+}
+
+/*
+ * Read the meta of 'page', or of its block's page 0 if it is a tail,
+ * without faulting on a bad pointer. False if either is unreadable.
+ */
+static bool read_block_meta(
+    const struct page *page, const struct page **out_block, ptr_t *out_meta
+)
+{
+    ptr_t meta;
+
+    if (!address_is_struct_page((ptr_t)page))
+        return false;
+    if (try_memcpy(&meta, &page->meta, sizeof(meta)) != EOK)
+        return false;
+
+    if (META_IS_TAIL(meta)) {
+        if (!address_is_struct_page(META_TO_BLOCK_PTR(meta)))
+            return false;
+
+        page = block_to_page(META_TO_BLOCK(meta));
+        if (try_memcpy(&meta, &page->meta, sizeof(meta)) != EOK)
+            return false;
+    }
+
+    *out_block = page;
+    *out_meta = meta;
+    return true;
+}
+
+static struct string describe_struct_page(
+    const void *addr, char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
+)
+{
+    const struct page *page, *block;
+    size_t offset;
+    ptr_t meta;
+
+    offset = ((ptr_t)addr - MEMORY_MAP_BASE) % sizeof(struct page);
+    page = addr - offset;
+
+    if (!read_block_meta(page, &block, &meta)) {
+        return str_printf(
+            out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH,
+            "unreadable struct page of pfn 0x%zx +0x%zx", page_to_pfn(page),
+            offset
+        );
+    }
+
+    return str_printf(
+        out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH,
+        "struct page of pfn 0x%zx +0x%zx, %s block pfn 0x%zx order %hhu",
+        page_to_pfn(page), offset, page_type_name(META_TO_TYPE(meta)),
+        page_to_pfn(block), META_TO_ORDER(meta)
+    );
+}
+
+static struct string page_describe(
+    const void *addr, char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
+)
+{
+    const struct page *block;
+    phys_addr_t phys;
+    ptr_t meta;
+
+    if (init_level_below(INIT_LEVEL_MEMORY_MAP_AVAILABLE))
+        return NULL_STR();
+
+    if (within_bounds((ptr_t)addr, MEMORY_MAP_BASE, MEMORY_MAP_END))
+        return describe_struct_page(addr, out_desc_buf);
+
+    if (!pointer_is_direct_map(addr))
+        return NULL_STR();
+
+    phys = virt_to_phys(addr);
+    if (!read_block_meta(phys_to_page(phys), &block, &meta)) {
+        return str_printf(
+            out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH,
+            "direct map %pPA without a page", &phys
+        );
+    }
+
+    return str_printf(
+        out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH,
+        "%s block pfn 0x%zx order %hhu +0x%zx",
+        page_type_name(META_TO_TYPE(meta)), page_to_pfn(block),
+        META_TO_ORDER(meta), (size_t)(phys - page_to_phys(block))
+    );
+}
+
+typedef struct string (*describer_t)(
+    const void *addr, char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
+);
+
+/*
+ * Each describer handles one kind of memory and returns NULL_STR() for an
+ * address that isn't its own. A report can come from any context with any
+ * address, so a describer only takes relaxed reads and copies through
+ * try_memcpy().
+ */
+static const describer_t s_describers[] = {
+    page_describe,
+};
+
+struct string mem_describe(
+    const void *addr, char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
+)
+{
+    struct string desc;
+    size_t i;
+
+    for (i = 0; i < ARRAY_SIZE(s_describers); i++) {
+        desc = s_describers[i](addr, out_desc_buf);
+        if (!str_is_null(desc))
+            return desc;
+    }
+
+    return str_printf(
+        out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH, "unknown memory"
+    );
+}

From d42fc3587cf6dc9363b22feefc83f156f17f39a5 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 23 Oct 2026 13:44:14 +0300
Subject: [PATCH 064/152] memory: describe kernel heap objects

Most objects live in the kernel heap, where the page alone says little.
Describe a heap address by its cache, the object it falls in and the
offset into it, or by the large allocation that holds it.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/memory/describe.c | 82 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)

diff --git a/kernel/memory/describe.c b/kernel/memory/describe.c
index 4be32df4..41576f34 100644
--- a/kernel/memory/describe.c
+++ b/kernel/memory/describe.c
@@ -2,6 +2,8 @@
 #include <common/helpers.h>
 #include <common/minmax.h>
 
+#include <memory/alloc_cache.h>
+#include <memory/buddy.h>
 #include <memory/describe.h>
 #include <memory/io.h>
 #include <memory/page.h>
@@ -127,6 +129,85 @@ static struct string page_describe(
     );
 }
 
+static struct string describe_slab(
+    const struct page *block, ptr_t meta, size_t offset,
+    char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
+)
+{
+    struct alloc_cache cache, *cache_ptr;
+    char name[32] = { 0 };
+    size_t slab_size, index;
+
+    // The slab can be freed and reused meanwhile, so trust nothing in it
+    if (try_memcpy(&cache_ptr, &block->kheap.cache, sizeof(cache_ptr)) != EOK)
+        goto out_unreadable;
+    if (try_memcpy(&cache, cache_ptr, sizeof(cache)) != EOK)
+        goto out_unreadable;
+    if (try_memcpy(name, cache.name, sizeof(name) - 1) != EOK)
+        goto out_unreadable;
+    if (cache.object_size == 0)
+        goto out_unreadable;
+
+    slab_size = order_to_bytes(META_TO_ORDER(meta));
+    index = offset / cache.object_size;
+
+    // The leftover bytes at the end of a slab that fit no object
+    if (index >= slab_size / cache.object_size) {
+        return str_printf(
+            out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH,
+            "kheap %s slab pfn 0x%zx tail +0x%zx", name, page_to_pfn(block),
+            offset
+        );
+    }
+
+    return str_printf(
+        out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH, "kheap %s object %p +0x%zx",
+        name, (char*)page_to_virt(block) + index * cache.object_size,
+        offset % cache.object_size
+    );
+
+out_unreadable:
+    return str_printf(
+        out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH,
+        "kheap slab pfn 0x%zx +0x%zx, unreadable cache", page_to_pfn(block),
+        offset
+    );
+}
+
+static struct string kheap_describe(
+    const void *addr, char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
+)
+{
+    const struct page *block;
+    phys_addr_t phys;
+    size_t offset;
+    ptr_t meta;
+
+    if (init_level_below(INIT_LEVEL_HEAP_AVAILABLE))
+        return NULL_STR();
+    if (!pointer_is_direct_map(addr))
+        return NULL_STR();
+
+    phys = virt_to_phys(addr);
+    if (!read_block_meta(phys_to_page(phys), &block, &meta))
+        return NULL_STR();
+
+    offset = phys - page_to_phys(block);
+
+    switch (META_TO_TYPE(meta)) {
+    case PAGE_TYPE_KHEAP:
+        return describe_slab(block, meta, offset, out_desc_buf);
+    case PAGE_TYPE_KHEAP_LARGE:
+        return str_printf(
+            out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH,
+            "kheap large allocation %p order %hhu +0x%zx",
+            page_to_virt(block), META_TO_ORDER(meta), offset
+        );
+    default:
+        return NULL_STR();
+    }
+}
+
 typedef struct string (*describer_t)(
     const void *addr, char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
 );
@@ -138,6 +219,7 @@ typedef struct string (*describer_t)(
  * try_memcpy().
  */
 static const describer_t s_describers[] = {
+    kheap_describe,
     page_describe,
 };
 

From bff5a165b3754858d45ccfcce11a477d019fc79d Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 23 Oct 2026 13:44:38 +0300
Subject: [PATCH 065/152] symbols: describe kernel image addresses

Describe an address in the kernel image by the function or variable it
falls in and the offset into it, now that the symbol table has data with
sizes.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/private/symbols.h |  6 +++++
 kernel/memory/describe.c         |  3 +++
 kernel/symbols.c                 | 46 ++++++++++++++++++++++++++++++++
 3 files changed, 55 insertions(+)

diff --git a/kernel/include/private/symbols.h b/kernel/include/private/symbols.h
index 43a5dd0f..ddd68b83 100644
--- a/kernel/include/private/symbols.h
+++ b/kernel/include/private/symbols.h
@@ -1,5 +1,7 @@
 #include <common/types.h>
 
+#include <memory/describe.h>
+
 // Number of kernel symbols
 extern const u32 g_symbol_count;
 
@@ -54,3 +56,7 @@ extern const u32 g_symbol_sizes[];
  * tables. This can be used for O(log N) lookups of name -> address.
  */
 extern const u8 g_symbol_name_index_to_address_index[];
+
+struct string symbol_describe(
+    const void *addr, char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
+);
diff --git a/kernel/memory/describe.c b/kernel/memory/describe.c
index 41576f34..26a9d433 100644
--- a/kernel/memory/describe.c
+++ b/kernel/memory/describe.c
@@ -8,6 +8,8 @@
 #include <memory/io.h>
 #include <memory/page.h>
 
+#include <private/symbols.h>
+
 #include <init_level.h>
 #include <unsafe_access.h>
 
@@ -219,6 +221,7 @@ typedef struct string (*describer_t)(
  * try_memcpy().
  */
 static const describer_t s_describers[] = {
+    symbol_describe,
     kheap_describe,
     page_describe,
 };
diff --git a/kernel/symbols.c b/kernel/symbols.c
index 6648ab9f..c685bf9a 100644
--- a/kernel/symbols.c
+++ b/kernel/symbols.c
@@ -1,4 +1,5 @@
 #include <common/string.h>
+#include <common/format.h>
 
 #include <leb128.h>
 #include <symbols.h>
@@ -115,3 +116,48 @@ struct string symbol_lookup_data_by_address(
         address, out_name_buf, out_offset_within, out_size, true
     );
 }
+
+struct string symbol_describe(
+    const void *addr, char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
+)
+{
+    char name_buf[MAX_SYMBOL_LENGTH];
+    size_t offset, symbol_size;
+    struct string name;
+    ptr_t address;
+
+    address = (ptr_t)addr;
+
+    if (address_is_kernel_code(address)) {
+        name = symbol_lookup_code_by_address(
+            address, name_buf, &offset, &symbol_size
+        );
+        if (str_is_null(name))
+            return name;
+
+        return str_printf(
+            out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH, "%pS+0x%zx/0x%zx", &name,
+            offset, symbol_size
+        );
+    }
+
+    if (address_is_kernel_data(address)) {
+        name = symbol_lookup_data_by_address(
+            address, name_buf, &offset, &symbol_size
+        );
+
+        // Padding, or an object the table has no size for
+        if (str_is_null(name)) {
+            return str_printf(
+                out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH, "unnamed kernel data"
+            );
+        }
+
+        return str_printf(
+            out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH, "%pS+0x%zx/0x%zx", &name,
+            offset, symbol_size
+        );
+    }
+
+    return NULL_STR();
+}

From a2f2f1cd0b9aae00cdecbb141189f5d8f44a13a0 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 23 Oct 2026 13:45:32 +0300
Subject: [PATCH 066/152] per_cpu: describe per-CPU variables

A per-CPU variable lives in its CPU's copy of the per-CPU section, which
to the page allocator is just boot memory. Rebase such an address onto
the section and describe it by the variable and the CPU.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/private/per_cpu.h |  7 +++++
 kernel/memory/describe.c         |  2 ++
 kernel/per_cpu.c                 | 49 ++++++++++++++++++++++++++++++++
 3 files changed, 58 insertions(+)
 create mode 100644 kernel/include/private/per_cpu.h

diff --git a/kernel/include/private/per_cpu.h b/kernel/include/private/per_cpu.h
new file mode 100644
index 00000000..6a0f269d
--- /dev/null
+++ b/kernel/include/private/per_cpu.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include <memory/describe.h>
+
+struct string per_cpu_describe(
+    const void *addr, char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
+);
diff --git a/kernel/memory/describe.c b/kernel/memory/describe.c
index 26a9d433..481ab574 100644
--- a/kernel/memory/describe.c
+++ b/kernel/memory/describe.c
@@ -8,6 +8,7 @@
 #include <memory/io.h>
 #include <memory/page.h>
 
+#include <private/per_cpu.h>
 #include <private/symbols.h>
 
 #include <init_level.h>
@@ -222,6 +223,7 @@ typedef struct string (*describer_t)(
  */
 static const describer_t s_describers[] = {
     symbol_describe,
+    per_cpu_describe,
     kheap_describe,
     page_describe,
 };
diff --git a/kernel/per_cpu.c b/kernel/per_cpu.c
index 612da683..de64c48f 100644
--- a/kernel/per_cpu.c
+++ b/kernel/per_cpu.c
@@ -10,10 +10,14 @@
 #include <panic.h>
 #include <free_after_init.h>
 #include <config.h>
+#include <symbols.h>
+#include <private/per_cpu.h>
 
 #include <common/string.h>
 #include <common/error.h>
 #include <common/align.h>
+#include <common/format.h>
+#include <common/minmax.h>
 
 #include <memory/io.h>
 
@@ -73,3 +77,48 @@ static error_t INIT_CODE per_cpu_setup(void)
     return EOK;
 }
 INIT_CALL_AT(PER_CPU_AVAILABLE, per_cpu_setup);
+
+struct string per_cpu_describe(
+    const void *addr, char out_desc_buf[static MAX_MEM_DESCRIPTION_LENGTH]
+)
+{
+    char name_buf[MAX_SYMBOL_LENGTH];
+    size_t offset, symbol_size, per_cpu_size;
+    struct string name;
+    ptr_t begin, template;
+    cpu_t cpu;
+
+    if (init_level_below(INIT_LEVEL_PER_CPU_AVAILABLE))
+        return NULL_STR();
+
+    begin = SECTION_MARKER_BEGIN_ADDR(PER_CPU_SECTION);
+    per_cpu_size = PAGE_ROUND_UP(SECTION_SIZE(PER_CPU_SECTION));
+
+    /*
+     * Every CPU, the boot CPU included, runs on a copy of the section made by
+     * per_cpu_setup(). Map the address back onto the section to find the
+     * variable.
+     */
+    for_each_present_cpu(cpu) {
+        template = (ptr_t)addr - g_per_cpu_offset[cpu];
+        if (!within_bounds(template, begin, begin + per_cpu_size))
+            continue;
+
+        name = symbol_lookup_data_by_address(
+            template, name_buf, &offset, &symbol_size
+        );
+        if (!str_is_null(name)) {
+            return str_printf(
+                out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH,
+                "%pS+0x%zx/0x%zx on CPU %u", &name, offset, symbol_size, cpu
+            );
+        }
+
+        return str_printf(
+            out_desc_buf, MAX_MEM_DESCRIPTION_LENGTH,
+            "per-CPU area of CPU %u +0x%zx", cpu, template - begin
+        );
+    }
+
+    return NULL_STR();
+}

From 263b4530ab97a531cd7ce5eb01f2225f676e14db Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 11 Nov 2026 23:51:16 +0300
Subject: [PATCH 067/152] symbols: zero the offset and size when a lookup finds
 nothing

This stops GCC from producing false -Wmaybe-uninitialized warnings with
LTO enabled.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/symbols.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/kernel/symbols.c b/kernel/symbols.c
index c685bf9a..d0f1fd43 100644
--- a/kernel/symbols.c
+++ b/kernel/symbols.c
@@ -91,11 +91,27 @@ static struct string do_lookup_symbol(
     );
 }
 
+/*
+ * This silences a false positive -Wmaybe-uninitialized warning that GCC
+ * otherwise produces if LTO is enabled.
+ */
+static inline void clear_lookup_results(
+    size_t *out_offset_within, size_t *out_size
+)
+{
+    if (out_offset_within)
+        *out_offset_within = 0;
+    if (out_size)
+        *out_size = 0;
+}
+
 struct string symbol_lookup_code_by_address(
     ptr_t address, char out_name_buf[static MAX_SYMBOL_LENGTH],
     size_t *out_offset_within, size_t *out_size
 )
 {
+    clear_lookup_results(out_offset_within, out_size);
+
     if (unlikely(!address_is_kernel_code(address)))
         return NULL_STR();
 
@@ -109,6 +125,8 @@ struct string symbol_lookup_data_by_address(
     size_t *out_offset_within, size_t *out_size
 )
 {
+    clear_lookup_results(out_offset_within, out_size);
+
     if (unlikely(!address_is_kernel_data(address)))
         return NULL_STR();
 

From 51724d61ea4ac7b584e8a5c1a0429a5f3b603cdb Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 23 Oct 2026 13:50:11 +0300
Subject: [PATCH 068/152] common/format: add %pMEM

Print what mem_describe() knows about an address, so a report can say
which object a pointer is in rather than just where it points.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/common/format.c | 10 ++++++++++
 scripts/lint.py        |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/kernel/common/format.c b/kernel/common/format.c
index d61e95f6..4d7d05ea 100644
--- a/kernel/common/format.c
+++ b/kernel/common/format.c
@@ -9,6 +9,7 @@
 
 #include <symbols.h>
 #include <pci/address.h>
+#include <memory/describe.h>
 
 struct fmt_buf_state {
     char *buffer;
@@ -475,6 +476,15 @@ static MAYBE_NERR(int) do_vsnprintf(
                 continue;
             }
 
+            if (consume(&fmt, STR("MEM"))) {
+                char text[MAX_MEM_DESCRIPTION_LENGTH];
+                struct string desc;
+
+                desc = mem_describe(va_arg(vlist, void*), text);
+                write_str(fb_state, desc);
+                continue;
+            }
+
             if (consume(&fmt, STR("V"))) {
                 struct nested_printf *npf;
                 va_list nested_vlist;
diff --git a/scripts/lint.py b/scripts/lint.py
index 61282d6a..d940540a 100755
--- a/scripts/lint.py
+++ b/scripts/lint.py
@@ -323,7 +323,7 @@ class ScanResult(NamedTuple):
 
 PRINTF_SOURCE = "kernel/common/format.c"
 
-POINTER_EXTENSIONS = ("RSM", "SM", "S", "E", "PCI", "PA", "V")
+POINTER_EXTENSIONS = ("RSM", "SM", "S", "E", "PCI", "PA", "MEM", "V")
 POINTER_EXTENSION_TYPES = {
     "PA": ("phys_addr_t",),
     "E": ("error_t",),

From 76b98c9429c685b0f16dd91ec14a4b33aa92295e Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 14:46:35 +0300
Subject: [PATCH 069/152] refcount: add refcount32 and refcount64

The buddy allocator counts page references by hand, and tasks need the
same. Add a reference count that catches an increment on 0, an underflow
and an overflow, reports it and parks the count at a saturated value, so
the object leaks instead of being freed while in use. A 64-bit count
cannot overflow in practice and skips that check.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/CMakeLists.txt            |  1 +
 kernel/include/detail/refcount.h | 94 ++++++++++++++++++++++++++++++++
 kernel/include/refcount.h        | 18 ++++++
 kernel/refcount.c                | 35 ++++++++++++
 tests/CMakeLists.txt             |  3 +
 tests/test-cases/files.cmake     |  1 +
 tests/test-cases/test_refcount.c | 77 ++++++++++++++++++++++++++
 7 files changed, 229 insertions(+)
 create mode 100644 kernel/include/detail/refcount.h
 create mode 100644 kernel/include/refcount.h
 create mode 100644 kernel/refcount.c
 create mode 100644 tests/test-cases/test_refcount.c

diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt
index 5a9b6f71..3e0aaaa5 100644
--- a/kernel/CMakeLists.txt
+++ b/kernel/CMakeLists.txt
@@ -101,6 +101,7 @@ ultra_sources(
     entry.c
     panic.c
     bug.c
+    refcount.c
     console.c
     log.c
     symbols.c
diff --git a/kernel/include/detail/refcount.h b/kernel/include/detail/refcount.h
new file mode 100644
index 00000000..425c91f4
--- /dev/null
+++ b/kernel/include/detail/refcount.h
@@ -0,0 +1,94 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/helpers.h>
+#include <common/atomic.h>
+#include <common/bit.h>
+
+/*
+ * A count above MAX is broken. Once caught it is parked at SATURATED,
+ * halfway into the broken range, so racing increments and decrements
+ * can't bring it back to 0 or into the valid range. The object leaks
+ * instead of being freed while in use.
+ */
+#define REFCOUNT32_MAX MAKE_BIT_MASK_U32(30, 0)
+#define REFCOUNT32_SATURATED MAKE_BIT_MASK_U32(31, 30)
+
+#define REFCOUNT64_MAX MAKE_BIT_MASK_U64(62, 0)
+#define REFCOUNT64_SATURATED MAKE_BIT_MASK_U64(63, 62)
+
+/*
+ * At one increment per nanosecond, a 64-bit count takes 292 years to pass
+ * MAX, so it is only checked for use-after-free.
+ */
+#define REFCOUNT_CAN_OVERFLOW(width) ((width) < 64)
+
+enum refcount_op : u8 {
+    REFCOUNT_OP_INC,
+    REFCOUNT_OP_DEC,
+};
+
+/*
+ * set() and read() are relaxed: the count is set before the object is
+ * published, and a read is only a snapshot. dec_and_test() releases so
+ * this owner's writes happen before the free, and acquires on the final
+ * reference so the freeing thread sees every owner's writes.
+ */
+#define MAKE_REFCOUNT_CLASS(width)                                        \
+    struct refcount##width {                                              \
+        u##width value;                                                   \
+    };                                                                    \
+                                                                          \
+    void refcount##width##_saturate(                                      \
+        struct refcount##width *ref, u##width old, enum refcount_op op    \
+    );                                                                    \
+                                                                          \
+    static inline void refcount##width##_set(                             \
+        struct refcount##width *ref, u##width value                       \
+    )                                                                     \
+    {                                                                     \
+        atomic_store_relaxed(&ref->value, value);                         \
+    }                                                                     \
+                                                                          \
+    static inline u##width refcount##width##_read(                        \
+        const struct refcount##width *ref                                 \
+    )                                                                     \
+    {                                                                     \
+        return atomic_load_relaxed(&ref->value);                          \
+    }                                                                     \
+                                                                          \
+    static inline void refcount##width##_inc(struct refcount##width *ref) \
+    {                                                                     \
+        u##width old;                                                     \
+                                                                          \
+        old = atomic_fetch_add(&ref->value, 1, MO_RELAXED);               \
+        if (unlikely(old == 0 || (REFCOUNT_CAN_OVERFLOW(width) &&         \
+                                  old >= REFCOUNT##width##_MAX)))         \
+            refcount##width##_saturate(ref, old, REFCOUNT_OP_INC);        \
+    }                                                                     \
+                                                                          \
+    static inline bool refcount##width##_dec_and_test(                    \
+        struct refcount##width *ref                                       \
+    )                                                                     \
+    {                                                                     \
+        u##width old;                                                     \
+                                                                          \
+        old = atomic_fetch_sub(&ref->value, 1, MO_RELEASE);               \
+        if (old == 1) {                                                   \
+            barrier_acquire();                                            \
+            return true;                                                  \
+        }                                                                 \
+                                                                          \
+        if (unlikely(old == 0 || (REFCOUNT_CAN_OVERFLOW(width) &&         \
+                                  old > REFCOUNT##width##_MAX)))          \
+            refcount##width##_saturate(ref, old, REFCOUNT_OP_DEC);        \
+                                                                          \
+        return false;                                                     \
+    }
+
+MAKE_REFCOUNT_CLASS(32)
+MAKE_REFCOUNT_CLASS(64)
+
+#define REFCOUNT_DISPATCH(ref, op) (_Generic(*(ref), \
+    struct refcount32: refcount32_##op,              \
+    struct refcount64: refcount64_##op))
diff --git a/kernel/include/refcount.h b/kernel/include/refcount.h
new file mode 100644
index 00000000..dbd7a2be
--- /dev/null
+++ b/kernel/include/refcount.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <detail/refcount.h>
+
+/*
+ * An opaque 32/64 bit structure that represents a reference count.
+ *
+ * Use refcount_* helpers to manipulate, all fields are considered internal.
+ */
+struct refcount32;
+struct refcount64;
+
+#define REFCOUNT_INIT(n) { .value = (n) }
+
+#define refcount_set(ref, value) REFCOUNT_DISPATCH(ref, set)(ref, value)
+#define refcount_read(ref) REFCOUNT_DISPATCH(ref, read)(ref)
+#define refcount_inc(ref) REFCOUNT_DISPATCH(ref, inc)(ref)
+#define refcount_dec_and_test(ref) REFCOUNT_DISPATCH(ref, dec_and_test)(ref)
diff --git a/kernel/refcount.c b/kernel/refcount.c
new file mode 100644
index 00000000..459ec4b0
--- /dev/null
+++ b/kernel/refcount.c
@@ -0,0 +1,35 @@
+#include <refcount.h>
+
+#include <bug.h>
+
+static void refcount_report(
+    const void *ref, u64 old, u64 max, enum refcount_op op
+)
+{
+    const char *what;
+
+    // Already saturated, reported when it got there
+    if (old > max)
+        return;
+
+    if (old != 0)
+        what = "overflow, leaking the object";
+    else if (op == REFCOUNT_OP_INC)
+        what = "increment on 0, use-after-free";
+    else
+        what = "underflow, use-after-free";
+
+    WARN_WITH_MSG("refcount %p (%pMEM): %s", ref, ref, what);
+}
+
+#define MAKE_REFCOUNT_SATURATE(width)                                   \
+    void refcount##width##_saturate(                                    \
+        struct refcount##width *ref, u##width old, enum refcount_op op  \
+    )                                                                   \
+    {                                                                   \
+        atomic_store_relaxed(&ref->value, REFCOUNT##width##_SATURATED); \
+        refcount_report(ref, old, REFCOUNT##width##_MAX, op);           \
+    }
+
+MAKE_REFCOUNT_SATURATE(32)
+MAKE_REFCOUNT_SATURATE(64)
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index d9507ae5..b2bbbf54 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -189,6 +189,8 @@ KERNEL_FILE(INCLUDE_FILE "config_helpers.h")
 KERNEL_FILE(INCLUDE_FILE "log.h")
 KERNEL_FILE(INCLUDE_FILE "bug.h")
 KERNEL_FILE(INCLUDE_PATH "detail" INCLUDE_FILE "bug.h")
+KERNEL_FILE(INCLUDE_FILE "refcount.h" SOURCE_FILE "refcount.c")
+KERNEL_FILE(INCLUDE_PATH "detail" INCLUDE_FILE "refcount.h")
 KERNEL_FILE(INCLUDE_FILE "panic.h")
 KERNEL_FILE(INCLUDE_FILE "linker.h")
 KERNEL_FILE(INCLUDE_FILE "linker_common.ld.h")
@@ -252,6 +254,7 @@ add_executable(
     kernel-source/locking/spinlock.c kernel-source/memory/units.c
     kernel-source/common/bit.c kernel-source/locking/mutex.c
     kernel-source/common/string_container.c kernel-source/common/conversions.c
+    kernel-source/refcount.c
 )
 set_property(
     DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
diff --git a/tests/test-cases/files.cmake b/tests/test-cases/files.cmake
index c51f8103..13680b85 100644
--- a/tests/test-cases/files.cmake
+++ b/tests/test-cases/files.cmake
@@ -3,6 +3,7 @@ add_test_cases(
     test_parameter.c
     test_log_ring.c
     test_bit.c
+    test_refcount.c
     test_console.c
     test_string.c
     test_minmax.c
diff --git a/tests/test-cases/test_refcount.c b/tests/test-cases/test_refcount.c
new file mode 100644
index 00000000..18095051
--- /dev/null
+++ b/tests/test-cases/test_refcount.c
@@ -0,0 +1,77 @@
+#include <refcount.h>
+
+#include <test_harness.h>
+
+#define ASSERT_GET_PUT(ref) do {                  \
+        refcount_set(ref, 1);                     \
+        refcount_inc(ref);                        \
+        ASSERT_EQ(refcount_read(ref), 2);         \
+        ASSERT_FALSE(refcount_dec_and_test(ref)); \
+        ASSERT_TRUE(refcount_dec_and_test(ref));  \
+        ASSERT_EQ(refcount_read(ref), 0);         \
+    } while (0)
+
+TEST_CASE(refcount_get_put)
+{
+    struct refcount32 ref32;
+    struct refcount64 ref64;
+
+    ASSERT_GET_PUT(&ref32);
+    ASSERT_GET_PUT(&ref64);
+}
+
+TEST_CASE(refcount_init_and_const_read)
+{
+    struct refcount32 ref32 = REFCOUNT_INIT(1);
+    const struct refcount64 ref64 = REFCOUNT_INIT(3);
+
+    ASSERT_EQ(refcount_read(&ref32), 1);
+    ASSERT_EQ(refcount_read(&ref64), 3);
+}
+
+TEST_CASE(refcount_inc_on_zero_saturates)
+{
+    struct refcount32 ref32 = REFCOUNT_INIT(0);
+    struct refcount64 ref64 = REFCOUNT_INIT(0);
+
+    refcount_inc(&ref32);
+    refcount_inc(&ref64);
+    ASSERT_EQ(refcount_read(&ref32), REFCOUNT32_SATURATED);
+    ASSERT_EQ(refcount_read(&ref64), REFCOUNT64_SATURATED);
+
+    ASSERT_FALSE(refcount_dec_and_test(&ref32));
+    ASSERT_FALSE(refcount_dec_and_test(&ref64));
+}
+
+TEST_CASE(refcount_underflow_saturates)
+{
+    struct refcount32 ref32 = REFCOUNT_INIT(0);
+    struct refcount64 ref64 = REFCOUNT_INIT(0);
+
+    ASSERT_FALSE(refcount_dec_and_test(&ref32));
+    ASSERT_FALSE(refcount_dec_and_test(&ref64));
+    ASSERT_EQ(refcount_read(&ref32), REFCOUNT32_SATURATED);
+    ASSERT_EQ(refcount_read(&ref64), REFCOUNT64_SATURATED);
+}
+
+TEST_CASE(refcount32_overflow_saturates)
+{
+    struct refcount32 ref = REFCOUNT_INIT(REFCOUNT32_MAX);
+
+    refcount_inc(&ref);
+    ASSERT_EQ(refcount_read(&ref), REFCOUNT32_SATURATED);
+
+    // A saturated count stays put
+    refcount_inc(&ref);
+    ASSERT_EQ(refcount_read(&ref), REFCOUNT32_SATURATED);
+    ASSERT_FALSE(refcount_dec_and_test(&ref));
+    ASSERT_EQ(refcount_read(&ref), REFCOUNT32_SATURATED);
+}
+
+TEST_CASE(refcount64_skips_the_overflow_check)
+{
+    struct refcount64 ref = REFCOUNT_INIT(REFCOUNT64_MAX);
+
+    refcount_inc(&ref);
+    ASSERT_EQ(refcount_read(&ref), REFCOUNT64_MAX + 1);
+}

From f3514bdcd3b785f43ecab4bd2179cdcfc67cfd5c Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 14:46:35 +0300
Subject: [PATCH 070/152] memory: use refcount32 for refcounted pages

The page reference count was only checked by MM_BUG_ON. Use refcount32,
which checks it in every build.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/memory/page.h |  5 +++--
 kernel/memory/buddy.c        | 32 +++-----------------------------
 2 files changed, 6 insertions(+), 31 deletions(-)

diff --git a/kernel/include/memory/page.h b/kernel/include/memory/page.h
index ac3c0dba..249b9b3e 100644
--- a/kernel/include/memory/page.h
+++ b/kernel/include/memory/page.h
@@ -40,6 +40,7 @@
 #include <memory/io.h>
 #include <memory/bug.h>
 #include <arch_constants.h>
+#include <refcount.h>
 #include <locking/spinlock.h>
 
 /*
@@ -171,7 +172,7 @@ struct ptdesc_page {
 
 // Base type for every reference counted "page_type"
 struct refcounted_page {
-    u32 num_references;
+    struct refcount32 refcount;
 };
 
 struct anon_page {
@@ -364,7 +365,7 @@ static inline u32 block_num_references(const struct page_block *block)
     const struct page *page = block_to_page(block);
 
     MM_BUG_ON(!page_type_is_refcounted(page_type(page)));
-    return atomic_load_relaxed(&page->refcounted.num_references);
+    return refcount_read(&page->refcounted.refcount);
 }
 
 static inline bool block_is_kernel_heap(const struct page_block *block)
diff --git a/kernel/memory/buddy.c b/kernel/memory/buddy.c
index 5412d157..8287bad5 100644
--- a/kernel/memory/buddy.c
+++ b/kernel/memory/buddy.c
@@ -155,7 +155,7 @@ static struct page_block *prep_block(
         page_set_meta(&page[i], META_MAKE_TAIL(block));
 
     if (page_type_is_refcounted(type))
-        atomic_store_relaxed(&page->refcounted.num_references, 1);
+        refcount_set(&page->refcounted.refcount, 1);
 
     if (behavior & ALLOC_ZEROED)
         memzero(page_to_virt(page), PAGE_SIZE << order);
@@ -220,38 +220,12 @@ void free_frozen_block(struct page_block *block)
 
 void block_ref(struct page_block *block)
 {
-    struct page *page = block_to_page(block);
-    u32 old;
-
-    MM_BUG_ON(!page_type_is_refcounted(page_type(page)));
-
-    old = atomic_fetch_add(
-        &page->refcounted.num_references, 1, MO_RELAXED
-    );
-    MM_BUG_ON(old == 0);
+    refcount_inc(&block_refcounted(block)->refcount);
 }
 
-/*
- * Release decrement so this owner's writes happen before the free,
- * acquire fence on the final reference so the freeing thread observes
- * every owner's writes before the block is reused.
- */
 static bool block_drop_reference(struct page_block *block)
 {
-    struct refcounted_page *refcounted;
-    u32 old;
-
-    refcounted = block_refcounted(block);
-    old = atomic_fetch_sub(
-        &refcounted->num_references, 1, MO_RELEASE
-    );
-    MM_BUG_ON(old == 0);
-
-    if (old != 1)
-        return false;
-
-    barrier_acquire();
-    return true;
+    return refcount_dec_and_test(&block_refcounted(block)->refcount);
 }
 
 void block_unref(struct page_block *block)

From 5ca384a3fe38721b31119f7eb8d6254f5bab1646 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 14:53:45 +0300
Subject: [PATCH 071/152] sched: introduce task allocation and kernel stacks

A task gets a guarded 16 KiB stack and is reference counted, the last
task_unref() frees both.

The boot CPU's idle task takes its stack bounds from the loader's memory
map, since that is the stack the kernel runs on.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/sched/task.h  |  2 +
 kernel/include/private/sched/task.h |  5 ++
 kernel/include/sched/task.h         |  3 ++
 kernel/sched/task.c                 | 72 +++++++++++++++++++++++++++++
 tests/CMakeLists.txt                |  2 +-
 tests/sched_alloc.c                 | 39 ++++++++++++++++
 tests/test-cases/test_sched.c       | 40 ++++++++++++++++
 7 files changed, 162 insertions(+), 1 deletion(-)
 create mode 100644 tests/sched_alloc.c

diff --git a/kernel/include/detail/sched/task.h b/kernel/include/detail/sched/task.h
index 6d3e853e..2db958fc 100644
--- a/kernel/include/detail/sched/task.h
+++ b/kernel/include/detail/sched/task.h
@@ -7,6 +7,7 @@
 #include <config.h>
 #include <sched/task.h>
 
+#include <detail/refcount.h>
 #include <detail/sched/rr.h>
 #include <arch/detail/task.h>
 
@@ -31,6 +32,7 @@ struct task {
 #ifndef ARCH_HAS_CUSTOM_PREEMPT_COUNT
     u32 preempt_count;
 #endif
+    struct refcount32 refcount;
 
     cpu_t cpu;
     u8 prio;
diff --git a/kernel/include/private/sched/task.h b/kernel/include/private/sched/task.h
index 43967bf1..b4fc720f 100644
--- a/kernel/include/private/sched/task.h
+++ b/kernel/include/private/sched/task.h
@@ -2,6 +2,7 @@
 
 #include <common/atomic.h>
 
+#include <memory/units.h>
 #include <per_cpu.h>
 #include <detail/sched/task.h>
 
@@ -11,6 +12,8 @@
 #define TASK_PRIO_LOWEST 31
 #define TASK_NUM_PRIOS (TASK_PRIO_LOWEST + 1)
 
+#define TASK_STACK_SIZE (16 * KiB)
+
 DECLARE_PER_CPU(struct task, g_this_cpu_idle_task);
 
 static inline bool task_test_and_set_flag(
@@ -24,3 +27,5 @@ static inline void task_clear_flag(struct task *task, enum task_flags flag)
 {
     atomic_fetch_and(&task->flags, ~flag, MO_RELEASE);
 }
+
+struct task *task_alloc(void);
diff --git a/kernel/include/sched/task.h b/kernel/include/sched/task.h
index feeafe37..0931a0b9 100644
--- a/kernel/include/sched/task.h
+++ b/kernel/include/sched/task.h
@@ -6,4 +6,7 @@
 // A thread of execution, 'current' is the one running on this CPU
 struct task;
 
+void task_ref(struct task *task);
+void task_unref(struct task *task);
+
 const char *task_name(const struct task *task);
diff --git a/kernel/sched/task.c b/kernel/sched/task.c
index cee3f6dd..efb768f7 100644
--- a/kernel/sched/task.c
+++ b/kernel/sched/task.c
@@ -2,12 +2,20 @@
 
 #include <common/format.h>
 
+#include <boot/boot.h>
+#include <memory/alloc.h>
+#include <memory/valloc.h>
+#include <memory/io.h>
+
+#include <bug.h>
+#include <refcount.h>
 #include <smp.h>
 #include <preempt.h>
 #include <init_level.h>
 #include <free_after_init.h>
 
 DEFINE_PER_CPU(struct task, g_this_cpu_idle_task) = {
+    .refcount = REFCOUNT_INIT(1),
     .name = "idle/0",
 #ifndef ARCH_HAS_CUSTOM_PREEMPT_COUNT
     .preempt_count = PREEMPT_DISABLE_DEPTH(1),
@@ -32,3 +40,67 @@ const char *task_name(const struct task *task)
 {
     return task->name;
 }
+
+static bool INIT_CODE is_kernel_stack(struct ultra_memory_map_entry *mme)
+{
+    return mme->type == ULTRA_MEMORY_TYPE_KERNEL_STACK;
+}
+
+static void INIT_CODE boot_task_set_stack(
+    phys_addr_t start, phys_addr_t end, u32 type, void *user
+)
+{
+    struct task *task = user;
+
+    UNREFERENCED_PARAMETER(type);
+
+    task->stack = phys_to_virt(start);
+    task->stack_size = end - start;
+}
+
+/*
+ * The boot CPU keeps running on the stack the loader set up, which becomes
+ * the stack of its idle task. It lives in the direct map and has no guard.
+ */
+static error_t INIT_CODE boot_task_find_stack(void)
+{
+    for_each_memory_map_range(boot_task_set_stack, is_kernel_stack, current);
+    BUG_ON(current->stack == nullptr);
+
+    return EOK;
+}
+INIT_CALL_POST(BOOT_INFO_AVAILABLE, boot_task_find_stack);
+
+struct task *task_alloc(void)
+{
+    struct task *task;
+
+    task = alloc(sizeof(*task), ALLOC_GENERIC_ZEROED);
+    if (unlikely(task == nullptr))
+        return nullptr;
+
+    task->stack = valloc(TASK_STACK_SIZE, ALLOC_GENERIC);
+    if (unlikely(task->stack == nullptr)) {
+        free(task);
+        return nullptr;
+    }
+
+    task->stack_size = TASK_STACK_SIZE;
+    refcount_set(&task->refcount, 1);
+
+    return task;
+}
+
+void task_ref(struct task *task)
+{
+    refcount_inc(&task->refcount);
+}
+
+void task_unref(struct task *task)
+{
+    if (!refcount_dec_and_test(&task->refcount))
+        return;
+
+    vrelease(task->stack);
+    free(task);
+}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index b2bbbf54..d680e178 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -250,7 +250,7 @@ include("test-cases/files.cmake")
 add_executable(
     run_tests
     run_tests.cpp kernel_helpers.cpp test_harness.cpp memory_map.c ${TEST_SOURCES}
-    task_state.c
+    task_state.c sched_alloc.c
     kernel-source/locking/spinlock.c kernel-source/memory/units.c
     kernel-source/common/bit.c kernel-source/locking/mutex.c
     kernel-source/common/string_container.c kernel-source/common/conversions.c
diff --git a/tests/sched_alloc.c b/tests/sched_alloc.c
new file mode 100644
index 00000000..6c27bea7
--- /dev/null
+++ b/tests/sched_alloc.c
@@ -0,0 +1,39 @@
+#include <stdlib.h>
+#include <string.h>
+
+#include <memory/alloc_behavior.h>
+
+// Counting allocators that test_sched.c substitutes for the kernel's
+int g_sched_test_live_tasks, g_sched_test_live_stacks;
+
+void *sched_test_alloc(size_t size, enum alloc_behavior behavior)
+{
+    void *ptr;
+
+    ptr = malloc(size);
+    if (ptr != nullptr && (behavior & ALLOC_ZEROED))
+        memset(ptr, 0, size);
+
+    g_sched_test_live_tasks++;
+    return ptr;
+}
+
+void sched_test_free(void *ptr)
+{
+    g_sched_test_live_tasks--;
+    free(ptr);
+}
+
+void *sched_test_valloc(size_t size, enum alloc_behavior behavior)
+{
+    (void)behavior;
+
+    g_sched_test_live_stacks++;
+    return malloc(size);
+}
+
+void sched_test_vrelease(void *ptr)
+{
+    g_sched_test_live_stacks--;
+    free(ptr);
+}
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 5adc6df1..3f580ac0 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -1,3 +1,19 @@
+#include <string.h>
+
+#include <memory/alloc_behavior.h>
+
+extern int g_sched_test_live_tasks, g_sched_test_live_stacks;
+
+void *sched_test_alloc(size_t size, enum alloc_behavior behavior);
+void sched_test_free(void *ptr);
+void *sched_test_valloc(size_t size, enum alloc_behavior behavior);
+void sched_test_vrelease(void *ptr);
+
+#define alloc sched_test_alloc
+#define free sched_test_free
+#define valloc sched_test_valloc
+#define vrelease sched_test_vrelease
+
 #include <kernel-source/sched/task.c>
 #include <kernel-source/sched/core.c>
 #include <kernel-source/sched/rr.c>
@@ -149,3 +165,27 @@ TEST_CASE(sched_rr_requeue_goes_to_the_tail)
 
     finish(rq);
 }
+
+TEST_CASE(task_alloc_and_put)
+{
+    struct task *task;
+
+    g_sched_test_live_tasks = 0;
+    g_sched_test_live_stacks = 0;
+
+    task = task_alloc();
+    ASSERT_NOT_NULL(task);
+    ASSERT_NOT_NULL(task->stack);
+    ASSERT_EQ(task->stack_size, TASK_STACK_SIZE);
+    ASSERT_EQ(g_sched_test_live_tasks, 1);
+    ASSERT_EQ(g_sched_test_live_stacks, 1);
+
+    task_ref(task);
+    task_unref(task);
+    ASSERT_EQ(g_sched_test_live_tasks, 1);
+    ASSERT_EQ(g_sched_test_live_stacks, 1);
+
+    task_unref(task);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+    ASSERT_EQ(g_sched_test_live_stacks, 0);
+}

From 7d786a9cb0b9cd9f5933ae8444d4e43241a7f90c Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 14:57:41 +0300
Subject: [PATCH 072/152] sched: introduce schedule() and context switching

More scheduling primitives & building blocks.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/CMakeLists.txt     |   1 +
 kernel/arch/x86_64/task.S             |  10 ++
 kernel/arch/x86_64/task.c             |  35 +++++
 kernel/include/detail/sched/sched.h   |  13 ++
 kernel/include/detail/sched/task.h    |   5 +
 kernel/include/private/arch/task.h    |  16 ++
 kernel/include/private/sched/core.h   |  11 ++
 kernel/include/private/sched/sched.h  |  25 ++++
 kernel/include/private/sched/task.h   |  27 ++++
 kernel/include/sched/task.h           |   2 +
 kernel/sched/core.c                   | 123 +++++++++++++++
 kernel/sched/idle.c                   |   8 +
 kernel/sched/rr.c                     |  10 ++
 kernel/sched/task.c                   |  14 ++
 tests/CMakeLists.txt                  |   6 +-
 tests/arch_task.c                     |  34 +++++
 tests/synthetic-include/irq_helpers.h |  10 ++
 tests/test-cases/test_sched.c         | 207 ++++++++++++++++++++++++++
 18 files changed, 556 insertions(+), 1 deletion(-)
 create mode 100644 kernel/arch/x86_64/task.c
 create mode 100644 kernel/include/detail/sched/sched.h
 create mode 100644 kernel/include/private/sched/sched.h
 create mode 100644 tests/arch_task.c

diff --git a/kernel/arch/x86_64/CMakeLists.txt b/kernel/arch/x86_64/CMakeLists.txt
index 2a1ac382..a1e290cc 100644
--- a/kernel/arch/x86_64/CMakeLists.txt
+++ b/kernel/arch/x86_64/CMakeLists.txt
@@ -24,6 +24,7 @@ ultra_sources(
     earlycon.c
     pci_type1.c
     tsc.c
+    task.c
     task.S
 )
 setup_asm_offsets(asm_offsets.c)
diff --git a/kernel/arch/x86_64/task.S b/kernel/arch/x86_64/task.S
index 3f7ad731..4f988228 100644
--- a/kernel/arch/x86_64/task.S
+++ b/kernel/arch/x86_64/task.S
@@ -5,6 +5,7 @@
 
 ASM_PRELUDE
 
+.extern task_entry
 .extern g_this_cpu_task
 
 ASM_GLOBAL_FUNCTION arch_switch_to
@@ -29,3 +30,12 @@ ASM_GLOBAL_FUNCTION arch_switch_to
     mov rax, rdi
     ret
 ASM_FUNCTION_END arch_switch_to
+
+ASM_GLOBAL_FUNCTION x86_task_entry_asm
+    UNWIND_HINT_UNDEFINED(rip)
+
+    mov rdi, rax
+    mov rsi, rbx
+    mov rdx, r12
+    call task_entry
+ASM_FUNCTION_END x86_task_entry_asm
diff --git a/kernel/arch/x86_64/task.c b/kernel/arch/x86_64/task.c
new file mode 100644
index 00000000..db4da18c
--- /dev/null
+++ b/kernel/arch/x86_64/task.c
@@ -0,0 +1,35 @@
+#include <arch/registers.h>
+
+#include <private/arch/task.h>
+#include <detail/sched/task.h>
+
+#include <common/helpers.h>
+
+struct x86_switch_frame {
+    reg_t r15;
+    reg_t r14;
+    reg_t r13;
+    reg_t r12;
+    reg_t rbx;
+    reg_t rbp;
+    reg_t return_address;
+};
+
+void x86_task_entry_asm(void);
+
+void arch_task_frame_init(struct task *task, task_fn_t fn, void *user)
+{
+    struct x86_switch_frame *frame;
+    void *top;
+
+    top = task->stack + task->stack_size;
+
+    frame = top - sizeof(*frame);
+    *frame = (struct x86_switch_frame) {
+        .r12 = (reg_t)user,
+        .rbx = (reg_t)fn,
+        .return_address = (reg_t)x86_task_entry_asm,
+    };
+
+    task->arch.sp = (virt_addr_t)frame;
+}
diff --git a/kernel/include/detail/sched/sched.h b/kernel/include/detail/sched/sched.h
new file mode 100644
index 00000000..d090b212
--- /dev/null
+++ b/kernel/include/detail/sched/sched.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <common/types.h>
+
+/*
+ * Switch to the next task. A caller that isn't TASK_STATE_RUNNING is taken off
+ * the runqueue and blocks until someone wakes them up. A running caller only
+ * switches if something should run first.
+ *
+ * NOTE: A blocked caller may be woken up for a reason other than the one it
+ *       waits for, so it has to check its condition again after this returns.
+ */
+void schedule(void);
diff --git a/kernel/include/detail/sched/task.h b/kernel/include/detail/sched/task.h
index 2db958fc..8731123e 100644
--- a/kernel/include/detail/sched/task.h
+++ b/kernel/include/detail/sched/task.h
@@ -5,6 +5,7 @@
 #include <common/bit.h>
 
 #include <config.h>
+#include <locking/spinlock.h>
 #include <sched/task.h>
 
 #include <detail/refcount.h>
@@ -27,12 +28,16 @@ enum task_flags : u32 {
 #endif
 };
 
+enum task_state : u32;
+
 struct task {
     enum task_flags flags;
 #ifndef ARCH_HAS_CUSTOM_PREEMPT_COUNT
     u32 preempt_count;
 #endif
+    enum task_state state;
     struct refcount32 refcount;
+    struct spinlock lock;
 
     cpu_t cpu;
     u8 prio;
diff --git a/kernel/include/private/arch/task.h b/kernel/include/private/arch/task.h
index 32770974..38e5796c 100644
--- a/kernel/include/private/arch/task.h
+++ b/kernel/include/private/arch/task.h
@@ -1,10 +1,26 @@
 #pragma once
 
+#include <common/attributes.h>
+
 #include <sched/task.h>
 
+/*
+ * Prepare the stack of a task that has never run, so that the first
+ * arch_switch_to() into it calls task_entry() with 'fn' and 'user'.
+ */
+void arch_task_frame_init(struct task *task, task_fn_t fn, void *user);
+
 /*
  * Perform a context switch from 'prev' to 'next'. Once the CPU is
  * yielded back to 'prev', the return value contains the task that
  * performed the switch.
  */
 struct task *arch_switch_to(struct task *prev, struct task *next);
+
+/*
+ * The first generic code a new task runs, called from the frame set up by
+ * arch_task_frame_init(). The architecture must pass the return value of
+ * the arch_switch_to() call that first spun up the task as 'prev'.
+ */
+NORETURN
+void task_entry(struct task *prev, task_fn_t fn, void *user);
diff --git a/kernel/include/private/sched/core.h b/kernel/include/private/sched/core.h
index 82e06c61..09648145 100644
--- a/kernel/include/private/sched/core.h
+++ b/kernel/include/private/sched/core.h
@@ -5,12 +5,16 @@
 
 #include <per_cpu.h>
 #include <linker.h>
+#include <preempt.h>
 #include <locking/spinlock.h>
 
+#include <private/sched/sched.h>
 #include <private/sched/rr.h>
 #include <private/sched/classes.h>
 #include <private/sched/task.h>
 
+#define PREEMPT_COUNT_AT_SWITCH PREEMPT_DISABLE_DEPTH(2)
+
 struct rq;
 
 /*
@@ -37,6 +41,9 @@ struct sched_class {
      * nullptr if the class has nothing runnable.
      */
     struct task *(*pick)(struct rq *rq);
+
+    // Called when rq->curr yields the CPU to its peers but stays runnable
+    void (*yield)(struct rq *rq);
 };
 
 #define DEFINE_SCHED_CLASS(name)                                      \
@@ -56,6 +63,7 @@ struct rq {
      * running task. The idle task is not accounted here.
      */
     u32 nr_running;
+    u64 nr_switches;
 
     struct rr_rq rr;
 };
@@ -69,3 +77,6 @@ DECLARE_PER_CPU(struct rq, g_this_cpu_rq);
 void rq_enqueue(struct rq *rq, struct task *task);
 void rq_dequeue(struct rq *rq, struct task *task);
 struct task *rq_pick_next(struct rq *rq);
+
+void schedule_idle(void);
+void sched_finish_switch(struct task *prev);
diff --git a/kernel/include/private/sched/sched.h b/kernel/include/private/sched/sched.h
new file mode 100644
index 00000000..cdb79b4e
--- /dev/null
+++ b/kernel/include/private/sched/sched.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#include <common/atomic.h>
+#include <common/types.h>
+
+#include <private/sched/task.h>
+#include <detail/sched/sched.h>
+
+/*
+ * The store is followed by a full barrier, so that a waiter can set its
+ * state, check its condition and then schedule() without missing a wakeup
+ * that makes the condition true in between.
+ */
+#define set_current_state(new_state) ({ \
+    task_set_state(current, new_state); \
+    barrier_full();                     \
+})
+
+/*
+ * Stay runnable but let the other tasks of the same priority run first.
+ *
+ * NOTE: This is not a way to wait for another task, and tasks of a lower
+ *       priority never get to run.
+ */
+void yield(void);
diff --git a/kernel/include/private/sched/task.h b/kernel/include/private/sched/task.h
index b4fc720f..89d5ddfe 100644
--- a/kernel/include/private/sched/task.h
+++ b/kernel/include/private/sched/task.h
@@ -1,6 +1,8 @@
 #pragma once
 
+#include <common/attributes.h>
 #include <common/atomic.h>
+#include <common/bit.h>
 
 #include <memory/units.h>
 #include <per_cpu.h>
@@ -14,8 +16,30 @@
 
 #define TASK_STACK_SIZE (16 * KiB)
 
+enum task_state : u32 {
+    TASK_STATE_RUNNING = 0,
+    TASK_STATE_UNINTERRUPTIBLE = BIT_U32(0),
+
+    /*
+     * A task is TASK_STATE_NEW until it first runs and TASK_STATE_DEAD once
+     * it exits
+     */
+    TASK_STATE_NEW = BIT_U32(8),
+    TASK_STATE_DEAD = BIT_U32(9),
+};
+
 DECLARE_PER_CPU(struct task, g_this_cpu_idle_task);
 
+static inline enum task_state task_state(struct task *task)
+{
+    return atomic_load_relaxed(&task->state);
+}
+
+static inline void task_set_state(struct task *task, enum task_state state)
+{
+    atomic_store_relaxed(&task->state, state);
+}
+
 static inline bool task_test_and_set_flag(
     struct task *task, enum task_flags flag
 )
@@ -29,3 +53,6 @@ static inline void task_clear_flag(struct task *task, enum task_flags flag)
 }
 
 struct task *task_alloc(void);
+
+NORETURN
+void task_exit(void);
diff --git a/kernel/include/sched/task.h b/kernel/include/sched/task.h
index 0931a0b9..ce67be7f 100644
--- a/kernel/include/sched/task.h
+++ b/kernel/include/sched/task.h
@@ -3,6 +3,8 @@
 // Arch-specific way to fetch 'current'
 #include <arch/detail/current.h>
 
+typedef void (*task_fn_t)(void *user);
+
 // A thread of execution, 'current' is the one running on this CPU
 struct task;
 
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 6ad8a222..eb13c7f1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1,11 +1,16 @@
 #include <private/sched/core.h>
 #include <private/sched/rr.h>
 #include <private/sched/idle.h>
+#include <private/sched/task.h>
+#include <private/arch/task.h>
 
 #include <common/helpers.h>
+#include <common/atomic.h>
 
 #include <bug.h>
 #include <linker.h>
+#include <preempt.h>
+#include <irq_helpers.h>
 #include <smp.h>
 #include <init_level.h>
 #include <free_after_init.h>
@@ -48,6 +53,124 @@ struct task *rq_pick_next(struct rq *rq)
     BUG();
 }
 
+/*
+ * The second half of a switch, run by the task that was switched to. The
+ * runqueue lock was taken by 'prev' and is released here, and every task
+ * gets here with the preempt count at two: one from whoever called
+ * do_schedule() and one for that lock.
+ */
+void sched_finish_switch(struct task *prev)
+{
+    struct rq *rq;
+    enum task_state prev_state;
+
+    rq = this_rq();
+    BUG_ON(preempt_count() != PREEMPT_COUNT_AT_SWITCH);
+
+    prev_state = task_state(prev);
+
+    spin_unlock(&rq->lock);
+    local_irq_enable();
+
+    /*
+     * A task cannot free its own stack that it's running on, so free it here
+     * now that we're off it.
+     */
+    if (prev_state == TASK_STATE_DEAD)
+        task_unref(prev);
+}
+
+/*
+ * Called with preemption disabled exactly once and interrupts enabled.
+ * A task that is not runnable anymore is taken off the runqueue here,
+ * and only here, so that a wakeup arriving before this point finds it
+ * still queued.
+ */
+static void do_schedule(void)
+{
+    struct rq *rq;
+    struct task *prev, *next;
+    enum task_state prev_state;
+
+    BUG_ON(in_hard_irq());
+    BUG_ON(preempt_count() != PREEMPT_DISABLE_DEPTH(1));
+    BUG_ON(!local_irq_enabled());
+
+    local_irq_disable();
+
+    rq = this_rq();
+    spin_lock(&rq->lock);
+
+    prev = rq->curr;
+    prev_state = task_state(prev);
+
+    if (prev_state != TASK_STATE_RUNNING) {
+        BUG_ON(prev == rq->idle);
+        rq_dequeue(rq, prev);
+    }
+
+    next = rq_pick_next(rq);
+    if (next == prev) {
+        spin_unlock(&rq->lock);
+        local_irq_enable();
+        return;
+    }
+
+    rq->nr_switches++;
+    rq->curr = next;
+
+    prev = arch_switch_to(prev, next);
+    sched_finish_switch(prev);
+}
+
+void schedule(void)
+{
+    preempt_disable();
+    do_schedule();
+    preempt_enable();
+}
+
+void schedule_idle(void)
+{
+    /*
+     * The idle task runs at a preempt depth of one for its whole life,
+     * so the level schedule() adds around this call is already there.
+     */
+    do_schedule();
+}
+
+void yield(void)
+{
+    struct rq *rq;
+    irq_state_t irq_state;
+
+    preempt_disable();
+
+    rq = this_rq();
+
+    irq_state = spin_lock_irq_save(&rq->lock);
+    current->sched_class->yield(rq);
+    spin_unlock_irq_restore(&rq->lock, irq_state);
+
+    do_schedule();
+    preempt_enable();
+}
+
+void task_exit(void)
+{
+    struct task *task = current;
+    irq_state_t irq_state;
+
+    preempt_disable();
+
+    irq_state = spin_lock_irq_save(&task->lock);
+    task_set_state(task, TASK_STATE_DEAD);
+    spin_unlock_irq_restore(&task->lock, irq_state);
+
+    do_schedule();
+    BUG();
+}
+
 static void rq_init(struct rq *rq, cpu_t cpu)
 {
     struct task *idle;
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 592a749b..49b5fab3 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -26,8 +26,16 @@ static struct task *idle_pick(struct rq *rq)
     return rq->idle;
 }
 
+static void idle_yield(struct rq *rq)
+{
+    UNREFERENCED_PARAMETER(rq);
+
+    BUG();
+}
+
 DEFINE_SCHED_CLASS(idle) = {
     .enqueue = idle_enqueue,
     .dequeue = idle_dequeue,
     .pick = idle_pick,
+    .yield = idle_yield,
 };
diff --git a/kernel/sched/rr.c b/kernel/sched/rr.c
index 6fcd38b8..b531c8bb 100644
--- a/kernel/sched/rr.c
+++ b/kernel/sched/rr.c
@@ -43,8 +43,18 @@ static struct task *rr_pick(struct rq *rq)
     return list_first_entry(&rr->queues[prio], struct task, rr.link);
 }
 
+static void rr_yield(struct rq *rq)
+{
+    struct task *curr = rq->curr;
+    struct rr_rq *rr = &rq->rr;
+
+    list_remove(&curr->rr.link);
+    list_insert_prev(&rr->queues[curr->prio], &curr->rr.link);
+}
+
 DEFINE_SCHED_CLASS(rr) = {
     .enqueue = rr_enqueue,
     .dequeue = rr_dequeue,
     .pick = rr_pick,
+    .yield = rr_yield,
 };
diff --git a/kernel/sched/task.c b/kernel/sched/task.c
index efb768f7..03063745 100644
--- a/kernel/sched/task.c
+++ b/kernel/sched/task.c
@@ -1,4 +1,6 @@
 #include <private/sched/task.h>
+#include <private/arch/task.h>
+#include <private/sched/core.h>
 
 #include <common/format.h>
 
@@ -86,6 +88,9 @@ struct task *task_alloc(void)
     }
 
     task->stack_size = TASK_STACK_SIZE;
+#ifndef ARCH_HAS_CUSTOM_PREEMPT_COUNT
+    task->preempt_count = PREEMPT_COUNT_AT_SWITCH;
+#endif
     refcount_set(&task->refcount, 1);
 
     return task;
@@ -104,3 +109,12 @@ void task_unref(struct task *task)
     vrelease(task->stack);
     free(task);
 }
+
+void task_entry(struct task *prev, task_fn_t fn, void *user)
+{
+    sched_finish_switch(prev);
+    preempt_enable();
+
+    fn(user);
+    task_exit();
+}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index d680e178..e2fc2147 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -206,11 +206,14 @@ KERNEL_FILE(
 )
 KERNEL_FILE(INCLUDE_PATH "detail/sched" INCLUDE_FILE "task.h")
 KERNEL_FILE(INCLUDE_PATH "private/sched" INCLUDE_FILE "task.h")
+KERNEL_FILE(INCLUDE_PATH "private/arch" INCLUDE_FILE "task.h")
 KERNEL_FILE(
     INCLUDE_PATH "private/sched" INCLUDE_FILE "core.h"
     SOURCE_PATH "sched" SOURCE_FILE "core.c"
 )
 KERNEL_FILE(INCLUDE_PATH "private/sched" INCLUDE_FILE "classes.h")
+KERNEL_FILE(INCLUDE_PATH "private/sched" INCLUDE_FILE "sched.h")
+KERNEL_FILE(INCLUDE_PATH "detail/sched" INCLUDE_FILE "sched.h")
 KERNEL_FILE(
     INCLUDE_PATH "private/sched" INCLUDE_FILE "rr.h"
     SOURCE_PATH "sched" SOURCE_FILE "rr.c"
@@ -249,8 +252,9 @@ include("test-cases/files.cmake")
 
 add_executable(
     run_tests
-    run_tests.cpp kernel_helpers.cpp test_harness.cpp memory_map.c ${TEST_SOURCES}
+    run_tests.cpp kernel_helpers.cpp test_harness.cpp memory_map.c arch_task.c
     task_state.c sched_alloc.c
+    ${TEST_SOURCES}
     kernel-source/locking/spinlock.c kernel-source/memory/units.c
     kernel-source/common/bit.c kernel-source/locking/mutex.c
     kernel-source/common/string_container.c kernel-source/common/conversions.c
diff --git a/tests/arch_task.c b/tests/arch_task.c
new file mode 100644
index 00000000..340761db
--- /dev/null
+++ b/tests/arch_task.c
@@ -0,0 +1,34 @@
+#include <private/arch/task.h>
+#include <private/sched/task.h>
+
+// Userspace stand-in for the arch task switching primitives, built on ucontext
+static __thread struct task *s_switched_from;
+
+static void entry_trampoline(void)
+{
+    struct task *self = g_test_current;
+
+    task_entry(s_switched_from, self->arch.fn, self->arch.user);
+}
+
+void arch_task_frame_init(struct task *task, task_fn_t fn, void *user)
+{
+    getcontext(&task->arch.context);
+
+    task->arch.context.uc_stack.ss_sp = task->stack;
+    task->arch.context.uc_stack.ss_size = task->stack_size;
+    task->arch.context.uc_link = nullptr;
+    task->arch.fn = fn;
+    task->arch.user = user;
+
+    makecontext(&task->arch.context, entry_trampoline, 0);
+}
+
+struct task *arch_switch_to(struct task *prev, struct task *next)
+{
+    s_switched_from = prev;
+    g_test_current = next;
+
+    swapcontext(&prev->arch.context, &next->arch.context);
+    return s_switched_from;
+}
diff --git a/tests/synthetic-include/irq_helpers.h b/tests/synthetic-include/irq_helpers.h
index 3620b3ac..a614b69a 100644
--- a/tests/synthetic-include/irq_helpers.h
+++ b/tests/synthetic-include/irq_helpers.h
@@ -31,3 +31,13 @@ static inline void local_irq_state_restore(irq_state_t state)
 {
     g_test_irqs_enabled = state;
 }
+
+static inline void arch_local_irq_enable(void)
+{
+    g_test_irqs_enabled = true;
+}
+
+static inline void arch_local_irq_disable(void)
+{
+    g_test_irqs_enabled = false;
+}
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 3f580ac0..27519b99 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -19,6 +19,8 @@ void sched_test_vrelease(void *ptr);
 #include <kernel-source/sched/rr.c>
 #include <kernel-source/sched/idle.c>
 
+#include <private/arch/task.h>
+
 #include <test_harness.h>
 
 #define NUM_TEST_TASKS 8
@@ -189,3 +191,208 @@ TEST_CASE(task_alloc_and_put)
     ASSERT_EQ(g_sched_test_live_tasks, 0);
     ASSERT_EQ(g_sched_test_live_stacks, 0);
 }
+
+#define MAX_LOG 64
+
+struct runner {
+    int id;
+    int rounds;
+};
+
+static int s_log[MAX_LOG];
+static int s_log_length;
+
+static void log_id(int id)
+{
+    if (s_log_length < MAX_LOG)
+        s_log[s_log_length++] = id;
+}
+
+static struct rq *switching_test_begin(void)
+{
+    g_num_present_cpus = 1;
+    g_per_cpu_offset[0] = 0;
+
+    memset(&g_this_cpu_rq, 0, sizeof(g_this_cpu_rq));
+    sched_init();
+
+    g_test_current = &g_this_cpu_idle_task;
+    current->preempt_count = PREEMPT_DISABLE_DEPTH(1);
+    g_test_irqs_enabled = true;
+
+    g_sched_test_live_tasks = 0;
+    g_sched_test_live_stacks = 0;
+    s_log_length = 0;
+
+    return cpu_rq(0);
+}
+
+static void switching_test_end(void)
+{
+    current->preempt_count = 0;
+}
+
+static struct task *spawn(task_fn_t fn, void *user, u8 prio)
+{
+    struct rq *rq = cpu_rq(0);
+    struct task *task;
+
+    task = task_alloc();
+    task->prio = prio;
+    task->sched_class = &SCHED_CLASS_INSTANCE(rr);
+    arch_task_frame_init(task, fn, user);
+
+    spin_lock(&rq->lock);
+    rq_enqueue(rq, task);
+    spin_unlock(&rq->lock);
+
+    return task;
+}
+
+static void yield_between_rounds(void *user)
+{
+    struct runner *runner = user;
+    int i;
+
+    for (i = 0; i < runner->rounds; i++) {
+        log_id(runner->id);
+        yield();
+    }
+}
+
+static void run_to_completion(void *user)
+{
+    struct runner *runner = user;
+    int i;
+
+    for (i = 0; i < runner->rounds; i++)
+        log_id(runner->id);
+}
+
+static void log_and_sleep(void *user)
+{
+    struct runner *runner = user;
+
+    for (;;) {
+        log_id(runner->id);
+        set_current_state(TASK_STATE_UNINTERRUPTIBLE);
+        schedule();
+    }
+}
+
+TEST_CASE(sched_idle_with_nothing_to_run)
+{
+    struct rq *rq;
+
+    rq = switching_test_begin();
+
+    schedule_idle();
+
+    ASSERT_EQ(current, rq->idle);
+    ASSERT_EQ(rq->nr_switches, 0);
+    ASSERT_EQ(preempt_count(), PREEMPT_DISABLE_DEPTH(1));
+    ASSERT(local_irq_enabled());
+
+    switching_test_end();
+}
+
+TEST_CASE(sched_runs_a_task_until_it_exits)
+{
+    struct runner runner = { .id = 7, .rounds = 3 };
+    struct rq *rq;
+
+    rq = switching_test_begin();
+    spawn(run_to_completion, &runner, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(current, rq->idle);
+    ASSERT_EQ(rq->curr, rq->idle);
+    ASSERT_EQ(s_log_length, 3);
+    ASSERT_EQ(rq->nr_running, 0);
+    ASSERT_EQ(rq->nr_switches, 2);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+    ASSERT_EQ(g_sched_test_live_stacks, 0);
+    ASSERT_EQ(preempt_count(), PREEMPT_DISABLE_DEPTH(1));
+    ASSERT(local_irq_enabled());
+    ASSERT(!spin_is_locked(&rq->lock));
+
+    switching_test_end();
+}
+
+TEST_CASE(sched_yield_round_robins)
+{
+    struct runner runners[3];
+    struct rq *rq;
+    int i;
+
+    rq = switching_test_begin();
+
+    for (i = 0; i < 3; i++) {
+        runners[i] = (struct runner) { .id = i, .rounds = 3 };
+        spawn(yield_between_rounds, &runners[i], TASK_PRIO_DEFAULT);
+    }
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 9);
+    for (i = 0; i < 9; i++)
+        ASSERT_EQ(s_log[i], i % 3);
+
+    ASSERT_EQ(rq->nr_running, 0);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+    ASSERT_EQ(g_sched_test_live_stacks, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(sched_higher_priority_runs_first)
+{
+    struct runner low = { .id = 1, .rounds = 2 };
+    struct runner high = { .id = 2, .rounds = 2 };
+    struct rq *rq;
+
+    rq = switching_test_begin();
+
+    spawn(yield_between_rounds, &low, TASK_PRIO_LOWEST);
+    spawn(yield_between_rounds, &high, TASK_PRIO_HIGHEST);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 4);
+    ASSERT_EQ(s_log[0], 2);
+    ASSERT_EQ(s_log[1], 2);
+    ASSERT_EQ(s_log[2], 1);
+    ASSERT_EQ(s_log[3], 1);
+    ASSERT_EQ(rq->nr_running, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(sched_sleeping_task_leaves_the_runqueue)
+{
+    struct runner sleeper = { .id = 1 };
+    struct runner other = { .id = 2, .rounds = 2 };
+    struct task *task;
+    struct rq *rq;
+
+    rq = switching_test_begin();
+
+    task = spawn(log_and_sleep, &sleeper, TASK_PRIO_HIGHEST);
+    spawn(run_to_completion, &other, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 3);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 2);
+    ASSERT_EQ(s_log[2], 2);
+
+    ASSERT(!task->on_rq);
+    ASSERT_EQ(task_state(task), TASK_STATE_UNINTERRUPTIBLE);
+    ASSERT_EQ(rq->nr_running, 0);
+    ASSERT_EQ(g_sched_test_live_tasks, 1);
+
+    task_unref(task);
+    switching_test_end();
+}

From 24789251caf9da5b1d8fb9a48cb35ed8f8588008 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:29:57 +0300
Subject: [PATCH 073/152] sched: introduce wake_up_task()

A sleeping task needs a way back onto a runqueue. wake_up_task() makes a
task in one of the given states runnable, and new tasks are started
through it as well.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/sched/task.h   |   1 +
 kernel/include/private/sched/sched.h |   7 ++
 kernel/sched/core.c                  |  67 +++++++++++++++
 kernel/sched/task.c                  |   1 +
 tests/test-cases/test_sched.c        | 117 +++++++++++++++++++++++++--
 5 files changed, 188 insertions(+), 5 deletions(-)

diff --git a/kernel/include/detail/sched/task.h b/kernel/include/detail/sched/task.h
index 8731123e..2bc5e7e2 100644
--- a/kernel/include/detail/sched/task.h
+++ b/kernel/include/detail/sched/task.h
@@ -42,6 +42,7 @@ struct task {
     cpu_t cpu;
     u8 prio;
     bool on_rq;
+    bool on_cpu;
     const struct sched_class *sched_class;
 
     union {
diff --git a/kernel/include/private/sched/sched.h b/kernel/include/private/sched/sched.h
index cdb79b4e..5a26f563 100644
--- a/kernel/include/private/sched/sched.h
+++ b/kernel/include/private/sched/sched.h
@@ -16,6 +16,13 @@
     barrier_full();                     \
 })
 
+/*
+ * Make 'task' runnable if it is in one of the states in 'state_mask'.
+ * Returns whether it was. Callable from any context that may take a
+ * spinlock, the caller must hold a reference to the task.
+ */
+bool wake_up_task(struct task *task, enum task_state state_mask);
+
 /*
  * Stay runnable but let the other tasks of the same priority run first.
  *
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index eb13c7f1..4d58bef2 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -15,6 +15,8 @@
 #include <init_level.h>
 #include <free_after_init.h>
 
+#include <arch/cpu_helpers.h>
+
 DEFINE_PER_CPU(struct rq, g_this_cpu_rq);
 
 void rq_enqueue(struct rq *rq, struct task *task)
@@ -68,6 +70,7 @@ void sched_finish_switch(struct task *prev)
     BUG_ON(preempt_count() != PREEMPT_COUNT_AT_SWITCH);
 
     prev_state = task_state(prev);
+    atomic_store_release(&prev->on_cpu, false);
 
     spin_unlock(&rq->lock);
     local_irq_enable();
@@ -118,11 +121,73 @@ static void do_schedule(void)
 
     rq->nr_switches++;
     rq->curr = next;
+    atomic_store_relaxed(&next->on_cpu, true);
 
     prev = arch_switch_to(prev, next);
     sched_finish_switch(prev);
 }
 
+bool wake_up_task(struct task *task, enum task_state state_mask)
+{
+    struct rq *rq;
+    irq_state_t irq_state;
+    bool woken = false;
+
+    irq_state = spin_lock_irq_save(&task->lock);
+    /*
+     * Pairs with the barrier in set_current_state(), the sleeper stores
+     * its state and then checks its condition, the waker stores the
+     * condition and then loads the state here, so one of the two is
+     * guaranteed to see the other's store
+     */
+    barrier_full();
+
+    if (!(task_state(task) & state_mask))
+        goto out;
+
+    rq = task_rq(task);
+    spin_lock(&rq->lock);
+
+    /*
+     * Still queued, so its own schedule() call, whenever it gets there,
+     * will find it runnable and keep it queued.
+     */
+    if (task->on_rq) {
+        task_set_state(task, TASK_STATE_RUNNING);
+        spin_unlock(&rq->lock);
+
+        woken = true;
+        goto out;
+    }
+
+    spin_unlock(&rq->lock);
+
+    // Its CPU may still be in the middle of switching away from it
+    while (atomic_load_acquire(&task->on_cpu))
+        arch_cpu_relax();
+
+    rq = task_rq(task);
+    spin_lock(&rq->lock);
+
+    /*
+     * Give the task its own reference so it cannot be freed via an external
+     * task_unref() while it's running. This pairs with task_unref() inside
+     * sched_finish_switch() when we detect that this task has exited.
+     */
+    if (task_state(task) == TASK_STATE_NEW)
+        task_ref(task);
+
+    task_set_state(task, TASK_STATE_RUNNING);
+    rq_enqueue(rq, task);
+
+    spin_unlock(&rq->lock);
+    woken = true;
+
+out:
+    spin_unlock_irq_restore(&task->lock, irq_state);
+    return woken;
+}
+
 void schedule(void)
 {
     preempt_disable();
@@ -182,6 +247,8 @@ static void rq_init(struct rq *rq, cpu_t cpu)
     spin_lock_init(&rq->lock);
     rr_rq_init(&rq->rr);
 
+    idle->on_cpu = true;
+
     rq->idle = idle;
     rq->curr = idle;
 }
diff --git a/kernel/sched/task.c b/kernel/sched/task.c
index 03063745..9b0c4dd5 100644
--- a/kernel/sched/task.c
+++ b/kernel/sched/task.c
@@ -92,6 +92,7 @@ struct task *task_alloc(void)
     task->preempt_count = PREEMPT_COUNT_AT_SWITCH;
 #endif
     refcount_set(&task->refcount, 1);
+    task->state = TASK_STATE_NEW;
 
     return task;
 }
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 27519b99..6ce504c4 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -234,7 +234,6 @@ static void switching_test_end(void)
 
 static struct task *spawn(task_fn_t fn, void *user, u8 prio)
 {
-    struct rq *rq = cpu_rq(0);
     struct task *task;
 
     task = task_alloc();
@@ -242,10 +241,8 @@ static struct task *spawn(task_fn_t fn, void *user, u8 prio)
     task->sched_class = &SCHED_CLASS_INSTANCE(rr);
     arch_task_frame_init(task, fn, user);
 
-    spin_lock(&rq->lock);
-    rq_enqueue(rq, task);
-    spin_unlock(&rq->lock);
-
+    wake_up_task(task, TASK_STATE_NEW);
+    task_unref(task);
     return task;
 }
 
@@ -396,3 +393,113 @@ TEST_CASE(sched_sleeping_task_leaves_the_runqueue)
     task_unref(task);
     switching_test_end();
 }
+
+struct sleeper {
+    struct task *task;
+    struct task *to_wake;
+    int id;
+    bool woke_self;
+    bool wake_result;
+};
+
+static void sleep_once(void *user)
+{
+    struct sleeper *sleeper = user;
+
+    log_id(sleeper->id);
+
+    set_current_state(TASK_STATE_UNINTERRUPTIBLE);
+    schedule();
+
+    log_id(sleeper->id);
+}
+
+static void wake_other(void *user)
+{
+    struct sleeper *sleeper = user;
+
+    log_id(sleeper->id);
+    sleeper->wake_result = wake_up_task(
+        sleeper->to_wake, TASK_STATE_UNINTERRUPTIBLE
+    );
+    log_id(sleeper->id);
+}
+
+static void wake_self_before_schedule(void *user)
+{
+    struct sleeper *sleeper = user;
+
+    set_current_state(TASK_STATE_UNINTERRUPTIBLE);
+    sleeper->woke_self = wake_up_task(current, TASK_STATE_UNINTERRUPTIBLE);
+    schedule();
+
+    log_id(sleeper->id);
+}
+
+TEST_CASE(sched_new_task_is_woken_once)
+{
+    struct runner runner = { .id = 1, .rounds = 1 };
+    struct task *task;
+    struct rq *rq;
+
+    rq = switching_test_begin();
+
+    task = spawn(run_to_completion, &runner, TASK_PRIO_DEFAULT);
+    ASSERT(task->on_rq);
+    ASSERT_EQ(task_state(task), TASK_STATE_RUNNING);
+    ASSERT_EQ(rq->nr_running, 1);
+
+    ASSERT(!wake_up_task(task, TASK_STATE_NEW));
+    ASSERT(!wake_up_task(task, TASK_STATE_UNINTERRUPTIBLE));
+    ASSERT_EQ(rq->nr_running, 1);
+
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 1);
+
+    switching_test_end();
+}
+
+TEST_CASE(sched_wakeup_resumes_a_sleeper)
+{
+    struct sleeper sleeper = { .id = 1 };
+    struct sleeper waker = { .id = 2 };
+    struct rq *rq;
+
+    rq = switching_test_begin();
+
+    sleeper.task = spawn(sleep_once, &sleeper, TASK_PRIO_DEFAULT);
+    waker.to_wake = sleeper.task;
+    waker.task = spawn(wake_other, &waker, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT(waker.wake_result);
+    ASSERT_EQ(s_log_length, 4);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 2);
+    ASSERT_EQ(s_log[2], 2);
+    ASSERT_EQ(s_log[3], 1);
+
+    ASSERT_EQ(rq->nr_running, 0);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(sched_wakeup_before_schedule_is_not_lost)
+{
+    struct sleeper sleeper = { .id = 1 };
+    struct rq *rq;
+
+    rq = switching_test_begin();
+
+    spawn(wake_self_before_schedule, &sleeper, TASK_PRIO_DEFAULT);
+    schedule_idle();
+
+    ASSERT(sleeper.woke_self);
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT_EQ(rq->nr_running, 0);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}

From eb233afa510ccf0ef5aa620255377056428988a8 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:31:04 +0300
Subject: [PATCH 074/152] per_cpu: add this_cpu_and() and this_cpu_or()

The need_resched bit in the x86_64 preempt count has to be set and
cleared without racing interrupts. A single instruction on x86_64 like
the rest of the family.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/include/arch/detail/per_cpu.h | 10 ++++++++++
 kernel/include/per_cpu.h                         |  2 ++
 tests/synthetic-include/per_cpu.h                |  2 ++
 3 files changed, 14 insertions(+)

diff --git a/kernel/arch/x86_64/include/arch/detail/per_cpu.h b/kernel/arch/x86_64/include/arch/detail/per_cpu.h
index fbf40b67..a513c23d 100644
--- a/kernel/arch/x86_64/include/arch/detail/per_cpu.h
+++ b/kernel/arch/x86_64/include/arch/detail/per_cpu.h
@@ -42,6 +42,16 @@
 #define this_cpu_sub_32(val, x) X86_THIS_CPU_MODIFY("subl", "ri", u32, val, x)
 #define this_cpu_sub_64(val, x) X86_THIS_CPU_MODIFY("subq", "re", u64, val, x)
 
+#define this_cpu_and_8(val, x) X86_THIS_CPU_MODIFY("andb", "qi", u8, val, x)
+#define this_cpu_and_16(val, x) X86_THIS_CPU_MODIFY("andw", "ri", u16, val, x)
+#define this_cpu_and_32(val, x) X86_THIS_CPU_MODIFY("andl", "ri", u32, val, x)
+#define this_cpu_and_64(val, x) X86_THIS_CPU_MODIFY("andq", "re", u64, val, x)
+
+#define this_cpu_or_8(val, x) X86_THIS_CPU_MODIFY("orb", "qi", u8, val, x)
+#define this_cpu_or_16(val, x) X86_THIS_CPU_MODIFY("orw", "ri", u16, val, x)
+#define this_cpu_or_32(val, x) X86_THIS_CPU_MODIFY("orl", "ri", u32, val, x)
+#define this_cpu_or_64(val, x) X86_THIS_CPU_MODIFY("orq", "re", u64, val, x)
+
 /*
  * Don't use xchg here, which implies the lock prefix when dealing with a
  * memory operand. We don't need such semantics for per-cpu private variables,
diff --git a/kernel/include/per_cpu.h b/kernel/include/per_cpu.h
index 09c2ce5f..ffc6d77e 100644
--- a/kernel/include/per_cpu.h
+++ b/kernel/include/per_cpu.h
@@ -112,6 +112,8 @@
 
 #define this_cpu_add(var, x) DISPATCH_CPU_WRITE_OP(var, add, x)
 #define this_cpu_sub(var, x) DISPATCH_CPU_WRITE_OP(var, sub, x)
+#define this_cpu_and(var, x) DISPATCH_CPU_WRITE_OP(var, and, x)
+#define this_cpu_or(var, x) DISPATCH_CPU_WRITE_OP(var, or, x)
 
 #define this_cpu_inc(var) this_cpu_add(var, 1)
 #define this_cpu_dec(var) this_cpu_sub(var, 1)
diff --git a/tests/synthetic-include/per_cpu.h b/tests/synthetic-include/per_cpu.h
index b477f436..ad5e0e28 100644
--- a/tests/synthetic-include/per_cpu.h
+++ b/tests/synthetic-include/per_cpu.h
@@ -22,6 +22,8 @@ extern ptr_t g_per_cpu_offset[];
 
 #define this_cpu_add(var, x) ((void)((var) += (x)))
 #define this_cpu_sub(var, x) ((void)((var) -= (x)))
+#define this_cpu_and(var, x) ((void)((var) &= (x)))
+#define this_cpu_or(var, x) ((void)((var) |= (x)))
 
 #define this_cpu_inc(var) this_cpu_add(var, 1)
 #define this_cpu_dec(var) this_cpu_sub(var, 1)

From 47b79a0070466fb5b8091a1f2bf0b93dc2a9ed83 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:32:28 +0300
Subject: [PATCH 075/152] sched: introduce need_resched and wakeup preemption

A woken task of higher priority than whatever is currently running on
its CPU's runqueue cannot take over from inside wake_up_task(), since
that holds the runqueue lock with interrupts disabled and may be called
from an interrupt handler. Instead, we set TASK_FLAG_NEED_RESCHED on the
running task of that runqueue (the waker itself, unless an interrupt
handler did the waking), and preempt_enable() switches to the woken
task once the preempt count drops back to zero. On x86_64 we also keep
the flag in the per-CPU preempt count, inverted, so that a single subl
can test whether a preemption is pending.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .../arch/x86_64/include/arch/detail/preempt.h |  29 +++
 kernel/include/detail/preempt.h               |  35 ++++
 kernel/include/detail/sched/task.h            |   7 +
 kernel/include/preempt.h                      |   4 +-
 kernel/include/private/sched/classes.h        |   2 +
 kernel/include/private/sched/core.h           |  16 ++
 kernel/sched/core.c                           |  82 +++++++--
 kernel/sched/idle.c                           |   9 +
 kernel/sched/rr.c                             |   7 +
 tests/test-cases/test_preempt.c               |  31 ++++
 tests/test-cases/test_sched.c                 | 166 ++++++++++++++++++
 11 files changed, 375 insertions(+), 13 deletions(-)

diff --git a/kernel/arch/x86_64/include/arch/detail/preempt.h b/kernel/arch/x86_64/include/arch/detail/preempt.h
index 8cd2f8cd..16f1b464 100644
--- a/kernel/arch/x86_64/include/arch/detail/preempt.h
+++ b/kernel/arch/x86_64/include/arch/detail/preempt.h
@@ -32,3 +32,32 @@ static ALWAYS_INLINE void preempt_count_sub(u32 value)
 {
     this_cpu_sub(g_this_cpu_preempt_count, value);
 }
+
+// True if nothing keeps preemption off anymore and a reschedule was pending
+static ALWAYS_INLINE bool preempt_count_sub_and_test(u32 value)
+{
+    bool is_zero;
+
+    asm volatile(
+        "subl %[value], %%gs:%[count]"
+        : "=@ccz" (is_zero), [count] "+m" (g_this_cpu_preempt_count)
+        : [value] "ri" (value)
+    );
+
+    return is_zero;
+}
+
+static ALWAYS_INLINE bool preempt_should_resched(void)
+{
+    return this_cpu_read(g_this_cpu_preempt_count) == 0;
+}
+
+static ALWAYS_INLINE void preempt_set_need_resched(void)
+{
+    this_cpu_and(g_this_cpu_preempt_count, ~X86_PREEMPT_RESCHED_NOT_NEEDED);
+}
+
+static ALWAYS_INLINE void preempt_clear_need_resched(void)
+{
+    this_cpu_or(g_this_cpu_preempt_count, X86_PREEMPT_RESCHED_NOT_NEEDED);
+}
diff --git a/kernel/include/detail/preempt.h b/kernel/include/detail/preempt.h
index 1ebe040e..3a88e559 100644
--- a/kernel/include/detail/preempt.h
+++ b/kernel/include/detail/preempt.h
@@ -42,4 +42,39 @@ static ALWAYS_INLINE void preempt_count_sub(u32 value)
     count = atomic_load_relaxed(&current->preempt_count);
     atomic_store_relaxed(&current->preempt_count, count - value);
 }
+
+static ALWAYS_INLINE bool preempt_should_resched(void)
+{
+    return atomic_load_relaxed(&current->preempt_count) == 0 &&
+           task_test_flag(current, TASK_FLAG_NEED_RESCHED);
+}
+
+static ALWAYS_INLINE bool preempt_count_sub_and_test(u32 value)
+{
+    struct task *task = current;
+    u32 count;
+
+    count = atomic_load_relaxed(&task->preempt_count) - value;
+    atomic_store_relaxed(&task->preempt_count, count);
+    if (count != 0)
+        return false;
+
+    /*
+     * The count must be stored before the flag is read since an IRQ after the
+     * store sees zero and preempts on its own way out, and an IRQ before sets
+     * the flag that is read here.
+     */
+    compiler_barrier();
+    return task_test_flag(task, TASK_FLAG_NEED_RESCHED);
+}
+
+static ALWAYS_INLINE void preempt_set_need_resched(void)
+{
+}
+
+static ALWAYS_INLINE void preempt_clear_need_resched(void)
+{
+}
 #endif
+
+void preempt_schedule(void);
diff --git a/kernel/include/detail/sched/task.h b/kernel/include/detail/sched/task.h
index 2bc5e7e2..657fb384 100644
--- a/kernel/include/detail/sched/task.h
+++ b/kernel/include/detail/sched/task.h
@@ -3,6 +3,7 @@
 #include <common/types.h>
 #include <common/helpers.h>
 #include <common/bit.h>
+#include <common/atomic.h>
 
 #include <config.h>
 #include <locking/spinlock.h>
@@ -23,6 +24,7 @@ struct sched_class;
 
 enum task_flags : u32 {
     TASK_FLAG_NONE = 0,
+    TASK_FLAG_NEED_RESCHED = BIT_U32(0),
 #if IS_ENABLED(UBSAN)
     TASK_FLAG_IN_UBSAN_REPORT = BIT_U32(31),
 #endif
@@ -55,3 +57,8 @@ struct task {
 
     struct arch_task arch;
 };
+
+static inline bool task_test_flag(struct task *task, enum task_flags flag)
+{
+    return atomic_load_relaxed(&task->flags) & flag;
+}
diff --git a/kernel/include/preempt.h b/kernel/include/preempt.h
index 163f88f4..9c81442a 100644
--- a/kernel/include/preempt.h
+++ b/kernel/include/preempt.h
@@ -14,7 +14,9 @@ static ALWAYS_INLINE void preempt_disable(void)
 static ALWAYS_INLINE void preempt_enable(void)
 {
     compiler_barrier();
-    preempt_count_sub(PREEMPT_DISABLE_UNIT);
+
+    if (unlikely(preempt_count_sub_and_test(PREEMPT_DISABLE_UNIT)))
+        preempt_schedule();
 }
 
 // True while executing in hardware interrupt context on this CPU
diff --git a/kernel/include/private/sched/classes.h b/kernel/include/private/sched/classes.h
index b2c861f1..574260aa 100644
--- a/kernel/include/private/sched/classes.h
+++ b/kernel/include/private/sched/classes.h
@@ -19,3 +19,5 @@
 #define SCHED_CLASSES \
     SCHED_CLASS(rr)   \
     SCHED_CLASS(idle)
+
+#define SCHED_CLASS_IS_ABOVE(class, other) ((ptr_t)(class) < (ptr_t)(other))
diff --git a/kernel/include/private/sched/core.h b/kernel/include/private/sched/core.h
index 09648145..7461c21c 100644
--- a/kernel/include/private/sched/core.h
+++ b/kernel/include/private/sched/core.h
@@ -44,6 +44,16 @@ struct sched_class {
 
     // Called when rq->curr yields the CPU to its peers but stays runnable
     void (*yield)(struct rq *rq);
+
+    /*
+     * Called when 'task' wakes up on 'rq' and is of the same class as
+     * rq->curr. The callback is expected to decide whether 'task' is more
+     * important than rq->curr and call resched_curr() if that's the case.
+     *
+     * NOTE: For a 'task' of a class ranked above that of rq->curr,
+     *       resched_curr() is invoked automatically.
+     */
+    void (*wakeup_preempt)(struct rq *rq, struct task *task);
 };
 
 #define DEFINE_SCHED_CLASS(name)                                      \
@@ -74,9 +84,15 @@ DECLARE_PER_CPU(struct rq, g_this_cpu_rq);
 #define cpu_rq(cpu) per_cpu_ptr(&g_this_cpu_rq, cpu)
 #define task_rq(task) cpu_rq((task)->cpu)
 
+static inline bool need_resched(void)
+{
+    return task_test_flag(current, TASK_FLAG_NEED_RESCHED);
+}
+
 void rq_enqueue(struct rq *rq, struct task *task);
 void rq_dequeue(struct rq *rq, struct task *task);
 struct task *rq_pick_next(struct rq *rq);
+void resched_curr(struct rq *rq);
 
 void schedule_idle(void);
 void sched_finish_switch(struct task *prev);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 4d58bef2..ae0eedd5 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -55,6 +55,27 @@ struct task *rq_pick_next(struct rq *rq)
     BUG();
 }
 
+void resched_curr(struct rq *rq)
+{
+    BUG_ON(!spin_is_locked(&rq->lock));
+
+    if (task_test_and_set_flag(rq->curr, TASK_FLAG_NEED_RESCHED))
+        return;
+
+    BUG_ON(rq != this_rq());
+    preempt_set_need_resched();
+}
+
+static void rq_wakeup_preempt(struct rq *rq, struct task *task)
+{
+    struct task *curr = rq->curr;
+
+    if (task->sched_class == curr->sched_class)
+        task->sched_class->wakeup_preempt(rq, task);
+    else if (SCHED_CLASS_IS_ABOVE(task->sched_class, curr->sched_class))
+        resched_curr(rq);
+}
+
 /*
  * The second half of a switch, run by the task that was switched to. The
  * runqueue lock was taken by 'prev' and is released here, and every task
@@ -89,7 +110,7 @@ void sched_finish_switch(struct task *prev)
  * and only here, so that a wakeup arriving before this point finds it
  * still queued.
  */
-static void do_schedule(void)
+static void do_schedule(bool preempted)
 {
     struct rq *rq;
     struct task *prev, *next;
@@ -107,12 +128,21 @@ static void do_schedule(void)
     prev = rq->curr;
     prev_state = task_state(prev);
 
-    if (prev_state != TASK_STATE_RUNNING) {
+    /*
+     * The only way for a task to get dequeued is if it yielded execution itself
+     * (aka wasn't preempted) AND it set its state to something else besides
+     * RUNNING, e.g. UNINTERRUPTIBLE or DEAD.
+     */
+    if (!preempted && prev_state != TASK_STATE_RUNNING) {
         BUG_ON(prev == rq->idle);
         rq_dequeue(rq, prev);
     }
 
     next = rq_pick_next(rq);
+
+    task_clear_flag(prev, TASK_FLAG_NEED_RESCHED);
+    preempt_clear_need_resched();
+
     if (next == prev) {
         spin_unlock(&rq->lock);
         local_irq_enable();
@@ -154,6 +184,10 @@ bool wake_up_task(struct task *task, enum task_state state_mask)
      */
     if (task->on_rq) {
         task_set_state(task, TASK_STATE_RUNNING);
+
+        if (!task->on_cpu)
+            rq_wakeup_preempt(rq, task);
+
         spin_unlock(&rq->lock);
 
         woken = true;
@@ -179,6 +213,7 @@ bool wake_up_task(struct task *task, enum task_state state_mask)
 
     task_set_state(task, TASK_STATE_RUNNING);
     rq_enqueue(rq, task);
+    rq_wakeup_preempt(rq, task);
 
     spin_unlock(&rq->lock);
     woken = true;
@@ -188,20 +223,43 @@ bool wake_up_task(struct task *task, enum task_state state_mask)
     return woken;
 }
 
+static ALWAYS_INLINE void preempt_enable_no_resched(void)
+{
+    compiler_barrier();
+    preempt_count_sub(PREEMPT_DISABLE_UNIT);
+}
+
 void schedule(void)
 {
-    preempt_disable();
-    do_schedule();
-    preempt_enable();
+    do {
+        preempt_disable();
+        do_schedule(false);
+        preempt_enable_no_resched();
+    } while (need_resched());
 }
 
 void schedule_idle(void)
 {
-    /*
-     * The idle task runs at a preempt depth of one for its whole life,
-     * so the level schedule() adds around this call is already there.
-     */
-    do_schedule();
+    do {
+        /*
+         * The idle task runs at a preempt depth of one for its whole life,
+         * so the level schedule() adds around this call is already there.
+         */
+        do_schedule(false);
+    } while (need_resched());
+}
+
+// Invoked by the outermost preempt_enable() if there was a pending reschedule
+void preempt_schedule(void)
+{
+    if (!local_irq_enabled())
+        return;
+
+    do {
+        preempt_disable();
+        do_schedule(true);
+        preempt_enable_no_resched();
+    } while (need_resched());
 }
 
 void yield(void)
@@ -217,7 +275,7 @@ void yield(void)
     current->sched_class->yield(rq);
     spin_unlock_irq_restore(&rq->lock, irq_state);
 
-    do_schedule();
+    do_schedule(false);
     preempt_enable();
 }
 
@@ -232,7 +290,7 @@ void task_exit(void)
     task_set_state(task, TASK_STATE_DEAD);
     spin_unlock_irq_restore(&task->lock, irq_state);
 
-    do_schedule();
+    do_schedule(false);
     BUG();
 }
 
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 49b5fab3..ec5ca9bd 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -33,9 +33,18 @@ static void idle_yield(struct rq *rq)
     BUG();
 }
 
+static void idle_wakeup_preempt(struct rq *rq, struct task *task)
+{
+    UNREFERENCED_PARAMETER(rq);
+    UNREFERENCED_PARAMETER(task);
+
+    BUG();
+}
+
 DEFINE_SCHED_CLASS(idle) = {
     .enqueue = idle_enqueue,
     .dequeue = idle_dequeue,
     .pick = idle_pick,
     .yield = idle_yield,
+    .wakeup_preempt = idle_wakeup_preempt,
 };
diff --git a/kernel/sched/rr.c b/kernel/sched/rr.c
index b531c8bb..ad46ce5d 100644
--- a/kernel/sched/rr.c
+++ b/kernel/sched/rr.c
@@ -52,9 +52,16 @@ static void rr_yield(struct rq *rq)
     list_insert_prev(&rr->queues[curr->prio], &curr->rr.link);
 }
 
+static void rr_wakeup_preempt(struct rq *rq, struct task *task)
+{
+    if (task->prio < rq->curr->prio)
+        resched_curr(rq);
+}
+
 DEFINE_SCHED_CLASS(rr) = {
     .enqueue = rr_enqueue,
     .dequeue = rr_dequeue,
     .pick = rr_pick,
     .yield = rr_yield,
+    .wakeup_preempt = rr_wakeup_preempt,
 };
diff --git a/tests/test-cases/test_preempt.c b/tests/test-cases/test_preempt.c
index 4d3d55c2..13cf2f57 100644
--- a/tests/test-cases/test_preempt.c
+++ b/tests/test-cases/test_preempt.c
@@ -1,4 +1,5 @@
 #include <preempt.h>
+#include <private/sched/task.h>
 #include <private/arch/preempt.h>
 
 #include <test_harness.h>
@@ -56,3 +57,33 @@ TEST_CASE(preempt_hard_irq_independent_of_depth)
     preempt_enable();
     ASSERT_EQ(preempt_count(), base);
 }
+
+TEST_CASE(preempt_reaching_zero_alone_does_not_reschedule)
+{
+    u32 base;
+
+    base = preempt_count();
+    preempt_count_add(PREEMPT_DISABLE_UNIT);
+    ASSERT(!preempt_count_sub_and_test(PREEMPT_DISABLE_UNIT));
+    ASSERT(!preempt_should_resched());
+    ASSERT_EQ(preempt_count(), base);
+}
+
+TEST_CASE(preempt_need_resched_takes_effect_at_zero)
+{
+    u32 base;
+
+    base = preempt_count();
+    preempt_count_add(PREEMPT_DISABLE_DEPTH(2));
+    task_test_and_set_flag(current, TASK_FLAG_NEED_RESCHED);
+    ASSERT_EQ(preempt_count(), base + PREEMPT_DISABLE_DEPTH(2));
+    ASSERT(!preempt_should_resched());
+
+    ASSERT(!preempt_count_sub_and_test(PREEMPT_DISABLE_UNIT));
+    ASSERT(preempt_count_sub_and_test(PREEMPT_DISABLE_UNIT));
+    ASSERT(preempt_should_resched());
+
+    task_clear_flag(current, TASK_FLAG_NEED_RESCHED);
+    ASSERT(!preempt_should_resched());
+    ASSERT_EQ(preempt_count(), base);
+}
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 6ce504c4..2c952991 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -218,7 +218,9 @@ static struct rq *switching_test_begin(void)
 
     g_test_current = &g_this_cpu_idle_task;
     current->preempt_count = PREEMPT_DISABLE_DEPTH(1);
+    task_clear_flag(current, TASK_FLAG_NEED_RESCHED);
     g_test_irqs_enabled = true;
+    g_this_cpu_idle_task.flags = 0;
 
     g_sched_test_live_tasks = 0;
     g_sched_test_live_stacks = 0;
@@ -503,3 +505,167 @@ TEST_CASE(sched_wakeup_before_schedule_is_not_lost)
 
     switching_test_end();
 }
+
+struct spawner {
+    int id;
+    u8 child_prio;
+    bool disable_preemption;
+    struct runner child;
+};
+
+static void spawn_child_between_logs(void *user)
+{
+    struct spawner *spawner = user;
+
+    log_id(spawner->id);
+
+    if (spawner->disable_preemption)
+        preempt_disable();
+
+    spawn(run_to_completion, &spawner->child, spawner->child_prio);
+    log_id(spawner->id);
+
+    if (spawner->disable_preemption)
+        preempt_enable();
+
+    log_id(spawner->id);
+}
+
+TEST_CASE(sched_waking_idle_cpu_requests_a_reschedule)
+{
+    struct runner runner = { .id = 1, .rounds = 1 };
+    struct rq *rq;
+
+    rq = switching_test_begin();
+    ASSERT(!need_resched());
+
+    spawn(run_to_completion, &runner, TASK_PRIO_DEFAULT);
+    ASSERT(need_resched());
+    ASSERT_EQ(current, rq->idle);
+
+    schedule_idle();
+    ASSERT(!need_resched());
+    ASSERT_EQ(s_log_length, 1);
+
+    switching_test_end();
+}
+
+TEST_CASE(sched_higher_priority_wakeup_preempts)
+{
+    struct spawner spawner = {
+        .id = 1,
+        .child_prio = TASK_PRIO_HIGHEST,
+        .child = { .id = 2, .rounds = 1 },
+    };
+
+    switching_test_begin();
+    spawn(spawn_child_between_logs, &spawner, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 4);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 2);
+    ASSERT_EQ(s_log[2], 1);
+    ASSERT_EQ(s_log[3], 1);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(sched_equal_priority_wakeup_does_not_preempt)
+{
+    struct spawner spawner = {
+        .id = 1,
+        .child_prio = TASK_PRIO_DEFAULT,
+        .child = { .id = 2, .rounds = 1 },
+    };
+
+    switching_test_begin();
+    spawn(spawn_child_between_logs, &spawner, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 4);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 1);
+    ASSERT_EQ(s_log[2], 1);
+    ASSERT_EQ(s_log[3], 2);
+
+    switching_test_end();
+}
+
+TEST_CASE(sched_preemption_waits_for_preempt_enable)
+{
+    struct spawner spawner = {
+        .id = 1,
+        .child_prio = TASK_PRIO_HIGHEST,
+        .disable_preemption = true,
+        .child = { .id = 2, .rounds = 1 },
+    };
+
+    switching_test_begin();
+    spawn(spawn_child_between_logs, &spawner, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 4);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 1);
+    ASSERT_EQ(s_log[2], 2);
+    ASSERT_EQ(s_log[3], 1);
+
+    switching_test_end();
+}
+
+struct preempted_sleeper {
+    struct task *task;
+    struct runner unused;
+    bool woken_by_child;
+};
+
+static void wake_parent(void *user)
+{
+    struct preempted_sleeper *sleeper = user;
+
+    log_id(2);
+    sleeper->woken_by_child = wake_up_task(
+        sleeper->task, TASK_STATE_UNINTERRUPTIBLE
+    );
+}
+
+static void get_preempted_while_going_to_sleep(void *user)
+{
+    struct preempted_sleeper *sleeper = user;
+
+    log_id(1);
+
+    set_current_state(TASK_STATE_UNINTERRUPTIBLE);
+    spawn(wake_parent, sleeper, TASK_PRIO_HIGHEST);
+    schedule();
+
+    log_id(1);
+}
+
+TEST_CASE(sched_preempted_sleeper_stays_queued)
+{
+    struct preempted_sleeper sleeper = { 0 };
+    struct rq *rq;
+
+    rq = switching_test_begin();
+
+    sleeper.task = spawn(
+        get_preempted_while_going_to_sleep, &sleeper, TASK_PRIO_DEFAULT
+    );
+    schedule_idle();
+
+    ASSERT(sleeper.woken_by_child);
+    ASSERT_EQ(s_log_length, 3);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 2);
+    ASSERT_EQ(s_log[2], 1);
+    ASSERT_EQ(rq->nr_running, 0);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}

From 1b6afd4a509f2c390ce8dde7de12c4f4fe804c20 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:35:10 +0300
Subject: [PATCH 076/152] sched: allow preemption on return from an interrupt

A handler cannot switch tasks, it can only ask for a reschedule,
typically by waking somebody. Act on it right before returning to kernel
code that had interrupts and preemption enabled, when all that is left
of the interrupt is the saved registers.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/include/arch/private/idt.h |   6 +-
 kernel/arch/x86_64/include/arch/registers.h   |   5 +
 kernel/arch/x86_64/irq.c                      |   4 +-
 kernel/arch/x86_64/lapic_domain.c             |   2 +-
 kernel/include/private/arch/preempt.h         |  16 ++-
 kernel/sched/core.c                           |  20 +++
 tests/synthetic-include/arch/registers.h      |  12 ++
 tests/synthetic-include/irq_helpers.h         |   5 +
 tests/test-cases/test_preempt.c               |   7 +-
 tests/test-cases/test_sched.c                 | 117 ++++++++++++++++++
 10 files changed, 186 insertions(+), 8 deletions(-)
 create mode 100644 tests/synthetic-include/arch/registers.h

diff --git a/kernel/arch/x86_64/include/arch/private/idt.h b/kernel/arch/x86_64/include/arch/private/idt.h
index 3e68b2df..6bdbd5b2 100644
--- a/kernel/arch/x86_64/include/arch/private/idt.h
+++ b/kernel/arch/x86_64/include/arch/private/idt.h
@@ -63,8 +63,10 @@
     void X86_IRQ_DISPATCH(struct registers *regs); \
     void X86_IRQ_DISPATCH(struct registers *regs)
 
+struct registers;
+
 void fixed_vector_enter(u8 vector);
-void fixed_vector_exit(void);
+void fixed_vector_exit(struct registers *regs);
 
 #define FIXED_VECTOR_HANDLER(x, vector)                 \
     static void CONCAT(do_, x)(struct registers *regs); \
@@ -73,7 +75,7 @@ void fixed_vector_exit(void);
     {                                                   \
         fixed_vector_enter(vector);                     \
         CONCAT(do_, x)(regs);                           \
-        fixed_vector_exit();                            \
+        fixed_vector_exit(regs);                        \
     }                                                   \
     static void CONCAT(do_, x)(struct registers *regs)
 
diff --git a/kernel/arch/x86_64/include/arch/registers.h b/kernel/arch/x86_64/include/arch/registers.h
index 2434b680..ca8d967a 100644
--- a/kernel/arch/x86_64/include/arch/registers.h
+++ b/kernel/arch/x86_64/include/arch/registers.h
@@ -58,3 +58,8 @@ static inline void registers_set_return_value(struct registers *regs, u64 value)
 {
     regs->rax = value;
 }
+
+static inline irq_state_t registers_get_irq_state(struct registers *regs)
+{
+    return regs->flags;
+}
diff --git a/kernel/arch/x86_64/irq.c b/kernel/arch/x86_64/irq.c
index 3d6776f2..28a00d9b 100644
--- a/kernel/arch/x86_64/irq.c
+++ b/kernel/arch/x86_64/irq.c
@@ -23,9 +23,9 @@ void fixed_vector_enter(u8 vector)
     this_cpu_inc(s_num_fixed_invocations[vector - VECTOR_FIXED_FIRST]);
 }
 
-void fixed_vector_exit(void)
+void fixed_vector_exit(struct registers *regs)
 {
-    hard_irq_exit();
+    hard_irq_exit(regs);
 }
 
 // No fixed vector handlers are wired up yet, every slot lands here
diff --git a/kernel/arch/x86_64/lapic_domain.c b/kernel/arch/x86_64/lapic_domain.c
index 03900097..4257da55 100644
--- a/kernel/arch/x86_64/lapic_domain.c
+++ b/kernel/arch/x86_64/lapic_domain.c
@@ -56,7 +56,7 @@ IRQ_HANDLER {
     irq_deliver(irq);
 
 out:
-    hard_irq_exit();
+    hard_irq_exit(regs);
 }
 
 // The domain's private bookkeeping for one allocated interrupt
diff --git a/kernel/include/private/arch/preempt.h b/kernel/include/private/arch/preempt.h
index 34b2a43e..9934e708 100644
--- a/kernel/include/private/arch/preempt.h
+++ b/kernel/include/private/arch/preempt.h
@@ -5,6 +5,9 @@
 #include <bug.h>
 #include <preempt.h>
 
+struct registers;
+void preempt_schedule_irq(struct registers *regs);
+
 /*
  * The hook arch-specific interrupt handling code calls upon entry into an
  * IRQ handler (NMI and exceptions never call this).
@@ -17,7 +20,18 @@ static ALWAYS_INLINE void hard_irq_enter(void)
     preempt_count_add(PREEMPT_HARD_IRQ_UNIT);
 }
 
-static ALWAYS_INLINE void hard_irq_exit(void)
+/*
+ * The hook arch-specific interrupt handling code calls at the end of every
+ * IRQ (NMI and exceptions never call this), after the EOI and with interrupts
+ * still disabled.
+ *
+ * This leaves hard IRQ context, then checks whether a preemption is pending
+ * and if it's allowed to actually perform it (the interrupted context had IRQs
+ * enabled & preemption was not explicitly disabled), and if so, performs a
+ * context switch.
+ */
+static ALWAYS_INLINE void hard_irq_exit(struct registers *regs)
 {
     preempt_count_sub(PREEMPT_HARD_IRQ_UNIT);
+    preempt_schedule_irq(regs);
 }
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index ae0eedd5..a1292796 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3,6 +3,7 @@
 #include <private/sched/idle.h>
 #include <private/sched/task.h>
 #include <private/arch/task.h>
+#include <private/arch/preempt.h>
 
 #include <common/helpers.h>
 #include <common/atomic.h>
@@ -16,6 +17,7 @@
 #include <free_after_init.h>
 
 #include <arch/cpu_helpers.h>
+#include <arch/registers.h>
 
 DEFINE_PER_CPU(struct rq, g_this_cpu_rq);
 
@@ -262,6 +264,24 @@ void preempt_schedule(void)
     } while (need_resched());
 }
 
+void preempt_schedule_irq(struct registers *regs)
+{
+    if (!arch_local_irq_is_enabled_state(registers_get_irq_state(regs)))
+        return;
+    if (!preempt_should_resched())
+        return;
+
+    do {
+        preempt_disable();
+        local_irq_enable();
+
+        do_schedule(true);
+
+        local_irq_disable();
+        preempt_enable_no_resched();
+    } while (need_resched());
+}
+
 void yield(void)
 {
     struct rq *rq;
diff --git a/tests/synthetic-include/arch/registers.h b/tests/synthetic-include/arch/registers.h
new file mode 100644
index 00000000..c0b65d4a
--- /dev/null
+++ b/tests/synthetic-include/arch/registers.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include <common/types.h>
+
+struct registers {
+    irq_state_t irq_state;
+};
+
+static inline irq_state_t registers_get_irq_state(struct registers *regs)
+{
+    return regs->irq_state;
+}
diff --git a/tests/synthetic-include/irq_helpers.h b/tests/synthetic-include/irq_helpers.h
index a614b69a..ad3b4075 100644
--- a/tests/synthetic-include/irq_helpers.h
+++ b/tests/synthetic-include/irq_helpers.h
@@ -32,6 +32,11 @@ static inline void local_irq_state_restore(irq_state_t state)
     g_test_irqs_enabled = state;
 }
 
+static inline bool arch_local_irq_is_enabled_state(irq_state_t state)
+{
+    return state;
+}
+
 static inline void arch_local_irq_enable(void)
 {
     g_test_irqs_enabled = true;
diff --git a/tests/test-cases/test_preempt.c b/tests/test-cases/test_preempt.c
index 13cf2f57..aa3f2e19 100644
--- a/tests/test-cases/test_preempt.c
+++ b/tests/test-cases/test_preempt.c
@@ -1,6 +1,7 @@
 #include <preempt.h>
 #include <private/sched/task.h>
 #include <private/arch/preempt.h>
+#include <arch/registers.h>
 
 #include <test_harness.h>
 
@@ -23,6 +24,7 @@ TEST_CASE(preempt_disable_nests)
 
 TEST_CASE(preempt_hard_irq_context)
 {
+    struct registers regs = { .irq_state = false };
     u32 base = preempt_count();
 
     ASSERT(!in_hard_irq());
@@ -31,13 +33,14 @@ TEST_CASE(preempt_hard_irq_context)
     ASSERT(in_hard_irq());
     ASSERT_EQ(preempt_count(), base + PREEMPT_HARD_IRQ_DEPTH(1));
 
-    hard_irq_exit();
+    hard_irq_exit(&regs);
     ASSERT(!in_hard_irq());
     ASSERT_EQ(preempt_count(), base);
 }
 
 TEST_CASE(preempt_hard_irq_independent_of_depth)
 {
+    struct registers regs = { .irq_state = false };
     u32 base = preempt_count();
 
     preempt_disable();
@@ -51,7 +54,7 @@ TEST_CASE(preempt_hard_irq_independent_of_depth)
         base + PREEMPT_DISABLE_DEPTH(2) + PREEMPT_HARD_IRQ_DEPTH(1)
     );
     preempt_enable();
-    hard_irq_exit();
+    hard_irq_exit(&regs);
 
     ASSERT(!in_hard_irq());
     preempt_enable();
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 2c952991..2aa5aca4 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -20,6 +20,7 @@ void sched_test_vrelease(void *ptr);
 #include <kernel-source/sched/idle.c>
 
 #include <private/arch/task.h>
+#include <private/arch/preempt.h>
 
 #include <test_harness.h>
 
@@ -669,3 +670,119 @@ TEST_CASE(sched_preempted_sleeper_stays_queued)
 
     switching_test_end();
 }
+
+struct interrupted {
+    int id;
+    bool irqs_were_enabled;
+    bool preemption_disabled;
+    struct runner child;
+};
+
+/*
+ * Emulates a handler that wakes a task: the wakeup runs in hard
+ * interrupt context with interrupts off, then the exit path decides
+ * whether to switch.
+ */
+static void take_fake_interrupt(struct interrupted *ctx)
+{
+    struct registers regs = { .irq_state = ctx->irqs_were_enabled };
+    irq_state_t irq_state;
+
+    irq_state = local_irq_state_save();
+    hard_irq_enter();
+    spawn(run_to_completion, &ctx->child, TASK_PRIO_HIGHEST);
+    hard_irq_exit(&regs);
+
+    local_irq_state_restore(irq_state);
+}
+
+static void get_interrupted(void *user)
+{
+    struct interrupted *ctx = user;
+    irq_state_t irq_state = 0;
+
+    log_id(ctx->id);
+
+    if (ctx->preemption_disabled)
+        preempt_disable();
+    if (!ctx->irqs_were_enabled)
+        irq_state = local_irq_state_save();
+
+    take_fake_interrupt(ctx);
+    log_id(ctx->id);
+
+    if (!ctx->irqs_were_enabled)
+        local_irq_state_restore(irq_state);
+    if (ctx->preemption_disabled)
+        preempt_enable();
+
+    log_id(ctx->id);
+}
+
+TEST_CASE(sched_interrupt_exit_preempts)
+{
+    struct interrupted ctx = {
+        .id = 1,
+        .irqs_were_enabled = true,
+        .child = { .id = 2, .rounds = 1 },
+    };
+
+    switching_test_begin();
+    spawn(get_interrupted, &ctx, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 4);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 2);
+    ASSERT_EQ(s_log[2], 1);
+    ASSERT_EQ(s_log[3], 1);
+    ASSERT(local_irq_enabled());
+
+    switching_test_end();
+}
+
+TEST_CASE(sched_interrupt_exit_respects_preempt_disable)
+{
+    struct interrupted ctx = {
+        .id = 1,
+        .irqs_were_enabled = true,
+        .preemption_disabled = true,
+        .child = { .id = 2, .rounds = 1 },
+    };
+
+    switching_test_begin();
+    spawn(get_interrupted, &ctx, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 4);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 1);
+    ASSERT_EQ(s_log[2], 2);
+    ASSERT_EQ(s_log[3], 1);
+
+    switching_test_end();
+}
+
+TEST_CASE(sched_interrupt_exit_respects_disabled_interrupts)
+{
+    struct interrupted ctx = {
+        .id = 1,
+        .irqs_were_enabled = false,
+        .child = { .id = 2, .rounds = 1 },
+    };
+
+    switching_test_begin();
+    spawn(get_interrupted, &ctx, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 4);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 1);
+    ASSERT_EQ(s_log[2], 1);
+    ASSERT_EQ(s_log[3], 2);
+
+    switching_test_end();
+}

From a59ce433746f2f052d1c51fe599d5ee8cca10415 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:36:38 +0300
Subject: [PATCH 077/152] kthread: introduce kthread_create() and kthread_run()

Kernel code needs threads of its own. A kernel thread runs a function
with one argument and exits when it returns. kthread_run() also starts
it.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/sched/kthread.h | 38 ++++++++++++++++++++
 kernel/sched/CMakeLists.txt    |  1 +
 kernel/sched/kthread.c         | 65 ++++++++++++++++++++++++++++++++++
 tests/CMakeLists.txt           |  4 +++
 tests/test-cases/test_sched.c  | 55 ++++++++++++++++++++++++++++
 5 files changed, 163 insertions(+)
 create mode 100644 kernel/include/sched/kthread.h
 create mode 100644 kernel/sched/kthread.c

diff --git a/kernel/include/sched/kthread.h b/kernel/include/sched/kthread.h
new file mode 100644
index 00000000..00a35ca1
--- /dev/null
+++ b/kernel/include/sched/kthread.h
@@ -0,0 +1,38 @@
+#pragma once
+
+#include <stdarg.h>
+
+#include <common/attributes.h>
+#include <common/error.h>
+
+#include <sched/task.h>
+
+/*
+ * Create a kernel thread that runs fn(user) once it is started with
+ * kthread_start(), and exits when the function returns. Use task_unref()
+ * to release.
+ */
+PRINTF_DECL(3, 4)
+struct task *kthread_create(
+    task_fn_t fn, void *user, const char *name_fmt, ...
+);
+
+struct task *kthread_vcreate(
+    task_fn_t fn, void *user, const char *name_fmt, va_list vlist
+);
+
+/*
+ * Start a previously created thread.
+ *
+ * NOTE: the thread gets a reference of its own when started, so the caller
+ *       still owns one exclusive reference at this point from the initial
+ *       kthread_create() call.
+ */
+void kthread_start(struct task *task);
+
+/*
+ * A shorthand for kthread_create()+kthread_start(), the caller gets no
+ * reference to the task.
+ */
+PRINTF_DECL(3, 4)
+error_t kthread_run(task_fn_t fn, void *user, const char *name_fmt, ...);
diff --git a/kernel/sched/CMakeLists.txt b/kernel/sched/CMakeLists.txt
index 815dd636..b27967d6 100644
--- a/kernel/sched/CMakeLists.txt
+++ b/kernel/sched/CMakeLists.txt
@@ -3,4 +3,5 @@ ultra_sources(
     core.c
     rr.c
     idle.c
+    kthread.c
 )
diff --git a/kernel/sched/kthread.c b/kernel/sched/kthread.c
new file mode 100644
index 00000000..20488bf2
--- /dev/null
+++ b/kernel/sched/kthread.c
@@ -0,0 +1,65 @@
+#include <sched/kthread.h>
+
+#include <common/format.h>
+
+#include <private/sched/task.h>
+#include <private/sched/core.h>
+#include <private/sched/rr.h>
+#include <private/arch/task.h>
+
+struct task *kthread_vcreate(
+    task_fn_t fn, void *user, const char *name_fmt, va_list vlist
+)
+{
+    struct task *task;
+
+    task = task_alloc();
+    if (unlikely(task == nullptr))
+        return nullptr;
+
+    vsnprintf(task->name, sizeof(task->name), name_fmt, vlist);
+
+    task->cpu = current->cpu;
+    task->prio = TASK_PRIO_DEFAULT;
+    task->sched_class = &SCHED_CLASS_INSTANCE(rr);
+    arch_task_frame_init(task, fn, user);
+
+    return task;
+}
+
+struct task *kthread_create(
+    task_fn_t fn, void *user, const char *name_fmt, ...
+)
+{
+    struct task *task;
+    va_list vlist;
+
+    va_start(vlist, name_fmt);
+    task = kthread_vcreate(fn, user, name_fmt, vlist);
+    va_end(vlist);
+
+    return task;
+}
+
+void kthread_start(struct task *task)
+{
+    wake_up_task(task, TASK_STATE_NEW);
+}
+
+error_t kthread_run(task_fn_t fn, void *user, const char *name_fmt, ...)
+{
+    struct task *task;
+    va_list vlist;
+
+    va_start(vlist, name_fmt);
+    task = kthread_vcreate(fn, user, name_fmt, vlist);
+    va_end(vlist);
+
+    if (unlikely(task == nullptr))
+        return ENOMEM;
+
+    kthread_start(task);
+    task_unref(task);
+
+    return EOK;
+}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index e2fc2147..0398d738 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -222,6 +222,10 @@ KERNEL_FILE(
     INCLUDE_PATH "private/sched" INCLUDE_FILE "idle.h"
     SOURCE_PATH "sched" SOURCE_FILE "idle.c"
 )
+KERNEL_FILE(
+    INCLUDE_PATH "sched" INCLUDE_FILE "kthread.h"
+    SOURCE_PATH "sched" SOURCE_FILE "kthread.c"
+)
 KERNEL_FILE(
     INCLUDE_PATH "locking" INCLUDE_FILE "spinlock.h"
     SOURCE_PATH "locking" SOURCE_FILE "spinlock.c"
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 2aa5aca4..2ea20e70 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -18,6 +18,7 @@ void sched_test_vrelease(void *ptr);
 #include <kernel-source/sched/core.c>
 #include <kernel-source/sched/rr.c>
 #include <kernel-source/sched/idle.c>
+#include <kernel-source/sched/kthread.c>
 
 #include <private/arch/task.h>
 #include <private/arch/preempt.h>
@@ -786,3 +787,57 @@ TEST_CASE(sched_interrupt_exit_respects_disabled_interrupts)
 
     switching_test_end();
 }
+
+TEST_CASE(kthread_create_then_start)
+{
+    struct runner runner = { .id = 5, .rounds = 2 };
+    struct task *task;
+    struct rq *rq;
+
+    rq = switching_test_begin();
+
+    task = kthread_create(run_to_completion, &runner, "worker/%d", 3);
+    ASSERT_NOT_NULL(task);
+    ASSERT_STR_EQ(task->name, "worker/3");
+    ASSERT_EQ(task_state(task), TASK_STATE_NEW);
+    ASSERT_EQ(task->prio, TASK_PRIO_DEFAULT);
+    ASSERT_EQ(task->sched_class, &SCHED_CLASS_INSTANCE(rr));
+    ASSERT(!task->on_rq);
+    ASSERT_EQ(rq->nr_running, 0);
+
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 0);
+
+    kthread_start(task);
+    ASSERT_EQ(rq->nr_running, 1);
+
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 2);
+    ASSERT_EQ(task_state(task), TASK_STATE_DEAD);
+    ASSERT_EQ(g_sched_test_live_tasks, 1);
+
+    task_unref(task);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+    ASSERT_EQ(g_sched_test_live_stacks, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(kthread_run_runs_and_frees)
+{
+    struct runner runner = { .id = 9, .rounds = 3 };
+
+    switching_test_begin();
+
+    ASSERT_EQ(kthread_run(run_to_completion, &runner, "runner"), EOK);
+    ASSERT_EQ(g_sched_test_live_tasks, 1);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 3);
+    ASSERT_EQ(s_log[0], 9);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+    ASSERT_EQ(g_sched_test_live_stacks, 0);
+
+    switching_test_end();
+}

From c7fd27d8d476fff25a511bae20797b2f4d4db8d4 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:53:03 +0300
Subject: [PATCH 078/152] kthread: add kthread_exit()

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/sched/kthread.h | 4 ++++
 kernel/sched/kthread.c         | 5 +++++
 2 files changed, 9 insertions(+)

diff --git a/kernel/include/sched/kthread.h b/kernel/include/sched/kthread.h
index 00a35ca1..c83066f2 100644
--- a/kernel/include/sched/kthread.h
+++ b/kernel/include/sched/kthread.h
@@ -36,3 +36,7 @@ void kthread_start(struct task *task);
  */
 PRINTF_DECL(3, 4)
 error_t kthread_run(task_fn_t fn, void *user, const char *name_fmt, ...);
+
+// Exit the current kernel thread
+NORETURN
+void kthread_exit(void);
diff --git a/kernel/sched/kthread.c b/kernel/sched/kthread.c
index 20488bf2..b029e19b 100644
--- a/kernel/sched/kthread.c
+++ b/kernel/sched/kthread.c
@@ -63,3 +63,8 @@ error_t kthread_run(task_fn_t fn, void *user, const char *name_fmt, ...)
 
     return EOK;
 }
+
+void kthread_exit(void)
+{
+    task_exit();
+}

From 5a93c558325ebd42b84f887f84bbe4b9c2a96e6f Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:39:11 +0300
Subject: [PATCH 079/152] sched: make the boot thread the idle task

Interrupts can only be enabled once there is somewhere to switch to.
Raising the new SCHEDULER_ONLINE level enables them for the first time,
and the rest of the initialization moves into an init thread, since the
idle task must never block.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/include/arch/cpu_helpers.h | 10 ++++++++
 kernel/entry.c                                | 25 +++++++++++++++----
 kernel/include/detail/init_level.h            | 10 ++++++++
 kernel/include/private/sched/sched.h          |  4 +++
 kernel/sched/core.c                           |  7 ++++++
 kernel/sched/idle.c                           | 23 +++++++++++++++++
 scripts/baremetal_ci/config.example.json      |  2 +-
 scripts/baremetal_ci/criteria.json            |  2 +-
 tests/synthetic-include/arch/cpu_helpers.h    |  7 ++++++
 tests/test-cases/test_sched.c                 |  5 ++++
 10 files changed, 88 insertions(+), 7 deletions(-)

diff --git a/kernel/arch/x86_64/include/arch/cpu_helpers.h b/kernel/arch/x86_64/include/arch/cpu_helpers.h
index ebf22dd2..376b3e57 100644
--- a/kernel/arch/x86_64/include/arch/cpu_helpers.h
+++ b/kernel/arch/x86_64/include/arch/cpu_helpers.h
@@ -7,6 +7,16 @@ static inline void arch_cpu_relax(void)
     asm volatile("pause" ::: "memory");
 }
 
+/*
+ * Called with interrupts disabled, returns with them enabled once one
+ * has been taken. sti only takes effect after the next instruction, so
+ * nothing can be delivered between the two and get slept through.
+ */
+static inline void arch_cpu_idle_wait(void)
+{
+    asm volatile("sti; hlt" ::: "memory");
+}
+
 static NORETURN inline void arch_cpu_halt(void)
 {
     for (;;)
diff --git a/kernel/entry.c b/kernel/entry.c
index a0c7608f..8c5d7fa6 100644
--- a/kernel/entry.c
+++ b/kernel/entry.c
@@ -7,8 +7,6 @@
 #include <boot/boot.h>
 #include <boot/ultra_protocol.h>
 
-#include <arch/cpu_helpers.h>
-
 #include <free_after_init.h>
 #include <log.h>
 #include <bug.h>
@@ -16,8 +14,11 @@
 #include <param.h>
 #include <config.h>
 #include <init_level.h>
+#include <sched/kthread.h>
+#include <panic.h>
 
 #include <private/param.h>
+#include <private/sched/sched.h>
 
 struct boot_context g_boot_ctx;
 ptr_t g_direct_map_base;
@@ -128,14 +129,28 @@ static error_t INIT_CODE boot_info_init(void)
 }
 INIT_CALL_AT(BOOT_INFO_AVAILABLE, boot_info_init);
 
+static void INIT_CODE threaded_init(void *user)
+{
+    UNREFERENCED_PARAMETER(user);
+
+    init_level_raise(INIT_LEVEL_GENERIC_MODULES_INITIALIZED);
+
+    print(LOG_NOTICE"Nothing left to do, idling\n");
+}
+
 void INIT_CODE entry(struct ultra_boot_context *ctx)
 {
+    error_t ret;
+
     print(LOG_INFO"%s\n", g_build_banner);
 
     s_loader_ctx = ctx;
 
-    init_level_raise(INIT_LEVEL_IRQS_AVAILABLE);
+    init_level_raise(INIT_LEVEL_SCHEDULER_ONLINE);
+
+    ret = kthread_run(threaded_init, nullptr, "init");
+    if (is_error(ret))
+        panic("Unable to start the init thread: %pE", &ret);
 
-    print(LOG_NOTICE"Nothing left to do, halting!\n");
-    arch_cpu_halt();
+    sched_idle_loop();
 }
diff --git a/kernel/include/detail/init_level.h b/kernel/include/detail/init_level.h
index 9ccd108c..40768e00 100644
--- a/kernel/include/detail/init_level.h
+++ b/kernel/include/detail/init_level.h
@@ -50,6 +50,12 @@
      */ \
     INIT_LEVEL(IRQS_AVAILABLE) \
     ARCH_INIT_LEVELS_AFTER_IRQS_AVAILABLE \
+    /*
+     * Interrupts are enabled and tasks run. The rest of kernel init
+     * happens in a thread of its own that is allowed to sleep
+     */ \
+    INIT_LEVEL(SCHEDULER_ONLINE) \
+    ARCH_INIT_LEVELS_AFTER_SCHEDULER_ONLINE \
     INIT_LEVEL(GENERIC_MODULES_INITIALIZED) \
     ARCH_INIT_LEVELS_AFTER_GENERIC_MODULES_INITIALIZED
 
@@ -109,6 +115,10 @@
 #define ARCH_INIT_LEVELS_AFTER_IRQS_AVAILABLE
 #endif
 
+#ifndef ARCH_INIT_LEVELS_AFTER_SCHEDULER_ONLINE
+#define ARCH_INIT_LEVELS_AFTER_SCHEDULER_ONLINE
+#endif
+
 #ifndef ARCH_INIT_LEVELS_AFTER_GENERIC_MODULES_INITIALIZED
 #define ARCH_INIT_LEVELS_AFTER_GENERIC_MODULES_INITIALIZED
 #endif
diff --git a/kernel/include/private/sched/sched.h b/kernel/include/private/sched/sched.h
index 5a26f563..0d8512a4 100644
--- a/kernel/include/private/sched/sched.h
+++ b/kernel/include/private/sched/sched.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <common/atomic.h>
+#include <common/attributes.h>
 #include <common/types.h>
 
 #include <private/sched/task.h>
@@ -30,3 +31,6 @@ bool wake_up_task(struct task *task, enum task_state state_mask);
  *       priority never get to run.
  */
 void yield(void);
+
+NORETURN
+void sched_idle_loop(void);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index a1292796..c301fb0c 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -341,3 +341,10 @@ static error_t INIT_CODE sched_init(void)
     return EOK;
 }
 INIT_CALL_POST(PER_CPU_AVAILABLE, sched_init);
+
+static error_t INIT_CODE sched_online(void)
+{
+    local_irq_enable();
+    return EOK;
+}
+INIT_CALL_AT(SCHEDULER_ONLINE, sched_online);
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index ec5ca9bd..c0e9e0d3 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -4,6 +4,9 @@
 #include <common/helpers.h>
 
 #include <bug.h>
+#include <irq_helpers.h>
+
+#include <arch/cpu_helpers.h>
 
 static void idle_enqueue(struct rq *rq, struct task *task)
 {
@@ -48,3 +51,23 @@ DEFINE_SCHED_CLASS(idle) = {
     .yield = idle_yield,
     .wakeup_preempt = idle_wakeup_preempt,
 };
+
+/*
+ * The function every CPU's idle task runs. The check and the wait
+ * are done with interrupts disabled so that a wakeup arriving in
+ * between ends the wait instead of being slept through.
+ */
+void sched_idle_loop(void)
+{
+    for (;;) {
+        local_irq_disable();
+
+        if (need_resched()) {
+            local_irq_enable();
+            schedule_idle();
+            continue;
+        }
+
+        arch_cpu_idle_wait();
+    }
+}
diff --git a/scripts/baremetal_ci/config.example.json b/scripts/baremetal_ci/config.example.json
index c33b8530..dee40edb 100644
--- a/scripts/baremetal_ci/config.example.json
+++ b/scripts/baremetal_ci/config.example.json
@@ -41,7 +41,7 @@
       "mock_lines": [
         "Booted via network (PXE)",
         "Starting ultra kernel",
-        "Nothing left to do, halting!"
+        "Nothing left to do, idling"
       ]
     }
   ]
diff --git a/scripts/baremetal_ci/criteria.json b/scripts/baremetal_ci/criteria.json
index 23c6755b..b2633aae 100644
--- a/scripts/baremetal_ci/criteria.json
+++ b/scripts/baremetal_ci/criteria.json
@@ -1,5 +1,5 @@
 {
-  "success": "Nothing left to do, halting!",
+  "success": "Nothing left to do, idling",
   "failures": [
     "Kernel panic:",
     "BUG!",
diff --git a/tests/synthetic-include/arch/cpu_helpers.h b/tests/synthetic-include/arch/cpu_helpers.h
index 67729fba..f90a342d 100644
--- a/tests/synthetic-include/arch/cpu_helpers.h
+++ b/tests/synthetic-include/arch/cpu_helpers.h
@@ -8,3 +8,10 @@ static inline void arch_cpu_relax(void)
     if (g_cpu_relax_hook)
         g_cpu_relax_hook();
 }
+
+void test_cpu_idle_wait(void);
+
+static inline void arch_cpu_idle_wait(void)
+{
+    test_cpu_idle_wait();
+}
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 2ea20e70..d683b816 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -23,6 +23,11 @@ void sched_test_vrelease(void *ptr);
 #include <private/arch/task.h>
 #include <private/arch/preempt.h>
 
+void test_cpu_idle_wait(void)
+{
+    g_test_irqs_enabled = true;
+}
+
 #include <test_harness.h>
 
 #define NUM_TEST_TASKS 8

From f900c3171442bae31c8f3a71d983c01ff9c44baf Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:49:33 +0300
Subject: [PATCH 080/152] sched: add might_sleep()

A function that may block must not be called from atomic context, nor
by a task that has already marked itself as sleeping, since blocking
would overwrite that state. might_sleep() catches such callers whether
they end up blocking or not.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/sched/sched.h |  9 +++++++++
 kernel/sched/core.c          | 12 ++++++++++++
 tests/CMakeLists.txt         |  1 +
 3 files changed, 22 insertions(+)
 create mode 100644 kernel/include/sched/sched.h

diff --git a/kernel/include/sched/sched.h b/kernel/include/sched/sched.h
new file mode 100644
index 00000000..6e48fd32
--- /dev/null
+++ b/kernel/include/sched/sched.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <common/types.h>
+
+/*
+ * Call from anything that may sleep, whether or not it ends up doing
+ * so, to catch callers that are not allowed to.
+ */
+void might_sleep(void);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index c301fb0c..3d3c2ca7 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5,6 +5,8 @@
 #include <private/arch/task.h>
 #include <private/arch/preempt.h>
 
+#include <sched/sched.h>
+
 #include <common/helpers.h>
 #include <common/atomic.h>
 
@@ -106,6 +108,16 @@ void sched_finish_switch(struct task *prev)
         task_unref(prev);
 }
 
+void might_sleep(void)
+{
+    if (init_level_below(INIT_LEVEL_SCHEDULER_ONLINE))
+        return;
+
+    BUG_ON(preempt_count() != 0);
+    BUG_ON(!local_irq_enabled());
+    BUG_ON(task_state(current) != TASK_STATE_RUNNING);
+}
+
 /*
  * Called with preemption disabled exactly once and interrupts enabled.
  * A task that is not runnable anymore is taken off the runqueue here,
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 0398d738..586ca27b 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -207,6 +207,7 @@ KERNEL_FILE(
 KERNEL_FILE(INCLUDE_PATH "detail/sched" INCLUDE_FILE "task.h")
 KERNEL_FILE(INCLUDE_PATH "private/sched" INCLUDE_FILE "task.h")
 KERNEL_FILE(INCLUDE_PATH "private/arch" INCLUDE_FILE "task.h")
+KERNEL_FILE(INCLUDE_PATH "sched" INCLUDE_FILE "sched.h")
 KERNEL_FILE(
     INCLUDE_PATH "private/sched" INCLUDE_FILE "core.h"
     SOURCE_PATH "sched" SOURCE_FILE "core.c"

From dfb665a893e99b89223f19cf62459ff2ed23f65d Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:49:33 +0300
Subject: [PATCH 081/152] sched: introduce wait queues

Sleeping until something happens needs a list for the waker to walk.
Waking an entry runs its callback, which by default wakes the task that
queued it. A wakeup stops at the first exclusive entry that accepts it,
so a resource only one waiter can take does not wake all of them.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/sched/wait_queue.h  |  32 +++
 kernel/include/private/sched/wait_queue.h |  19 ++
 kernel/include/sched/wait_queue.h         |  98 +++++++++
 kernel/sched/CMakeLists.txt               |   1 +
 kernel/sched/wait_queue.c                 | 108 ++++++++++
 tests/CMakeLists.txt                      |   6 +
 tests/test-cases/test_sched.c             | 252 ++++++++++++++++++++++
 7 files changed, 516 insertions(+)
 create mode 100644 kernel/include/detail/sched/wait_queue.h
 create mode 100644 kernel/include/private/sched/wait_queue.h
 create mode 100644 kernel/include/sched/wait_queue.h
 create mode 100644 kernel/sched/wait_queue.c

diff --git a/kernel/include/detail/sched/wait_queue.h b/kernel/include/detail/sched/wait_queue.h
new file mode 100644
index 00000000..f48225b4
--- /dev/null
+++ b/kernel/include/detail/sched/wait_queue.h
@@ -0,0 +1,32 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/list.h>
+
+#include <locking/spinlock.h>
+
+struct task;
+struct wait_entry;
+
+enum wait_entry_flags : u32;
+
+/*
+ * Decides what waking the entry means. Returns whether the entry took
+ * the wakeup, only those count against the exclusive ones woken.
+ * Called with the queue locked.
+ */
+typedef bool (*wait_wake_fn_t)(struct wait_entry *entry);
+
+bool wait_entry_wake_and_remove(struct wait_entry *entry);
+
+struct wait_entry {
+    enum wait_entry_flags flags;
+    struct task *task;
+    wait_wake_fn_t wake;
+    struct list_link link;
+};
+
+struct wait_queue {
+    struct spinlock lock;
+    struct list_link entries;
+};
diff --git a/kernel/include/private/sched/wait_queue.h b/kernel/include/private/sched/wait_queue.h
new file mode 100644
index 00000000..7b7fffcb
--- /dev/null
+++ b/kernel/include/private/sched/wait_queue.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include <common/types.h>
+
+struct wait_queue;
+struct wait_entry;
+
+void wait_queue_prepare_unlocked(
+    struct wait_queue *wq, struct wait_entry *entry
+);
+void wait_queue_remove_unlocked(
+    struct wait_queue *wq, struct wait_entry *entry
+);
+
+/*
+ * Wakes every plain waiter and 'num_exclusive' exclusive ones, all of
+ * them if zero.
+ */
+void wait_queue_wake_unlocked(struct wait_queue *wq, u32 num_exclusive);
diff --git a/kernel/include/sched/wait_queue.h b/kernel/include/sched/wait_queue.h
new file mode 100644
index 00000000..95a3e1a1
--- /dev/null
+++ b/kernel/include/sched/wait_queue.h
@@ -0,0 +1,98 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/list.h>
+#include <common/bit.h>
+
+#include <sched/sched.h>
+#include <sched/task.h>
+#include <locking/spinlock.h>
+
+#include <detail/sched/sched.h>
+#include <detail/sched/wait_queue.h>
+
+/*
+ * A list of tasks waiting for something, initialized with
+ * WAIT_QUEUE_INIT() or wait_queue_init(). All fields are internal.
+ */
+struct wait_queue;
+
+#define WAIT_QUEUE_INIT(name) (struct wait_queue) { \
+    .lock = SPINLOCK_INIT((name).lock),             \
+    .entries = LIST_INIT((name).entries),           \
+}
+#define DEFINE_WAIT_QUEUE(name) struct wait_queue name = WAIT_QUEUE_INIT(name)
+
+void wait_queue_init(struct wait_queue *wq);
+
+/*
+ * A task's place on a wait queue, set up with WAIT_ENTRY_INIT() or
+ * wait_entry_init() and queued by wait_queue_prepare(). All fields are
+ * internal.
+ */
+struct wait_entry;
+
+enum wait_entry_flags : u32 {
+    // An ordinary (shared) waiter
+    WAIT_ENTRY_FLAG_NONE = 0,
+
+    /*
+     * An exclusive waiter, a normal wakeup stops at the first such entry that
+     * accepts it
+     */
+    WAIT_ENTRY_FLAG_EXCLUSIVE = BIT_U32(0),
+};
+
+#define WAIT_ENTRY_INIT(name, entry_flags) (struct wait_entry) { \
+    .flags = (entry_flags),                                      \
+    .task = current,                                             \
+    .wake = wait_entry_wake_and_remove,                          \
+    .link = LIST_INIT((name).link),                              \
+}
+#define DEFINE_WAIT_ENTRY(name, entry_flags)                    \
+    struct wait_entry name = WAIT_ENTRY_INIT(name, entry_flags)
+
+void wait_entry_init(struct wait_entry *entry, enum wait_entry_flags flags);
+
+/*
+ * Queue 'entry' if it is not queued yet and mark the current task as
+ * sleeping. Check the condition after this and before schedule(), so
+ * that a wakeup in between is not lost.
+ */
+void wait_queue_prepare(struct wait_queue *wq, struct wait_entry *entry);
+
+/*
+ * Ends the wait once the condition holds. Marks the current task running
+ * again and takes 'entry' off the queue. Call it exactly once after the
+ * last wait_queue_prepare(), whether or not the task ever slept.
+ */
+void wait_queue_finish(struct wait_queue *wq, struct wait_entry *entry);
+
+// Wake every shared and at most one exclusive waiter
+void wait_queue_wake(struct wait_queue *wq);
+
+// Wake everyone
+void wait_queue_wake_all(struct wait_queue *wq);
+
+// Sleep until 'condition' is true, it is rechecked after every wakeup
+#define wait_event(wq, condition) do {                        \
+    struct wait_entry wait_event_entry;                       \
+                                                              \
+    might_sleep();                                            \
+                                                              \
+    if (condition)                                            \
+        break;                                                \
+                                                              \
+    wait_entry_init(&wait_event_entry, WAIT_ENTRY_FLAG_NONE); \
+                                                              \
+    for (;;) {                                                \
+        wait_queue_prepare(wq, &wait_event_entry);            \
+                                                              \
+        if (condition)                                        \
+            break;                                            \
+                                                              \
+        schedule();                                           \
+    }                                                         \
+                                                              \
+    wait_queue_finish(wq, &wait_event_entry);                 \
+} while (0)
diff --git a/kernel/sched/CMakeLists.txt b/kernel/sched/CMakeLists.txt
index b27967d6..9cbbc83f 100644
--- a/kernel/sched/CMakeLists.txt
+++ b/kernel/sched/CMakeLists.txt
@@ -4,4 +4,5 @@ ultra_sources(
     rr.c
     idle.c
     kthread.c
+    wait_queue.c
 )
diff --git a/kernel/sched/wait_queue.c b/kernel/sched/wait_queue.c
new file mode 100644
index 00000000..a029cd18
--- /dev/null
+++ b/kernel/sched/wait_queue.c
@@ -0,0 +1,108 @@
+#include <sched/wait_queue.h>
+
+#include <private/sched/sched.h>
+#include <private/sched/wait_queue.h>
+
+#include <bug.h>
+
+bool wait_entry_wake_and_remove(struct wait_entry *entry)
+{
+    if (!wake_up_task(entry->task, TASK_STATE_UNINTERRUPTIBLE))
+        return false;
+
+    list_remove(&entry->link);
+    return true;
+}
+
+void wait_queue_init(struct wait_queue *wq)
+{
+    *wq = WAIT_QUEUE_INIT(*wq);
+}
+
+void wait_entry_init(struct wait_entry *entry, enum wait_entry_flags flags)
+{
+    *entry = WAIT_ENTRY_INIT(*entry, flags);
+}
+
+void wait_queue_prepare_unlocked(
+    struct wait_queue *wq, struct wait_entry *entry
+)
+{
+    BUG_ON(!spin_is_locked(&wq->lock));
+
+    if (list_is_empty(&entry->link)) {
+        if (entry->flags & WAIT_ENTRY_FLAG_EXCLUSIVE)
+            list_insert_prev(&wq->entries, &entry->link);
+        else
+            list_insert_next(&wq->entries, &entry->link);
+    }
+
+    set_current_state(TASK_STATE_UNINTERRUPTIBLE);
+}
+
+void wait_queue_prepare(struct wait_queue *wq, struct wait_entry *entry)
+{
+    irq_state_t irq_state;
+
+    irq_state = spin_lock_irq_save(&wq->lock);
+    wait_queue_prepare_unlocked(wq, entry);
+    spin_unlock_irq_restore(&wq->lock, irq_state);
+}
+
+void wait_queue_remove_unlocked(
+    struct wait_queue *wq, struct wait_entry *entry
+)
+{
+    BUG_ON(!spin_is_locked(&wq->lock));
+    list_remove(&entry->link);
+}
+
+void wait_queue_finish(struct wait_queue *wq, struct wait_entry *entry)
+{
+    irq_state_t irq_state;
+
+    set_current_state(TASK_STATE_RUNNING);
+
+    irq_state = spin_lock_irq_save(&wq->lock);
+    wait_queue_remove_unlocked(wq, entry);
+    spin_unlock_irq_restore(&wq->lock, irq_state);
+}
+
+void wait_queue_wake_unlocked(struct wait_queue *wq, u32 num_exclusive)
+{
+    struct wait_entry *entry, *next;
+    enum wait_entry_flags flags;
+
+    BUG_ON(!spin_is_locked(&wq->lock));
+
+    list_for_each_entry_safe(entry, next, &wq->entries, link) {
+        flags = entry->flags;
+
+        if (!entry->wake(entry))
+            continue;
+        if (!(flags & WAIT_ENTRY_FLAG_EXCLUSIVE))
+            continue;
+
+        if (--num_exclusive == 0)
+            break;
+    }
+}
+
+static void do_wake(struct wait_queue *wq, u32 num_exclusive)
+{
+    irq_state_t irq_state;
+
+    irq_state = spin_lock_irq_save(&wq->lock);
+    wait_queue_wake_unlocked(wq, num_exclusive);
+    spin_unlock_irq_restore(&wq->lock, irq_state);
+}
+
+void wait_queue_wake(struct wait_queue *wq)
+{
+    do_wake(wq, 1);
+}
+
+void wait_queue_wake_all(struct wait_queue *wq)
+{
+    do_wake(wq, 0);
+}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 586ca27b..dd0fe933 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -223,6 +223,12 @@ KERNEL_FILE(
     INCLUDE_PATH "private/sched" INCLUDE_FILE "idle.h"
     SOURCE_PATH "sched" SOURCE_FILE "idle.c"
 )
+KERNEL_FILE(
+    INCLUDE_PATH "sched" INCLUDE_FILE "wait_queue.h"
+    SOURCE_PATH "sched" SOURCE_FILE "wait_queue.c"
+)
+KERNEL_FILE(INCLUDE_PATH "detail/sched" INCLUDE_FILE "wait_queue.h")
+KERNEL_FILE(INCLUDE_PATH "private/sched" INCLUDE_FILE "wait_queue.h")
 KERNEL_FILE(
     INCLUDE_PATH "sched" INCLUDE_FILE "kthread.h"
     SOURCE_PATH "sched" SOURCE_FILE "kthread.c"
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index d683b816..64a78a0d 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -19,6 +19,7 @@ void sched_test_vrelease(void *ptr);
 #include <kernel-source/sched/rr.c>
 #include <kernel-source/sched/idle.c>
 #include <kernel-source/sched/kthread.c>
+#include <kernel-source/sched/wait_queue.c>
 
 #include <private/arch/task.h>
 #include <private/arch/preempt.h>
@@ -846,3 +847,254 @@ TEST_CASE(kthread_run_runs_and_frees)
 
     switching_test_end();
 }
+
+struct waiter {
+    struct wait_queue *wq;
+    bool *condition;
+    int id;
+    bool exclusive;
+};
+
+static void wait_for_condition(void *user)
+{
+    struct waiter *waiter = user;
+
+    wait_event(waiter->wq, *waiter->condition);
+    log_id(waiter->id);
+}
+
+static void wait_once(void *user)
+{
+    struct waiter *waiter = user;
+    struct wait_entry entry;
+    enum wait_entry_flags flags = WAIT_ENTRY_FLAG_NONE;
+
+    if (waiter->exclusive)
+        flags = WAIT_ENTRY_FLAG_EXCLUSIVE;
+
+    wait_entry_init(&entry, flags);
+    wait_queue_prepare(waiter->wq, &entry);
+    schedule();
+    wait_queue_finish(waiter->wq, &entry);
+
+    log_id(waiter->id);
+}
+
+static int count_ids(int id)
+{
+    int i, count = 0;
+
+    for (i = 0; i < s_log_length; i++) {
+        if (s_log[i] == id)
+            count++;
+    }
+
+    return count;
+}
+
+TEST_CASE(wait_event_returns_right_away_if_true)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    bool condition = true;
+    struct waiter waiter = { .wq = &wq, .condition = &condition, .id = 1 };
+
+    switching_test_begin();
+    spawn(wait_for_condition, &waiter, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT(list_is_empty(&wq.entries));
+
+    switching_test_end();
+}
+
+TEST_CASE(wait_event_sleeps_until_woken_with_the_condition)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    bool condition = false;
+    struct waiter waiter = { .wq = &wq, .condition = &condition, .id = 1 };
+    struct task *task;
+
+    switching_test_begin();
+    task = spawn(wait_for_condition, &waiter, TASK_PRIO_DEFAULT);
+    task_ref(task);
+
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 0);
+    ASSERT_EQ(task_state(task), TASK_STATE_UNINTERRUPTIBLE);
+    ASSERT(!list_is_empty(&wq.entries));
+
+    wait_queue_wake(&wq);
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 0);
+    ASSERT_EQ(task_state(task), TASK_STATE_UNINTERRUPTIBLE);
+
+    condition = true;
+    wait_queue_wake(&wq);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT_EQ(task_state(task), TASK_STATE_DEAD);
+    ASSERT(list_is_empty(&wq.entries));
+
+    task_unref(task);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(wait_queue_wake_wakes_every_plain_waiter)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    struct waiter waiters[3];
+    int i;
+
+    switching_test_begin();
+
+    for (i = 0; i < 3; i++) {
+        waiters[i] = (struct waiter) { .wq = &wq, .id = i };
+        spawn(wait_once, &waiters[i], TASK_PRIO_DEFAULT);
+    }
+
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 0);
+
+    wait_queue_wake(&wq);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 3);
+    ASSERT(list_is_empty(&wq.entries));
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(wait_queue_wake_wakes_one_exclusive_waiter)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    struct waiter waiters[3];
+    int i;
+
+    switching_test_begin();
+
+    for (i = 0; i < 3; i++) {
+        waiters[i] = (struct waiter) { .wq = &wq, .id = i, .exclusive = true };
+        spawn(wait_once, &waiters[i], TASK_PRIO_DEFAULT);
+    }
+
+    schedule_idle();
+
+    for (i = 0; i < 3; i++) {
+        wait_queue_wake(&wq);
+        schedule_idle();
+
+        ASSERT_EQ(s_log_length, i + 1);
+        ASSERT_EQ(s_log[i], i);
+    }
+
+    ASSERT(list_is_empty(&wq.entries));
+
+    switching_test_end();
+}
+
+TEST_CASE(wait_queue_wake_all_ignores_exclusivity)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    struct waiter waiters[3];
+    int i;
+
+    switching_test_begin();
+
+    for (i = 0; i < 3; i++) {
+        waiters[i] = (struct waiter) { .wq = &wq, .id = i, .exclusive = true };
+        spawn(wait_once, &waiters[i], TASK_PRIO_DEFAULT);
+    }
+
+    schedule_idle();
+
+    wait_queue_wake_all(&wq);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 3);
+
+    switching_test_end();
+}
+
+TEST_CASE(wait_queue_plain_waiters_come_before_exclusive_ones)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    struct waiter waiters[4];
+    int i;
+
+    switching_test_begin();
+
+    for (i = 0; i < 4; i++) {
+        waiters[i] = (struct waiter) {
+            .wq = &wq, .id = i % 2, .exclusive = i % 2 == 0,
+        };
+        spawn(wait_once, &waiters[i], TASK_PRIO_DEFAULT);
+    }
+
+    schedule_idle();
+
+    wait_queue_wake(&wq);
+    schedule_idle();
+
+    ASSERT_EQ(count_ids(1), 2);
+    ASSERT_EQ(count_ids(0), 1);
+
+    wait_queue_wake(&wq);
+    schedule_idle();
+
+    ASSERT_EQ(count_ids(0), 2);
+    ASSERT(list_is_empty(&wq.entries));
+
+    switching_test_end();
+}
+
+struct counting_entry {
+    struct wait_entry entry;
+    int calls;
+    bool result;
+};
+
+static bool count_the_wake(struct wait_entry *entry)
+{
+    struct counting_entry *counting;
+
+    counting = container_of(entry, struct counting_entry, entry);
+    counting->calls++;
+
+    return counting->result;
+}
+
+TEST_CASE(wait_queue_entry_with_its_own_callback)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    struct counting_entry first = { .result = false };
+    struct counting_entry second = { .result = true };
+
+    switching_test_begin();
+
+    first.entry.flags = WAIT_ENTRY_FLAG_EXCLUSIVE;
+    first.entry.wake = count_the_wake;
+    second.entry.flags = WAIT_ENTRY_FLAG_EXCLUSIVE;
+    second.entry.wake = count_the_wake;
+
+    list_insert_prev(&wq.entries, &first.entry.link);
+    list_insert_prev(&wq.entries, &second.entry.link);
+
+    wait_queue_wake(&wq);
+    ASSERT_EQ(first.calls, 1);
+    ASSERT_EQ(second.calls, 1);
+
+    second.result = false;
+    first.result = true;
+
+    wait_queue_wake(&wq);
+    ASSERT_EQ(first.calls, 2);
+    ASSERT_EQ(second.calls, 1);
+
+    switching_test_end();
+}

From d349b435e41d872d4f1a4edde7369332896b6465 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:51:15 +0300
Subject: [PATCH 082/152] sched: introduce completions

A bare wakeup that comes before anybody waits is lost. A completion
keeps a count on top of a wait queue, so the waiter finds it already
done.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/sched/completion.h |  10 ++
 kernel/include/sched/completion.h        |  29 ++++++
 kernel/sched/CMakeLists.txt              |   1 +
 kernel/sched/completion.c                |  67 +++++++++++++
 tests/CMakeLists.txt                     |   5 +
 tests/test-cases/test_sched.c            | 122 +++++++++++++++++++++++
 6 files changed, 234 insertions(+)
 create mode 100644 kernel/include/detail/sched/completion.h
 create mode 100644 kernel/include/sched/completion.h
 create mode 100644 kernel/sched/completion.c

diff --git a/kernel/include/detail/sched/completion.h b/kernel/include/detail/sched/completion.h
new file mode 100644
index 00000000..eb966ca0
--- /dev/null
+++ b/kernel/include/detail/sched/completion.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <common/types.h>
+
+#include <detail/sched/wait_queue.h>
+
+struct completion {
+    u32 done;
+    struct wait_queue wq;
+};
diff --git a/kernel/include/sched/completion.h b/kernel/include/sched/completion.h
new file mode 100644
index 00000000..cce6ada4
--- /dev/null
+++ b/kernel/include/sched/completion.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include <common/types.h>
+
+#include <sched/wait_queue.h>
+#include <detail/sched/completion.h>
+
+/*
+ * Something to wait for that happens once, or a counted number of
+ * times. A complete() that comes before the wait is remembered, so
+ * the two may happen in either order.
+ */
+struct completion;
+
+#define COMPLETION_INIT(name) (struct completion) { \
+    .wq = WAIT_QUEUE_INIT((name).wq),               \
+}
+#define DEFINE_COMPLETION(name) struct completion name = COMPLETION_INIT(name)
+
+void completion_init(struct completion *completion);
+
+// Let one waiter through. Callable from any context that may take a spinlock
+void complete(struct completion *completion);
+
+// Let every waiter through, including future ones
+void complete_all(struct completion *completion);
+
+// Sleep until a complete() is called on this object
+void wait_for_completion(struct completion *completion);
diff --git a/kernel/sched/CMakeLists.txt b/kernel/sched/CMakeLists.txt
index 9cbbc83f..1a2200cf 100644
--- a/kernel/sched/CMakeLists.txt
+++ b/kernel/sched/CMakeLists.txt
@@ -5,4 +5,5 @@ ultra_sources(
     idle.c
     kthread.c
     wait_queue.c
+    completion.c
 )
diff --git a/kernel/sched/completion.c b/kernel/sched/completion.c
new file mode 100644
index 00000000..345d34e7
--- /dev/null
+++ b/kernel/sched/completion.c
@@ -0,0 +1,67 @@
+#include <sched/completion.h>
+#include <private/sched/wait_queue.h>
+
+#define DONE_PERMANENT UINT32_MAX
+
+void completion_init(struct completion *completion)
+{
+    *completion = COMPLETION_INIT(*completion);
+}
+
+void complete(struct completion *completion)
+{
+    struct wait_queue *wq = &completion->wq;
+    irq_state_t irq_state;
+
+    irq_state = spin_lock_irq_save(&wq->lock);
+
+    if (completion->done != DONE_PERMANENT)
+        completion->done++;
+    wait_queue_wake_unlocked(wq, 1);
+
+    spin_unlock_irq_restore(&wq->lock, irq_state);
+}
+
+void complete_all(struct completion *completion)
+{
+    struct wait_queue *wq = &completion->wq;
+    irq_state_t irq_state;
+
+    irq_state = spin_lock_irq_save(&wq->lock);
+
+    completion->done = DONE_PERMANENT;
+    wait_queue_wake_unlocked(wq, 0);
+
+    spin_unlock_irq_restore(&wq->lock, irq_state);
+}
+
+void wait_for_completion(struct completion *completion)
+{
+    struct wait_queue *wq = &completion->wq;
+    struct wait_entry entry;
+    irq_state_t irq_state;
+
+    might_sleep();
+
+    irq_state = spin_lock_irq_save(&wq->lock);
+
+    if (completion->done == 0) {
+        wait_entry_init(&entry, WAIT_ENTRY_FLAG_EXCLUSIVE);
+
+        do {
+            wait_queue_prepare_unlocked(wq, &entry);
+            spin_unlock_irq_restore(&wq->lock, irq_state);
+
+            schedule();
+
+            irq_state = spin_lock_irq_save(&wq->lock);
+        } while (completion->done == 0);
+
+        wait_queue_remove_unlocked(wq, &entry);
+    }
+
+    if (completion->done != DONE_PERMANENT)
+        completion->done--;
+
+    spin_unlock_irq_restore(&wq->lock, irq_state);
+}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index dd0fe933..ff9c49bd 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -229,6 +229,11 @@ KERNEL_FILE(
 )
 KERNEL_FILE(INCLUDE_PATH "detail/sched" INCLUDE_FILE "wait_queue.h")
 KERNEL_FILE(INCLUDE_PATH "private/sched" INCLUDE_FILE "wait_queue.h")
+KERNEL_FILE(
+    INCLUDE_PATH "sched" INCLUDE_FILE "completion.h"
+    SOURCE_PATH "sched" SOURCE_FILE "completion.c"
+)
+KERNEL_FILE(INCLUDE_PATH "detail/sched" INCLUDE_FILE "completion.h")
 KERNEL_FILE(
     INCLUDE_PATH "sched" INCLUDE_FILE "kthread.h"
     SOURCE_PATH "sched" SOURCE_FILE "kthread.c"
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 64a78a0d..7869b52b 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -20,6 +20,7 @@ void sched_test_vrelease(void *ptr);
 #include <kernel-source/sched/idle.c>
 #include <kernel-source/sched/kthread.c>
 #include <kernel-source/sched/wait_queue.c>
+#include <kernel-source/sched/completion.c>
 
 #include <private/arch/task.h>
 #include <private/arch/preempt.h>
@@ -1098,3 +1099,124 @@ TEST_CASE(wait_queue_entry_with_its_own_callback)
 
     switching_test_end();
 }
+
+struct completion_waiter {
+    struct completion *completion;
+    int id;
+};
+
+static void wait_then_log(void *user)
+{
+    struct completion_waiter *waiter = user;
+
+    wait_for_completion(waiter->completion);
+    log_id(waiter->id);
+}
+
+TEST_CASE(completion_done_before_the_wait)
+{
+    DEFINE_COMPLETION(completion);
+    struct completion_waiter waiter = { .completion = &completion, .id = 1 };
+
+    switching_test_begin();
+
+    complete(&completion);
+    ASSERT_EQ(completion.done, 1);
+
+    spawn(wait_then_log, &waiter, TASK_PRIO_DEFAULT);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT_EQ(completion.done, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(completion_wakes_one_waiter_per_complete)
+{
+    DEFINE_COMPLETION(completion);
+    struct completion_waiter waiters[3];
+    int i;
+
+    switching_test_begin();
+
+    for (i = 0; i < 3; i++) {
+        waiters[i] = (struct completion_waiter) {
+            .completion = &completion, .id = i,
+        };
+        spawn(wait_then_log, &waiters[i], TASK_PRIO_DEFAULT);
+    }
+
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 0);
+
+    for (i = 0; i < 3; i++) {
+        complete(&completion);
+        schedule_idle();
+
+        ASSERT_EQ(s_log_length, i + 1);
+        ASSERT_EQ(s_log[i], i);
+        ASSERT_EQ(completion.done, 0);
+    }
+
+    ASSERT(list_is_empty(&completion.wq.entries));
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(completion_complete_all_stays_done)
+{
+    DEFINE_COMPLETION(completion);
+    struct completion_waiter waiters[3];
+    struct completion_waiter late = { .completion = &completion, .id = 9 };
+    int i;
+
+    switching_test_begin();
+
+    for (i = 0; i < 3; i++) {
+        waiters[i] = (struct completion_waiter) {
+            .completion = &completion, .id = i,
+        };
+        spawn(wait_then_log, &waiters[i], TASK_PRIO_DEFAULT);
+    }
+
+    schedule_idle();
+
+    complete_all(&completion);
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 3);
+
+    spawn(wait_then_log, &late, TASK_PRIO_DEFAULT);
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 4);
+    ASSERT_EQ(s_log[3], 9);
+
+    switching_test_end();
+}
+
+TEST_CASE(completion_from_interrupt_context)
+{
+    DEFINE_COMPLETION(completion);
+    struct completion_waiter waiter = { .completion = &completion, .id = 1 };
+    struct registers regs = { .irq_state = false };
+    irq_state_t irq_state;
+
+    switching_test_begin();
+
+    spawn(wait_then_log, &waiter, TASK_PRIO_DEFAULT);
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 0);
+
+    irq_state = local_irq_state_save();
+    hard_irq_enter();
+    complete(&completion);
+    hard_irq_exit(&regs);
+    local_irq_state_restore(irq_state);
+
+    ASSERT(need_resched());
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 1);
+
+    switching_test_end();
+}

From 6254ddc8faa945e60e93e9506eafb02eb1d6ef72 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:53:03 +0300
Subject: [PATCH 083/152] kthread: introduce kthread_stop() and
 kthread_should_stop()

The owner of a kernel thread needs a way to end it. kthread_stop() asks
the thread to exit and waits until it has, the thread checks
kthread_should_stop() when it wakes. A thread that was never started
exits without running its function.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/sched/kthread.h  |  10 +++
 kernel/include/detail/sched/task.h     |   7 ++
 kernel/include/private/sched/kthread.h |   6 ++
 kernel/include/private/sched/task.h    |  10 +++
 kernel/include/sched/kthread.h         |  18 ++++-
 kernel/sched/core.c                    |   4 +
 kernel/sched/kthread.c                 |  39 ++++++++-
 tests/CMakeLists.txt                   |   2 +
 tests/test-cases/test_sched.c          | 106 +++++++++++++++++++++++++
 9 files changed, 200 insertions(+), 2 deletions(-)
 create mode 100644 kernel/include/detail/sched/kthread.h
 create mode 100644 kernel/include/private/sched/kthread.h

diff --git a/kernel/include/detail/sched/kthread.h b/kernel/include/detail/sched/kthread.h
new file mode 100644
index 00000000..ef6ab988
--- /dev/null
+++ b/kernel/include/detail/sched/kthread.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <sched/task.h>
+#include <detail/sched/completion.h>
+
+struct kthread {
+    task_fn_t fn;
+    void *user;
+    struct completion exited;
+};
diff --git a/kernel/include/detail/sched/task.h b/kernel/include/detail/sched/task.h
index 657fb384..62bd1af9 100644
--- a/kernel/include/detail/sched/task.h
+++ b/kernel/include/detail/sched/task.h
@@ -10,6 +10,7 @@
 #include <sched/task.h>
 
 #include <detail/refcount.h>
+#include <detail/sched/kthread.h>
 #include <detail/sched/rr.h>
 #include <arch/detail/task.h>
 
@@ -25,6 +26,8 @@ struct sched_class;
 enum task_flags : u32 {
     TASK_FLAG_NONE = 0,
     TASK_FLAG_NEED_RESCHED = BIT_U32(0),
+    TASK_FLAG_KTHREAD = BIT_U32(1),
+    TASK_FLAG_KTHREAD_SHOULD_STOP = BIT_U32(2),
 #if IS_ENABLED(UBSAN)
     TASK_FLAG_IN_UBSAN_REPORT = BIT_U32(31),
 #endif
@@ -55,6 +58,10 @@ struct task {
     void *stack;
     size_t stack_size;
 
+    union {
+        struct kthread kthread;
+    };
+
     struct arch_task arch;
 };
 
diff --git a/kernel/include/private/sched/kthread.h b/kernel/include/private/sched/kthread.h
new file mode 100644
index 00000000..da7aaa17
--- /dev/null
+++ b/kernel/include/private/sched/kthread.h
@@ -0,0 +1,6 @@
+#pragma once
+
+struct task;
+
+// Called by task_exit() for a kthread once it is marked dead
+void kthread_did_exit(struct task *task);
diff --git a/kernel/include/private/sched/task.h b/kernel/include/private/sched/task.h
index 89d5ddfe..cad8c9fc 100644
--- a/kernel/include/private/sched/task.h
+++ b/kernel/include/private/sched/task.h
@@ -47,11 +47,21 @@ static inline bool task_test_and_set_flag(
     return atomic_fetch_or(&task->flags, flag, MO_ACQ_REL) & flag;
 }
 
+static inline void task_set_flag(struct task *task, enum task_flags flag)
+{
+    atomic_fetch_or(&task->flags, flag, MO_RELEASE);
+}
+
 static inline void task_clear_flag(struct task *task, enum task_flags flag)
 {
     atomic_fetch_and(&task->flags, ~flag, MO_RELEASE);
 }
 
+static inline bool task_is_kthread(struct task *task)
+{
+    return task_test_flag(task, TASK_FLAG_KTHREAD);
+}
+
 struct task *task_alloc(void);
 
 NORETURN
diff --git a/kernel/include/sched/kthread.h b/kernel/include/sched/kthread.h
index c83066f2..f016b074 100644
--- a/kernel/include/sched/kthread.h
+++ b/kernel/include/sched/kthread.h
@@ -6,11 +6,12 @@
 #include <common/error.h>
 
 #include <sched/task.h>
+#include <detail/sched/task.h>
 
 /*
  * Create a kernel thread that runs fn(user) once it is started with
  * kthread_start(), and exits when the function returns. Use task_unref()
- * to release.
+ * or kthread_stop() to release.
  */
 PRINTF_DECL(3, 4)
 struct task *kthread_create(
@@ -40,3 +41,18 @@ error_t kthread_run(task_fn_t fn, void *user, const char *name_fmt, ...);
 // Exit the current kernel thread
 NORETURN
 void kthread_exit(void);
+
+/*
+ * Ask a kernel thread to stop and wait until it has exited. The thread
+ * has to cooperate by checking kthread_should_stop() whenever it is
+ * woken.
+ *
+ * NOTE: This consumes the caller reference handed out by kthread_create().
+ */
+void kthread_stop(struct task *task);
+
+// True once kthread_stop() asked the current thread to return
+static inline bool kthread_should_stop(void)
+{
+    return task_test_flag(current, TASK_FLAG_KTHREAD_SHOULD_STOP);
+}
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3d3c2ca7..df4e5045 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1,6 +1,7 @@
 #include <private/sched/core.h>
 #include <private/sched/rr.h>
 #include <private/sched/idle.h>
+#include <private/sched/kthread.h>
 #include <private/sched/task.h>
 #include <private/arch/task.h>
 #include <private/arch/preempt.h>
@@ -322,6 +323,9 @@ void task_exit(void)
     task_set_state(task, TASK_STATE_DEAD);
     spin_unlock_irq_restore(&task->lock, irq_state);
 
+    if (task_is_kthread(task))
+        kthread_did_exit(task);
+
     do_schedule(false);
     BUG();
 }
diff --git a/kernel/sched/kthread.c b/kernel/sched/kthread.c
index b029e19b..df0815c7 100644
--- a/kernel/sched/kthread.c
+++ b/kernel/sched/kthread.c
@@ -1,12 +1,29 @@
 #include <sched/kthread.h>
 
 #include <common/format.h>
+#include <common/helpers.h>
+#include <common/atomic.h>
+
+#include <sched/completion.h>
 
 #include <private/sched/task.h>
 #include <private/sched/core.h>
 #include <private/sched/rr.h>
+#include <private/sched/kthread.h>
 #include <private/arch/task.h>
 
+static void kthread_entry(void *user)
+{
+    struct kthread *kthread = &current->kthread;
+
+    UNREFERENCED_PARAMETER(user);
+
+    if (kthread_should_stop())
+        return;
+
+    kthread->fn(kthread->user);
+}
+
 struct task *kthread_vcreate(
     task_fn_t fn, void *user, const char *name_fmt, va_list vlist
 )
@@ -22,7 +39,13 @@ struct task *kthread_vcreate(
     task->cpu = current->cpu;
     task->prio = TASK_PRIO_DEFAULT;
     task->sched_class = &SCHED_CLASS_INSTANCE(rr);
-    arch_task_frame_init(task, fn, user);
+    task->flags |= TASK_FLAG_KTHREAD;
+    task->kthread.fn = fn;
+    task->kthread.user = user;
+
+    completion_init(&task->kthread.exited);
+
+    arch_task_frame_init(task, kthread_entry, nullptr);
 
     return task;
 }
@@ -68,3 +91,17 @@ void kthread_exit(void)
 {
     task_exit();
 }
+
+void kthread_did_exit(struct task *task)
+{
+    complete_all(&task->kthread.exited);
+}
+
+void kthread_stop(struct task *task)
+{
+    task_set_flag(task, TASK_FLAG_KTHREAD_SHOULD_STOP);
+    wake_up_task(task, TASK_STATE_NEW | TASK_STATE_UNINTERRUPTIBLE);
+    wait_for_completion(&task->kthread.exited);
+
+    task_unref(task);
+}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index ff9c49bd..a70774c7 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -238,6 +238,8 @@ KERNEL_FILE(
     INCLUDE_PATH "sched" INCLUDE_FILE "kthread.h"
     SOURCE_PATH "sched" SOURCE_FILE "kthread.c"
 )
+KERNEL_FILE(INCLUDE_PATH "detail/sched" INCLUDE_FILE "kthread.h")
+KERNEL_FILE(INCLUDE_PATH "private/sched" INCLUDE_FILE "kthread.h")
 KERNEL_FILE(
     INCLUDE_PATH "locking" INCLUDE_FILE "spinlock.h"
     SOURCE_PATH "locking" SOURCE_FILE "spinlock.c"
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 7869b52b..ef7f7791 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -1220,3 +1220,109 @@ TEST_CASE(completion_from_interrupt_context)
 
     switching_test_end();
 }
+
+struct worker {
+    int id;
+    int iterations;
+};
+
+static void work_until_stopped(void *user)
+{
+    struct worker *worker = user;
+
+    for (;;) {
+        set_current_state(TASK_STATE_UNINTERRUPTIBLE);
+
+        if (kthread_should_stop())
+            break;
+
+        worker->iterations++;
+        schedule();
+    }
+
+    set_current_state(TASK_STATE_RUNNING);
+    log_id(worker->id);
+}
+
+struct stopper {
+    struct task *target;
+    int id;
+};
+
+static void stop_target(void *user)
+{
+    struct stopper *stopper = user;
+
+    kthread_stop(stopper->target);
+    log_id(stopper->id);
+}
+
+TEST_CASE(kthread_stop_waits_for_a_sleeping_thread)
+{
+    struct worker worker = { .id = 1 };
+    struct stopper stopper = { .id = 2 };
+
+    switching_test_begin();
+
+    stopper.target = kthread_create(work_until_stopped, &worker, "worker");
+    kthread_start(stopper.target);
+
+    schedule_idle();
+    ASSERT_EQ(worker.iterations, 1);
+    ASSERT_EQ(s_log_length, 0);
+
+    ASSERT_EQ(kthread_run(stop_target, &stopper, "stopper"), EOK);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 2);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 2);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+    ASSERT_EQ(g_sched_test_live_stacks, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(kthread_stop_before_start_never_runs_it)
+{
+    struct runner runner = { .id = 1, .rounds = 3 };
+    struct stopper stopper = { .id = 2 };
+
+    switching_test_begin();
+
+    stopper.target = kthread_create(run_to_completion, &runner, "unstarted");
+
+    ASSERT_EQ(kthread_run(stop_target, &stopper, "stopper"), EOK);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT_EQ(s_log[0], 2);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+    ASSERT_EQ(g_sched_test_live_stacks, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(kthread_stop_after_it_exited)
+{
+    struct runner runner = { .id = 1, .rounds = 1 };
+    struct stopper stopper = { .id = 2 };
+
+    switching_test_begin();
+
+    stopper.target = kthread_create(run_to_completion, &runner, "short");
+    kthread_start(stopper.target);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT_EQ(task_state(stopper.target), TASK_STATE_DEAD);
+
+    ASSERT_EQ(kthread_run(stop_target, &stopper, "stopper"), EOK);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 2);
+    ASSERT_EQ(s_log[1], 2);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}

From 50688909ec2a4d7b88c76f912efaef2c9703885c Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:53:03 +0300
Subject: [PATCH 084/152] seq: introduce seq_t

A generic type for representing sequence numbers so that users don't
have to go into the header to read the struct fields or signatures to
figure out which one they're supposed to be using.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/locking/seq.h | 12 ++++++++----
 kernel/time/keeper.c         |  2 +-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/kernel/include/locking/seq.h b/kernel/include/locking/seq.h
index 95dc4ffe..0eb157dc 100644
--- a/kernel/include/locking/seq.h
+++ b/kernel/include/locking/seq.h
@@ -8,8 +8,10 @@
 
 #include <irq_helpers.h>
 
+typedef u32 seq_t;
+
 struct seqcount {
-    u32 sequence;
+    seq_t sequence;
 };
 
 #define SEQCOUNT_INIT(name) (struct seqcount) { .sequence = 0 }
@@ -47,9 +49,9 @@ static inline void seqcount_write_end_irq_restore(
     local_irq_state_restore(state);
 }
 
-static inline u32 seqcount_read_begin(const struct seqcount *s)
+static inline seq_t seqcount_read_begin(const struct seqcount *s)
 {
-    u32 seq;
+    seq_t seq;
 
     for (;;) {
         seq = atomic_load_acquire(&s->sequence);
@@ -63,7 +65,9 @@ static inline u32 seqcount_read_begin(const struct seqcount *s)
     }
 }
 
-static inline bool seqcount_read_retry(const struct seqcount *s, u32 start_seq)
+static inline bool seqcount_read_retry(
+    const struct seqcount *s, seq_t start_seq
+)
 {
     barrier_acquire();
     return atomic_load_relaxed(&s->sequence) != start_seq;
diff --git a/kernel/time/keeper.c b/kernel/time/keeper.c
index 87c9a2e0..3c74d702 100644
--- a/kernel/time/keeper.c
+++ b/kernel/time/keeper.c
@@ -93,7 +93,7 @@ void time_keeper_set_counter_device(struct counter_device *cd)
 
 u64 ns_since_boot(void)
 {
-    u32 seq;
+    seq_t seq;
     u64 base, ns_frac, prev_cycles, now, delta_cycles, delta_ns;
     struct counter_device *cd;
 

From 7839ebdb18e8f927b726f7d068c58469f773e9a8 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:09:46 +0300
Subject: [PATCH 085/152] seq: introduce struct seqcount_latch

A latch keeps two copies of some data and a sequence count whose low bit
selects the copy a reader may use, so a reader never waits for a writer.
It is a type of its own wrapping a sequence count, since the plain
operations don't work on it: seqcount_read_begin() spins while the count
is odd, which a latch reader must never do.

Add the type and the operations to flip the count, begin a read, pick
the copy and check whether the read has to be retried.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/locking/seq.h | 48 ++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/kernel/include/locking/seq.h b/kernel/include/locking/seq.h
index 0eb157dc..2de9d198 100644
--- a/kernel/include/locking/seq.h
+++ b/kernel/include/locking/seq.h
@@ -73,6 +73,54 @@ static inline bool seqcount_read_retry(
     return atomic_load_relaxed(&s->sequence) != start_seq;
 }
 
+/*
+ * A sequence count used as a latch protects two copies of the same
+ * data. The writer flips, updates copy 0, flips again and updates
+ * copy 1. A reader picks the copy seqcount_latch_copy() names, which
+ * is the one not being written, and retries with
+ * seqcount_latch_read_retry(). It never waits for the writer, so it
+ * may interrupt one on its own CPU. Writers must exclude each other.
+ */
+struct seqcount_latch {
+    struct seqcount seqcount;
+};
+
+#define SEQCOUNT_LATCH_INIT(name) \
+    (struct seqcount_latch) { .seqcount = SEQCOUNT_INIT((name).seqcount) }
+
+static inline void seqcount_latch_init(struct seqcount_latch *latch)
+{
+    *latch = SEQCOUNT_LATCH_INIT(*latch);
+}
+
+static inline void seqcount_latch_flip(struct seqcount_latch *latch)
+{
+    struct seqcount *s = &latch->seqcount;
+
+    barrier_release();
+    atomic_store_relaxed(&s->sequence, atomic_load_relaxed(&s->sequence) + 1);
+    barrier_release();
+}
+
+static inline seq_t seqcount_latch_read_begin(
+    const struct seqcount_latch *latch
+)
+{
+    return atomic_load_acquire(&latch->seqcount.sequence);
+}
+
+static inline size_t seqcount_latch_copy(seq_t seq)
+{
+    return seq & 1;
+}
+
+static inline bool seqcount_latch_read_retry(
+    const struct seqcount_latch *latch, seq_t start_seq
+)
+{
+    return seqcount_read_retry(&latch->seqcount, start_seq);
+}
+
 struct seqlock {
     struct seqcount seqcount;
     struct spinlock lock;

From 0c186a3c9cef63eb9c2114215f7fe2c8221c4b6d Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:09:46 +0300
Subject: [PATCH 086/152] time: don't make ns_since_boot() wait for an update
 in progress

A reader spins while an update is in progress, which never ends when it
interrupted the update on its own CPU. An NMI or an exception inside the
update does exactly that, since both log.

Publish two copies of the keeper's base instead and have readers take
the one not being written.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/time/keeper.c         | 115 ++++++++++++++++++++++-------------
 tests/test-cases/test_time.c |  79 +++++++++++++++++++++++-
 2 files changed, 152 insertions(+), 42 deletions(-)

diff --git a/kernel/time/keeper.c b/kernel/time/keeper.c
index 3c74d702..50dffa65 100644
--- a/kernel/time/keeper.c
+++ b/kernel/time/keeper.c
@@ -11,22 +11,54 @@
 
 #include <arch/cpu_helpers.h>
 
+struct keeper_base {
+    struct counter_device *cd;
+    u64 ns;
+    u64 ns_frac;
+    u64 prev_cycles;
+};
+
 static struct keeper {
     struct spinlock lock;
-    struct seqcount time_seq;
+    struct seqcount_latch time_seq;
     struct counter_device *cd;
 
     u64 ns;
     u64 ns_frac;
     u64 prev_cycles;
+
+    // At least one of these is always consistent (time_seq decides which)
+    struct keeper_base published[2];
 } s_ctx = {
     .lock = SPINLOCK_INIT(s_ctx.lock),
-    .time_seq = SEQCOUNT_INIT(s_ctx.time_seq),
+    .time_seq = SEQCOUNT_LATCH_INIT(s_ctx.time_seq),
 };
 
+static void base_store(struct keeper_base *base)
+{
+    atomic_store_relaxed(&base->cd, s_ctx.cd);
+    atomic_store_relaxed(&base->ns, s_ctx.ns);
+    atomic_store_relaxed(&base->ns_frac, s_ctx.ns_frac);
+    atomic_store_relaxed(&base->prev_cycles, s_ctx.prev_cycles);
+}
+
+static void publish(void)
+{
+    seqcount_latch_flip(&s_ctx.time_seq);
+    base_store(&s_ctx.published[0]);
+
+    seqcount_latch_flip(&s_ctx.time_seq);
+    base_store(&s_ctx.published[1]);
+}
+
+static struct keeper_base *read_published(seq_t seq)
+{
+    return &s_ctx.published[seqcount_latch_copy(seq)];
+}
+
 static void do_tick(void)
 {
-    u64 cycles, delta_cycles, delta_ns, ns_frac = s_ctx.ns_frac;
+    u64 cycles, delta_cycles, delta_ns;
     struct counter_device *cd = s_ctx.cd;
 
     if (unlikely(cd == nullptr))
@@ -34,18 +66,14 @@ static void do_tick(void)
 
     cycles = counter_device_read(cd);
     delta_cycles = counter_device_delta(cd, s_ctx.prev_cycles, cycles);
-    delta_ns = counter_device_cycles_to_ns_frac(cd, delta_cycles, &ns_frac);
-
-    seqcount_write_begin(&s_ctx.time_seq);
-
-    atomic_store_relaxed(
-        &s_ctx.ns,
-        atomic_load_relaxed(&s_ctx.ns) + delta_ns
+    delta_ns = counter_device_cycles_to_ns_frac(
+        cd, delta_cycles, &s_ctx.ns_frac
     );
-    atomic_store_relaxed(&s_ctx.ns_frac, ns_frac);
-    atomic_store_relaxed(&s_ctx.prev_cycles, cycles);
 
-    seqcount_write_end(&s_ctx.time_seq);
+    s_ctx.ns += delta_ns;
+    s_ctx.prev_cycles = cycles;
+
+    publish();
 }
 
 void time_keeper_tick(void)
@@ -60,64 +88,69 @@ void time_keeper_tick(void)
 void time_keeper_set_counter_device(struct counter_device *cd)
 {
     irq_state_t state;
-    u64 cycles = 0;
+    u64 new_cycles = 0;
 
     state = spin_lock_irq_save(&s_ctx.lock);
-    seqcount_write_begin(&s_ctx.time_seq);
+
+    /*
+     * The new counter is sampled before the old one is read for the last
+     * time. Whatever passes in between is counted twice, which moves time
+     * forward a little. The other order would lose it, and a reader in
+     * that window would have seen a later time than the ones after it.
+     */
+    if (cd)
+        new_cycles = counter_device_read(cd);
 
     // If we already had a device, accumulate its final time before dumping it
     if (s_ctx.cd) {
-        u64 old_cycles, delta_cycles, ns_frac = s_ctx.ns_frac;
+        u64 old_cycles, delta_cycles;
 
         old_cycles = counter_device_read(s_ctx.cd);
         delta_cycles = counter_device_delta(
             s_ctx.cd, s_ctx.prev_cycles, old_cycles
         );
-        atomic_store_relaxed(
-            &s_ctx.ns,
-            atomic_load_relaxed(&s_ctx.ns) +
-            counter_device_cycles_to_ns_frac(s_ctx.cd, delta_cycles, &ns_frac)
+        s_ctx.ns += counter_device_cycles_to_ns_frac(
+            s_ctx.cd, delta_cycles, &s_ctx.ns_frac
         );
-        atomic_store_relaxed(&s_ctx.ns_frac, 0);
-    }
 
-    if (cd)
-        cycles = counter_device_read(cd);
+        // The fraction is in units of the old device's shift
+        s_ctx.ns_frac = 0;
+    }
 
     atomic_store_relaxed(&s_ctx.cd, cd);
-    atomic_store_relaxed(&s_ctx.prev_cycles, cycles);
+    s_ctx.prev_cycles = new_cycles;
 
-    seqcount_write_end(&s_ctx.time_seq);
+    publish();
     spin_unlock_irq_restore(&s_ctx.lock, state);
 }
 
 u64 ns_since_boot(void)
 {
     seq_t seq;
-    u64 base, ns_frac, prev_cycles, now, delta_cycles, delta_ns;
+    u64 ns, ns_frac, prev_cycles, now, delta_cycles;
+    const struct keeper_base *base;
     struct counter_device *cd;
 
     do {
-        seq = seqcount_read_begin(&s_ctx.time_seq);
+        seq = seqcount_latch_read_begin(&s_ctx.time_seq);
+        base = read_published(seq);
 
-        base = atomic_load_relaxed(&s_ctx.ns);
+        ns = atomic_load_relaxed(&base->ns);
+        ns_frac = atomic_load_relaxed(&base->ns_frac);
+        cd = atomic_load_relaxed(&base->cd);
+        prev_cycles = atomic_load_relaxed(&base->prev_cycles);
+        barrier_acquire();
 
         // No device installed, time is frozen at the accumulated value
-        cd = atomic_load_relaxed(&s_ctx.cd);
-        if (cd == nullptr)
-            return base;
+        if (cd != nullptr)
+            now = counter_device_read(cd);
+    } while (seqcount_latch_read_retry(&s_ctx.time_seq, seq));
 
-        ns_frac = atomic_load_relaxed(&s_ctx.ns_frac);
-        prev_cycles = atomic_load_relaxed(&s_ctx.prev_cycles);
-        barrier_acquire();
-
-        now = counter_device_read(cd);
-    } while (seqcount_read_retry(&s_ctx.time_seq, seq));
+    if (cd == nullptr)
+        return ns;
 
     delta_cycles = counter_device_delta(cd, prev_cycles, now);
-    delta_ns = counter_device_cycles_to_ns_frac(cd, delta_cycles, &ns_frac);
-
-    return base + delta_ns;
+    return ns + counter_device_cycles_to_ns_frac(cd, delta_cycles, &ns_frac);
 }
 
 void delay_ns(u64 ns)
diff --git a/tests/test-cases/test_time.c b/tests/test-cases/test_time.c
index 88932086..edc720a5 100644
--- a/tests/test-cases/test_time.c
+++ b/tests/test-cases/test_time.c
@@ -36,11 +36,12 @@ static void time_test_reset(void)
     s_active_cd = nullptr;
 
     spin_lock_init(&s_ctx.lock);
-    seqcount_init(&s_ctx.time_seq);
+    seqcount_latch_init(&s_ctx.time_seq);
     s_ctx.cd = nullptr;
     s_ctx.ns = 0;
     s_ctx.ns_frac = 0;
     s_ctx.prev_cycles = 0;
+    memset(s_ctx.published, 0, sizeof(s_ctx.published));
 
     g_counter_value = 0;
 }
@@ -452,3 +453,79 @@ TEST_TEARDOWN()
     // ns_since_boot() is used by other tests as well, so make sure its clean
     time_test_reset();
 }
+
+static u64 g_seen_during_switch;
+static bool g_switch_pending;
+
+// The first read is the keeper sampling the device it is switching to
+static u64 test_counter_read_during_switch(struct counter_device *dev)
+{
+    UNREFERENCED_PARAMETER(dev);
+
+    if (g_switch_pending) {
+        g_switch_pending = false;
+
+        g_counter_value += 400;
+        g_seen_during_switch = ns_since_boot();
+    }
+
+    return 0;
+}
+
+TEST_CASE(keeper_time_does_not_step_back_across_a_device_switch)
+{
+    struct counter_device old_dev = TEST_CD("old", COUNTER_MASK(64), GOOD);
+    struct counter_device new_dev = {
+        .name = "new",
+        .read = test_counter_read_during_switch,
+        .mask = COUNTER_MASK(64),
+        .rating = COUNTER_DEVICE_RATING_GREAT,
+    };
+
+    time_test_reset();
+    g_counter_value = 0;
+    g_seen_during_switch = 0;
+    g_switch_pending = true;
+
+    ASSERT_EQ(counter_device_register(&old_dev, ONE_NS_PER_CYCLE_HZ), EOK);
+    g_counter_value = 1000;
+
+    ASSERT_EQ(counter_device_register(&new_dev, ONE_NS_PER_CYCLE_HZ), EOK);
+    ASSERT_EQ((uintptr_t)s_ctx.cd, (uintptr_t)&new_dev);
+
+    ASSERT_EQ(g_seen_during_switch, 1400);
+    ASSERT(ns_since_boot() >= g_seen_during_switch);
+
+    counter_device_unregister(&new_dev);
+    counter_device_unregister(&old_dev);
+}
+
+TEST_CASE(keeper_read_does_not_wait_for_an_update_in_progress)
+{
+    struct counter_device dev = TEST_CD("test", COUNTER_MASK(64), GOOD);
+    u64 before;
+
+    time_test_reset();
+    g_counter_value = 0;
+
+    ASSERT_EQ(counter_device_register(&dev, ONE_NS_PER_CYCLE_HZ), EOK);
+    g_counter_value = 1000;
+    time_keeper_tick();
+    g_counter_value = 1500;
+
+    before = ns_since_boot();
+    ASSERT_EQ(before, 1500);
+
+    seqcount_latch_flip(&s_ctx.time_seq);
+    s_ctx.published[0].ns = 0xDEAD0000;
+    s_ctx.published[0].prev_cycles = 7;
+    ASSERT_EQ(ns_since_boot(), before);
+
+    s_ctx.published[0] = s_ctx.published[1];
+    seqcount_latch_flip(&s_ctx.time_seq);
+    s_ctx.published[1].ns = 0xDEAD0000;
+    ASSERT_EQ(ns_since_boot(), before);
+
+    s_ctx.published[1] = s_ctx.published[0];
+    counter_device_unregister(&dev);
+}

From b5768230a0126a2ea0b236f6dc6a998aafd5f2de Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Mon, 5 Oct 2026 00:15:15 +0300
Subject: [PATCH 087/152] time: add frequency_to_human()

A helper for converting a hertz value into something nice and printable
in logs.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/tsc.c      |  7 +++++--
 kernel/include/time/units.h   | 21 +++++++++++++++++++
 kernel/time/CMakeLists.txt    |  1 +
 kernel/time/counter_device.c  | 21 +++++++------------
 kernel/time/units.c           | 23 +++++++++++++++++++++
 tests/CMakeLists.txt          |  7 +++++--
 tests/test-cases/files.cmake  |  1 +
 tests/test-cases/test_units.c | 39 +++++++++++++++++++++++++++++++++++
 8 files changed, 102 insertions(+), 18 deletions(-)
 create mode 100644 kernel/time/units.c
 create mode 100644 tests/test-cases/test_units.c

diff --git a/kernel/arch/x86_64/tsc.c b/kernel/arch/x86_64/tsc.c
index 1b6ecf28..e36819c6 100644
--- a/kernel/arch/x86_64/tsc.c
+++ b/kernel/arch/x86_64/tsc.c
@@ -55,6 +55,8 @@ static void INIT_CODE tsc_set_frequency(
     u64 hz, bool precise, const char *reported_by
 )
 {
+    struct human_repr freq;
+
     if (s_tsc_ctx.hz) {
         if (!precise)
             // One imprecise guess is good enough
@@ -71,9 +73,10 @@ static void INIT_CODE tsc_set_frequency(
     s_tsc_ctx.hz = hz;
     s_tsc_ctx.reported_by = reported_by;
 
+    frequency_to_human(hz, &freq);
     pr_info(
-        "frequency set to %llu.%03llu MHz (%s, reported by %s)\n",
-        hz / MHZ, hz % MHZ, precise ? "precise" : "approximate",
+        "frequency set to " HUMAN_REPR_FMT " (%s, reported by %s)\n",
+        HUMAN_REPR_ARG(freq), precise ? "precise" : "approximate",
         reported_by
     );
 }
diff --git a/kernel/include/time/units.h b/kernel/include/time/units.h
index c21370da..7f1f1b58 100644
--- a/kernel/include/time/units.h
+++ b/kernel/include/time/units.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <common/types.h>
+
 #define HZ 1ull
 #define KHZ (HZ * 1000)
 #define MHZ (KHZ * 1000)
@@ -20,3 +22,22 @@
 #define MS_PER_HOUR (60ull * MS_PER_MIN)
 #define MS_PER_DAY  (24ull * MS_PER_HOUR)
 #define MS_PER_YEAR (365ull * MS_PER_DAY)
+
+/*
+ * A quantity broken down for display: 'value' and 'thousandths' are the
+ * integral and fractional parts of the quantity expressed in 'unit'.
+ */
+struct human_repr {
+    u64 value;
+    u64 thousandths;
+    const char *unit;
+};
+
+#define HUMAN_REPR_FMT "%llu.%03llu %s"
+#define HUMAN_REPR_ARG(x) (x).value, (x).thousandths, (x).unit
+
+/*
+ * Express 'hz' in the largest unit it reaches, up to GHz, e.g. 2995200000
+ * becomes { 2, 995, "GHz" }. The fraction is truncated.
+ */
+void frequency_to_human(u64 hz, struct human_repr *out_human_repr);
diff --git a/kernel/time/CMakeLists.txt b/kernel/time/CMakeLists.txt
index 4a04a234..fb219f33 100644
--- a/kernel/time/CMakeLists.txt
+++ b/kernel/time/CMakeLists.txt
@@ -1,4 +1,5 @@
 ultra_sources(
     counter_device.c
     keeper.c
+    units.c
 )
diff --git a/kernel/time/counter_device.c b/kernel/time/counter_device.c
index fdc167d6..ec99eb38 100644
--- a/kernel/time/counter_device.c
+++ b/kernel/time/counter_device.c
@@ -6,6 +6,7 @@
 
 #include <private/time/counter_device.h>
 #include <private/time/keeper.h>
+#include <time/units.h>
 
 #include <log.h>
 #include <locking/spinlock.h>
@@ -86,21 +87,12 @@ static void calculate_thresholds(struct counter_device *dev)
 
 static void announce_new_counter(struct counter_device *dev, u64 hz, bool best)
 {
+    struct human_repr freq;
     u64 ms;
-    char freq_str[32], wrap_str[32];
-#define snprintf_freq(...) snprintf(freq_str, sizeof(freq_str), __VA_ARGS__)
+    char wrap_str[32];
 #define snprintf_wrap(...) snprintf(wrap_str, sizeof(wrap_str), __VA_ARGS__)
 
-    if (hz >= GHZ) {
-        snprintf_freq("%llu.%03llu GHz", hz / GHZ, (hz % GHZ) / MHZ);
-    } else if (hz >= MHZ) {
-        snprintf_freq("%llu.%03llu MHz", hz / MHZ, (hz % MHZ) / KHZ);
-    } else if (hz >= KHZ) {
-        snprintf_freq("%llu.%03llu KHz", hz / KHZ, hz % KHZ);
-    } else {
-        snprintf_freq("%llu Hz", hz);
-    }
-
+    frequency_to_human(hz, &freq);
     ms = dev->max_unattended_ns / NS_PER_MS;
 
     if (ms >= MS_PER_HOUR) {
@@ -114,8 +106,9 @@ static void announce_new_counter(struct counter_device *dev, u64 hz, bool best)
     }
 
     pr_info(
-        "%s @ %s [rated %u], unattended up to %s%s\n",
-        dev->name, freq_str, dev->rating, wrap_str, best ? " [best]" : ""
+        "%s @ " HUMAN_REPR_FMT " [rated %u], unattended up to %s%s\n",
+        dev->name, HUMAN_REPR_ARG(freq), dev->rating, wrap_str,
+        best ? " [best]" : ""
     );
 }
 
diff --git a/kernel/time/units.c b/kernel/time/units.c
new file mode 100644
index 00000000..b4191380
--- /dev/null
+++ b/kernel/time/units.c
@@ -0,0 +1,23 @@
+#include <common/helpers.h>
+
+#include <time/units.h>
+
+static const char *const s_units[] = {
+    "Hz", "kHz", "MHz", "GHz"
+};
+
+void frequency_to_human(u64 hz, struct human_repr *out_human_repr)
+{
+    u64 rem = 0;
+    size_t i = 0;
+
+    while (i < (ARRAY_SIZE(s_units) - 1) && hz >= KHZ) {
+        rem = hz % KHZ;
+        hz /= KHZ;
+        i++;
+    }
+
+    out_human_repr->value = hz;
+    out_human_repr->thousandths = rem;
+    out_human_repr->unit = s_units[i];
+}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index a70774c7..13710932 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -174,7 +174,10 @@ KERNEL_FILE(
     SOURCE_PATH "time" SOURCE_FILE "keeper.c"
     INCLUDE_PATH "private/time" INCLUDE_FILE "keeper.h"
 )
-KERNEL_FILE(INCLUDE_PATH "time" INCLUDE_FILE "units.h")
+KERNEL_FILE(
+    SOURCE_PATH "time" SOURCE_FILE "units.c"
+    INCLUDE_PATH "time" INCLUDE_FILE "units.h"
+)
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "page.h")
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "bug.h")
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "alloc_behavior.h")
@@ -276,7 +279,7 @@ add_executable(
     kernel-source/locking/spinlock.c kernel-source/memory/units.c
     kernel-source/common/bit.c kernel-source/locking/mutex.c
     kernel-source/common/string_container.c kernel-source/common/conversions.c
-    kernel-source/refcount.c
+    kernel-source/refcount.c kernel-source/time/units.c
 )
 set_property(
     DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
diff --git a/tests/test-cases/files.cmake b/tests/test-cases/files.cmake
index 13680b85..9730969b 100644
--- a/tests/test-cases/files.cmake
+++ b/tests/test-cases/files.cmake
@@ -22,4 +22,5 @@ add_test_cases(
     test_irq.c
     test_conversions.c
     test_types.c
+    test_units.c
 )
diff --git a/tests/test-cases/test_units.c b/tests/test-cases/test_units.c
new file mode 100644
index 00000000..3ab67b08
--- /dev/null
+++ b/tests/test-cases/test_units.c
@@ -0,0 +1,39 @@
+#include <time/units.h>
+
+#include <test_harness.h>
+
+static void check(u64 hz, u64 value, u64 thousandths, const char *unit)
+{
+    struct human_repr freq;
+
+    frequency_to_human(hz, &freq);
+
+    ASSERT_EQ(freq.value, value);
+    ASSERT_EQ(freq.thousandths, thousandths);
+    ASSERT_STR_EQ(freq.unit, unit);
+}
+
+TEST_CASE(frequency_below_a_kilohertz_stays_in_hertz)
+{
+    check(0, 0, 0, "Hz");
+    check(999, 999, 0, "Hz");
+}
+
+TEST_CASE(frequency_is_expressed_in_the_largest_unit_it_reaches)
+{
+    check(1000, 1, 0, "kHz");
+    check(32768, 32, 768, "kHz");
+    check(24 * MHZ, 24, 0, "MHz");
+    check(2995200000, 2, 995, "GHz");
+}
+
+TEST_CASE(frequency_fraction_is_truncated)
+{
+    check(1000145999, 1, 0, "GHz");
+    check(1999999, 1, 999, "MHz");
+}
+
+TEST_CASE(frequency_stops_at_gigahertz)
+{
+    check(12345 * GHZ, 12345, 0, "GHz");
+}

From 49c0cb2984e770dce6c72e6b95ff10fd79b36443 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Mon, 5 Oct 2026 00:15:15 +0300
Subject: [PATCH 088/152] time: add duration_to_human()

A helper for converting a nanosecond value into something nice and
printable in logs.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/time/units.h   |  6 ++++++
 kernel/time/counter_device.c  | 24 +++++-----------------
 kernel/time/units.c           | 29 ++++++++++++++++++++++++++
 tests/test-cases/test_units.c | 38 +++++++++++++++++++++++++++++++++++
 4 files changed, 78 insertions(+), 19 deletions(-)

diff --git a/kernel/include/time/units.h b/kernel/include/time/units.h
index 7f1f1b58..09b9c00b 100644
--- a/kernel/include/time/units.h
+++ b/kernel/include/time/units.h
@@ -41,3 +41,9 @@ struct human_repr {
  * becomes { 2, 995, "GHz" }. The fraction is truncated.
  */
 void frequency_to_human(u64 hz, struct human_repr *out_human_repr);
+
+/*
+ * Express 'ns' in the largest unit it reaches, from ns up to h, e.g.
+ * 90000000000 becomes { 1, 500, "min" }. The fraction is truncated.
+ */
+void duration_to_human(u64 ns, struct human_repr *out_human_repr);
diff --git a/kernel/time/counter_device.c b/kernel/time/counter_device.c
index ec99eb38..e343e955 100644
--- a/kernel/time/counter_device.c
+++ b/kernel/time/counter_device.c
@@ -1,7 +1,6 @@
 #define MSG_FMT(msg) "time: " msg
 
 #include <common/minmax.h>
-#include <common/format.h>
 #include <common/list.h>
 
 #include <private/time/counter_device.h>
@@ -87,28 +86,15 @@ static void calculate_thresholds(struct counter_device *dev)
 
 static void announce_new_counter(struct counter_device *dev, u64 hz, bool best)
 {
-    struct human_repr freq;
-    u64 ms;
-    char wrap_str[32];
-#define snprintf_wrap(...) snprintf(wrap_str, sizeof(wrap_str), __VA_ARGS__)
+    struct human_repr freq, unattended;
 
     frequency_to_human(hz, &freq);
-    ms = dev->max_unattended_ns / NS_PER_MS;
-
-    if (ms >= MS_PER_HOUR) {
-        snprintf_wrap("%llu hours", ms / MS_PER_HOUR);
-    } else if (ms >= MS_PER_MIN) {
-        snprintf_wrap("%llu mins", ms / MS_PER_MIN);
-    } else if (ms >= MS_PER_SEC) {
-        snprintf_wrap("%llu.%03llu secs", ms / MS_PER_SEC, ms % MS_PER_SEC);
-    } else {
-        snprintf_wrap("%llu ms", ms);
-    }
+    duration_to_human(dev->max_unattended_ns, &unattended);
 
     pr_info(
-        "%s @ " HUMAN_REPR_FMT " [rated %u], unattended up to %s%s\n",
-        dev->name, HUMAN_REPR_ARG(freq), dev->rating, wrap_str,
-        best ? " [best]" : ""
+        "%s @ " HUMAN_REPR_FMT " [rated %u], unattended up to "
+        HUMAN_REPR_FMT "%s\n", dev->name, HUMAN_REPR_ARG(freq), dev->rating,
+        HUMAN_REPR_ARG(unattended), best ? " [best]" : ""
     );
 }
 
diff --git a/kernel/time/units.c b/kernel/time/units.c
index b4191380..10337bc8 100644
--- a/kernel/time/units.c
+++ b/kernel/time/units.c
@@ -21,3 +21,32 @@ void frequency_to_human(u64 hz, struct human_repr *out_human_repr)
     out_human_repr->thousandths = rem;
     out_human_repr->unit = s_units[i];
 }
+
+struct duration_unit {
+    const char *name;
+    u64 ns;
+};
+
+static const struct duration_unit s_duration_units[] = {
+    { "ns", 1 },
+    { "us", NS_PER_US },
+    { "ms", NS_PER_MS },
+    { "s", NS_PER_SEC },
+    { "min", MS_PER_MIN * NS_PER_MS },
+    { "h", MS_PER_HOUR * NS_PER_MS },
+};
+
+void duration_to_human(u64 ns, struct human_repr *out_human_repr)
+{
+    const struct duration_unit *unit;
+    size_t i = 0;
+
+    while (i < (ARRAY_SIZE(s_duration_units) - 1) &&
+           ns >= s_duration_units[i + 1].ns)
+        i++;
+
+    unit = &s_duration_units[i];
+    out_human_repr->value = ns / unit->ns;
+    out_human_repr->thousandths = ((ns % unit->ns) * 1000) / unit->ns;
+    out_human_repr->unit = unit->name;
+}
diff --git a/tests/test-cases/test_units.c b/tests/test-cases/test_units.c
index 3ab67b08..28765db8 100644
--- a/tests/test-cases/test_units.c
+++ b/tests/test-cases/test_units.c
@@ -37,3 +37,41 @@ TEST_CASE(frequency_stops_at_gigahertz)
 {
     check(12345 * GHZ, 12345, 0, "GHz");
 }
+
+static void check_duration(
+    u64 ns, u64 value, u64 thousandths, const char *unit
+)
+{
+    struct human_repr duration;
+
+    duration_to_human(ns, &duration);
+
+    ASSERT_EQ(duration.value, value);
+    ASSERT_EQ(duration.thousandths, thousandths);
+    ASSERT_STR_EQ(duration.unit, unit);
+}
+
+TEST_CASE(duration_below_a_microsecond_stays_in_nanoseconds)
+{
+    check_duration(0, 0, 0, "ns");
+    check_duration(999, 999, 0, "ns");
+}
+
+TEST_CASE(duration_is_expressed_in_the_largest_unit_it_reaches)
+{
+    check_duration(1500, 1, 500, "us");
+    check_duration(2343 * NS_PER_MS, 2, 343, "s");
+    check_duration(90 * NS_PER_SEC, 1, 500, "min");
+    check_duration(3 * MS_PER_HOUR * NS_PER_MS, 3, 0, "h");
+}
+
+TEST_CASE(duration_fraction_is_truncated)
+{
+    check_duration(1999999, 1, 999, "ms");
+    check_duration(549800 * NS_PER_MS, 9, 163, "min");
+}
+
+TEST_CASE(duration_stops_at_hours)
+{
+    check_duration(48 * MS_PER_HOUR * NS_PER_MS, 48, 0, "h");
+}

From c1d083a0d8710b3d3bcfc9dbe8e23f7ec176280f Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 11 Nov 2026 00:13:43 +0300
Subject: [PATCH 089/152] time: add unit conversion helpers

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/tsc.c    | 3 +--
 kernel/include/time/units.h | 7 +++++++
 kernel/log.c                | 4 ++--
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/kernel/arch/x86_64/tsc.c b/kernel/arch/x86_64/tsc.c
index e36819c6..f0cbeae1 100644
--- a/kernel/arch/x86_64/tsc.c
+++ b/kernel/arch/x86_64/tsc.c
@@ -165,7 +165,6 @@ static void INIT_CODE early_tsc_calibrate(void)
 {
     u64 tsc_start, tsc_end, tsc_delta;
     u64 ref_start, ref_now, ref_delta, ref_ns;
-    u64 target_ns = EARLY_CALIBRATE_MS * NS_PER_MS;
     struct counter_device *ref_cd;
 
     ref_cd = current_counter_device();
@@ -190,7 +189,7 @@ static void INIT_CODE early_tsc_calibrate(void)
         }
 
         ref_ns = counter_device_cycles_to_ns(ref_cd, ref_delta);
-    } while (ref_ns < target_ns);
+    } while (ref_ns < MS_TO_NS(EARLY_CALIBRATE_MS));
 
     tsc_end = rdtsc();
     tsc_delta = tsc_end - tsc_start;
diff --git a/kernel/include/time/units.h b/kernel/include/time/units.h
index 09b9c00b..463b9507 100644
--- a/kernel/include/time/units.h
+++ b/kernel/include/time/units.h
@@ -23,6 +23,13 @@
 #define MS_PER_DAY  (24ull * MS_PER_HOUR)
 #define MS_PER_YEAR (365ull * MS_PER_DAY)
 
+#define MS_TO_NS(ms) ((ms) * NS_PER_MS)
+#define SEC_TO_NS(sec) ((sec) * NS_PER_SEC)
+
+#define NS_TO_US(ns) ((ns) / NS_PER_US)
+#define NS_TO_MS(ns) ((ns) / NS_PER_MS)
+#define NS_TO_SEC(ns) ((ns) / NS_PER_SEC)
+
 /*
  * A quantity broken down for display: 'value' and 'thousandths' are the
  * integral and fractional parts of the quantity expressed in 'unit'.
diff --git a/kernel/log.c b/kernel/log.c
index d8da650f..69377ed4 100644
--- a/kernel/log.c
+++ b/kernel/log.c
@@ -242,8 +242,8 @@ static bool log_flush_console(struct console *con)
     if (rec.level > s_log_level)
         return true;
 
-    sec = rec.timestamp_ns / NS_PER_SEC;
-    usec = (rec.timestamp_ns % NS_PER_SEC) / 1000;
+    sec = NS_TO_SEC(rec.timestamp_ns);
+    usec = NS_TO_US(rec.timestamp_ns % NS_PER_SEC);
     stamp_len = snprintf(
         stamp, sizeof(stamp), "[%5llu.%06llu] ", sec, usec
     );

From b17c79d795ffeb1e43c681f0401b1022e5de983d Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 4 Nov 2026 16:08:24 +0300
Subject: [PATCH 090/152] common/attributes: add MUST_CHECK

Makes the compiler warn when a caller ignores the return value of a
function.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/common/attributes.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/kernel/include/common/attributes.h b/kernel/include/common/attributes.h
index 9b012ccb..2c74f03e 100644
--- a/kernel/include/common/attributes.h
+++ b/kernel/include/common/attributes.h
@@ -33,6 +33,7 @@
 #define USED __attribute__((used))
 #define WEAK __attribute__((weak))
 #define UNUSED_DECL __attribute__((unused))
+#define MUST_CHECK __attribute__((warn_unused_result))
 #define FALLTHROUGH __attribute__((fallthrough))
 
 #define SECTION_VAR(section, qualifiers, type) \

From 57b74916164578b8dda8d343c09fe5bf13765863 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 4 Nov 2026 15:57:25 +0300
Subject: [PATCH 091/152] common: add check_{add,sub,mul}_overflow()

Wrappers around the compiler's overflow builtins. They compute the exact
result, store it wrapped to the destination type and report whether it
fit, whatever the types of the operands.

Ignoring the report leaves the caller with the wrapped value, so the
result goes through a MUST_CHECK inline function. The attribute cannot
be put on a macro.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/common/overflow.h | 20 +++++++++++++
 tests/CMakeLists.txt             |  1 +
 tests/test-cases/files.cmake     |  1 +
 tests/test-cases/test_overflow.c | 50 ++++++++++++++++++++++++++++++++
 4 files changed, 72 insertions(+)
 create mode 100644 kernel/include/common/overflow.h
 create mode 100644 tests/test-cases/test_overflow.c

diff --git a/kernel/include/common/overflow.h b/kernel/include/common/overflow.h
new file mode 100644
index 00000000..e4fe7979
--- /dev/null
+++ b/kernel/include/common/overflow.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <common/attributes.h>
+#include <common/helpers.h>
+
+static ALWAYS_INLINE MUST_CHECK bool must_check_overflow(bool overflow)
+{
+    return unlikely(overflow);
+}
+
+/*
+ * Compute 'a + b', 'a - b' or 'a * b' and store the result in '*result'.
+ * Returns true if there was an overflow, false otherwise.
+ */
+#define check_add_overflow(a, b, result) \
+    must_check_overflow(__builtin_add_overflow(a, b, result))
+#define check_sub_overflow(a, b, result) \
+    must_check_overflow(__builtin_sub_overflow(a, b, result))
+#define check_mul_overflow(a, b, result) \
+    must_check_overflow(__builtin_mul_overflow(a, b, result))
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 13710932..8d156598 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -104,6 +104,7 @@ KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "helpers.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "types.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "attributes.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "minmax.h")
+KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "overflow.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "align.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "atomic.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "ctype.h")
diff --git a/tests/test-cases/files.cmake b/tests/test-cases/files.cmake
index 9730969b..6bf9d4ac 100644
--- a/tests/test-cases/files.cmake
+++ b/tests/test-cases/files.cmake
@@ -7,6 +7,7 @@ add_test_cases(
     test_console.c
     test_string.c
     test_minmax.c
+    test_overflow.c
     test_smbios.c
     test_list.c
     test_slist.c
diff --git a/tests/test-cases/test_overflow.c b/tests/test-cases/test_overflow.c
new file mode 100644
index 00000000..61a30c3a
--- /dev/null
+++ b/tests/test-cases/test_overflow.c
@@ -0,0 +1,50 @@
+#include <common/overflow.h>
+#include <common/types.h>
+#include <common/bit.h>
+
+#include <test_harness.h>
+
+TEST_CASE(check_add_overflow_reports_a_wrap)
+{
+    u64 result;
+
+    ASSERT_FALSE(check_add_overflow(UNSIGNED_MAX(u64) - 1, 1u, &result));
+    ASSERT_EQ(result, UNSIGNED_MAX(u64));
+
+    ASSERT_TRUE(check_add_overflow(UNSIGNED_MAX(u64), 2u, &result));
+    ASSERT_EQ(result, 1u);
+}
+
+TEST_CASE(check_sub_overflow_reports_a_wrap)
+{
+    u32 result;
+
+    ASSERT_FALSE(check_sub_overflow(5u, 5u, &result));
+    ASSERT_EQ(result, 0u);
+
+    ASSERT_TRUE(check_sub_overflow(5u, 6u, &result));
+    ASSERT_EQ(result, UNSIGNED_MAX(u32));
+}
+
+TEST_CASE(check_mul_overflow_reports_a_wrap)
+{
+    u32 result;
+
+    ASSERT_FALSE(check_mul_overflow(0x10000u, 0xFFFFu, &result));
+    ASSERT_EQ(result, 0xFFFF0000u);
+
+    ASSERT_TRUE(check_mul_overflow(0x10000u, 0x10000u, &result));
+    ASSERT_EQ(result, 0u);
+}
+
+TEST_CASE(check_overflow_judges_by_the_result_type)
+{
+    u8 narrow;
+    i32 value;
+
+    ASSERT_TRUE(check_add_overflow(200ull, 100ull, &narrow));
+    ASSERT_EQ(narrow, 44);
+
+    ASSERT_FALSE(check_sub_overflow(0u, 1u, &value));
+    ASSERT_EQ(value, -1);
+}

From 216147f09d77070c65bf85d571e254312bae7c07 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 4 Nov 2026 16:08:24 +0300
Subject: [PATCH 092/152] memory/boot_alloc: catch every overflow of the
 allocation size

allocate_within() treated 'bytes_to_allocate <= page_count' as an
overflow, which misses a size that wraps to above the page count:
3 << 51 pages wrap to 1 << 63 bytes. Check the multiplication with
check_mul_overflow() instead.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/memory/boot_alloc.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/kernel/memory/boot_alloc.c b/kernel/memory/boot_alloc.c
index 97f4b563..47519040 100644
--- a/kernel/memory/boot_alloc.c
+++ b/kernel/memory/boot_alloc.c
@@ -4,6 +4,7 @@
 #include <common/string.h>
 #include <common/minmax.h>
 #include <common/align.h>
+#include <common/overflow.h>
 
 #include <boot/boot.h>
 #include <boot/ultra_protocol.h>
@@ -338,14 +339,15 @@ static INIT_CODE error_t allocate_within(
 )
 {
     phys_addr_t range_begin;
-    size_t bytes_to_allocate;;
+    size_t bytes_to_allocate;
     ssize_t mr_idx;
     struct memory_range *picked_mr = nullptr;
     struct memory_range allocated_mr;
+    bool overflow;
 
-    bytes_to_allocate = page_count * PAGE_SIZE;
+    overflow = check_mul_overflow(page_count, PAGE_SIZE, &bytes_to_allocate);
     BUG_ON_WITH_MSG(
-        bytes_to_allocate <= page_count,
+        page_count == 0 || overflow,
         "invalid allocation size (%zu pages)", page_count
     );
 

From 66c7dc73c471b41e1e1bff4fa892e9262a164dd4 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 4 Nov 2026 16:08:24 +0300
Subject: [PATCH 093/152] memory: use check_add_overflow() for open-coded
 overflow checks

No functional change.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/memory/boot_alloc.c |  9 ++++-----
 kernel/memory/valloc.c     | 14 ++++++--------
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/kernel/memory/boot_alloc.c b/kernel/memory/boot_alloc.c
index 47519040..86b344cf 100644
--- a/kernel/memory/boot_alloc.c
+++ b/kernel/memory/boot_alloc.c
@@ -338,7 +338,7 @@ static INIT_CODE error_t allocate_within(
     phys_addr_t *out_addr
 )
 {
-    phys_addr_t range_begin;
+    phys_addr_t range_begin, range_end;
     size_t bytes_to_allocate;
     ssize_t mr_idx;
     struct memory_range *picked_mr = nullptr;
@@ -355,12 +355,11 @@ static INIT_CODE error_t allocate_within(
     if (lower_limit >= upper_limit)
         goto out_invalid_allocation;
 
-    // search gap is too small
-    if (lower_limit + bytes_to_allocate > upper_limit)
+    if (check_add_overflow(lower_limit, bytes_to_allocate, &range_end))
         goto out_invalid_allocation;
 
-    // overflow
-    if (lower_limit + bytes_to_allocate < lower_limit)
+    // search gap is too small
+    if (range_end > upper_limit)
         goto out_invalid_allocation;
 
     mr_idx = find_range(lower_limit, ALLOW_ONE_ABOVE_YES);
diff --git a/kernel/memory/valloc.c b/kernel/memory/valloc.c
index 038b598c..876c5805 100644
--- a/kernel/memory/valloc.c
+++ b/kernel/memory/valloc.c
@@ -7,6 +7,7 @@
 #include <common/minmax.h>
 #include <common/string.h>
 #include <common/bit.h>
+#include <common/overflow.h>
 
 #include <bug.h>
 #include <panic.h>
@@ -304,16 +305,16 @@ static bool area_fits(
     size_t size, size_t align, virt_addr_t *out_start
 )
 {
-    virt_addr_t start;
+    virt_addr_t start, end;
 
     start = MAX(area->start, vstart);
     start = ALIGN_UP(start, align);
 
     // The alignment, or the run itself, wrapped around the address space
-    if (start < area->start || start + size < start)
+    if (start < area->start || check_add_overflow(start, size, &end))
         return false;
 
-    if (start + size > area->end || start + size > vend)
+    if (end > area->end || end > vend)
         return false;
 
     *out_start = start;
@@ -492,11 +493,8 @@ static struct varea *varea_alloc_within(
     if (unlikely(size == 0))
         return nullptr;
 
-    if (guarded) {
-        if (unlikely(size + PAGE_SIZE < size))
-            return nullptr;
-        size += PAGE_SIZE;
-    }
+    if (guarded && check_add_overflow(size, PAGE_SIZE, &size))
+        return nullptr;
 
     area = alloc_from_cache(&s_varea_cache, behavior);
     info = alloc_from_cache(&s_vallocation_cache, behavior | ALLOC_ZEROED);

From b80216e5a417523164e004c38c9a48996e702270 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 5 Nov 2026 16:40:19 +0300
Subject: [PATCH 094/152] time: add ns_from_now()

A deadline is computed as ns_since_boot() + ns, which wraps around for
a huge timeout and puts the deadline in the past, so the wait ends right
away. ns_from_now() saturates to NO_DEADLINE instead, so passing
UINT64_MAX means waiting forever.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/time/time.h   | 14 ++++++++++++++
 kernel/time/keeper.c         |  2 +-
 tests/test-cases/test_time.c | 14 ++++++++++++++
 3 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/kernel/include/time/time.h b/kernel/include/time/time.h
index 2d24cb1d..ad24ffdb 100644
--- a/kernel/include/time/time.h
+++ b/kernel/include/time/time.h
@@ -1,7 +1,21 @@
 #pragma once
 
 #include <common/types.h>
+#include <common/overflow.h>
+
+#define NO_DEADLINE UINT64_MAX
 
 u64 ns_since_boot(void);
 
+// A time point 'ns' nanoseconds in the future
+static inline u64 ns_from_now(u64 ns)
+{
+    u64 deadline_ns;
+
+    if (check_add_overflow(ns_since_boot(), ns, &deadline_ns))
+        return NO_DEADLINE;
+
+    return deadline_ns;
+}
+
 void delay_ns(u64);
diff --git a/kernel/time/keeper.c b/kernel/time/keeper.c
index 50dffa65..94f11108 100644
--- a/kernel/time/keeper.c
+++ b/kernel/time/keeper.c
@@ -161,7 +161,7 @@ void delay_ns(u64 ns)
         // TODO: maybe add an arch_early_delay_ns helper
         return;
 
-    deadline = ns_since_boot() + ns;
+    deadline = ns_from_now(ns);
     while (ns_since_boot() < deadline)
         arch_cpu_relax();
 }
diff --git a/tests/test-cases/test_time.c b/tests/test-cases/test_time.c
index edc720a5..bad32da9 100644
--- a/tests/test-cases/test_time.c
+++ b/tests/test-cases/test_time.c
@@ -268,6 +268,20 @@ TEST_CASE(keeper_without_device_is_zero)
     ASSERT_EQ(ns_since_boot(), 0);
 }
 
+TEST_CASE(ns_from_now_saturates)
+{
+    struct counter_device dev = TEST_CD("tsc", COUNTER_MASK(64), IDEAL);
+
+    time_test_reset();
+    counter_device_register(&dev, ONE_NS_PER_CYCLE_HZ);
+
+    g_counter_value = 1000;
+    ASSERT_EQ(ns_from_now(500), 1500);
+    ASSERT_EQ(ns_from_now(UNSIGNED_MAX(u64)), NO_DEADLINE);
+
+    counter_device_unregister(&dev);
+}
+
 TEST_CASE(keeper_accumulates_across_ticks)
 {
     struct counter_device dev = TEST_CD("tsc", COUNTER_MASK(64), IDEAL);

From ac3e1ffdc8b02e741a6596f2d00d07a68e56d345 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:59:54 +0300
Subject: [PATCH 095/152] time: introduce the per-CPU timer queue

This adds the ability to schedule a piece of code to run at a given
point in time via the new timer api.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/time/timer.h  |  18 ++
 kernel/include/private/time/timer.h |   3 +
 kernel/include/time/timer.h         |  59 +++++++
 kernel/time/CMakeLists.txt          |   1 +
 kernel/time/timer.c                 | 247 ++++++++++++++++++++++++++++
 5 files changed, 328 insertions(+)
 create mode 100644 kernel/include/detail/time/timer.h
 create mode 100644 kernel/include/private/time/timer.h
 create mode 100644 kernel/include/time/timer.h
 create mode 100644 kernel/time/timer.c

diff --git a/kernel/include/detail/time/timer.h b/kernel/include/detail/time/timer.h
new file mode 100644
index 00000000..a8457b84
--- /dev/null
+++ b/kernel/include/detail/time/timer.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/rb_tree.h>
+
+struct timer_queue;
+enum timer_flags : u32;
+enum timer_result : u32;
+
+struct timer {
+    u64 deadline_ns;
+
+    enum timer_result (*fn)(struct timer *timer);
+
+    struct rb_node node;
+    struct timer_queue *queue;
+    enum timer_flags flags;
+};
diff --git a/kernel/include/private/time/timer.h b/kernel/include/private/time/timer.h
new file mode 100644
index 00000000..4b21e67f
--- /dev/null
+++ b/kernel/include/private/time/timer.h
@@ -0,0 +1,3 @@
+#pragma once
+
+void timer_queue_expire(void);
diff --git a/kernel/include/time/timer.h b/kernel/include/time/timer.h
new file mode 100644
index 00000000..7cc04313
--- /dev/null
+++ b/kernel/include/time/timer.h
@@ -0,0 +1,59 @@
+#pragma once
+
+#include <common/types.h>
+
+#include <detail/time/timer.h>
+
+/*
+ * A callback run at a point in monotonic time, in hard interrupt
+ * context on the CPU that armed the timer. Set up with TIMER_INIT() or
+ * timer_init(), all fields are internal.
+ */
+struct timer;
+
+// The result of a timer callback
+enum timer_result : u32 {
+    // Don't re-arm, this timer is done (or has re-armed itself already)
+    TIMER_RESULT_DONE = 0,
+};
+
+typedef enum timer_result (*timer_fn_t)(struct timer *timer);
+
+#define TIMER_INIT(name, timer_fn) (struct timer) { \
+    .fn = (timer_fn),                               \
+}
+#define DEFINE_TIMER(name, timer_fn)               \
+    struct timer name = TIMER_INIT(name, timer_fn)
+
+void timer_init(struct timer *timer, timer_fn_t fn);
+
+/*
+ * Arm the timer to run its callback once at an absolute 'deadline_ns' in
+ * ns_since_boot() time. If 'deadline_ns' is in the past, the callback is
+ * invoked as soon as possible. To run it periodically, arm the timer again
+ * from its callback, e.g. for timer_deadline() plus the period. Callable
+ * from any context that may take a spinlock, including the callback itself.
+ *
+ * NOTE: The timer is armed on the calling CPU, unless its callback is running
+ *       on another CPU at that moment, in which case it stays there. Arming a
+ *       timer that is already armed changes its deadline.
+ */
+void timer_arm(struct timer *timer, u64 deadline_ns);
+
+// The deadline the timer was last armed with
+static inline u64 timer_deadline(const struct timer *timer)
+{
+    return timer->deadline_ns;
+}
+
+/*
+ * Cancel a timer. The callback is guaranteed to not be running once this
+ * returns.
+ */
+void timer_cancel(struct timer *timer);
+
+/*
+ * Cancel a timer. The callback may still be running once this function returns,
+ * this is indicated by the return value.
+ */
+bool timer_cancel_nosync(struct timer *timer);
diff --git a/kernel/time/CMakeLists.txt b/kernel/time/CMakeLists.txt
index fb219f33..3091f502 100644
--- a/kernel/time/CMakeLists.txt
+++ b/kernel/time/CMakeLists.txt
@@ -1,5 +1,6 @@
 ultra_sources(
     counter_device.c
     keeper.c
+    timer.c
     units.c
 )
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
new file mode 100644
index 00000000..9f880f2c
--- /dev/null
+++ b/kernel/time/timer.c
@@ -0,0 +1,247 @@
+#include <private/time/timer.h>
+
+#include <time/timer.h>
+#include <time/time.h>
+#include <time/units.h>
+
+#include <common/atomic.h>
+#include <common/bit.h>
+
+#include <bug.h>
+#include <per_cpu.h>
+#include <preempt.h>
+#include <locking/spinlock.h>
+#include <irq_helpers.h>
+
+#include <arch/cpu_helpers.h>
+
+enum timer_flags : u32 {
+    TIMER_FLAG_QUEUED = BIT_U32(0),
+    TIMER_FLAG_RUNNING = BIT_U32(1),
+};
+
+struct timer_queue {
+    struct spinlock lock;
+    struct rb_root_cached timers;
+    bool in_interrupt;
+};
+
+static DEFINE_PER_CPU(struct timer_queue, s_queue);
+
+/*
+ * A sentinel value that is written into timer->queue while it's being moved
+ * between queues. We do this because we can't take both queue locks at the
+ * same time since that may cause a deadlock.
+ */
+static struct timer_queue s_moving_queue;
+
+static bool timer_expires_before(
+    const struct rb_node *new_node, const struct rb_node *old_node
+)
+{
+    const struct timer *new_timer, *old_timer;
+
+    new_timer = rb_entry(new_node, struct timer, node);
+    old_timer = rb_entry(old_node, struct timer, node);
+
+    return new_timer->deadline_ns < old_timer->deadline_ns;
+}
+
+static struct timer *queue_first(struct timer_queue *queue)
+{
+    struct rb_node *node;
+
+    node = rb_first_cached(&queue->timers);
+    if (node == nullptr)
+        return nullptr;
+
+    return rb_entry(node, struct timer, node);
+}
+
+static void queue_remove(struct timer_queue *queue, struct timer *timer)
+{
+    rb_node_remove_cached(&timer->node, &queue->timers);
+    timer->flags &= ~TIMER_FLAG_QUEUED;
+}
+
+static struct timer_queue *timer_lock_queue(
+    struct timer *timer, irq_state_t *out_irq_state
+)
+{
+    struct timer_queue *queue;
+    irq_state_t irq_state;
+
+    for (;;) {
+        queue = atomic_load_acquire(&timer->queue);
+        if (queue == nullptr)
+            return nullptr;
+        if (queue == &s_moving_queue) {
+            arch_cpu_relax();
+            continue;
+        }
+
+        irq_state = spin_lock_irq_save(&queue->lock);
+
+        // It may have been armed on another CPU while we took the lock
+        if (likely(queue == atomic_load_relaxed(&timer->queue))) {
+            *out_irq_state = irq_state;
+            return queue;
+        }
+
+        spin_unlock_irq_restore(&queue->lock, irq_state);
+    }
+}
+
+/*
+ * Lock and return the queue a timer armed on this CPU goes on. A timer
+ * whose callback runs on another CPU stays on that CPU's queue.
+ */
+static struct timer_queue *timer_lock_queue_for_arm(struct timer *timer)
+{
+    struct timer_queue *queue, *old_queue;
+
+    queue = this_cpu_ptr(&s_queue);
+
+    for (;;) {
+        old_queue = atomic_load_acquire(&timer->queue);
+        if (old_queue == &s_moving_queue) {
+            arch_cpu_relax();
+            continue;
+        }
+
+        if (old_queue == nullptr) {
+            spin_lock(&queue->lock);
+
+            // Another CPU may arm this fresh timer at the same time
+            if (atomic_cmpxchg_acq_rel(&timer->queue, &old_queue, queue))
+                return queue;
+
+            spin_unlock(&queue->lock);
+            continue;
+        }
+
+        spin_lock(&old_queue->lock);
+        if (old_queue != atomic_load_relaxed(&timer->queue)) {
+            spin_unlock(&old_queue->lock);
+            continue;
+        }
+
+        if (old_queue == queue)
+            return queue;
+
+        if (timer->flags & TIMER_FLAG_RUNNING) {
+            // This timer is already running on another CPU, so leave it there
+            BUG_ON(!old_queue->in_interrupt);
+            return old_queue;
+        }
+
+        if (timer->flags & TIMER_FLAG_QUEUED)
+            queue_remove(old_queue, timer);
+
+        /*
+         * Once the old lock is dropped nothing covers the timer, so the other
+         * CPUs wait until it is on the new queue.
+         */
+        atomic_store_relaxed(&timer->queue, &s_moving_queue);
+        spin_unlock(&old_queue->lock);
+
+        spin_lock(&queue->lock);
+        atomic_store_release(&timer->queue, queue);
+        return queue;
+    }
+}
+
+void timer_init(struct timer *timer, timer_fn_t fn)
+{
+    *timer = TIMER_INIT(*timer, fn);
+}
+
+static void queue_insert(
+    struct timer_queue *queue, struct timer *timer, u64 deadline_ns
+)
+{
+    if (timer->flags & TIMER_FLAG_QUEUED)
+        queue_remove(queue, timer);
+
+    timer->deadline_ns = deadline_ns;
+
+    rb_node_insert_cached(&timer->node, &queue->timers, timer_expires_before);
+    timer->flags |= TIMER_FLAG_QUEUED;
+}
+
+void timer_arm(struct timer *timer, u64 deadline_ns)
+{
+    struct timer_queue *queue;
+    irq_state_t irq_state;
+
+    irq_state = local_irq_state_save();
+    queue = timer_lock_queue_for_arm(timer);
+    queue_insert(queue, timer, deadline_ns);
+
+    spin_unlock(&queue->lock);
+    local_irq_state_restore(irq_state);
+}
+
+bool timer_cancel_nosync(struct timer *timer)
+{
+    struct timer_queue *queue;
+    irq_state_t irq_state;
+    bool running;
+
+    queue = timer_lock_queue(timer, &irq_state);
+    if (queue == nullptr)
+        return true;
+
+    if (timer->flags & TIMER_FLAG_QUEUED)
+        queue_remove(queue, timer);
+    running = timer->flags & TIMER_FLAG_RUNNING;
+
+    spin_unlock_irq_restore(&queue->lock, irq_state);
+    return !running;
+}
+
+void timer_cancel(struct timer *timer)
+{
+    while (!timer_cancel_nosync(timer)) {
+        BUG_ON(in_hard_irq());
+        arch_cpu_relax();
+    }
+}
+
+static void queue_run_due(struct timer_queue *queue, u64 now_ns)
+{
+    struct timer *timer;
+
+    for (;;) {
+        timer = queue_first(queue);
+        if (timer == nullptr)
+            break;
+        if (timer->deadline_ns > now_ns)
+            break;
+
+        queue_remove(queue, timer);
+        timer->flags |= TIMER_FLAG_RUNNING;
+        spin_unlock(&queue->lock);
+
+        timer->fn(timer);
+
+        spin_lock(&queue->lock);
+        timer->flags &= ~TIMER_FLAG_RUNNING;
+    }
+}
+
+void timer_queue_expire(void)
+{
+    struct timer_queue *queue;
+
+    queue = this_cpu_ptr(&s_queue);
+    BUG_ON(!in_hard_irq());
+
+    spin_lock(&queue->lock);
+    queue->in_interrupt = true;
+
+    queue_run_due(queue, ns_since_boot());
+
+    queue->in_interrupt = false;
+    spin_unlock(&queue->lock);
+}

From 61a61f3e000e0dda1d865fd641f609f06c8f6d9a Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:59:54 +0300
Subject: [PATCH 096/152] common/minmax: add CLAMP()

A helper for clamping a value between a given min and max bounds.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/common/helpers.h | 10 ++++++
 kernel/include/common/minmax.h  | 49 +++++++++++++++++++++++++++++
 tests/test-cases/test_minmax.c  | 56 +++++++++++++++++++++++++++++++++
 3 files changed, 115 insertions(+)

diff --git a/kernel/include/common/helpers.h b/kernel/include/common/helpers.h
index 9578a081..fc06045a 100644
--- a/kernel/include/common/helpers.h
+++ b/kernel/include/common/helpers.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <limits.h>
+
 #define DO_CONCAT(x, y) x##y
 #define CONCAT(x, y) DO_CONCAT(x, y)
 
@@ -113,6 +115,14 @@
 #define IS_SIGNED_TYPE(x) (((typeof(x))-1) < ((typeof(x))1))
 #define IS_UNSIGNED_TYPE(x) (!IS_SIGNED_TYPE(x))
 
+#define IS_UNSIGNED_OR_NON_NEGATIVE_CONSTANT(x)                      \
+    CHOOSE_EXPR(                                                     \
+        IS_UNSIGNED_TYPE(x), 1,                                      \
+        CHOOSE_EXPR(                                                 \
+            IS_CONSTEXPR(x), !((x) >> (sizeof(x) * CHAR_BIT - 1)), 0 \
+        )                                                            \
+    )
+
 #ifdef __has_builtin
     #define HAS_BUILTIN(x) __has_builtin(x)
 #else
diff --git a/kernel/include/common/minmax.h b/kernel/include/common/minmax.h
index a11eb118..ee1eaaf1 100644
--- a/kernel/include/common/minmax.h
+++ b/kernel/include/common/minmax.h
@@ -21,6 +21,55 @@
 #define MIN(x, y) DO_COMPARE(x, y, <)
 #define MAX(x, y) DO_COMPARE(x, y, >)
 
+#define CLAMP_TYPE(value, low, high) typeof((value) + (low) + (high))
+
+#define CLAMP_SIGNS_MATCH(value, low, high)           \
+    ((IS_SIGNED_TYPE(value) && IS_SIGNED_TYPE(low) && \
+      IS_SIGNED_TYPE(high)) ||                        \
+     (IS_UNSIGNED_OR_NON_NEGATIVE_CONSTANT(value) &&  \
+      IS_UNSIGNED_OR_NON_NEGATIVE_CONSTANT(low) &&    \
+      IS_UNSIGNED_OR_NON_NEGATIVE_CONSTANT(high)))
+
+#define CLAMP_LIMIT(x, low, high) \
+    CHOOSE_EXPR(IS_CONSTEXPR(low) && IS_CONSTEXPR(high), (x), 0)
+
+#define CLAMP_CHECKS(value, low, high) (                             \
+    EMBED_STATIC_ASSERT(                                             \
+        CLAMP_SIGNS_MATCH(value, low, high),                         \
+        "CLAMP() arguments must be all signed or all unsigned"       \
+    ) +                                                              \
+    EMBED_STATIC_ASSERT(                                             \
+        CLAMP_LIMIT(low, low, high) <= CLAMP_LIMIT(high, low, high), \
+        "CLAMP() low limit must not be above the high limit"         \
+    ))
+
+#define CLAMP_COMPARE(value, low, high) \
+    COMPARE(COMPARE(value, low, >), high, <)
+
+#define RUNTIME_CLAMP(value, low, high, uv, ul, uh) ({ \
+    CLAMP_TYPE(value, low, high) uv = (value);         \
+    CLAMP_TYPE(value, low, high) ul = (low);           \
+    CLAMP_TYPE(value, low, high) uh = (high);          \
+                                                       \
+    (void)CLAMP_CHECKS(value, low, high);              \
+    CLAMP_COMPARE(uv, ul, uh);                         \
+})
+
+#define DO_CLAMP(value, low, high)                                           \
+    CHOOSE_EXPR(                                                             \
+        IS_CONSTEXPR(value) && IS_CONSTEXPR(low) && IS_CONSTEXPR(high),      \
+        (CLAMP_TYPE(value, low, high))(                                      \
+            CLAMP_CHECKS(value, low, high) + CLAMP_COMPARE(value, low, high) \
+        ),                                                                   \
+        RUNTIME_CLAMP(                                                       \
+            value, low, high, CONCAT(uv, __COUNTER__),                       \
+            CONCAT(ul, __COUNTER__), CONCAT(uh, __COUNTER__)                 \
+        )                                                                    \
+    )
+
+// Clamp a 'value' to be between [low, high]
+#define CLAMP(value, low, high) DO_CLAMP(value, low, high)
+
 // Add 0u to keep u8 or a u16 from being promoted to a signed int
 #define BOUNDS_TYPE(value, begin, end) typeof((value) + (begin) + (end) + 0u)
 
diff --git a/tests/test-cases/test_minmax.c b/tests/test-cases/test_minmax.c
index 7c6618ed..8b7e19ae 100644
--- a/tests/test-cases/test_minmax.c
+++ b/tests/test-cases/test_minmax.c
@@ -59,3 +59,59 @@ TEST_CASE(within_bounds_evaluates_each_argument_once)
     ASSERT_EQ(begin, 1);
     ASSERT_EQ(end, 11);
 }
+
+TEST_CASE(clamp_limits_a_value_to_the_range)
+{
+    ASSERT_EQ(CLAMP(5u, 10u, 20u), 10u);
+    ASSERT_EQ(CLAMP(15u, 10u, 20u), 15u);
+    ASSERT_EQ(CLAMP(25u, 10u, 20u), 20u);
+}
+
+TEST_CASE(clamp_evaluates_each_argument_once)
+{
+    u32 value, low, high;
+
+    value = 5;
+    low = 0;
+    high = 10;
+
+    ASSERT_EQ(CLAMP(value++, low++, high++), 5);
+    ASSERT_EQ(value, 6);
+    ASSERT_EQ(low, 1);
+    ASSERT_EQ(high, 11);
+}
+
+TEST_CASE(non_negative_constants_of_any_width)
+{
+    STATIC_ASSERT(IS_UNSIGNED_OR_NON_NEGATIVE_CONSTANT((i128)1 << 100), "");
+    STATIC_ASSERT(IS_UNSIGNED_OR_NON_NEGATIVE_CONSTANT((i128)1 << 63), "");
+    STATIC_ASSERT(!IS_UNSIGNED_OR_NON_NEGATIVE_CONSTANT(-((i128)1 << 64)), "");
+    STATIC_ASSERT(!IS_UNSIGNED_OR_NON_NEGATIVE_CONSTANT((i8)-1), "");
+    STATIC_ASSERT(IS_UNSIGNED_OR_NON_NEGATIVE_CONSTANT((u128)1 << 127), "");
+}
+
+TEST_CASE(clamp_accepts_non_negative_constants_for_an_unsigned_value)
+{
+    u32 value;
+
+    value = 30;
+
+    ASSERT_EQ(CLAMP(value, 0, 20), 20u);
+    ASSERT_EQ(CLAMP((u8)5, 10, 20), 10);
+}
+
+TEST_CASE(clamp_of_signed_values)
+{
+    i32 value;
+
+    value = -30;
+
+    ASSERT_EQ(CLAMP(value, -20, 20), -20);
+    ASSERT_EQ(CLAMP(-value, -20, 20), 20);
+}
+
+TEST_CASE(clamp_of_constants_is_a_constant)
+{
+    STATIC_ASSERT(CLAMP(30u, 10u, 20u) == 20u, "clamped to the high limit");
+    STATIC_ASSERT(CLAMP(-30, -20, 20) == -20, "clamped to the low limit");
+}

From e1e8e05f9126d5719ebaaf320a0466aebf2fea0a Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 5 Nov 2026 22:16:43 +0300
Subject: [PATCH 097/152] common/overflow: add saturating_sub()

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/common/overflow.h | 19 +++++++++++++++++++
 tests/test-cases/test_overflow.c | 17 +++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/kernel/include/common/overflow.h b/kernel/include/common/overflow.h
index e4fe7979..f12b3520 100644
--- a/kernel/include/common/overflow.h
+++ b/kernel/include/common/overflow.h
@@ -18,3 +18,22 @@ static ALWAYS_INLINE MUST_CHECK bool must_check_overflow(bool overflow)
     must_check_overflow(__builtin_sub_overflow(a, b, result))
 #define check_mul_overflow(a, b, result) \
     must_check_overflow(__builtin_mul_overflow(a, b, result))
+
+/*
+ * Compute 'a - b', or 0 if 'b' is greater than 'a'. The result has the type
+ * of 'a'. Both must be unsigned, except that 'b' may also be a non-negative
+ * constant.
+ */
+#define saturating_sub(a, b) ({                                         \
+    typeof(a) saturating_sub_result;                                    \
+                                                                        \
+    STATIC_ASSERT(                                                      \
+        IS_UNSIGNED_TYPE(a) && IS_UNSIGNED_OR_NON_NEGATIVE_CONSTANT(b), \
+        "saturating_sub() only supports unsigned numbers"               \
+    );                                                                  \
+                                                                        \
+    if (check_sub_overflow(a, b, &saturating_sub_result))               \
+        saturating_sub_result = 0;                                      \
+                                                                        \
+    saturating_sub_result;                                              \
+})
diff --git a/tests/test-cases/test_overflow.c b/tests/test-cases/test_overflow.c
index 61a30c3a..77c0af5c 100644
--- a/tests/test-cases/test_overflow.c
+++ b/tests/test-cases/test_overflow.c
@@ -48,3 +48,20 @@ TEST_CASE(check_overflow_judges_by_the_result_type)
     ASSERT_FALSE(check_sub_overflow(0u, 1u, &value));
     ASSERT_EQ(value, -1);
 }
+
+TEST_CASE(saturating_sub_stops_at_zero)
+{
+    u8 narrow = 10;
+    u32 calls = 0;
+
+    ASSERT_EQ(saturating_sub(10u, 3u), 7u);
+    ASSERT_EQ(saturating_sub(3u, 10u), 0u);
+    ASSERT_EQ(saturating_sub(UNSIGNED_MAX(u64), 1u), UNSIGNED_MAX(u64) - 1);
+    ASSERT_EQ(saturating_sub(5u, UNSIGNED_MAX(u64)), 0u);
+
+    ASSERT_EQ(saturating_sub(narrow, 300u), 0);
+    ASSERT(ARE_SAME_TYPE(saturating_sub(narrow, 1u), u8));
+
+    ASSERT_EQ(saturating_sub(10u, (calls++, 3u)), 7u);
+    ASSERT_EQ(calls, 1);
+}

From 06f2253c4d3c0b42412cc504fe216fa590f32c4c Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:59:54 +0300
Subject: [PATCH 098/152] time: introduce timer event devices

A timer event device raises an interrupt at a requested time, the
counterpart of a counter device. The queue arms its CPU's device for the
earliest timer, event_device_register() lets the core turn deadlines
into the device's ticks and clamp them to its range, and the driver's
interrupt handler expires the queue through event_device_interrupt().
The queue remembers the deadline the device is armed for and leaves
the device alone when the earliest deadline has not changed.

No device exists yet, so nothing fires.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/private/time/event_device.h |   8 +
 kernel/include/private/time/timer.h        |   4 +
 kernel/include/time/event_device.h         |  40 ++
 kernel/time/CMakeLists.txt                 |   1 +
 kernel/time/event_device.c                 |  78 ++++
 kernel/time/timer.c                        |  68 +++
 tests/CMakeLists.txt                       |  11 +
 tests/test-cases/files.cmake               |   1 +
 tests/test-cases/test_timer.c              | 505 +++++++++++++++++++++
 9 files changed, 716 insertions(+)
 create mode 100644 kernel/include/private/time/event_device.h
 create mode 100644 kernel/include/time/event_device.h
 create mode 100644 kernel/time/event_device.c
 create mode 100644 tests/test-cases/test_timer.c

diff --git a/kernel/include/private/time/event_device.h b/kernel/include/private/time/event_device.h
new file mode 100644
index 00000000..ee8ddfc2
--- /dev/null
+++ b/kernel/include/private/time/event_device.h
@@ -0,0 +1,8 @@
+#pragma once
+
+#include <common/types.h>
+
+struct event_device;
+
+// Arm 'dev' for 'deadline_ns', or as soon as it can if that has passed
+void event_device_arm(struct event_device *dev, u64 deadline_ns);
diff --git a/kernel/include/private/time/timer.h b/kernel/include/private/time/timer.h
index 4b21e67f..151c8841 100644
--- a/kernel/include/private/time/timer.h
+++ b/kernel/include/private/time/timer.h
@@ -1,3 +1,7 @@
 #pragma once
 
+struct event_device;
+
+// Called by event_device_register() on the CPU of the device
+void timer_queue_set_event_device(struct event_device *dev);
 void timer_queue_expire(void);
diff --git a/kernel/include/time/event_device.h b/kernel/include/time/event_device.h
new file mode 100644
index 00000000..62bea807
--- /dev/null
+++ b/kernel/include/time/event_device.h
@@ -0,0 +1,40 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/error.h>
+
+/*
+ * A device that is able to generate one-shot interrupts at a point of time
+ * in the future.
+ */
+struct event_device {
+    // Name of this device for pretty-printing
+    const char *name;
+
+    /*
+     * A mandatory callback that makes the device raise one interrupt 'ticks'
+     * ticks from now, replacing a request if one was pending. The supplied
+     * 'ticks' value is guaranteed to be within [min_ticks, max_ticks].
+     */
+    void (*arm)(struct event_device *dev, u64 ticks);
+
+    // A mandatory callback that cancels a previous arm, if there is one
+    void (*disarm)(struct event_device *dev);
+
+    /*
+     * The range that describes the number of ticks a device->arm() request is
+     * able to accept and serve. The timer core clamps the requested deadline to
+     * this range.
+     */
+    u64 min_ticks;
+    u64 max_ticks;
+
+    // Computed by event_device_register()
+    u64 ns_to_ticks_mult;
+};
+
+// Make 'dev' ticking at 'hz' the event device of the calling CPU
+error_t event_device_register(struct event_device *dev, u64 hz);
+
+// The callback all event devices are expected to call on every interrupt
+void event_device_interrupt(struct event_device *dev);
diff --git a/kernel/time/CMakeLists.txt b/kernel/time/CMakeLists.txt
index 3091f502..b839a910 100644
--- a/kernel/time/CMakeLists.txt
+++ b/kernel/time/CMakeLists.txt
@@ -1,5 +1,6 @@
 ultra_sources(
     counter_device.c
+    event_device.c
     keeper.c
     timer.c
     units.c
diff --git a/kernel/time/event_device.c b/kernel/time/event_device.c
new file mode 100644
index 00000000..8447a2be
--- /dev/null
+++ b/kernel/time/event_device.c
@@ -0,0 +1,78 @@
+#define MSG_FMT(msg) "event-device: " msg
+
+#include <private/time/event_device.h>
+#include <private/time/timer.h>
+
+#include <time/event_device.h>
+#include <time/time.h>
+#include <time/units.h>
+
+#include <common/minmax.h>
+#include <common/overflow.h>
+
+#include <bug.h>
+#include <log.h>
+#include <per_cpu.h>
+#include <irq_helpers.h>
+
+#define NS_TO_TICKS_SHIFT 32
+
+static DEFINE_PER_CPU(struct event_device *, s_device);
+
+static u64 ns_to_ticks_mult(u64 hz)
+{
+    u64 mult;
+
+    mult = (hz / NS_PER_SEC) << NS_TO_TICKS_SHIFT;
+    mult += ((hz % NS_PER_SEC) << NS_TO_TICKS_SHIFT) / NS_PER_SEC;
+
+    return mult;
+}
+
+error_t event_device_register(struct event_device *dev, u64 hz)
+{
+    struct human_repr freq;
+    irq_state_t irq_state;
+
+    if (dev->arm == nullptr || dev->disarm == nullptr)
+        return EINVAL;
+    if (hz == 0 || dev->min_ticks > dev->max_ticks)
+        return EINVAL;
+
+    dev->ns_to_ticks_mult = ns_to_ticks_mult(hz);
+
+    irq_state = local_irq_state_save();
+    this_cpu_write(s_device, dev);
+    timer_queue_set_event_device(dev);
+    local_irq_state_restore(irq_state);
+
+    frequency_to_human(hz, &freq);
+    pr_info("%s @ " HUMAN_REPR_FMT "\n", dev->name, HUMAN_REPR_ARG(freq));
+    return EOK;
+}
+
+void event_device_arm(struct event_device *dev, u64 deadline_ns)
+{
+    u64 delta_ns, ticks;
+
+    delta_ns = saturating_sub(deadline_ns, ns_since_boot());
+
+    /*
+     * It's possible for a deadline to be much further away than what this
+     * device can accept, but this is fine as we just clamp it to its maximum
+     * here and then the timer core reprograms it again once it gets an
+     * interrupt earlier and finds out there's nothing due.
+     */
+    ticks = CLAMP(
+        ((u128)delta_ns * dev->ns_to_ticks_mult) >> NS_TO_TICKS_SHIFT,
+        dev->min_ticks, dev->max_ticks
+    );
+
+    dev->arm(dev, ticks);
+}
+
+void event_device_interrupt(struct event_device *dev)
+{
+    BUG_ON(this_cpu_read(s_device) != dev);
+    timer_queue_expire();
+}
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 9f880f2c..674b57be 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -1,6 +1,8 @@
 #include <private/time/timer.h>
+#include <private/time/event_device.h>
 
 #include <time/timer.h>
+#include <time/event_device.h>
 #include <time/time.h>
 #include <time/units.h>
 
@@ -23,6 +25,10 @@ enum timer_flags : u32 {
 struct timer_queue {
     struct spinlock lock;
     struct rb_root_cached timers;
+    struct event_device *dev;
+
+    // The deadline the device is currently armed for, or NO_DEADLINE
+    u64 armed_ns;
     bool in_interrupt;
 };
 
@@ -64,6 +70,41 @@ static void queue_remove(struct timer_queue *queue, struct timer *timer)
     timer->flags &= ~TIMER_FLAG_QUEUED;
 }
 
+static u64 queue_next_deadline(struct timer_queue *queue)
+{
+    struct timer *first;
+
+    first = queue_first(queue);
+    if (first == nullptr)
+        return NO_DEADLINE;
+
+    return first->deadline_ns;
+}
+
+static void queue_arm_device(struct timer_queue *queue, u64 deadline_ns)
+{
+    struct event_device *dev = queue->dev;
+
+    queue->armed_ns = deadline_ns;
+
+    if (deadline_ns == NO_DEADLINE)
+        dev->disarm(dev);
+    else
+        event_device_arm(dev, deadline_ns);
+}
+
+static void queue_arm(struct timer_queue *queue)
+{
+    u64 deadline_ns;
+
+    if (queue->dev == nullptr)
+        return;
+
+    deadline_ns = queue_next_deadline(queue);
+    if (deadline_ns != queue->armed_ns)
+        queue_arm_device(queue, deadline_ns);
+}
+
 static struct timer_queue *timer_lock_queue(
     struct timer *timer, irq_state_t *out_irq_state
 )
@@ -156,6 +197,12 @@ void timer_init(struct timer *timer, timer_fn_t fn)
     *timer = TIMER_INIT(*timer, fn);
 }
 
+// The interrupt handler re-arms the device at the end, after the last callback
+static bool queue_should_defer_arming(struct timer_queue *queue)
+{
+    return queue->in_interrupt;
+}
+
 static void queue_insert(
     struct timer_queue *queue, struct timer *timer, u64 deadline_ns
 )
@@ -178,6 +225,9 @@ void timer_arm(struct timer *timer, u64 deadline_ns)
     queue = timer_lock_queue_for_arm(timer);
     queue_insert(queue, timer, deadline_ns);
 
+    if (queue_first(queue) == timer && !queue_should_defer_arming(queue))
+        queue_arm(queue);
+
     spin_unlock(&queue->lock);
     local_irq_state_restore(irq_state);
 }
@@ -208,6 +258,22 @@ void timer_cancel(struct timer *timer)
     }
 }
 
+void timer_queue_set_event_device(struct event_device *dev)
+{
+    struct timer_queue *queue;
+    irq_state_t irq_state;
+
+    irq_state = local_irq_state_save();
+    queue = this_cpu_ptr(&s_queue);
+
+    spin_lock(&queue->lock);
+    queue->dev = dev;
+    queue_arm_device(queue, queue_next_deadline(queue));
+    spin_unlock(&queue->lock);
+
+    local_irq_state_restore(irq_state);
+}
+
 static void queue_run_due(struct timer_queue *queue, u64 now_ns)
 {
     struct timer *timer;
@@ -239,8 +305,10 @@ void timer_queue_expire(void)
 
     spin_lock(&queue->lock);
     queue->in_interrupt = true;
+    queue->armed_ns = NO_DEADLINE;
 
     queue_run_due(queue, ns_since_boot());
+    queue_arm(queue);
 
     queue->in_interrupt = false;
     spin_unlock(&queue->lock);
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 8d156598..ae76062a 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -179,6 +179,17 @@ KERNEL_FILE(
     SOURCE_PATH "time" SOURCE_FILE "units.c"
     INCLUDE_PATH "time" INCLUDE_FILE "units.h"
 )
+KERNEL_FILE(
+    SOURCE_PATH "time" SOURCE_FILE "timer.c"
+    INCLUDE_PATH "time" INCLUDE_FILE "timer.h"
+)
+KERNEL_FILE(INCLUDE_PATH "detail/time" INCLUDE_FILE "timer.h")
+KERNEL_FILE(INCLUDE_PATH "private/time" INCLUDE_FILE "timer.h")
+KERNEL_FILE(
+    SOURCE_PATH "time" SOURCE_FILE "event_device.c"
+    INCLUDE_PATH "time" INCLUDE_FILE "event_device.h"
+)
+KERNEL_FILE(INCLUDE_PATH "private/time" INCLUDE_FILE "event_device.h")
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "page.h")
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "bug.h")
 KERNEL_FILE(INCLUDE_PATH "memory" INCLUDE_FILE "alloc_behavior.h")
diff --git a/tests/test-cases/files.cmake b/tests/test-cases/files.cmake
index 6bf9d4ac..87df2075 100644
--- a/tests/test-cases/files.cmake
+++ b/tests/test-cases/files.cmake
@@ -12,6 +12,7 @@ add_test_cases(
     test_list.c
     test_slist.c
     test_time.c
+    test_timer.c
     test_rbtree.c
     test_cpu_mask.c
     test_cpu_mask_out_of_line.c
diff --git a/tests/test-cases/test_timer.c b/tests/test-cases/test_timer.c
new file mode 100644
index 00000000..265ce57e
--- /dev/null
+++ b/tests/test-cases/test_timer.c
@@ -0,0 +1,505 @@
+#include <common/types.h>
+
+u64 timer_test_now(void);
+
+#define ns_since_boot timer_test_now
+
+#include <kernel-source/time/timer.c>
+#include <kernel-source/time/event_device.c>
+#include <private/arch/preempt.h>
+#include <arch/registers.h>
+
+#include <test_harness.h>
+
+#define NOT_ARMED UINT64_MAX
+#define MAX_FIRED 16
+
+static u64 s_now;
+static u64 s_armed_for;
+static int s_num_arms;
+static int s_num_disarms;
+
+static struct timer *s_fired[MAX_FIRED];
+static int s_num_fired;
+
+u64 timer_test_now(void)
+{
+    return s_now;
+}
+
+static void fake_arm(struct event_device *dev, u64 ticks)
+{
+    UNREFERENCED_PARAMETER(dev);
+
+    s_armed_for = s_now + ticks;
+    s_num_arms++;
+}
+
+static void fake_disarm(struct event_device *dev)
+{
+    UNREFERENCED_PARAMETER(dev);
+
+    s_armed_for = NOT_ARMED;
+    s_num_disarms++;
+}
+
+static struct event_device s_fake_device = {
+    .name = "fake",
+    .arm = fake_arm,
+    .disarm = fake_disarm,
+    .max_ticks = UINT64_MAX,
+};
+
+static void reset_state(void)
+{
+    memset(&s_queue, 0, sizeof(s_queue));
+
+    s_now = 0;
+    s_armed_for = NOT_ARMED;
+    s_num_arms = 0;
+    s_num_disarms = 0;
+    s_num_fired = 0;
+
+    event_device_register(&s_fake_device, NS_PER_SEC);
+}
+
+static void take_interrupt_at(u64 now)
+{
+    struct registers regs = { .irq_state = false };
+    irq_state_t irq_state;
+
+    s_now = now;
+    s_armed_for = NOT_ARMED;
+
+    irq_state = local_irq_state_save();
+    hard_irq_enter();
+    event_device_interrupt(&s_fake_device);
+    hard_irq_exit(&regs);
+    local_irq_state_restore(irq_state);
+}
+
+static enum timer_result record(struct timer *timer)
+{
+    if (s_num_fired < MAX_FIRED)
+        s_fired[s_num_fired++] = timer;
+
+    return TIMER_RESULT_DONE;
+}
+
+TEST_CASE(timer_empty_queue_disarms_the_device)
+{
+    reset_state();
+
+    ASSERT_EQ(s_armed_for, NOT_ARMED);
+    ASSERT_EQ(s_num_disarms, 1);
+    ASSERT_EQ(s_num_arms, 0);
+}
+
+TEST_CASE(timer_fires_in_deadline_order)
+{
+    struct timer timers[3];
+    int i;
+
+    reset_state();
+
+    for (i = 0; i < 3; i++)
+        timer_init(&timers[i], record);
+
+    timer_arm(&timers[0], 300);
+    timer_arm(&timers[1], 100);
+    timer_arm(&timers[2], 200);
+
+    take_interrupt_at(1000);
+
+    ASSERT_EQ(s_num_fired, 3);
+    ASSERT_EQ(s_fired[0], &timers[1]);
+    ASSERT_EQ(s_fired[1], &timers[2]);
+    ASSERT_EQ(s_fired[2], &timers[0]);
+    ASSERT_EQ(s_armed_for, NOT_ARMED);
+}
+
+TEST_CASE(timer_does_not_fire_early)
+{
+    struct timer early, late;
+
+    reset_state();
+
+    timer_init(&early, record);
+    timer_init(&late, record);
+    timer_arm(&early, 100);
+    timer_arm(&late, 200);
+
+    take_interrupt_at(99);
+    ASSERT_EQ(s_num_fired, 0);
+    ASSERT_EQ(s_armed_for, 100);
+
+    take_interrupt_at(100);
+    ASSERT_EQ(s_num_fired, 1);
+    ASSERT_EQ(s_fired[0], &early);
+    ASSERT_EQ(s_armed_for, 200);
+
+    take_interrupt_at(250);
+    ASSERT_EQ(s_num_fired, 2);
+    ASSERT_EQ(s_armed_for, NOT_ARMED);
+}
+
+TEST_CASE(timer_arms_the_device_for_a_new_earliest_only)
+{
+    struct timer first, earlier, later;
+
+    reset_state();
+
+    timer_init(&first, record);
+    timer_init(&earlier, record);
+    timer_init(&later, record);
+
+    timer_arm(&first, 500);
+    ASSERT_EQ(s_armed_for, 500);
+    ASSERT_EQ(s_num_arms, 1);
+
+    timer_arm(&later, 900);
+    ASSERT_EQ(s_armed_for, 500);
+    ASSERT_EQ(s_num_arms, 1);
+
+    timer_arm(&earlier, 200);
+    ASSERT_EQ(s_armed_for, 200);
+    ASSERT_EQ(s_num_arms, 2);
+}
+
+TEST_CASE(timer_cancel_keeps_it_from_firing)
+{
+    struct timer first, second;
+
+    reset_state();
+
+    timer_init(&first, record);
+    timer_init(&second, record);
+    timer_arm(&first, 100);
+    timer_arm(&second, 200);
+
+    ASSERT(timer_cancel_nosync(&first));
+    ASSERT_EQ(s_armed_for, 100);
+
+    take_interrupt_at(150);
+    ASSERT_EQ(s_num_fired, 0);
+    ASSERT_EQ(s_armed_for, 200);
+
+    timer_cancel(&second);
+    take_interrupt_at(300);
+    ASSERT_EQ(s_num_fired, 0);
+    ASSERT_EQ(s_armed_for, NOT_ARMED);
+
+    ASSERT(timer_cancel_nosync(&first));
+}
+
+TEST_CASE(timer_cancel_of_a_never_armed_timer)
+{
+    struct timer timer;
+
+    reset_state();
+    timer_init(&timer, record);
+
+    ASSERT(timer_cancel_nosync(&timer));
+    timer_cancel(&timer);
+}
+
+TEST_CASE(timer_rearm_moves_the_deadline)
+{
+    struct timer timer;
+
+    reset_state();
+    timer_init(&timer, record);
+
+    timer_arm(&timer, 100);
+    timer_arm(&timer, 400);
+    ASSERT_EQ(s_armed_for, 400);
+
+    take_interrupt_at(200);
+    ASSERT_EQ(s_num_fired, 0);
+
+    take_interrupt_at(400);
+    ASSERT_EQ(s_num_fired, 1);
+}
+
+TEST_CASE(timer_rearm_to_the_same_deadline_leaves_the_device_alone)
+{
+    struct timer timer;
+
+    reset_state();
+
+    timer_init(&timer, record);
+    timer_arm(&timer, 500);
+    timer_arm(&timer, 500);
+    ASSERT_EQ(s_armed_for, 500);
+    ASSERT_EQ(s_num_arms, 1);
+}
+
+TEST_CASE(timer_early_interrupt_arms_the_device_again)
+{
+    struct timer timer;
+
+    reset_state();
+
+    timer_init(&timer, record);
+    timer_arm(&timer, 500);
+    ASSERT_EQ(s_num_arms, 1);
+
+    take_interrupt_at(300);
+    ASSERT_EQ(s_num_fired, 0);
+    ASSERT_EQ(s_armed_for, 500);
+    ASSERT_EQ(s_num_arms, 2);
+}
+
+struct periodic {
+    struct timer timer;
+    u64 period_ns;
+    int runs;
+    bool cancel_result;
+};
+
+static enum timer_result rearm(struct timer *timer)
+{
+    struct periodic *periodic;
+
+    periodic = container_of(timer, struct periodic, timer);
+    periodic->runs++;
+
+    timer_arm(timer, timer_deadline(timer) + periodic->period_ns);
+
+    return TIMER_RESULT_DONE;
+}
+
+static enum timer_result cancel_self(struct timer *timer)
+{
+    struct periodic *periodic;
+
+    periodic = container_of(timer, struct periodic, timer);
+    periodic->runs++;
+    periodic->cancel_result = timer_cancel_nosync(timer);
+
+    return TIMER_RESULT_DONE;
+}
+
+TEST_CASE(timer_callback_can_rearm_itself)
+{
+    struct periodic periodic = { .period_ns = 100 };
+    int arms_before;
+
+    reset_state();
+
+    timer_init(&periodic.timer, rearm);
+    timer_arm(&periodic.timer, 100);
+
+    arms_before = s_num_arms;
+    take_interrupt_at(100);
+
+    ASSERT_EQ(periodic.runs, 1);
+    ASSERT_EQ(s_armed_for, 200);
+    ASSERT_EQ(s_num_arms, arms_before + 1);
+
+    take_interrupt_at(450);
+    ASSERT_EQ(periodic.runs, 4);
+    ASSERT_EQ(s_armed_for, 500);
+
+    ASSERT(timer_cancel_nosync(&periodic.timer));
+}
+
+TEST_CASE(timer_cancel_from_its_own_callback_reports_it_running)
+{
+    struct periodic periodic = { .cancel_result = true };
+
+    reset_state();
+
+    timer_init(&periodic.timer, cancel_self);
+    timer_arm(&periodic.timer, 100);
+
+    take_interrupt_at(100);
+
+    ASSERT_EQ(periodic.runs, 1);
+    ASSERT(!periodic.cancel_result);
+    ASSERT(timer_cancel_nosync(&periodic.timer));
+}
+
+TEST_CASE(timer_deadline_in_the_past_is_armed_for_now)
+{
+    struct timer timer;
+
+    reset_state();
+    s_now = 1000;
+
+    timer_init(&timer, record);
+    timer_arm(&timer, 400);
+    ASSERT_EQ(s_armed_for, 1000);
+
+    take_interrupt_at(1000);
+    ASSERT_EQ(s_num_fired, 1);
+}
+
+TEST_CASE(timer_registering_a_device_arms_what_is_queued)
+{
+    struct timer timer;
+
+    reset_state();
+    memset(&s_queue, 0, sizeof(s_queue));
+    s_armed_for = NOT_ARMED;
+
+    timer_init(&timer, record);
+    timer_arm(&timer, 700);
+    ASSERT_EQ(s_armed_for, NOT_ARMED);
+
+    event_device_register(&s_fake_device, NS_PER_SEC);
+    ASSERT_EQ(s_armed_for, 700);
+
+    ASSERT(timer_cancel_nosync(&timer));
+}
+
+TEST_CASE(timer_far_deadline_is_armed_for_the_longest_the_device_allows)
+{
+    static struct event_device s_fast_device = {
+        .name = "fast",
+        .arm = fake_arm,
+        .disarm = fake_disarm,
+        .max_ticks = UINT32_MAX,
+    };
+    struct timer timer;
+
+    reset_state();
+
+    // 2^62 ns at 4 GHz is 2^64 ticks, one more than a u64 holds
+    event_device_register(&s_fast_device, 4 * NS_PER_SEC);
+
+    timer_init(&timer, record);
+    timer_arm(&timer, 1ull << 62);
+    ASSERT_EQ(s_armed_for, UINT32_MAX);
+
+    ASSERT(timer_cancel_nosync(&timer));
+}
+
+#define NUM_SIMULTANEOUS_TIMERS 1000
+
+static int s_num_counted;
+
+static enum timer_result count(struct timer *timer)
+{
+    UNREFERENCED_PARAMETER(timer);
+    s_num_counted++;
+
+    return TIMER_RESULT_DONE;
+}
+
+TEST_CASE(timer_interrupt_runs_everything_that_is_due)
+{
+    static struct timer s_timers[NUM_SIMULTANEOUS_TIMERS];
+    int i;
+
+    reset_state();
+    s_num_counted = 0;
+
+    for (i = 0; i < NUM_SIMULTANEOUS_TIMERS; i++) {
+        timer_init(&s_timers[i], count);
+        timer_arm(&s_timers[i], 100);
+    }
+
+    take_interrupt_at(100);
+
+    ASSERT_EQ(s_num_counted, NUM_SIMULTANEOUS_TIMERS);
+    ASSERT_EQ(s_armed_for, NOT_ARMED);
+}
+
+static struct timer *s_moving_timer;
+static struct timer_queue *s_moving_to;
+static int s_num_relaxes;
+
+// Plays the CPU that finishes moving s_moving_timer to s_moving_to
+static void finish_the_move(void)
+{
+    atomic_store_release(&s_moving_timer->queue, s_moving_to);
+    s_num_relaxes++;
+    g_cpu_relax_hook = nullptr;
+}
+
+static void start_a_move(struct timer *timer, struct timer_queue *to)
+{
+    timer->queue = &s_moving_queue;
+    s_moving_timer = timer;
+    s_moving_to = to;
+    s_num_relaxes = 0;
+    g_cpu_relax_hook = finish_the_move;
+}
+
+TEST_CASE(timer_armed_on_another_queue_moves_to_this_one)
+{
+    struct timer_queue other = {};
+    struct timer timer;
+
+    reset_state();
+    timer_init(&timer, record);
+
+    timer.queue = &other;
+    rb_node_insert_cached(&timer.node, &other.timers, timer_expires_before);
+    timer.flags = TIMER_FLAG_QUEUED;
+
+    timer_arm(&timer, 500);
+
+    ASSERT(timer.queue == this_cpu_ptr(&s_queue));
+    ASSERT(queue_first(&other) == nullptr);
+    ASSERT(queue_first(this_cpu_ptr(&s_queue)) == &timer);
+    ASSERT_EQ(s_armed_for, 500);
+
+    ASSERT(timer_cancel_nosync(&timer));
+}
+
+TEST_CASE(timer_arm_waits_for_a_timer_moving_between_queues)
+{
+    struct timer_queue other = {};
+    struct timer timer;
+
+    reset_state();
+    timer_init(&timer, record);
+    start_a_move(&timer, &other);
+
+    timer_arm(&timer, 500);
+
+    ASSERT_EQ(s_num_relaxes, 1);
+    ASSERT(timer.queue == this_cpu_ptr(&s_queue));
+    ASSERT(queue_first(this_cpu_ptr(&s_queue)) == &timer);
+    ASSERT(queue_first(&other) == nullptr);
+
+    ASSERT(timer_cancel_nosync(&timer));
+}
+
+TEST_CASE(timer_cancel_waits_for_a_timer_moving_between_queues)
+{
+    struct timer timer;
+
+    reset_state();
+    timer_init(&timer, record);
+    timer_arm(&timer, 500);
+    start_a_move(&timer, this_cpu_ptr(&s_queue));
+
+    ASSERT(timer_cancel_nosync(&timer));
+
+    ASSERT_EQ(s_num_relaxes, 1);
+    ASSERT(queue_first(this_cpu_ptr(&s_queue)) == nullptr);
+}
+
+TEST_CASE(timer_rearmed_while_running_elsewhere_stays_there)
+{
+    struct timer_queue other = { .in_interrupt = true };
+    struct timer timer;
+    int arms_before;
+
+    reset_state();
+    timer_init(&timer, record);
+
+    timer.queue = &other;
+    timer.flags = TIMER_FLAG_RUNNING;
+
+    arms_before = s_num_arms;
+    timer_arm(&timer, 500);
+
+    ASSERT(timer.queue == &other);
+    ASSERT(queue_first(&other) == &timer);
+    ASSERT(queue_first(this_cpu_ptr(&s_queue)) == nullptr);
+    ASSERT_EQ(s_num_arms, arms_before);
+}

From 0309657d55fbc33b7f6c1274f408403330dc48ce Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:59:54 +0300
Subject: [PATCH 099/152] time: add timer_arm_with_slack()

A timer may let its callback run up to some slack after its deadline.
The event device is only armed for the deadline plus the slack, and an
interrupt runs every timer whose deadline has passed, so timers with
slack share interrupts instead of each arming the device for its own.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/time/timer.h |  1 +
 kernel/include/time/timer.h        | 14 ++++++++++-
 kernel/time/timer.c                | 20 ++++++++++++----
 tests/test-cases/test_timer.c      | 38 ++++++++++++++++++++++++++++++
 4 files changed, 67 insertions(+), 6 deletions(-)

diff --git a/kernel/include/detail/time/timer.h b/kernel/include/detail/time/timer.h
index a8457b84..86e904a4 100644
--- a/kernel/include/detail/time/timer.h
+++ b/kernel/include/detail/time/timer.h
@@ -9,6 +9,7 @@ enum timer_result : u32;
 
 struct timer {
     u64 deadline_ns;
+    u64 hard_deadline_ns;
 
     enum timer_result (*fn)(struct timer *timer);
 
diff --git a/kernel/include/time/timer.h b/kernel/include/time/timer.h
index 7cc04313..609a26b5 100644
--- a/kernel/include/time/timer.h
+++ b/kernel/include/time/timer.h
@@ -27,6 +27,15 @@ typedef enum timer_result (*timer_fn_t)(struct timer *timer);
 
 void timer_init(struct timer *timer, timer_fn_t fn);
 
+/*
+ * Same as timer_arm(timer, deadline_ns + slack_ns), except the callback may
+ * end up running earlier, but no earlier than 'deadline_ns'. This allows for
+ * interrupt coalescing in case another timer's deadline happens to arrive in
+ * that window. timer_deadline() reports the 'deadline_ns' value for such
+ * timers.
+ */
+void timer_arm_with_slack(struct timer *timer, u64 deadline_ns, u64 slack_ns);
+
 /*
  * Arm the timer to run its callback once at an absolute 'deadline_ns' in
  * ns_since_boot() time. If 'deadline_ns' is in the past, the callback is
@@ -38,7 +47,10 @@ void timer_init(struct timer *timer, timer_fn_t fn);
  *       on another CPU at that moment, in which case it stays there. Arming a
  *       timer that is already armed changes its deadline.
  */
-void timer_arm(struct timer *timer, u64 deadline_ns);
+static inline void timer_arm(struct timer *timer, u64 deadline_ns)
+{
+    timer_arm_with_slack(timer, deadline_ns, 0);
+}
 
 // The deadline the timer was last armed with
 static inline u64 timer_deadline(const struct timer *timer)
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 674b57be..11b416d5 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -8,6 +8,7 @@
 
 #include <common/atomic.h>
 #include <common/bit.h>
+#include <common/overflow.h>
 
 #include <bug.h>
 #include <per_cpu.h>
@@ -50,7 +51,7 @@ static bool timer_expires_before(
     new_timer = rb_entry(new_node, struct timer, node);
     old_timer = rb_entry(old_node, struct timer, node);
 
-    return new_timer->deadline_ns < old_timer->deadline_ns;
+    return new_timer->hard_deadline_ns < old_timer->hard_deadline_ns;
 }
 
 static struct timer *queue_first(struct timer_queue *queue)
@@ -78,7 +79,7 @@ static u64 queue_next_deadline(struct timer_queue *queue)
     if (first == nullptr)
         return NO_DEADLINE;
 
-    return first->deadline_ns;
+    return first->hard_deadline_ns;
 }
 
 static void queue_arm_device(struct timer_queue *queue, u64 deadline_ns)
@@ -204,26 +205,35 @@ static bool queue_should_defer_arming(struct timer_queue *queue)
 }
 
 static void queue_insert(
-    struct timer_queue *queue, struct timer *timer, u64 deadline_ns
+    struct timer_queue *queue, struct timer *timer,
+    u64 deadline_ns, u64 hard_deadline_ns
 )
 {
     if (timer->flags & TIMER_FLAG_QUEUED)
         queue_remove(queue, timer);
 
     timer->deadline_ns = deadline_ns;
+    timer->hard_deadline_ns = hard_deadline_ns;
 
     rb_node_insert_cached(&timer->node, &queue->timers, timer_expires_before);
     timer->flags |= TIMER_FLAG_QUEUED;
 }
 
-void timer_arm(struct timer *timer, u64 deadline_ns)
+void timer_arm_with_slack(struct timer *timer, u64 deadline_ns, u64 slack_ns)
 {
     struct timer_queue *queue;
     irq_state_t irq_state;
+    u64 hard_deadline_ns;
+
+    /*
+     * A hard deadline that wraps around would sit at the front of the queue
+     * and keep every timer behind it from running
+     */
+    BUG_ON(check_add_overflow(deadline_ns, slack_ns, &hard_deadline_ns));
 
     irq_state = local_irq_state_save();
     queue = timer_lock_queue_for_arm(timer);
-    queue_insert(queue, timer, deadline_ns);
+    queue_insert(queue, timer, deadline_ns, hard_deadline_ns);
 
     if (queue_first(queue) == timer && !queue_should_defer_arming(queue))
         queue_arm(queue);
diff --git a/tests/test-cases/test_timer.c b/tests/test-cases/test_timer.c
index 265ce57e..e70b5188 100644
--- a/tests/test-cases/test_timer.c
+++ b/tests/test-cases/test_timer.c
@@ -250,6 +250,44 @@ TEST_CASE(timer_early_interrupt_arms_the_device_again)
     ASSERT_EQ(s_num_arms, 2);
 }
 
+TEST_CASE(timer_with_slack_rides_an_earlier_interrupt)
+{
+    struct timer exact, relaxed;
+
+    reset_state();
+
+    timer_init(&exact, record);
+    timer_init(&relaxed, record);
+
+    timer_arm(&exact, 100);
+    timer_arm_with_slack(&relaxed, 90, 410);
+    ASSERT_EQ(s_armed_for, 100);
+    ASSERT_EQ(s_num_arms, 1);
+
+    take_interrupt_at(100);
+
+    ASSERT_EQ(s_num_fired, 2);
+    ASSERT_EQ(s_fired[0], &exact);
+    ASSERT_EQ(s_fired[1], &relaxed);
+    ASSERT_EQ(s_armed_for, NOT_ARMED);
+}
+
+TEST_CASE(timer_with_slack_alone_waits_until_the_slack_runs_out)
+{
+    struct timer relaxed;
+
+    reset_state();
+
+    timer_init(&relaxed, record);
+    timer_arm_with_slack(&relaxed, 90, 410);
+    ASSERT_EQ(s_armed_for, 500);
+    ASSERT_EQ(timer_deadline(&relaxed), 90);
+
+    take_interrupt_at(80);
+    ASSERT_EQ(s_num_fired, 0);
+    ASSERT_EQ(s_armed_for, 500);
+}
+
 struct periodic {
     struct timer timer;
     u64 period_ns;

From bda095c25ec93c27c34921acf8122c1528b01302 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 4 Nov 2026 03:42:14 +0300
Subject: [PATCH 100/152] time: add timer_advance()

A periodic timer advances its deadline from its callback and returns
TIMER_RESULT_REARM to be queued again. The deadline moves by whole
periods to the first one after the current time, so a timer that fell
behind skips the periods it missed instead of running once for each.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/time/timer.h   | 29 +++++++++--
 kernel/time/timer.c           | 55 +++++++++++++++++++-
 tests/test-cases/test_timer.c | 95 +++++++++++++++++++++++++++++++++++
 3 files changed, 175 insertions(+), 4 deletions(-)

diff --git a/kernel/include/time/timer.h b/kernel/include/time/timer.h
index 609a26b5..14310cc1 100644
--- a/kernel/include/time/timer.h
+++ b/kernel/include/time/timer.h
@@ -15,6 +15,13 @@ struct timer;
 enum timer_result : u32 {
     // Don't re-arm, this timer is done (or has re-armed itself already)
     TIMER_RESULT_DONE = 0,
+
+    /*
+     * Re-arm the timer again. The callback is expected to have called
+     * timer_advance() to set a new deadline before returning this. This is a
+     * no-op if the timer was already re-armed earlier.
+     */
+    TIMER_RESULT_REARM,
 };
 
 typedef enum timer_result (*timer_fn_t)(struct timer *timer);
@@ -39,9 +46,8 @@ void timer_arm_with_slack(struct timer *timer, u64 deadline_ns, u64 slack_ns);
 /*
  * Arm the timer to run its callback once at an absolute 'deadline_ns' in
  * ns_since_boot() time. If 'deadline_ns' is in the past, the callback is
- * invoked as soon as possible. To run it periodically, arm the timer again
- * from its callback, e.g. for timer_deadline() plus the period. Callable
- * from any context that may take a spinlock, including the callback itself.
+ * invoked as soon as possible. Callable from any context that may take a
+ * spinlock, including the callback itself.
  *
  * NOTE: The timer is armed on the calling CPU, unless its callback is running
  *       on another CPU at that moment, in which case it stays there. Arming a
@@ -58,6 +64,23 @@ static inline u64 timer_deadline(const struct timer *timer)
     return timer->deadline_ns;
 }
 
+/*
+ * Move the deadline of 'timer' forward by whole periods of 'period_ns', to
+ * the first one after the current time, and return how many periods it
+ * moved. The slack the timer was armed with is kept. Only callable from the
+ * timer's own callback. Return TIMER_RESULT_REARM from the callback to re-arm
+ * the timer at the new deadline.
+ *
+ * NOTE: - If the timer was armed again after its callback started, this
+ *         returns 0 and keeps the deadline it was armed with. The caller must
+ *         ensure timer_advance() never happens at the same time as the timer
+ *         is armed from a different CPU.
+ *       - timer_cancel_nosync() returning false does not stop a callback that
+ *         returns TIMER_RESULT_REARM from re-arming the timer. Use
+ *         timer_cancel() if the target timer re-arms itself from the callback.
+ */
+u64 timer_advance(struct timer *timer, u64 period_ns);
+
 /*
  * Cancel a timer. The callback is guaranteed to not be running once this
  * returns.
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 11b416d5..1da51fdf 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -219,6 +219,21 @@ static void queue_insert(
     timer->flags |= TIMER_FLAG_QUEUED;
 }
 
+static void queue_requeue(
+    struct timer_queue *queue, struct timer *timer, u64 now_ns
+)
+{
+    if (timer->flags & TIMER_FLAG_QUEUED)
+        return;
+
+    /*
+     * Make sure the callback didn't forget to advance the timer, otherwise it
+     * would run again and again within this pass and cause us to hang.
+     */
+    BUG_ON(timer->deadline_ns <= now_ns);
+    queue_insert(queue, timer, timer->deadline_ns, timer->hard_deadline_ns);
+}
+
 void timer_arm_with_slack(struct timer *timer, u64 deadline_ns, u64 slack_ns)
 {
     struct timer_queue *queue;
@@ -268,6 +283,40 @@ void timer_cancel(struct timer *timer)
     }
 }
 
+u64 timer_advance(struct timer *timer, u64 period_ns)
+{
+    u64 now_ns, num_periods, advance_ns;
+
+    BUG_ON(period_ns == 0);
+
+    /*
+     * It was armed again since its callback started, so that deadline stands.
+     * Moving the deadline of a queued timer would break the order of the queue.
+     */
+    if (timer->flags & TIMER_FLAG_QUEUED)
+        return 0;
+
+    now_ns = ns_since_boot();
+    if (timer->deadline_ns > now_ns)
+        return 0;
+
+    num_periods = (now_ns - timer->deadline_ns) / period_ns + 1;
+
+    /*
+     * A hard deadline that wraps around would sit at the front of the queue
+     * and keep every timer behind it from running
+     */
+    BUG_ON(check_mul_overflow(num_periods, period_ns, &advance_ns));
+    BUG_ON(check_add_overflow(
+        timer->hard_deadline_ns, advance_ns, &timer->hard_deadline_ns
+    ));
+
+    // The deadline is never after the hard deadline, so this can't overflow
+    timer->deadline_ns += advance_ns;
+
+    return num_periods;
+}
+
 void timer_queue_set_event_device(struct event_device *dev)
 {
     struct timer_queue *queue;
@@ -287,6 +336,7 @@ void timer_queue_set_event_device(struct event_device *dev)
 static void queue_run_due(struct timer_queue *queue, u64 now_ns)
 {
     struct timer *timer;
+    enum timer_result result;
 
     for (;;) {
         timer = queue_first(queue);
@@ -299,10 +349,13 @@ static void queue_run_due(struct timer_queue *queue, u64 now_ns)
         timer->flags |= TIMER_FLAG_RUNNING;
         spin_unlock(&queue->lock);
 
-        timer->fn(timer);
+        result = timer->fn(timer);
 
         spin_lock(&queue->lock);
         timer->flags &= ~TIMER_FLAG_RUNNING;
+
+        if (result == TIMER_RESULT_REARM)
+            queue_requeue(queue, timer, now_ns);
     }
 }
 
diff --git a/tests/test-cases/test_timer.c b/tests/test-cases/test_timer.c
index e70b5188..f13f4104 100644
--- a/tests/test-cases/test_timer.c
+++ b/tests/test-cases/test_timer.c
@@ -291,6 +291,7 @@ TEST_CASE(timer_with_slack_alone_waits_until_the_slack_runs_out)
 struct periodic {
     struct timer timer;
     u64 period_ns;
+    u64 num_periods;
     int runs;
     bool cancel_result;
 };
@@ -318,6 +319,23 @@ static enum timer_result cancel_self(struct timer *timer)
     return TIMER_RESULT_DONE;
 }
 
+static enum timer_result advance(struct timer *timer)
+{
+    struct periodic *periodic;
+
+    periodic = container_of(timer, struct periodic, timer);
+    periodic->runs++;
+    periodic->num_periods = timer_advance(timer, periodic->period_ns);
+
+    return TIMER_RESULT_REARM;
+}
+
+static enum timer_result arm_and_ask_to_rearm(struct timer *timer)
+{
+    timer_arm(timer, 500);
+    return TIMER_RESULT_REARM;
+}
+
 TEST_CASE(timer_callback_can_rearm_itself)
 {
     struct periodic periodic = { .period_ns = 100 };
@@ -358,6 +376,83 @@ TEST_CASE(timer_cancel_from_its_own_callback_reports_it_running)
     ASSERT(timer_cancel_nosync(&periodic.timer));
 }
 
+TEST_CASE(timer_advance_rearms_for_the_next_period)
+{
+    struct periodic periodic = { .period_ns = 100 };
+    int arms_before;
+
+    reset_state();
+
+    timer_init(&periodic.timer, advance);
+    timer_arm(&periodic.timer, 100);
+
+    arms_before = s_num_arms;
+    take_interrupt_at(100);
+
+    ASSERT_EQ(periodic.runs, 1);
+    ASSERT_EQ(periodic.num_periods, 1);
+    ASSERT_EQ(timer_deadline(&periodic.timer), 200);
+    ASSERT_EQ(s_armed_for, 200);
+    ASSERT_EQ(s_num_arms, arms_before + 1);
+
+    ASSERT(timer_cancel_nosync(&periodic.timer));
+}
+
+TEST_CASE(timer_advance_skips_the_periods_it_missed)
+{
+    struct periodic periodic = { .period_ns = 100 };
+
+    reset_state();
+
+    timer_init(&periodic.timer, advance);
+    timer_arm(&periodic.timer, 100);
+
+    take_interrupt_at(450);
+
+    ASSERT_EQ(periodic.runs, 1);
+    ASSERT_EQ(periodic.num_periods, 4);
+    ASSERT_EQ(timer_deadline(&periodic.timer), 500);
+    ASSERT_EQ(s_armed_for, 500);
+
+    ASSERT(timer_cancel_nosync(&periodic.timer));
+}
+
+TEST_CASE(timer_advance_keeps_the_slack)
+{
+    struct periodic periodic = { .period_ns = 100 };
+
+    reset_state();
+
+    timer_init(&periodic.timer, advance);
+    timer_arm_with_slack(&periodic.timer, 100, 50);
+
+    take_interrupt_at(150);
+
+    ASSERT_EQ(periodic.num_periods, 1);
+    ASSERT_EQ(timer_deadline(&periodic.timer), 200);
+    ASSERT_EQ(s_armed_for, 250);
+
+    ASSERT(timer_cancel_nosync(&periodic.timer));
+}
+
+TEST_CASE(timer_armed_while_its_callback_runs_ignores_the_rearm)
+{
+    struct timer timer;
+
+    reset_state();
+
+    timer_init(&timer, arm_and_ask_to_rearm);
+    timer_arm(&timer, 100);
+
+    take_interrupt_at(100);
+
+    ASSERT_EQ(timer_deadline(&timer), 500);
+    ASSERT_EQ(s_armed_for, 500);
+
+    ASSERT(timer_cancel_nosync(&timer));
+    ASSERT(queue_first(this_cpu_ptr(&s_queue)) == nullptr);
+}
+
 TEST_CASE(timer_deadline_in_the_past_is_armed_for_now)
 {
     struct timer timer;

From 0001ab55231fbdd97f27a43e0d208e6637725a76 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 15:59:54 +0300
Subject: [PATCH 101/152] time: back off when the timer interrupt cannot keep
 up

Timers that come due while the interrupt runs callbacks run in the same
interrupt, but only for a few passes. Past that the device is armed for
later, so a flood of due timers cannot keep the CPU in the interrupt.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/time/timer.c           |  44 +++++++++++-
 tests/test-cases/test_timer.c | 123 ++++++++++++++++++++++++++++++++++
 2 files changed, 164 insertions(+), 3 deletions(-)

diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 1da51fdf..6cf22787 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -8,6 +8,7 @@
 
 #include <common/atomic.h>
 #include <common/bit.h>
+#include <common/minmax.h>
 #include <common/overflow.h>
 
 #include <bug.h>
@@ -23,6 +24,9 @@ enum timer_flags : u32 {
     TIMER_FLAG_RUNNING = BIT_U32(1),
 };
 
+#define MAX_PASSES_PER_INTERRUPT 3
+#define MAX_BACKOFF_NS (100 * NS_PER_MS)
+
 struct timer_queue {
     struct spinlock lock;
     struct rb_root_cached timers;
@@ -31,6 +35,7 @@ struct timer_queue {
     // The deadline the device is currently armed for, or NO_DEADLINE
     u64 armed_ns;
     bool in_interrupt;
+    bool backing_off;
 };
 
 static DEFINE_PER_CPU(struct timer_queue, s_queue);
@@ -201,7 +206,11 @@ void timer_init(struct timer *timer, timer_fn_t fn)
 // The interrupt handler re-arms the device at the end, after the last callback
 static bool queue_should_defer_arming(struct timer_queue *queue)
 {
-    return queue->in_interrupt;
+    /*
+     * During a back-off the device stays armed for the end of it, a
+     * timer armed meanwhile runs then at the latest.
+     */
+    return queue->in_interrupt || queue->backing_off;
 }
 
 static void queue_insert(
@@ -362,16 +371,45 @@ static void queue_run_due(struct timer_queue *queue, u64 now_ns)
 void timer_queue_expire(void)
 {
     struct timer_queue *queue;
+    struct timer *first;
+    u64 entry_ns, now_ns, backoff_ns;
+    u32 pass = 0;
 
     queue = this_cpu_ptr(&s_queue);
     BUG_ON(!in_hard_irq());
 
     spin_lock(&queue->lock);
     queue->in_interrupt = true;
+    queue->backing_off = false;
     queue->armed_ns = NO_DEADLINE;
 
-    queue_run_due(queue, ns_since_boot());
-    queue_arm(queue);
+    entry_ns = ns_since_boot();
+    now_ns = entry_ns;
+
+    for (;;) {
+        queue_run_due(queue, now_ns);
+
+        first = queue_first(queue);
+        now_ns = ns_since_boot();
+
+        if (first == nullptr || first->hard_deadline_ns > now_ns) {
+            queue_arm(queue);
+            break;
+        }
+
+        if (++pass < MAX_PASSES_PER_INTERRUPT)
+            continue;
+
+        /*
+         * Timers come due faster than their callbacks finish. Leave the
+         * interrupt for as long as we have spent in it, so that something
+         * other than timer callbacks gets to run.
+         */
+        backoff_ns = MIN(now_ns - entry_ns, MAX_BACKOFF_NS);
+        queue_arm_device(queue, now_ns + backoff_ns);
+        queue->backing_off = true;
+        break;
+    }
 
     queue->in_interrupt = false;
     spin_unlock(&queue->lock);
diff --git a/tests/test-cases/test_timer.c b/tests/test-cases/test_timer.c
index f13f4104..1ad62f69 100644
--- a/tests/test-cases/test_timer.c
+++ b/tests/test-cases/test_timer.c
@@ -539,6 +539,129 @@ TEST_CASE(timer_interrupt_runs_everything_that_is_due)
     ASSERT_EQ(s_armed_for, NOT_ARMED);
 }
 
+static enum timer_result advance_clock_and_count(struct timer *timer)
+{
+    UNREFERENCED_PARAMETER(timer);
+
+    s_num_counted++;
+    s_now += 50;
+
+    return TIMER_RESULT_DONE;
+}
+
+TEST_CASE(timer_due_during_the_interrupt_runs_in_the_same_one)
+{
+    struct timer first, second;
+
+    reset_state();
+    s_num_counted = 0;
+
+    timer_init(&first, advance_clock_and_count);
+    timer_init(&second, count);
+    timer_arm(&first, 100);
+    timer_arm(&second, 120);
+
+    take_interrupt_at(100);
+
+    ASSERT_EQ(s_num_counted, 2);
+    ASSERT_EQ(s_armed_for, NOT_ARMED);
+}
+
+static enum timer_result fall_further_behind(struct timer *timer)
+{
+    s_num_counted++;
+    s_now += 50;
+
+    timer_arm(timer, timer_deadline(timer) + 10);
+
+    return TIMER_RESULT_DONE;
+}
+
+TEST_CASE(timer_interrupt_backs_off_when_it_cannot_keep_up)
+{
+    struct timer timer;
+    u64 entered_at = 100;
+
+    reset_state();
+    s_num_counted = 0;
+
+    timer_init(&timer, fall_further_behind);
+    timer_arm(&timer, 100);
+
+    take_interrupt_at(entered_at);
+
+    /*
+     * One run in the first pass, then 5 and 25 as every pass has
+     * more to catch up with than the one before it.
+     */
+    ASSERT_EQ(s_num_counted, 31);
+    ASSERT_EQ(s_armed_for, s_now + (s_now - entered_at));
+
+    ASSERT(timer_cancel_nosync(&timer));
+}
+
+TEST_CASE(timer_armed_during_a_backoff_does_not_cut_it_short)
+{
+    struct timer hog, overdue, later;
+    u64 backoff_ends_at;
+    int arms_before;
+
+    reset_state();
+    s_num_counted = 0;
+
+    timer_init(&hog, fall_further_behind);
+    timer_init(&overdue, count);
+    timer_init(&later, count);
+
+    timer_arm(&hog, 100);
+    take_interrupt_at(100);
+
+    backoff_ends_at = s_armed_for;
+    arms_before = s_num_arms;
+    ASSERT(timer_cancel_nosync(&hog));
+
+    timer_arm(&overdue, 1);
+    ASSERT_EQ(s_armed_for, backoff_ends_at);
+    ASSERT_EQ(s_num_arms, arms_before);
+
+    s_num_counted = 0;
+    take_interrupt_at(backoff_ends_at);
+    ASSERT_EQ(s_num_counted, 1);
+
+    timer_arm(&later, backoff_ends_at + 500);
+    ASSERT_EQ(s_armed_for, backoff_ends_at + 500);
+
+    ASSERT(timer_cancel_nosync(&later));
+}
+
+static enum timer_result fall_behind_by_a_second(struct timer *timer)
+{
+    s_num_counted++;
+    s_now += NS_PER_SEC;
+
+    timer_arm(timer, s_now);
+
+    return TIMER_RESULT_DONE;
+}
+
+TEST_CASE(timer_interrupt_backoff_is_capped)
+{
+    struct timer timer;
+
+    reset_state();
+    s_num_counted = 0;
+
+    timer_init(&timer, fall_behind_by_a_second);
+    timer_arm(&timer, 100);
+
+    take_interrupt_at(100);
+
+    ASSERT_EQ(s_num_counted, MAX_PASSES_PER_INTERRUPT);
+    ASSERT_EQ(s_armed_for, s_now + MAX_BACKOFF_NS);
+
+    ASSERT(timer_cancel_nosync(&timer));
+}
+
 static struct timer *s_moving_timer;
 static struct timer_queue *s_moving_to;
 static int s_num_relaxes;

From 9347ad64d142a80d0bc358e5341e56a45ac8b94b Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:03:04 +0300
Subject: [PATCH 102/152] arch/x86_64: introduce the LAPIC timer event device

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/CMakeLists.txt             |   1 +
 kernel/arch/x86_64/apic.c                     |   7 --
 kernel/arch/x86_64/apic_timer.c               | 109 ++++++++++++++++++
 .../arch/x86_64/include/arch/private/apic.h   |   3 +-
 .../x86_64/include/arch/private/apic_timer.h  |   5 +
 kernel/arch/x86_64/include/arch/private/idt.h |   3 +
 kernel/arch/x86_64/interrupts.S               |   3 +-
 kernel/arch/x86_64/tsc.c                      |   1 +
 8 files changed, 122 insertions(+), 10 deletions(-)
 create mode 100644 kernel/arch/x86_64/apic_timer.c
 create mode 100644 kernel/arch/x86_64/include/arch/private/apic_timer.h

diff --git a/kernel/arch/x86_64/CMakeLists.txt b/kernel/arch/x86_64/CMakeLists.txt
index a1e290cc..08568174 100644
--- a/kernel/arch/x86_64/CMakeLists.txt
+++ b/kernel/arch/x86_64/CMakeLists.txt
@@ -2,6 +2,7 @@ ultra_sources_no_lto(string.c)
 
 ultra_sources(
     apic.c
+    apic_timer.c
     vector_alloc.c
     lapic_domain.c
     xapic.c
diff --git a/kernel/arch/x86_64/apic.c b/kernel/arch/x86_64/apic.c
index b45262d2..b24bf6da 100644
--- a/kernel/arch/x86_64/apic.c
+++ b/kernel/arch/x86_64/apic.c
@@ -14,8 +14,6 @@
 #include <free_after_init.h>
 #include <init_level.h>
 
-static UNUSED_DECL u64 s_freq_hz;
-
 struct apic *g_apic = nullptr;
 enum apic_mode g_apic_mode = APIC_MODE_NONE;
 
@@ -68,11 +66,6 @@ void INIT_CODE apic_detect(void)
     g_bsp_apic_id = g_boot_cpu_apic_id;
 }
 
-void apic_set_known_frequency(u64 hz)
-{
-    s_freq_hz = hz;
-}
-
 void apic_eoi(void)
 {
     g_apic->write(APIC_REG_EOI, 0);
diff --git a/kernel/arch/x86_64/apic_timer.c b/kernel/arch/x86_64/apic_timer.c
new file mode 100644
index 00000000..790c2f7a
--- /dev/null
+++ b/kernel/arch/x86_64/apic_timer.c
@@ -0,0 +1,109 @@
+#define MSG_FMT(msg) "apic-timer: " msg
+
+#include <arch/private/apic.h>
+#include <arch/private/apic_timer.h>
+#include <arch/private/vectors.h>
+#include <arch/private/idt.h>
+#include <arch/registers.h>
+#include <arch/cpu_helpers.h>
+
+#include <time/event_device.h>
+#include <time/time.h>
+#include <time/units.h>
+#include <time/counter_device.h>
+
+#include <log.h>
+#include <init_level.h>
+#include <free_after_init.h>
+
+#define CALIBRATE_NS (20 * NS_PER_MS)
+
+#define TIMER_DIVISOR 2
+#define TIMER_DIVIDE_CONFIG CONCAT(APIC_TIMER_DIVIDE_BY_, TIMER_DIVISOR)
+
+static u64 s_freq_hz;
+
+void apic_set_known_frequency(u64 hz)
+{
+    s_freq_hz = hz;
+}
+
+static void apic_timer_arm(struct event_device *dev, u64 ticks)
+{
+    UNREFERENCED_PARAMETER(dev);
+
+    g_apic->write(APIC_REG_TIMER_INITIAL_COUNT, ticks);
+}
+
+static void apic_timer_disarm(struct event_device *dev)
+{
+    UNREFERENCED_PARAMETER(dev);
+
+    g_apic->write(APIC_REG_TIMER_INITIAL_COUNT, 0);
+}
+
+static struct event_device s_apic_timer = {
+    .name = "apic-timer",
+    .arm = apic_timer_arm,
+    .disarm = apic_timer_disarm,
+
+    // An initial count of 0 stops the timer
+    .min_ticks = 1,
+    .max_ticks = UNSIGNED_MAX(u32),
+};
+
+FIXED_VECTOR_HANDLER(X86_APIC_TIMER, VECTOR_APIC_TIMER)
+{
+    UNREFERENCED_PARAMETER(regs);
+
+    apic_eoi();
+    event_device_interrupt(&s_apic_timer);
+}
+
+static void INIT_CODE apic_timer_calibrate(void)
+{
+    u64 start_ns, elapsed_ns, ticks;
+
+    if (current_counter_device() == nullptr)
+        return;
+
+    g_apic->write(APIC_REG_LVT_TIMER, VECTOR_APIC_TIMER | APIC_LVT_MASKED);
+    g_apic->write(APIC_REG_TIMER_INITIAL_COUNT, UINT32_MAX);
+
+    start_ns = ns_since_boot();
+    do {
+        arch_cpu_relax();
+        elapsed_ns = ns_since_boot() - start_ns;
+    } while (elapsed_ns < CALIBRATE_NS);
+
+    ticks = UINT32_MAX - g_apic->read(APIC_REG_TIMER_CURRENT_COUNT);
+    g_apic->write(APIC_REG_TIMER_INITIAL_COUNT, 0);
+
+    s_freq_hz = (ticks * TIMER_DIVISOR * NS_PER_SEC) / elapsed_ns;
+}
+
+static error_t INIT_CODE apic_timer_init(void)
+{
+    const char *source = "known";
+    struct human_repr freq;
+
+    g_apic->write(APIC_REG_TIMER_DIVIDE_CONFIG, TIMER_DIVIDE_CONFIG);
+
+    if (s_freq_hz == 0) {
+        apic_timer_calibrate();
+        source = "calibrated";
+    }
+
+    if (unlikely(s_freq_hz == 0)) {
+        pr_warn("unable to calibrate!\n");
+        return EOK;
+    }
+
+    frequency_to_human(s_freq_hz, &freq);
+    pr_info(HUMAN_REPR_FMT " (%s)\n", HUMAN_REPR_ARG(freq), source);
+
+    g_apic->write(APIC_REG_LVT_TIMER, VECTOR_APIC_TIMER);
+
+    return event_device_register(&s_apic_timer, s_freq_hz / TIMER_DIVISOR);
+}
+INIT_CALL_POST(IRQS_AVAILABLE, apic_timer_init);
diff --git a/kernel/arch/x86_64/include/arch/private/apic.h b/kernel/arch/x86_64/include/arch/private/apic.h
index 6e348422..ba9ed3b4 100644
--- a/kernel/arch/x86_64/include/arch/private/apic.h
+++ b/kernel/arch/x86_64/include/arch/private/apic.h
@@ -55,6 +55,7 @@ enum apic_reg : u32 {
     APIC_REG_TIMER_INITIAL_COUNT = 0x380,
     APIC_REG_TIMER_CURRENT_COUNT = 0x390,
     APIC_REG_TIMER_DIVIDE_CONFIG = 0x3E0,
+        #define APIC_TIMER_DIVIDE_BY_2 0x0
 
     // x2APIC-only shorthand for a fixed IPI to self
     APIC_REG_SELF_IPI = 0x3F0,
@@ -106,5 +107,3 @@ apic_id_t apic_read_id(void);
 void apic_icr_write(u32 value, apic_id_t dest_apic_id);
 void apic_send_fixed_ipi(apic_id_t dest_apic_id, u8 vector);
 void apic_send_ipi_self(u8 vector);
-
-void apic_set_known_frequency(u64 hz);
diff --git a/kernel/arch/x86_64/include/arch/private/apic_timer.h b/kernel/arch/x86_64/include/arch/private/apic_timer.h
new file mode 100644
index 00000000..3d301f8f
--- /dev/null
+++ b/kernel/arch/x86_64/include/arch/private/apic_timer.h
@@ -0,0 +1,5 @@
+#pragma once
+
+#include <common/types.h>
+
+void apic_set_known_frequency(u64 hz);
diff --git a/kernel/arch/x86_64/include/arch/private/idt.h b/kernel/arch/x86_64/include/arch/private/idt.h
index 6bdbd5b2..5610a506 100644
--- a/kernel/arch/x86_64/include/arch/private/idt.h
+++ b/kernel/arch/x86_64/include/arch/private/idt.h
@@ -36,11 +36,14 @@
 #define X86_IRQ_DISPATCH irq_dispatch
 #define X86_FIXED_UNEXPECTED fixed_unexpected
 #define X86_APIC_ERROR apic_error
+#define X86_APIC_TIMER apic_timer
 #define X86_APIC_SPURIOUS apic_spurious
 
 #define X86_IRQ_DISPATCH_ASM CONCAT(X86_IRQ_DISPATCH, _asm)
 #define X86_FIXED_UNEXPECTED_ASM \
     CONCAT(handle_, CONCAT(X86_FIXED_UNEXPECTED, _asm))
+#define X86_APIC_TIMER_ASM \
+    CONCAT(handle_, CONCAT(X86_APIC_TIMER, _asm))
 #define X86_APIC_ERROR_ASM \
     CONCAT(handle_, CONCAT(X86_APIC_ERROR, _asm))
 #define X86_APIC_SPURIOUS_ASM \
diff --git a/kernel/arch/x86_64/interrupts.S b/kernel/arch/x86_64/interrupts.S
index f7f8fb44..9e12f975 100644
--- a/kernel/arch/x86_64/interrupts.S
+++ b/kernel/arch/x86_64/interrupts.S
@@ -209,6 +209,7 @@ ASM_FUNCTION_END handle_\name\()_asm
 
 MAKE_EXCEPTION_TRAMPOLINE X86_EXCEPTION_RSVD
 MAKE_EXCEPTION_TRAMPOLINE X86_FIXED_UNEXPECTED
+MAKE_EXCEPTION_TRAMPOLINE X86_APIC_TIMER
 MAKE_EXCEPTION_TRAMPOLINE X86_APIC_ERROR
 MAKE_EXCEPTION_TRAMPOLINE X86_APIC_SPURIOUS
 
@@ -277,7 +278,7 @@ MAKE_FIXED_UNEXPECTED_THUNK 9
 MAKE_FIXED_UNEXPECTED_THUNK 10
 MAKE_FIXED_UNEXPECTED_THUNK 11
 MAKE_FIXED_UNEXPECTED_THUNK 12
-MAKE_FIXED_UNEXPECTED_THUNK 13
+MAKE_FIXED_VECTOR_THUNK apic_timer, X86_APIC_TIMER_ASM, VECTOR_APIC_TIMER
 MAKE_FIXED_VECTOR_THUNK apic_error, X86_APIC_ERROR_ASM, VECTOR_APIC_ERROR
 MAKE_FIXED_VECTOR_THUNK apic_spurious, X86_APIC_SPURIOUS_ASM, \
                         VECTOR_APIC_SPURIOUS
diff --git a/kernel/arch/x86_64/tsc.c b/kernel/arch/x86_64/tsc.c
index f0cbeae1..d4d959dc 100644
--- a/kernel/arch/x86_64/tsc.c
+++ b/kernel/arch/x86_64/tsc.c
@@ -3,6 +3,7 @@
 #include <arch/private/cpu.h>
 #include <arch/private/tsc.h>
 #include <arch/private/apic.h>
+#include <arch/private/apic_timer.h>
 #include <arch/cpu_helpers.h>
 
 #include <log.h>

From 806adc8500e8b3a5785c638e77687b1d660f3239 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:07:10 +0300
Subject: [PATCH 103/152] arch/x86_64: add tsc_frequency()

The TSC deadline mode of the LAPIC timer needs the TSC frequency, which
the TSC code already knows.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/include/arch/private/tsc.h | 3 +++
 kernel/arch/x86_64/tsc.c                      | 5 +++++
 2 files changed, 8 insertions(+)

diff --git a/kernel/arch/x86_64/include/arch/private/tsc.h b/kernel/arch/x86_64/include/arch/private/tsc.h
index 270928f6..6c25b2c9 100644
--- a/kernel/arch/x86_64/include/arch/private/tsc.h
+++ b/kernel/arch/x86_64/include/arch/private/tsc.h
@@ -8,3 +8,6 @@
 #define TSC_PRECISE_RATING (TSC_APPROXIMATE_RATING + 2)
 
 void tsc_set_known_frequency(u64 hz, const char *reported_by);
+
+// Zero if the frequency is not known
+u64 tsc_frequency(void);
diff --git a/kernel/arch/x86_64/tsc.c b/kernel/arch/x86_64/tsc.c
index d4d959dc..227a21df 100644
--- a/kernel/arch/x86_64/tsc.c
+++ b/kernel/arch/x86_64/tsc.c
@@ -103,6 +103,11 @@ void INIT_CODE tsc_set_known_frequency(u64 hz, const char *reported_by)
     tsc_clocksource_register();
 }
 
+u64 tsc_frequency(void)
+{
+    return s_tsc_ctx.hz;
+}
+
 #define CPUID_TSC_AND_NOMINAL_CORE_CRYSTAL_CLOCK 0x15
 #define CPUID_PROCESSOR_FREQUENCY_INFORMATION 0x16
 

From aeb65da77aea8a6b87569dc15be77ab6adfb9f53 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:07:10 +0300
Subject: [PATCH 104/152] arch/x86_64: add the microcode revision to
 x86_cpu_info

Errata checks need the revision of the loaded microcode. Read it in the
vendor init hooks, which now get the x86_cpu_info they fill in, on Intel
and AMD.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/cpu.c                      | 47 +++++++++++++++++--
 kernel/arch/x86_64/include/arch/private/cpu.h |  2 +
 .../include/arch/private/msr_definitions.h    |  2 +
 3 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/kernel/arch/x86_64/cpu.c b/kernel/arch/x86_64/cpu.c
index 10f75562..a2b2f198 100644
--- a/kernel/arch/x86_64/cpu.c
+++ b/kernel/arch/x86_64/cpu.c
@@ -16,7 +16,27 @@ static reg_t s_cr4;
 struct x86_cpu_info g_cpu_info;
 DEFINE_PER_CPU(struct x86_cpu_info, g_this_cpu_info);
 
-static void intel_init(void)
+static u32 INIT_CODE intel_microcode_revision(void)
+{
+    u32 unused;
+    u64 value;
+    error_t ret;
+
+    // The MSR only reports the revision after a CPUID(1) for some reason
+    ret = wrmsr(MSR_IA32_BIOS_SIGN_ID, 0);
+    if (is_error(ret))
+        return 0;
+
+    cpuid_inline(1, &unused, &unused, &unused, &unused);
+
+    ret = rdmsr(MSR_IA32_BIOS_SIGN_ID, &value);
+    if (is_error(ret))
+        return 0;
+
+    return value >> 32;
+}
+
+static void INIT_CODE intel_init(struct x86_cpu_info *info)
 {
     /*
      * All x86_64 intel CPUs have constant TSC.
@@ -25,17 +45,36 @@ static void intel_init(void)
      * enter those, so for us it doesn't really matter.
      */
     all_cpus_enable(X86_FEATURE_TSC_RELIABLE);
+
+    info->microcode_revision = intel_microcode_revision();
+}
+
+static u32 INIT_CODE amd_microcode_revision(void)
+{
+    u64 value;
+    error_t ret;
+
+    ret = rdmsr(MSR_AMD64_PATCH_LEVEL, &value);
+    if (is_error(ret))
+        return 0;
+
+    return value & 0xFFFFFFFF;
+}
+
+static void INIT_CODE amd_init(struct x86_cpu_info *info)
+{
+    info->microcode_revision = amd_microcode_revision();
 }
 
 struct cpu_ident {
     char vendor_string[13];
     enum x86_cpu_vendor vendor;
-    void (*init)(void);
+    void (*init)(struct x86_cpu_info *info);
 };
 
 static const struct cpu_ident INIT_RODATA s_supported_cpus[] = {
     { "GenuineIntel", X86_VENDOR_INTEL, intel_init },
-    { "AuthenticAMD", X86_VENDOR_AMD,   nullptr    },
+    { "AuthenticAMD", X86_VENDOR_AMD,   amd_init   },
 };
 
 static void INIT_CODE cpu_vendor_detect(struct x86_cpu_info *info)
@@ -57,7 +96,7 @@ static void INIT_CODE cpu_vendor_detect(struct x86_cpu_info *info)
 
         info->vendor = this_id->vendor;
         if (this_id->init)
-            this_id->init();
+            this_id->init(info);
         break;
     }
 }
diff --git a/kernel/arch/x86_64/include/arch/private/cpu.h b/kernel/arch/x86_64/include/arch/private/cpu.h
index 8d5a6f33..b142dcbd 100644
--- a/kernel/arch/x86_64/include/arch/private/cpu.h
+++ b/kernel/arch/x86_64/include/arch/private/cpu.h
@@ -314,6 +314,8 @@ struct x86_cpu_info {
     u8 stepping;
     enum x86_cpu_vendor vendor;
 
+    u32 microcode_revision;
+
     u8 phys_bits;
     u8 virt_bits;
 
diff --git a/kernel/arch/x86_64/include/arch/private/msr_definitions.h b/kernel/arch/x86_64/include/arch/private/msr_definitions.h
index be506e52..c6c45393 100644
--- a/kernel/arch/x86_64/include/arch/private/msr_definitions.h
+++ b/kernel/arch/x86_64/include/arch/private/msr_definitions.h
@@ -5,4 +5,6 @@
 #define MSR_GS_BASE 0xC0000101
 #define MSR_KERNEL_GS_BASE 0xC0000102
 #define MSR_IA32_APIC_BASE  0x0000001B
+#define MSR_IA32_BIOS_SIGN_ID 0x0000008B
+#define MSR_AMD64_PATCH_LEVEL 0x0000008B
 #define MSR_IA32_PAT 0x00000277

From 7cb260b63c7f8e92f67768fdc4475bf089bcac38 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 5 Nov 2026 02:16:45 +0300
Subject: [PATCH 105/152] arch/x86_64: add Intel CPU model names

A model number only identifies a CPU together with its vendor and
family. Add X86_VFM(), which packs all three into one value, and name
the Intel models that errata checks are going to need, so that code
can say INTEL_BROADWELL instead of family 6, model 0x3D.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/include/arch/private/cpu.h | 14 ++++++++
 .../include/arch/private/intel_family.h       | 36 +++++++++++++++++++
 2 files changed, 50 insertions(+)
 create mode 100644 kernel/arch/x86_64/include/arch/private/intel_family.h

diff --git a/kernel/arch/x86_64/include/arch/private/cpu.h b/kernel/arch/x86_64/include/arch/private/cpu.h
index b142dcbd..e2231d4d 100644
--- a/kernel/arch/x86_64/include/arch/private/cpu.h
+++ b/kernel/arch/x86_64/include/arch/private/cpu.h
@@ -305,6 +305,20 @@ enum x86_cpu_vendor : u8 {
     X86_VENDOR_AMD,
 };
 
+#define X86_VFM_MODEL_MASK MAKE_BIT_MASK_U32(7, 0)
+#define X86_VFM_FAMILY_MASK MAKE_BIT_MASK_U32(15, 8)
+#define X86_VFM_VENDOR_MASK MAKE_BIT_MASK_U32(23, 16)
+
+/*
+ * A vendor, a family and a model number packed into a single value that
+ * identifies a CPU model. Model numbers are only unique within a family of
+ * one vendor.
+ */
+#define X86_VFM(vendor, family, model)               \
+    (BIT_FIELD_MAKE(X86_VFM_VENDOR_MASK, (vendor)) | \
+     BIT_FIELD_MAKE(X86_VFM_FAMILY_MASK, (family)) | \
+     BIT_FIELD_MAKE(X86_VFM_MODEL_MASK, (model)))
+
 struct x86_cpu_info {
     u32 max_cpuid;
     u32 max_extended_cpuid;
diff --git a/kernel/arch/x86_64/include/arch/private/intel_family.h b/kernel/arch/x86_64/include/arch/private/intel_family.h
new file mode 100644
index 00000000..258df666
--- /dev/null
+++ b/kernel/arch/x86_64/include/arch/private/intel_family.h
@@ -0,0 +1,36 @@
+#pragma once
+
+#include <arch/private/cpu.h>
+
+#define INTEL_VFM(family, model) X86_VFM(X86_VENDOR_INTEL, family, model)
+
+/*
+ * Models are named INTEL_<MICROARCH><SEGMENT>, where MICROARCH is the code name
+ * of the microarchitecture and SEGMENT is the market segment the CPU was
+ * designed for and tells apart the models that share it:
+ *
+ *  (none)    consumer/regular CPU
+ *  _MOBILE   mobile CPU
+ *  _GFX      CPU with extra graphics features (e.g. extra eDRAM)
+ *  _SERVER   server CPU
+ *  _MSERVER  micro server CPU
+ *
+ * NOTE: Later microarchitectures reuse some of these model numbers and differ
+ *       only by stepping, e.g. Whiskey Lake is INTEL_KABYLAKE_MOBILE.
+ */
+#define INTEL_HASWELL INTEL_VFM(6, 0x3C)
+#define INTEL_HASWELL_SERVER INTEL_VFM(6, 0x3F)
+#define INTEL_HASWELL_MOBILE INTEL_VFM(6, 0x45)
+#define INTEL_HASWELL_GFX INTEL_VFM(6, 0x46)
+
+#define INTEL_BROADWELL INTEL_VFM(6, 0x3D)
+#define INTEL_BROADWELL_GFX INTEL_VFM(6, 0x47)
+#define INTEL_BROADWELL_SERVER INTEL_VFM(6, 0x4F)
+#define INTEL_BROADWELL_MSERVER INTEL_VFM(6, 0x56)
+
+#define INTEL_SKYLAKE_MOBILE INTEL_VFM(6, 0x4E)
+#define INTEL_SKYLAKE_SERVER INTEL_VFM(6, 0x55)
+#define INTEL_SKYLAKE INTEL_VFM(6, 0x5E)
+
+#define INTEL_KABYLAKE_MOBILE INTEL_VFM(6, 0x8E)
+#define INTEL_KABYLAKE INTEL_VFM(6, 0x9E)

From 93cd542740f7ac32d87e2ec3200341ef491b7b3c Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 5 Nov 2026 02:17:11 +0300
Subject: [PATCH 106/152] arch/x86_64: add cpu_match()

Errata and model quirks are tables of CPU models, each entry with data
of its own, such as the microcode revision that fixes the erratum. Add
struct cpu_match_entry and cpu_match(), which returns the first entry of
such a table that matches the vendor, family, model and stepping of the
CPU, so that every check doesn't open-code the lookup.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/CMakeLists.txt             |  1 +
 kernel/arch/x86_64/cpu_match.c                | 23 ++++++++++
 .../x86_64/include/arch/private/cpu_match.h   | 45 +++++++++++++++++++
 3 files changed, 69 insertions(+)
 create mode 100644 kernel/arch/x86_64/cpu_match.c
 create mode 100644 kernel/arch/x86_64/include/arch/private/cpu_match.h

diff --git a/kernel/arch/x86_64/CMakeLists.txt b/kernel/arch/x86_64/CMakeLists.txt
index 08568174..553010ec 100644
--- a/kernel/arch/x86_64/CMakeLists.txt
+++ b/kernel/arch/x86_64/CMakeLists.txt
@@ -12,6 +12,7 @@ ultra_sources(
     entry.c
     entry.S
     cpu.c
+    cpu_match.c
     smp.c
     memory.c
     unwind.c
diff --git a/kernel/arch/x86_64/cpu_match.c b/kernel/arch/x86_64/cpu_match.c
new file mode 100644
index 00000000..e2c14de7
--- /dev/null
+++ b/kernel/arch/x86_64/cpu_match.c
@@ -0,0 +1,23 @@
+#include <arch/private/cpu_match.h>
+#include <arch/private/cpu.h>
+
+const struct cpu_match_entry *cpu_match(const struct cpu_match_entry *table)
+{
+    const struct cpu_match_entry *id;
+    u32 vfm;
+
+    vfm = X86_VFM(g_cpu_info.vendor, g_cpu_info.family, g_cpu_info.model);
+
+    // VFM of 0 means end-of-table
+    for (id = table; id->vfm != 0; id++) {
+        if (id->vfm != vfm)
+            continue;
+        if (g_cpu_info.stepping < id->first_stepping ||
+            g_cpu_info.stepping > id->last_stepping)
+            continue;
+
+        return id;
+    }
+
+    return nullptr;
+}
diff --git a/kernel/arch/x86_64/include/arch/private/cpu_match.h b/kernel/arch/x86_64/include/arch/private/cpu_match.h
new file mode 100644
index 00000000..555ebff3
--- /dev/null
+++ b/kernel/arch/x86_64/include/arch/private/cpu_match.h
@@ -0,0 +1,45 @@
+#pragma once
+
+#include <common/types.h>
+
+/*
+ * An entry of a table of CPU models for cpu_match(), defined with
+ * CPU_MATCH_VFM() or CPU_MATCH_VFM_WITH_STEPPINGS(). The table must be
+ * terminated with an empty entry, e.g. "{}".
+ */
+struct cpu_match_entry {
+    u32 vfm;
+    u8 first_stepping;
+    u8 last_stepping;
+
+    /*
+     * Private data not used by the matcher, e.g. the microcode revision that
+     * fixes an erratum or a pointer to model-specific properties for a
+     * workaround. The 'priv' parameter in CPU_MATCH_* macros accepts both
+     * integers and pointers.
+     */
+    union {
+        reg_t priv_data;
+        const void *priv_ptr;
+    };
+};
+
+#define CPU_STEPPING_MAX 0xF
+
+// Match steppings 'first' through 'last' of 'cpu_vfm'
+#define CPU_MATCH_VFM_WITH_STEPPINGS(cpu_vfm, first, last, priv) { \
+    .vfm = (cpu_vfm),                                              \
+    .first_stepping = (first),                                     \
+    .last_stepping = (last),                                       \
+    .priv_data = (reg_t)(priv),                                    \
+}
+
+// Match every stepping of 'cpu_vfm'
+#define CPU_MATCH_VFM(cpu_vfm, priv)                                \
+    CPU_MATCH_VFM_WITH_STEPPINGS(cpu_vfm, 0, CPU_STEPPING_MAX, priv)
+
+/*
+ * Returns the first entry in 'table' that matches the vendor, family, model and
+ * stepping of the boot CPU, or nullptr if no matches were found.
+ */
+const struct cpu_match_entry *cpu_match(const struct cpu_match_entry *table);

From 6a0bc55054465b62c020d25a14bfe0b55465117d Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:07:10 +0300
Subject: [PATCH 107/152] arch/x86_64: use TSC deadline mode for the LAPIC
 timer when available

Prefer the TSC deadline mode over the calibrated one-shot mode when the
CPU supports it, unless the microcode is affected by the erratum that
breaks the deadline timer after a write to IA32_TSC_ADJUST. The list of
affected models and the revisions that fix them come from Linux, see
commit bd9240a18edf.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/arch/x86_64/apic_timer.c               | 141 +++++++++++++++++-
 .../arch/x86_64/include/arch/private/apic.h   |   2 +
 .../include/arch/private/msr_definitions.h    |   1 +
 3 files changed, 142 insertions(+), 2 deletions(-)

diff --git a/kernel/arch/x86_64/apic_timer.c b/kernel/arch/x86_64/apic_timer.c
index 790c2f7a..b26bbc9f 100644
--- a/kernel/arch/x86_64/apic_timer.c
+++ b/kernel/arch/x86_64/apic_timer.c
@@ -2,8 +2,14 @@
 
 #include <arch/private/apic.h>
 #include <arch/private/apic_timer.h>
+#include <arch/private/cpu.h>
+#include <arch/private/cpu_match.h>
+#include <arch/private/hypervisor.h>
+#include <arch/private/msr.h>
+#include <arch/private/tsc.h>
 #include <arch/private/vectors.h>
 #include <arch/private/idt.h>
+#include <arch/private/intel_family.h>
 #include <arch/registers.h>
 #include <arch/cpu_helpers.h>
 
@@ -12,6 +18,8 @@
 #include <time/units.h>
 #include <time/counter_device.h>
 
+#include <common/overflow.h>
+
 #include <log.h>
 #include <init_level.h>
 #include <free_after_init.h>
@@ -22,6 +30,7 @@
 #define TIMER_DIVIDE_CONFIG CONCAT(APIC_TIMER_DIVIDE_BY_, TIMER_DIVISOR)
 
 static u64 s_freq_hz;
+static struct event_device *s_device;
 
 void apic_set_known_frequency(u64 hz)
 {
@@ -52,12 +61,42 @@ static struct event_device s_apic_timer = {
     .max_ticks = UNSIGNED_MAX(u32),
 };
 
+static void apic_deadline_arm(struct event_device *dev, u64 ticks)
+{
+    u64 tsc, deadline;
+
+    UNREFERENCED_PARAMETER(dev);
+
+    tsc = rdtsc();
+
+    if (check_add_overflow(tsc, ticks, &deadline))
+        deadline = UNSIGNED_MAX(u64);
+
+    wrmsr_unsafe(MSR_IA32_TSC_DEADLINE, deadline);
+}
+
+static void apic_deadline_disarm(struct event_device *dev)
+{
+    UNREFERENCED_PARAMETER(dev);
+
+    wrmsr_unsafe(MSR_IA32_TSC_DEADLINE, 0);
+}
+
+static struct event_device s_apic_deadline_timer = {
+    .name = "apic-tsc-deadline",
+    .arm = apic_deadline_arm,
+    .disarm = apic_deadline_disarm,
+
+    .min_ticks = 0,
+    .max_ticks = UNSIGNED_MAX(u64),
+};
+
 FIXED_VECTOR_HANDLER(X86_APIC_TIMER, VECTOR_APIC_TIMER)
 {
     UNREFERENCED_PARAMETER(regs);
 
     apic_eoi();
-    event_device_interrupt(&s_apic_timer);
+    event_device_interrupt(s_device);
 }
 
 static void INIT_CODE apic_timer_calibrate(void)
@@ -82,11 +121,108 @@ static void INIT_CODE apic_timer_calibrate(void)
     s_freq_hz = (ticks * TIMER_DIVISOR * NS_PER_SEC) / elapsed_ns;
 }
 
+/*
+ * These CPUs & microcode combinations apparently misbehave if the TSC_ADJUST
+ * MSR is ever written (which even the firmware itself is known to do upon
+ * leaving the SMM to hide stolen CPU cycles from the kernel).
+ */
+static INIT_RODATA const struct cpu_match_entry s_deadline_errata[] = {
+    CPU_MATCH_VFM(INTEL_HASWELL, 0x22),
+    CPU_MATCH_VFM_WITH_STEPPINGS(INTEL_HASWELL_SERVER, 0x2, 0x2, 0x3A),
+    CPU_MATCH_VFM_WITH_STEPPINGS(INTEL_HASWELL_SERVER, 0x4, 0x4, 0x0F),
+    CPU_MATCH_VFM(INTEL_HASWELL_MOBILE, 0x20),
+    CPU_MATCH_VFM(INTEL_HASWELL_GFX, 0x17),
+
+    CPU_MATCH_VFM(INTEL_BROADWELL, 0x25),
+    CPU_MATCH_VFM(INTEL_BROADWELL_GFX, 0x17),
+    CPU_MATCH_VFM(INTEL_BROADWELL_SERVER, 0x0B000020),
+    CPU_MATCH_VFM_WITH_STEPPINGS(INTEL_BROADWELL_MSERVER, 0x2, 0x2, 0x00000011),
+    CPU_MATCH_VFM_WITH_STEPPINGS(INTEL_BROADWELL_MSERVER, 0x3, 0x3, 0x0700000E),
+    CPU_MATCH_VFM_WITH_STEPPINGS(INTEL_BROADWELL_MSERVER, 0x4, 0x4, 0x0F00000C),
+    CPU_MATCH_VFM_WITH_STEPPINGS(INTEL_BROADWELL_MSERVER, 0x5, 0x5, 0x0E000003),
+
+    CPU_MATCH_VFM(INTEL_SKYLAKE_MOBILE, 0xB2),
+    CPU_MATCH_VFM_WITH_STEPPINGS(INTEL_SKYLAKE_SERVER, 0x3, 0x3, 0x01000136),
+    CPU_MATCH_VFM_WITH_STEPPINGS(INTEL_SKYLAKE_SERVER, 0x4, 0x4, 0x02000014),
+    CPU_MATCH_VFM(INTEL_SKYLAKE, 0xB2),
+
+    CPU_MATCH_VFM(INTEL_KABYLAKE_MOBILE, 0x52),
+    CPU_MATCH_VFM(INTEL_KABYLAKE, 0x52),
+    {},
+};
+
+static bool INIT_CODE deadline_mode_is_broken(void)
+{
+    const struct cpu_match_entry *erratum;
+    u32 revision;
+
+    if (in_hypervisor())
+        return false;
+
+    erratum = cpu_match(s_deadline_errata);
+    if (erratum == nullptr)
+        return false;
+
+    revision = g_cpu_info.microcode_revision;
+    if (revision >= erratum->priv_data)
+        return false;
+
+    pr_warn(
+        "TSC deadline mode is broken on microcode 0x%X, please update to "
+        "at least 0x%zX\n", revision, erratum->priv_data
+    );
+
+    return true;
+}
+
+static bool INIT_CODE apic_timer_try_deadline_mode(void)
+{
+    u64 hz;
+
+    if (!all_cpus_have(X86_FEATURE_TSC_DEADLINE))
+        return false;
+
+    hz = tsc_frequency();
+    if (hz == 0)
+        return false;
+
+    if (deadline_mode_is_broken())
+        return false;
+
+    g_apic->write(
+        APIC_REG_LVT_TIMER, VECTOR_APIC_TIMER | APIC_LVT_TIMER_TSC_DEADLINE
+    );
+
+    /*
+     * Intel 64 and IA-32 Architectures Software Developer's Manual, section
+     * TSC-Deadline Mode:
+     *
+     * "In xAPIC mode (in which the local-APIC registers are memory-mapped),
+     * software must order the memory-mapped write to the LVT entry that enables
+     * TSC-deadline mode and any subsequent WRMSR to the IA32_TSC_DEADLINE MSR.
+     * Software can assure proper ordering by executing the MFENCE instruction
+     * after the memory-mapped write and before any WRMSR. (In x2APIC mode, the
+     * WRMSR instruction is used to write to the LVT entry. The processor
+     * ensures the ordering of this write and any subsequent WRMSR to the
+     * deadline; no fencing is required.)"
+     */
+    if (g_apic_mode == APIC_MODE_X)
+        weak_wrmsr_fence();
+
+    pr_info("using TSC deadline mode\n");
+    s_device = &s_apic_deadline_timer;
+
+    return event_device_register(s_device, hz) == EOK;
+}
+
 static error_t INIT_CODE apic_timer_init(void)
 {
     const char *source = "known";
     struct human_repr freq;
 
+    if (apic_timer_try_deadline_mode())
+        return EOK;
+
     g_apic->write(APIC_REG_TIMER_DIVIDE_CONFIG, TIMER_DIVIDE_CONFIG);
 
     if (s_freq_hz == 0) {
@@ -103,7 +239,8 @@ static error_t INIT_CODE apic_timer_init(void)
     pr_info(HUMAN_REPR_FMT " (%s)\n", HUMAN_REPR_ARG(freq), source);
 
     g_apic->write(APIC_REG_LVT_TIMER, VECTOR_APIC_TIMER);
+    s_device = &s_apic_timer;
 
-    return event_device_register(&s_apic_timer, s_freq_hz / TIMER_DIVISOR);
+    return event_device_register(s_device, s_freq_hz / TIMER_DIVISOR);
 }
 INIT_CALL_POST(IRQS_AVAILABLE, apic_timer_init);
diff --git a/kernel/arch/x86_64/include/arch/private/apic.h b/kernel/arch/x86_64/include/arch/private/apic.h
index ba9ed3b4..37f05157 100644
--- a/kernel/arch/x86_64/include/arch/private/apic.h
+++ b/kernel/arch/x86_64/include/arch/private/apic.h
@@ -39,6 +39,8 @@ enum apic_reg : u32 {
         #define APIC_ICR2_XAPIC_DEST MAKE_BIT_MASK_U32(31, 24)
 
     APIC_REG_LVT_TIMER = 0x320,
+        #define APIC_LVT_TIMER_TSC_DEADLINE BIT_U32(18)
+
     APIC_REG_LVT_THERMAL = 0x330,
     APIC_REG_LVT_PERF = 0x340,
     APIC_REG_LVT_LINT0 = 0x350,
diff --git a/kernel/arch/x86_64/include/arch/private/msr_definitions.h b/kernel/arch/x86_64/include/arch/private/msr_definitions.h
index c6c45393..c7bd9f4e 100644
--- a/kernel/arch/x86_64/include/arch/private/msr_definitions.h
+++ b/kernel/arch/x86_64/include/arch/private/msr_definitions.h
@@ -8,3 +8,4 @@
 #define MSR_IA32_BIOS_SIGN_ID 0x0000008B
 #define MSR_AMD64_PATCH_LEVEL 0x0000008B
 #define MSR_IA32_PAT 0x00000277
+#define MSR_IA32_TSC_DEADLINE 0x000006E0

From 3486b3aaf4d72582b4eaa77efb3534353b22a31a Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:12:13 +0300
Subject: [PATCH 108/152] time: add a timer that ticks the time keeper

A counter must be re-read periodically to make sure we catch it before
it wraps around, e.g. the ACPI PM does so every 4.7 seconds. Add a timer
that does exactly that. The deadline is a range, so the read usually
shares an interrupt that happens anyway.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/private/time/keeper.h |   1 -
 kernel/time/keeper.c                 |  50 +++++++++++-
 tests/test-cases/test_time.c         | 118 ++++++++++++++++++++++++---
 3 files changed, 155 insertions(+), 14 deletions(-)

diff --git a/kernel/include/private/time/keeper.h b/kernel/include/private/time/keeper.h
index 69a91cc8..754857ce 100644
--- a/kernel/include/private/time/keeper.h
+++ b/kernel/include/private/time/keeper.h
@@ -2,5 +2,4 @@
 
 struct counter_device;
 
-void time_keeper_tick(void);
 void time_keeper_set_counter_device(struct counter_device *cd);
diff --git a/kernel/time/keeper.c b/kernel/time/keeper.c
index 94f11108..847b0299 100644
--- a/kernel/time/keeper.c
+++ b/kernel/time/keeper.c
@@ -5,9 +5,11 @@
 
 #include <bug.h>
 #include <init_level.h>
+#include <free_after_init.h>
 #include <locking/spinlock.h>
 #include <locking/seq.h>
 #include <time/time.h>
+#include <time/timer.h>
 
 #include <arch/cpu_helpers.h>
 
@@ -18,6 +20,8 @@ struct keeper_base {
     u64 prev_cycles;
 };
 
+static enum timer_result tick_timer_fn(struct timer *timer);
+
 static struct keeper {
     struct spinlock lock;
     struct seqcount_latch time_seq;
@@ -29,9 +33,12 @@ static struct keeper {
 
     // At least one of these is always consistent (time_seq decides which)
     struct keeper_base published[2];
+
+    struct timer tick_timer;
 } s_ctx = {
     .lock = SPINLOCK_INIT(s_ctx.lock),
     .time_seq = SEQCOUNT_LATCH_INIT(s_ctx.time_seq),
+    .tick_timer = TIMER_INIT(s_ctx.tick_timer, tick_timer_fn),
 };
 
 static void base_store(struct keeper_base *base)
@@ -76,14 +83,54 @@ static void do_tick(void)
     publish();
 }
 
-void time_keeper_tick(void)
+/*
+ * The counter must be read again before max_unattended_ns have passed.
+ * Any timer interrupt in the second half of that window will do.
+ */
+static void tick_timer_rearm(void)
+{
+    u64 max_ns;
+
+    if (init_level_below(INIT_LEVEL_PER_CPU_AVAILABLE))
+        return;
+
+    if (s_ctx.cd == nullptr) {
+        timer_cancel_nosync(&s_ctx.tick_timer);
+        return;
+    }
+
+    max_ns = s_ctx.cd->max_unattended_ns;
+    timer_arm_with_slack(&s_ctx.tick_timer, s_ctx.ns + max_ns / 2, max_ns / 2);
+}
+
+static enum timer_result tick_timer_fn(struct timer *timer)
+{
+    enum timer_result result = TIMER_RESULT_DONE;
+
+    spin_lock(&s_ctx.lock);
+    do_tick();
+
+    if (likely(s_ctx.cd != nullptr)) {
+        timer_advance(timer, s_ctx.cd->max_unattended_ns / 2);
+        result = TIMER_RESULT_REARM;
+    }
+
+    spin_unlock(&s_ctx.lock);
+    return result;
+}
+
+static error_t INIT_CODE tick_timer_init(void)
 {
     irq_state_t state;
 
     state = spin_lock_irq_save(&s_ctx.lock);
     do_tick();
+    tick_timer_rearm();
     spin_unlock_irq_restore(&s_ctx.lock, state);
+
+    return EOK;
 }
+INIT_CALL_POST(PER_CPU_AVAILABLE, tick_timer_init);
 
 void time_keeper_set_counter_device(struct counter_device *cd)
 {
@@ -121,6 +168,7 @@ void time_keeper_set_counter_device(struct counter_device *cd)
     s_ctx.prev_cycles = new_cycles;
 
     publish();
+    tick_timer_rearm();
     spin_unlock_irq_restore(&s_ctx.lock, state);
 }
 
diff --git a/tests/test-cases/test_time.c b/tests/test-cases/test_time.c
index bad32da9..b49f56dd 100644
--- a/tests/test-cases/test_time.c
+++ b/tests/test-cases/test_time.c
@@ -1,3 +1,17 @@
+#include <common/types.h>
+
+struct timer;
+
+void test_timer_arm_with_slack(
+    struct timer *timer, u64 deadline_ns, u64 slack_ns
+);
+bool test_timer_cancel_nosync(struct timer *timer);
+u64 test_timer_advance(struct timer *timer, u64 period_ns);
+
+#define timer_arm_with_slack test_timer_arm_with_slack
+#define timer_cancel_nosync test_timer_cancel_nosync
+#define timer_advance test_timer_advance
+
 #include <kernel-source/time/keeper.c>
 #undef MSG_FMT
 #include <kernel-source/time/counter_device.c>
@@ -29,6 +43,38 @@ static u64 test_counter_read(struct counter_device *dev)
         .rating = COUNTER_DEVICE_RATING_##r, \
     }
 
+static u64 s_tick_deadline_ns;
+static u64 s_tick_slack_ns;
+static u64 s_tick_period_ns;
+static bool s_tick_armed;
+
+void test_timer_arm_with_slack(
+    struct timer *timer, u64 deadline_ns, u64 slack_ns
+)
+{
+    UNREFERENCED_PARAMETER(timer);
+
+    s_tick_deadline_ns = deadline_ns;
+    s_tick_slack_ns = slack_ns;
+    s_tick_armed = true;
+}
+
+bool test_timer_cancel_nosync(struct timer *timer)
+{
+    UNREFERENCED_PARAMETER(timer);
+
+    s_tick_armed = false;
+    return true;
+}
+
+u64 test_timer_advance(struct timer *timer, u64 period_ns)
+{
+    UNREFERENCED_PARAMETER(timer);
+
+    s_tick_period_ns = period_ns;
+    return 1;
+}
+
 static void time_test_reset(void)
 {
     list_init(&s_counter_devices);
@@ -42,6 +88,8 @@ static void time_test_reset(void)
     s_ctx.ns_frac = 0;
     s_ctx.prev_cycles = 0;
     memset(s_ctx.published, 0, sizeof(s_ctx.published));
+    s_test_init_level = INIT_LEVEL_PLATFORM_INFO_AVAILABLE;
+    s_tick_armed = false;
 
     g_counter_value = 0;
 }
@@ -264,7 +312,7 @@ TEST_CASE(keeper_without_device_is_zero)
     ASSERT_EQ(ns_since_boot(), 0);
 
     // Ticking with no counter device installed is a harmless no-op.
-    time_keeper_tick();
+    tick_timer_fn(&s_ctx.tick_timer);
     ASSERT_EQ(ns_since_boot(), 0);
 }
 
@@ -290,11 +338,11 @@ TEST_CASE(keeper_accumulates_across_ticks)
     counter_device_register(&dev, ONE_NS_PER_CYCLE_HZ);
 
     g_counter_value = 1000;
-    time_keeper_tick();
+    tick_timer_fn(&s_ctx.tick_timer);
     ASSERT_EQ(ns_since_boot(), 1000);
 
     g_counter_value = 5000;
-    time_keeper_tick();
+    tick_timer_fn(&s_ctx.tick_timer);
     ASSERT_EQ(ns_since_boot(), 5000);
 }
 
@@ -307,7 +355,7 @@ TEST_CASE(keeper_reports_live_delta)
     counter_device_register(&dev, ONE_NS_PER_CYCLE_HZ);
 
     g_counter_value = 1000;
-    time_keeper_tick();
+    tick_timer_fn(&s_ctx.tick_timer);
 
     // No tick here: the extra 500 cycles are observed live
     g_counter_value = 1500;
@@ -324,7 +372,7 @@ TEST_CASE(keeper_carries_the_fraction_across_ticks)
 
     for (i = 1; i <= 1000; i++) {
         g_counter_value = i;
-        time_keeper_tick();
+        tick_timer_fn(&s_ctx.tick_timer);
     }
 
     ASSERT_EQ(ns_since_boot(), reference_cycles_to_ns(&dev, 1000));
@@ -343,7 +391,7 @@ TEST_CASE(keeper_time_does_not_step_back_across_a_tick)
 
     // The tick read the counter before the reader did, but published after it
     g_counter_value = 1;
-    time_keeper_tick();
+    tick_timer_fn(&s_ctx.tick_timer);
 
     g_counter_value = 2;
     ASSERT_EQ(ns_since_boot(), before);
@@ -362,7 +410,7 @@ TEST_CASE(keeper_banks_time_on_device_switch)
     counter_device_register(&a, ONE_NS_PER_CYCLE_HZ);
 
     g_counter_value = 2000;
-    time_keeper_tick();
+    tick_timer_fn(&s_ctx.tick_timer);
     ASSERT_EQ(ns_since_boot(), 2000);
 
     // Switching to the better device banks a's outstanding 1000 cycles
@@ -386,7 +434,7 @@ TEST_CASE(counter_unregister_repicks_best)
     ASSERT_EQ((uintptr_t)current_counter_device(), (uintptr_t)&great);
 
     g_counter_value = 1000;
-    time_keeper_tick();
+    tick_timer_fn(&s_ctx.tick_timer);
 
     // Dropping the active device falls back to the next best one
     counter_device_unregister(&great);
@@ -429,7 +477,7 @@ TEST_CASE(keeper_banking_ignores_new_counter_offset)
     counter_device_register(&a, ONE_NS_PER_CYCLE_HZ);
 
     g_counter_value = 2000;
-    time_keeper_tick();
+    tick_timer_fn(&s_ctx.tick_timer);
 
     counter_device_register(&b, ONE_NS_PER_CYCLE_HZ);
     ASSERT_EQ(ns_since_boot(), 2000);
@@ -453,12 +501,12 @@ TEST_CASE(keeper_ignores_counter_glitch)
     counter_device_register(&dev, 3579545);
 
     g_counter_value = 0x1000;
-    time_keeper_tick();
+    tick_timer_fn(&s_ctx.tick_timer);
     settled = ns_since_boot();
 
     // Counter jumps backwards, the resulting masked delta is rejected
     g_counter_value = 0x0FFF;
-    time_keeper_tick();
+    tick_timer_fn(&s_ctx.tick_timer);
     ASSERT_EQ(ns_since_boot(), settled);
 }
 
@@ -524,7 +572,7 @@ TEST_CASE(keeper_read_does_not_wait_for_an_update_in_progress)
 
     ASSERT_EQ(counter_device_register(&dev, ONE_NS_PER_CYCLE_HZ), EOK);
     g_counter_value = 1000;
-    time_keeper_tick();
+    tick_timer_fn(&s_ctx.tick_timer);
     g_counter_value = 1500;
 
     before = ns_since_boot();
@@ -543,3 +591,49 @@ TEST_CASE(keeper_read_does_not_wait_for_an_update_in_progress)
     s_ctx.published[1] = s_ctx.published[0];
     counter_device_unregister(&dev);
 }
+
+TEST_CASE(keeper_tick_timer_stays_off_until_usable)
+{
+    struct counter_device dev = TEST_CD("pm", COUNTER_MASK(24), GOOD);
+
+    time_test_reset();
+    g_counter_value = 0;
+
+    ASSERT_EQ(counter_device_register(&dev, ONE_NS_PER_CYCLE_HZ), EOK);
+    ASSERT(!s_tick_armed);
+
+    counter_device_unregister(&dev);
+}
+
+TEST_CASE(keeper_tick_timer_follows_the_counter)
+{
+    struct counter_device narrow = TEST_CD("pm", COUNTER_MASK(24), GOOD);
+    struct counter_device wide = TEST_CD("hpet", COUNTER_MASK(32), GREAT);
+
+    time_test_reset();
+    g_counter_value = 0;
+
+    s_test_init_level = INIT_LEVEL_PER_CPU_AVAILABLE;
+    ASSERT_EQ(tick_timer_init(), EOK);
+    ASSERT(!s_tick_armed);
+
+    ASSERT_EQ(counter_device_register(&narrow, ONE_NS_PER_CYCLE_HZ), EOK);
+    ASSERT(s_tick_armed);
+    ASSERT_EQ(s_tick_deadline_ns, narrow.max_unattended_ns / 2);
+    ASSERT_EQ(s_tick_slack_ns, narrow.max_unattended_ns / 2);
+
+    g_counter_value = 1000;
+    ASSERT_EQ(tick_timer_fn(&s_ctx.tick_timer), TIMER_RESULT_REARM);
+    ASSERT_EQ(s_tick_period_ns, narrow.max_unattended_ns / 2);
+
+    ASSERT_EQ(counter_device_register(&wide, ONE_NS_PER_CYCLE_HZ), EOK);
+    ASSERT_EQ(s_tick_deadline_ns, 1000 + wide.max_unattended_ns / 2);
+    ASSERT(wide.max_unattended_ns > narrow.max_unattended_ns);
+
+    counter_device_unregister(&wide);
+    ASSERT_EQ(s_tick_deadline_ns, 1000 + narrow.max_unattended_ns / 2);
+
+    counter_device_unregister(&narrow);
+    ASSERT(!s_tick_armed);
+    ASSERT_EQ(tick_timer_fn(&s_ctx.tick_timer), TIMER_RESULT_DONE);
+}

From 0975a4d3e00444ebb27ef89ecc87b74b32e7b8e3 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:15:40 +0300
Subject: [PATCH 109/152] sched: introduce schedule_until() and sleep_ns()

A primitive for sleeping for a bounded amount of time, or until
something else wakes up the caller. We also use this to implement
sleep_ns().

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/sched/sched.h |  12 ++
 kernel/include/sched/sched.h        |   3 +
 kernel/sched/core.c                 |  54 ++++++++
 tests/test-cases/test_sched.c       | 185 ++++++++++++++++++++++++++++
 4 files changed, 254 insertions(+)

diff --git a/kernel/include/detail/sched/sched.h b/kernel/include/detail/sched/sched.h
index d090b212..530b98ea 100644
--- a/kernel/include/detail/sched/sched.h
+++ b/kernel/include/detail/sched/sched.h
@@ -11,3 +11,15 @@
  *       waits for, so it has to check its condition again after this returns.
  */
 void schedule(void);
+
+/*
+ * Same as schedule(), except the caller is woken up after 'deadline_ns' instead
+ * of sleeping indefinitely. Returns true if the deadline passed, false if the
+ * caller was woken up by something else before the deadline. A 'deadline_ns'
+ * of NO_DEADLINE is the same as schedule().
+ *
+ * NOTE: The two may race. A caller woken up right before the deadline may get
+ *       true, and one woken up right after it may get false. A caller that
+ *       didn't sleep at all gets false.
+ */
+bool schedule_until(u64 deadline_ns);
diff --git a/kernel/include/sched/sched.h b/kernel/include/sched/sched.h
index 6e48fd32..63847731 100644
--- a/kernel/include/sched/sched.h
+++ b/kernel/include/sched/sched.h
@@ -7,3 +7,6 @@
  * so, to catch callers that are not allowed to.
  */
 void might_sleep(void);
+
+// Sleep until at least 'ns' nanoseconds have passed
+void sleep_ns(u64 ns);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index df4e5045..f390a556 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -19,6 +19,9 @@
 #include <init_level.h>
 #include <free_after_init.h>
 
+#include <time/time.h>
+#include <time/timer.h>
+
 #include <arch/cpu_helpers.h>
 #include <arch/registers.h>
 
@@ -295,6 +298,57 @@ void preempt_schedule_irq(struct registers *regs)
     } while (need_resched());
 }
 
+struct wakeup_timer {
+    struct timer timer;
+    struct task *task;
+    bool expired;
+};
+
+static enum timer_result wakeup_timer_fn(struct timer *timer)
+{
+    struct wakeup_timer *wakeup;
+
+    wakeup = container_of(timer, struct wakeup_timer, timer);
+
+    wakeup->expired = true;
+    wake_up_task(wakeup->task, TASK_STATE_UNINTERRUPTIBLE);
+
+    return TIMER_RESULT_DONE;
+}
+
+bool schedule_until(u64 deadline_ns)
+{
+    struct wakeup_timer wakeup;
+
+    if (deadline_ns == NO_DEADLINE) {
+        schedule();
+        return false;
+    }
+
+    wakeup.task = current;
+    wakeup.expired = false;
+    timer_init(&wakeup.timer, wakeup_timer_fn);
+    timer_arm(&wakeup.timer, deadline_ns);
+
+    schedule();
+
+    timer_cancel(&wakeup.timer);
+    return wakeup.expired;
+}
+
+void sleep_ns(u64 ns)
+{
+    u64 deadline_ns;
+
+    might_sleep();
+
+    deadline_ns = ns_from_now(ns);
+
+    do {
+        set_current_state(TASK_STATE_UNINTERRUPTIBLE);
+    } while (!schedule_until(deadline_ns));
+}
+
 void yield(void)
 {
     struct rq *rq;
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index ef7f7791..a9c0d79d 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -1,5 +1,6 @@
 #include <string.h>
 
+#include <common/types.h>
 #include <memory/alloc_behavior.h>
 
 extern int g_sched_test_live_tasks, g_sched_test_live_stacks;
@@ -9,6 +10,23 @@ void sched_test_free(void *ptr);
 void *sched_test_valloc(size_t size, enum alloc_behavior behavior);
 void sched_test_vrelease(void *ptr);
 
+struct timer;
+enum timer_result : u32;
+
+void sched_test_timer_init(
+    struct timer *timer, enum timer_result (*fn)(struct timer *timer)
+);
+void sched_test_timer_arm_with_slack(
+    struct timer *timer, u64 deadline_ns, u64 slack_ns
+);
+void sched_test_timer_cancel(struct timer *timer);
+u64 sched_test_now(void);
+
+#define timer_init sched_test_timer_init
+#define timer_arm_with_slack sched_test_timer_arm_with_slack
+#define timer_cancel sched_test_timer_cancel
+#define ns_since_boot sched_test_now
+
 #define alloc sched_test_alloc
 #define free sched_test_free
 #define valloc sched_test_valloc
@@ -1326,3 +1344,170 @@ TEST_CASE(kthread_stop_after_it_exited)
 
     switching_test_end();
 }
+
+#define MAX_PENDING_TIMERS 4
+
+static struct timer *s_pending_timers[MAX_PENDING_TIMERS];
+static u64 s_now;
+
+void sched_test_timer_init(
+    struct timer *timer, enum timer_result (*fn)(struct timer *timer)
+)
+{
+    *timer = (struct timer) {
+        .fn = fn,
+    };
+}
+
+void sched_test_timer_cancel(struct timer *timer)
+{
+    size_t i;
+
+    for (i = 0; i < MAX_PENDING_TIMERS; i++) {
+        if (s_pending_timers[i] == timer)
+            s_pending_timers[i] = nullptr;
+    }
+}
+
+void sched_test_timer_arm_with_slack(
+    struct timer *timer, u64 deadline_ns, u64 slack_ns
+)
+{
+    size_t i;
+
+    sched_test_timer_cancel(timer);
+
+    timer->deadline_ns = deadline_ns;
+    timer->hard_deadline_ns = deadline_ns + slack_ns;
+
+    for (i = 0; i < MAX_PENDING_TIMERS; i++) {
+        if (s_pending_timers[i] != nullptr)
+            continue;
+
+        s_pending_timers[i] = timer;
+        return;
+    }
+
+    BUG();
+}
+
+u64 sched_test_now(void)
+{
+    return s_now;
+}
+
+static int count_pending_timers(void)
+{
+    int count = 0;
+    size_t i;
+
+    for (i = 0; i < MAX_PENDING_TIMERS; i++) {
+        if (s_pending_timers[i] != nullptr)
+            count++;
+    }
+
+    return count;
+}
+
+// Moves the clock and runs what became due
+static void advance_time_to(u64 now)
+{
+    struct timer *timer;
+    struct registers regs = { .irq_state = false };
+    irq_state_t irq_state;
+    size_t i;
+
+    irq_state = local_irq_state_save();
+    hard_irq_enter();
+
+    s_now = now;
+
+    for (i = 0; i < MAX_PENDING_TIMERS; i++) {
+        timer = s_pending_timers[i];
+        if (timer == nullptr || timer->deadline_ns > now)
+            continue;
+
+        s_pending_timers[i] = nullptr;
+        timer->fn(timer);
+    }
+
+    hard_irq_exit(&regs);
+    local_irq_state_restore(irq_state);
+}
+
+static void timed_test_begin(void)
+{
+    switching_test_begin();
+
+    memset(s_pending_timers, 0, sizeof(s_pending_timers));
+    s_now = 1000;
+}
+
+struct timed_sleeper {
+    int id;
+    u64 ns;
+    struct wait_queue *wq;
+    struct completion *completion;
+    bool *condition;
+    bool result;
+};
+
+static void sleep_then_log(void *user)
+{
+    struct timed_sleeper *sleeper = user;
+
+    log_id(sleeper->id);
+    sleep_ns(sleeper->ns);
+    log_id(sleeper->id);
+}
+
+TEST_CASE(sleep_ns_sleeps_until_the_deadline)
+{
+    struct timed_sleeper sleeper = { .id = 1, .ns = 100 };
+
+    timed_test_begin();
+    spawn(sleep_then_log, &sleeper, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT_EQ(count_pending_timers(), 1);
+
+    advance_time_to(1099);
+    ASSERT(!need_resched());
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 1);
+
+    advance_time_to(1100);
+    ASSERT(need_resched());
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 2);
+
+    ASSERT_EQ(count_pending_timers(), 0);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(sleep_ns_goes_back_to_sleep_after_an_early_wakeup)
+{
+    struct timed_sleeper sleeper = { .id = 1, .ns = 100 };
+    struct task *task;
+
+    timed_test_begin();
+    task = spawn(sleep_then_log, &sleeper, TASK_PRIO_DEFAULT);
+    schedule_idle();
+
+    advance_time_to(1050);
+    ASSERT(wake_up_task(task, TASK_STATE_UNINTERRUPTIBLE));
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT_EQ(count_pending_timers(), 1);
+
+    advance_time_to(1100);
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 2);
+    ASSERT_EQ(count_pending_timers(), 0);
+
+    switching_test_end();
+}

From 1cd85deb0d8c6e57ec0caedaaf3ab3df20a2f25f Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:15:40 +0300
Subject: [PATCH 110/152] sched: add wait_event_timeout()

wait_event() with a deadline. The condition is checked once more after
the deadline, so an event arriving with it is not reported as a timeout.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/sched/wait_queue.h |  30 ++++++
 kernel/include/sched/wait_queue.h        |  32 +++----
 tests/test-cases/test_sched.c            | 111 +++++++++++++++++++++++
 3 files changed, 152 insertions(+), 21 deletions(-)

diff --git a/kernel/include/detail/sched/wait_queue.h b/kernel/include/detail/sched/wait_queue.h
index f48225b4..ec3e4c3b 100644
--- a/kernel/include/detail/sched/wait_queue.h
+++ b/kernel/include/detail/sched/wait_queue.h
@@ -30,3 +30,33 @@ struct wait_queue {
     struct spinlock lock;
     struct list_link entries;
 };
+
+// Same as wait_event_timeout(), but with an absolute 'deadline_ns'
+#define wait_event_until(wq, condition, deadline_ns) ({                    \
+    struct wait_entry wait_event_entry;                                    \
+    bool wait_event_ok, wait_event_timed_out = false;                      \
+    u64 wait_event_deadline_ns;                                            \
+                                                                           \
+    might_sleep();                                                         \
+                                                                           \
+    wait_event_ok = (condition);                                           \
+                                                                           \
+    if (!wait_event_ok) {                                                  \
+        wait_event_deadline_ns = (deadline_ns);                            \
+        wait_entry_init(&wait_event_entry, WAIT_ENTRY_FLAG_NONE);          \
+                                                                           \
+        for (;;) {                                                         \
+            wait_queue_prepare(wq, &wait_event_entry);                     \
+                                                                           \
+            wait_event_ok = (condition);                                   \
+            if (wait_event_ok || wait_event_timed_out)                     \
+                break;                                                     \
+                                                                           \
+            wait_event_timed_out = schedule_until(wait_event_deadline_ns); \
+        }                                                                  \
+                                                                           \
+        wait_queue_finish(wq, &wait_event_entry);                          \
+    }                                                                      \
+                                                                           \
+    wait_event_ok;                                                         \
+})
diff --git a/kernel/include/sched/wait_queue.h b/kernel/include/sched/wait_queue.h
index 95a3e1a1..22314e75 100644
--- a/kernel/include/sched/wait_queue.h
+++ b/kernel/include/sched/wait_queue.h
@@ -6,6 +6,7 @@
 
 #include <sched/sched.h>
 #include <sched/task.h>
+#include <time/time.h>
 #include <locking/spinlock.h>
 
 #include <detail/sched/sched.h>
@@ -75,24 +76,13 @@ void wait_queue_wake(struct wait_queue *wq);
 void wait_queue_wake_all(struct wait_queue *wq);
 
 // Sleep until 'condition' is true, it is rechecked after every wakeup
-#define wait_event(wq, condition) do {                        \
-    struct wait_entry wait_event_entry;                       \
-                                                              \
-    might_sleep();                                            \
-                                                              \
-    if (condition)                                            \
-        break;                                                \
-                                                              \
-    wait_entry_init(&wait_event_entry, WAIT_ENTRY_FLAG_NONE); \
-                                                              \
-    for (;;) {                                                \
-        wait_queue_prepare(wq, &wait_event_entry);            \
-                                                              \
-        if (condition)                                        \
-            break;                                            \
-                                                              \
-        schedule();                                           \
-    }                                                         \
-                                                              \
-    wait_queue_finish(wq, &wait_event_entry);                 \
-} while (0)
+#define wait_event(wq, condition)                        \
+    ((void)wait_event_until(wq, condition, NO_DEADLINE))
+
+/*
+ * Same as wait_event(), except the caller stops waiting once 'timeout_ns' have
+ * passed. Evaluates to true if 'condition' became true, false if it was still
+ * false when the timeout ran out.
+ */
+#define wait_event_timeout(wq, condition, timeout_ns)        \
+    wait_event_until(wq, condition, ns_from_now(timeout_ns))
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index a9c0d79d..a3e11362 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -1511,3 +1511,114 @@ TEST_CASE(sleep_ns_goes_back_to_sleep_after_an_early_wakeup)
 
     switching_test_end();
 }
+
+static void wait_event_with_timeout(void *user)
+{
+    struct timed_sleeper *sleeper = user;
+
+    sleeper->result = wait_event_timeout(
+        sleeper->wq, *sleeper->condition, sleeper->ns
+    );
+    log_id(sleeper->id);
+}
+
+TEST_CASE(wait_event_timeout_gives_up_at_the_deadline)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    bool condition = false;
+    struct timed_sleeper sleeper = {
+        .id = 1, .ns = 100, .wq = &wq, .condition = &condition,
+        .result = true,
+    };
+
+    timed_test_begin();
+    spawn(wait_event_with_timeout, &sleeper, TASK_PRIO_DEFAULT);
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 0);
+
+    wait_queue_wake_all(&wq);
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 0);
+
+    advance_time_to(1100);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT(!sleeper.result);
+    ASSERT(list_is_empty(&wq.entries));
+    ASSERT_EQ(count_pending_timers(), 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(wait_event_timeout_returns_when_the_condition_holds)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    bool condition = false;
+    struct timed_sleeper sleeper = {
+        .id = 1, .ns = 100, .wq = &wq, .condition = &condition,
+    };
+
+    timed_test_begin();
+    spawn(wait_event_with_timeout, &sleeper, TASK_PRIO_DEFAULT);
+    schedule_idle();
+
+    advance_time_to(1050);
+    condition = true;
+    wait_queue_wake_all(&wq);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT(sleeper.result);
+    ASSERT(list_is_empty(&wq.entries));
+    ASSERT_EQ(count_pending_timers(), 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(wait_event_timeout_takes_a_condition_met_at_the_deadline)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    bool condition = false;
+    struct timed_sleeper sleeper = {
+        .id = 1, .ns = 100, .wq = &wq, .condition = &condition,
+    };
+
+    timed_test_begin();
+    spawn(wait_event_with_timeout, &sleeper, TASK_PRIO_DEFAULT);
+    schedule_idle();
+
+    condition = true;
+    advance_time_to(1100);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT(sleeper.result);
+
+    switching_test_end();
+}
+
+TEST_CASE(wait_event_timeout_without_a_deadline_arms_no_timer)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    bool condition = false;
+    struct timed_sleeper sleeper = {
+        .id = 1, .ns = UNSIGNED_MAX(u64), .wq = &wq, .condition = &condition,
+    };
+
+    timed_test_begin();
+    spawn(wait_event_with_timeout, &sleeper, TASK_PRIO_DEFAULT);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 0);
+    ASSERT_EQ(count_pending_timers(), 0);
+
+    condition = true;
+    wait_queue_wake_all(&wq);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT(sleeper.result);
+
+    switching_test_end();
+}

From d1de3cea9930d3433457e14b0aa02163317f8941 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:15:40 +0300
Subject: [PATCH 111/152] sched: add wait_for_completion_timeout()

wait_for_completion() with a deadline. Like wait_event_timeout(), it
looks at the completion once more after the deadline.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/sched/completion.h |   7 ++
 kernel/sched/completion.c         |  28 ++++++--
 tests/test-cases/test_sched.c     | 102 ++++++++++++++++++++++++++++++
 3 files changed, 133 insertions(+), 4 deletions(-)

diff --git a/kernel/include/sched/completion.h b/kernel/include/sched/completion.h
index cce6ada4..cc969bd2 100644
--- a/kernel/include/sched/completion.h
+++ b/kernel/include/sched/completion.h
@@ -27,3 +27,10 @@ void complete_all(struct completion *completion);
 
 // Sleep until a complete() is called on this object
 void wait_for_completion(struct completion *completion);
+
+/*
+ * Same as wait_for_completion(), except the caller stops waiting once
+ * 'timeout_ns' have passed. Returns true if the caller was let through, false
+ * if it still wasn't when the timeout ran out.
+ */
+bool wait_for_completion_timeout(struct completion *completion, u64 timeout_ns);
diff --git a/kernel/sched/completion.c b/kernel/sched/completion.c
index 345d34e7..fab390ce 100644
--- a/kernel/sched/completion.c
+++ b/kernel/sched/completion.c
@@ -35,11 +35,12 @@ void complete_all(struct completion *completion)
     spin_unlock_irq_restore(&wq->lock, irq_state);
 }
 
-void wait_for_completion(struct completion *completion)
+static bool do_wait(struct completion *completion, u64 deadline_ns)
 {
     struct wait_queue *wq = &completion->wq;
     struct wait_entry entry;
     irq_state_t irq_state;
+    bool timed_out = false;
 
     might_sleep();
 
@@ -48,20 +49,39 @@ void wait_for_completion(struct completion *completion)
     if (completion->done == 0) {
         wait_entry_init(&entry, WAIT_ENTRY_FLAG_EXCLUSIVE);
 
-        do {
+        for (;;) {
             wait_queue_prepare_unlocked(wq, &entry);
             spin_unlock_irq_restore(&wq->lock, irq_state);
 
-            schedule();
+            timed_out = schedule_until(deadline_ns);
 
             irq_state = spin_lock_irq_save(&wq->lock);
-        } while (completion->done == 0);
+
+            if (completion->done != 0 || timed_out)
+                break;
+        }
 
         wait_queue_remove_unlocked(wq, &entry);
     }
 
+    if (completion->done == 0) {
+        spin_unlock_irq_restore(&wq->lock, irq_state);
+        return false;
+    }
+
     if (completion->done != DONE_PERMANENT)
         completion->done--;
 
     spin_unlock_irq_restore(&wq->lock, irq_state);
+    return true;
+}
+
+void wait_for_completion(struct completion *completion)
+{
+    do_wait(completion, NO_DEADLINE);
+}
+
+bool wait_for_completion_timeout(struct completion *completion, u64 timeout_ns)
+{
+    return do_wait(completion, ns_from_now(timeout_ns));
 }
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index a3e11362..46a813ac 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -1622,3 +1622,105 @@ TEST_CASE(wait_event_timeout_without_a_deadline_arms_no_timer)
 
     switching_test_end();
 }
+
+static void wait_for_completion_with_timeout(void *user)
+{
+    struct timed_sleeper *sleeper = user;
+
+    sleeper->result = wait_for_completion_timeout(
+        sleeper->completion, sleeper->ns
+    );
+    log_id(sleeper->id);
+}
+
+TEST_CASE(completion_timeout_gives_up_at_the_deadline)
+{
+    DEFINE_COMPLETION(completion);
+    struct timed_sleeper sleeper = {
+        .id = 1, .ns = 100, .completion = &completion, .result = true,
+    };
+
+    timed_test_begin();
+    spawn(wait_for_completion_with_timeout, &sleeper, TASK_PRIO_DEFAULT);
+    schedule_idle();
+    ASSERT_EQ(s_log_length, 0);
+
+    advance_time_to(1100);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT(!sleeper.result);
+    ASSERT(list_is_empty(&completion.wq.entries));
+    ASSERT_EQ(count_pending_timers(), 0);
+
+    complete(&completion);
+    ASSERT_EQ(completion.done, 1);
+
+    switching_test_end();
+}
+
+TEST_CASE(completion_timeout_returns_when_completed)
+{
+    DEFINE_COMPLETION(completion);
+    struct timed_sleeper sleeper = {
+        .id = 1, .ns = 100, .completion = &completion,
+    };
+
+    timed_test_begin();
+    spawn(wait_for_completion_with_timeout, &sleeper, TASK_PRIO_DEFAULT);
+    schedule_idle();
+
+    advance_time_to(1050);
+    complete(&completion);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT(sleeper.result);
+    ASSERT_EQ(completion.done, 0);
+    ASSERT_EQ(count_pending_timers(), 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(completion_timeout_takes_a_complete_at_the_deadline)
+{
+    DEFINE_COMPLETION(completion);
+    struct timed_sleeper sleeper = {
+        .id = 1, .ns = 100, .completion = &completion,
+    };
+
+    timed_test_begin();
+    spawn(wait_for_completion_with_timeout, &sleeper, TASK_PRIO_DEFAULT);
+    schedule_idle();
+
+    complete(&completion);
+    advance_time_to(1100);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT(sleeper.result);
+    ASSERT_EQ(completion.done, 0);
+    ASSERT_EQ(count_pending_timers(), 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(completion_timeout_does_not_wait_when_already_done)
+{
+    DEFINE_COMPLETION(completion);
+    struct timed_sleeper sleeper = {
+        .id = 1, .ns = 100, .completion = &completion,
+    };
+
+    timed_test_begin();
+    complete(&completion);
+
+    spawn(wait_for_completion_with_timeout, &sleeper, TASK_PRIO_DEFAULT);
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT(sleeper.result);
+    ASSERT_EQ(count_pending_timers(), 0);
+
+    switching_test_end();
+}

From c5ad366bcb050e362e78465fe6ffe2abd04c32e1 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:20:18 +0300
Subject: [PATCH 112/152] sched: add the switch_in and switch_out class
 operations

A class that accounts for the time its tasks run needs to know when one
starts and stops running. do_schedule() now calls switch_out() for
rq->curr before it picks the next task, and switch_in() once the task
picked is rq->curr. For that, rq->curr is now updated right after the
pick instead of right before the context switch.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/private/sched/core.h | 14 ++++++++++++++
 kernel/sched/core.c                 |  4 +++-
 kernel/sched/idle.c                 | 12 ++++++++++++
 kernel/sched/rr.c                   | 12 ++++++++++++
 4 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/kernel/include/private/sched/core.h b/kernel/include/private/sched/core.h
index 7461c21c..605a9f68 100644
--- a/kernel/include/private/sched/core.h
+++ b/kernel/include/private/sched/core.h
@@ -54,6 +54,20 @@ struct sched_class {
      *       resched_curr() is invoked automatically.
      */
     void (*wakeup_preempt)(struct rq *rq, struct task *task);
+
+    /*
+     * Called on every pass through schedule(), switch_out() for rq->curr
+     * before the next task is picked, then switch_in() once the task picked
+     * is rq->curr. A task runs from its switch_in() call until its
+     * switch_out() call.
+     *
+     * NOTE: The task picked may be rq->curr again, in which case it gets
+     *       switch_out() and switch_in() back to back without a context
+     *       switch. rq->curr may already be dequeued when switch_out() is
+     *       called.
+     */
+    void (*switch_in)(struct rq *rq);
+    void (*switch_out)(struct rq *rq);
 };
 
 #define DEFINE_SCHED_CLASS(name)                                      \
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f390a556..04711638 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -156,7 +156,10 @@ static void do_schedule(bool preempted)
         rq_dequeue(rq, prev);
     }
 
+    prev->sched_class->switch_out(rq);
     next = rq_pick_next(rq);
+    rq->curr = next;
+    next->sched_class->switch_in(rq);
 
     task_clear_flag(prev, TASK_FLAG_NEED_RESCHED);
     preempt_clear_need_resched();
@@ -168,7 +171,6 @@ static void do_schedule(bool preempted)
     }
 
     rq->nr_switches++;
-    rq->curr = next;
     atomic_store_relaxed(&next->on_cpu, true);
 
     prev = arch_switch_to(prev, next);
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index c0e9e0d3..6dfb3c01 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -44,12 +44,24 @@ static void idle_wakeup_preempt(struct rq *rq, struct task *task)
     BUG();
 }
 
+static void idle_switch_in(struct rq *rq)
+{
+    UNREFERENCED_PARAMETER(rq);
+}
+
+static void idle_switch_out(struct rq *rq)
+{
+    UNREFERENCED_PARAMETER(rq);
+}
+
 DEFINE_SCHED_CLASS(idle) = {
     .enqueue = idle_enqueue,
     .dequeue = idle_dequeue,
     .pick = idle_pick,
     .yield = idle_yield,
     .wakeup_preempt = idle_wakeup_preempt,
+    .switch_in = idle_switch_in,
+    .switch_out = idle_switch_out,
 };
 
 /*
diff --git a/kernel/sched/rr.c b/kernel/sched/rr.c
index ad46ce5d..f64732c8 100644
--- a/kernel/sched/rr.c
+++ b/kernel/sched/rr.c
@@ -58,10 +58,22 @@ static void rr_wakeup_preempt(struct rq *rq, struct task *task)
         resched_curr(rq);
 }
 
+static void rr_switch_in(struct rq *rq)
+{
+    UNREFERENCED_PARAMETER(rq);
+}
+
+static void rr_switch_out(struct rq *rq)
+{
+    UNREFERENCED_PARAMETER(rq);
+}
+
 DEFINE_SCHED_CLASS(rr) = {
     .enqueue = rr_enqueue,
     .dequeue = rr_dequeue,
     .pick = rr_pick,
     .yield = rr_yield,
     .wakeup_preempt = rr_wakeup_preempt,
+    .switch_in = rr_switch_in,
+    .switch_out = rr_switch_out,
 };

From 1c995a1b48be5cc85e48cd538c0917006dce4f15 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:20:18 +0300
Subject: [PATCH 113/152] time: add timer_arm_lazy()

timer_arm() reprograms the event device whenever the earliest deadline
in the queue changes, including when the first timer moves to a later
deadline. A timer that is re-armed to a later deadline much more often
than it fires, like the slice timer on every context switch, thus
reprograms the device every time, and under virtualization each of
those writes is a VM exit.

timer_arm_lazy() leaves the device armed for an earlier deadline
instead. The interrupt it raises finds nothing due and arms the device
for the new deadline, so a single extra interrupt replaces every
reprogram in between.

It is opt-in because it only pays off for such timers. A timer moved
from 1 ms to 4 ms only once still needs one reprogram for 4 ms either
way, timer_arm_lazy() just adds a pointless interrupt at 1 ms before
it. That interrupt may also wake an idle CPU.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/time/timer.h   |  8 ++++++
 kernel/time/timer.c           | 32 ++++++++++++++++++++--
 tests/test-cases/test_timer.c | 50 +++++++++++++++++++++++++++++++++++
 3 files changed, 88 insertions(+), 2 deletions(-)

diff --git a/kernel/include/time/timer.h b/kernel/include/time/timer.h
index 14310cc1..f33b504a 100644
--- a/kernel/include/time/timer.h
+++ b/kernel/include/time/timer.h
@@ -58,6 +58,14 @@ static inline void timer_arm(struct timer *timer, u64 deadline_ns)
     timer_arm_with_slack(timer, deadline_ns, 0);
 }
 
+/*
+ * Like timer_arm(), for a timer that is re-armed to a later deadline far
+ * more often than it fires. If the event device is already armed for an
+ * earlier point, it is left alone, and the interrupt it raises then arms
+ * the device for this timer.
+ */
+void timer_arm_lazy(struct timer *timer, u64 deadline_ns);
+
 // The deadline the timer was last armed with
 static inline u64 timer_deadline(const struct timer *timer)
 {
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 6cf22787..c6f63fc0 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -213,6 +213,22 @@ static bool queue_should_defer_arming(struct timer_queue *queue)
     return queue->in_interrupt || queue->backing_off;
 }
 
+static bool queue_should_arm(
+    struct timer_queue *queue, struct timer *timer, bool lazy
+)
+{
+    if (queue_first(queue) != timer)
+        return false;
+    if (queue_should_defer_arming(queue))
+        return false;
+
+    // The interrupt for the earlier deadline arms the device for this one
+    if (lazy && queue->armed_ns <= timer->hard_deadline_ns)
+        return false;
+
+    return true;
+}
+
 static void queue_insert(
     struct timer_queue *queue, struct timer *timer,
     u64 deadline_ns, u64 hard_deadline_ns
@@ -243,7 +259,9 @@ static void queue_requeue(
     queue_insert(queue, timer, timer->deadline_ns, timer->hard_deadline_ns);
 }
 
-void timer_arm_with_slack(struct timer *timer, u64 deadline_ns, u64 slack_ns)
+static void do_timer_arm(
+    struct timer *timer, u64 deadline_ns, u64 slack_ns, bool lazy
+)
 {
     struct timer_queue *queue;
     irq_state_t irq_state;
@@ -259,13 +277,23 @@ void timer_arm_with_slack(struct timer *timer, u64 deadline_ns, u64 slack_ns)
     queue = timer_lock_queue_for_arm(timer);
     queue_insert(queue, timer, deadline_ns, hard_deadline_ns);
 
-    if (queue_first(queue) == timer && !queue_should_defer_arming(queue))
+    if (queue_should_arm(queue, timer, lazy))
         queue_arm(queue);
 
     spin_unlock(&queue->lock);
     local_irq_state_restore(irq_state);
 }
 
+void timer_arm_with_slack(struct timer *timer, u64 deadline_ns, u64 slack_ns)
+{
+    do_timer_arm(timer, deadline_ns, slack_ns, false);
+}
+
+void timer_arm_lazy(struct timer *timer, u64 deadline_ns)
+{
+    do_timer_arm(timer, deadline_ns, 0, true);
+}
+
 bool timer_cancel_nosync(struct timer *timer)
 {
     struct timer_queue *queue;
diff --git a/tests/test-cases/test_timer.c b/tests/test-cases/test_timer.c
index 1ad62f69..f8acf9b5 100644
--- a/tests/test-cases/test_timer.c
+++ b/tests/test-cases/test_timer.c
@@ -759,3 +759,53 @@ TEST_CASE(timer_rearmed_while_running_elsewhere_stays_there)
     ASSERT(queue_first(this_cpu_ptr(&s_queue)) == nullptr);
     ASSERT_EQ(s_num_arms, arms_before);
 }
+
+TEST_CASE(timer_lazy_rearm_to_a_later_deadline_keeps_the_earlier_interrupt)
+{
+    struct timer timer;
+
+    reset_state();
+
+    timer_init(&timer, record);
+    timer_arm_lazy(&timer, 100);
+    ASSERT_EQ(s_armed_for, 100);
+
+    timer_arm_lazy(&timer, 400);
+    ASSERT_EQ(s_armed_for, 100);
+    ASSERT_EQ(s_num_arms, 1);
+
+    take_interrupt_at(100);
+    ASSERT_EQ(s_num_fired, 0);
+    ASSERT_EQ(s_armed_for, 400);
+
+    take_interrupt_at(400);
+    ASSERT_EQ(s_num_fired, 1);
+}
+
+TEST_CASE(timer_lazy_arm_after_a_cancel_keeps_the_earlier_interrupt)
+{
+    struct timer timer;
+
+    reset_state();
+
+    timer_init(&timer, record);
+    timer_arm_lazy(&timer, 100);
+    ASSERT(timer_cancel_nosync(&timer));
+
+    timer_arm_lazy(&timer, 400);
+    ASSERT_EQ(s_armed_for, 100);
+    ASSERT_EQ(s_num_arms, 1);
+}
+
+TEST_CASE(timer_lazy_rearm_to_an_earlier_deadline_arms_the_device)
+{
+    struct timer timer;
+
+    reset_state();
+
+    timer_init(&timer, record);
+    timer_arm_lazy(&timer, 400);
+    timer_arm_lazy(&timer, 100);
+    ASSERT_EQ(s_armed_for, 100);
+    ASSERT_EQ(s_num_arms, 2);
+}

From d6ce8976bd8e0cb4e9f1bbd34755b1b3e77b35f2 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:20:18 +0300
Subject: [PATCH 114/152] sched: add a per-runqueue slice timer

There is no periodic tick, so a class that wants to preempt a task after
a while arms a timer on the runqueue with rq_arm_slice(). The class of
the running task gets slice_expired when it fires. Every context switch
re-arms it, so it is armed with timer_arm_lazy().

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/private/sched/core.h |  9 ++++++++
 kernel/sched/core.c                 | 36 +++++++++++++++++++++++++++++
 kernel/sched/idle.c                 |  8 +++++++
 kernel/sched/rr.c                   |  6 +++++
 tests/test-cases/test_sched.c       | 15 ++++++++++++
 5 files changed, 74 insertions(+)

diff --git a/kernel/include/private/sched/core.h b/kernel/include/private/sched/core.h
index 605a9f68..d8ed688e 100644
--- a/kernel/include/private/sched/core.h
+++ b/kernel/include/private/sched/core.h
@@ -7,6 +7,7 @@
 #include <linker.h>
 #include <preempt.h>
 #include <locking/spinlock.h>
+#include <time/timer.h>
 
 #include <private/sched/sched.h>
 #include <private/sched/rr.h>
@@ -68,6 +69,9 @@ struct sched_class {
      */
     void (*switch_in)(struct rq *rq);
     void (*switch_out)(struct rq *rq);
+
+    // Called for rq->curr once the deadline of rq_arm_slice() expires
+    void (*slice_expired)(struct rq *rq);
 };
 
 #define DEFINE_SCHED_CLASS(name)                                      \
@@ -89,6 +93,8 @@ struct rq {
     u32 nr_running;
     u64 nr_switches;
 
+    struct timer slice_timer;
+
     struct rr_rq rr;
 };
 
@@ -108,5 +114,8 @@ void rq_dequeue(struct rq *rq, struct task *task);
 struct task *rq_pick_next(struct rq *rq);
 void resched_curr(struct rq *rq);
 
+void rq_arm_slice(struct rq *rq, u64 deadline_ns);
+void rq_cancel_slice(struct rq *rq);
+
 void schedule_idle(void);
 void sched_finish_switch(struct task *prev);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 04711638..8fb67c28 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -74,6 +74,41 @@ void resched_curr(struct rq *rq)
     preempt_set_need_resched();
 }
 
+void rq_arm_slice(struct rq *rq, u64 deadline_ns)
+{
+    BUG_ON(!spin_is_locked(&rq->lock));
+    BUG_ON(rq != this_rq());
+
+    timer_arm_lazy(&rq->slice_timer, deadline_ns);
+}
+
+void rq_cancel_slice(struct rq *rq)
+{
+    BUG_ON(!spin_is_locked(&rq->lock));
+    BUG_ON(rq != this_rq());
+
+    /*
+     * The slice timer is only ever armed on this CPU, so its callback runs
+     * here, in an interrupt, and can't be running while we hold rq->lock with
+     * interrupts disabled. timer_cancel() must not be used here, because it
+     * waits for a running callback, and the callback takes rq->lock.
+     */
+    timer_cancel_nosync(&rq->slice_timer);
+}
+
+static enum timer_result slice_timer_fn(struct timer *timer)
+{
+    struct rq *rq;
+
+    rq = container_of(timer, struct rq, slice_timer);
+
+    spin_lock(&rq->lock);
+    rq->curr->sched_class->slice_expired(rq);
+    spin_unlock(&rq->lock);
+
+    return TIMER_RESULT_DONE;
+}
+
 static void rq_wakeup_preempt(struct rq *rq, struct task *task)
 {
     struct task *curr = rq->curr;
@@ -396,6 +431,7 @@ static void rq_init(struct rq *rq, cpu_t cpu)
 
     spin_lock_init(&rq->lock);
     rr_rq_init(&rq->rr);
+    timer_init(&rq->slice_timer, slice_timer_fn);
 
     idle->on_cpu = true;
 
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 6dfb3c01..7d19b87d 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -54,6 +54,13 @@ static void idle_switch_out(struct rq *rq)
     UNREFERENCED_PARAMETER(rq);
 }
 
+static void idle_slice_expired(struct rq *rq)
+{
+    UNREFERENCED_PARAMETER(rq);
+
+    BUG();
+}
+
 DEFINE_SCHED_CLASS(idle) = {
     .enqueue = idle_enqueue,
     .dequeue = idle_dequeue,
@@ -62,6 +69,7 @@ DEFINE_SCHED_CLASS(idle) = {
     .wakeup_preempt = idle_wakeup_preempt,
     .switch_in = idle_switch_in,
     .switch_out = idle_switch_out,
+    .slice_expired = idle_slice_expired,
 };
 
 /*
diff --git a/kernel/sched/rr.c b/kernel/sched/rr.c
index f64732c8..41fdde7d 100644
--- a/kernel/sched/rr.c
+++ b/kernel/sched/rr.c
@@ -68,6 +68,11 @@ static void rr_switch_out(struct rq *rq)
     UNREFERENCED_PARAMETER(rq);
 }
 
+static void rr_slice_expired(struct rq *rq)
+{
+    UNREFERENCED_PARAMETER(rq);
+}
+
 DEFINE_SCHED_CLASS(rr) = {
     .enqueue = rr_enqueue,
     .dequeue = rr_dequeue,
@@ -76,4 +81,5 @@ DEFINE_SCHED_CLASS(rr) = {
     .wakeup_preempt = rr_wakeup_preempt,
     .switch_in = rr_switch_in,
     .switch_out = rr_switch_out,
+    .slice_expired = rr_slice_expired,
 };
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 46a813ac..427bca7a 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -19,12 +19,16 @@ void sched_test_timer_init(
 void sched_test_timer_arm_with_slack(
     struct timer *timer, u64 deadline_ns, u64 slack_ns
 );
+void sched_test_timer_arm_lazy(struct timer *timer, u64 deadline_ns);
 void sched_test_timer_cancel(struct timer *timer);
+bool sched_test_timer_cancel_nosync(struct timer *timer);
 u64 sched_test_now(void);
 
 #define timer_init sched_test_timer_init
 #define timer_arm_with_slack sched_test_timer_arm_with_slack
+#define timer_arm_lazy sched_test_timer_arm_lazy
 #define timer_cancel sched_test_timer_cancel
+#define timer_cancel_nosync sched_test_timer_cancel_nosync
 #define ns_since_boot sched_test_now
 
 #define alloc sched_test_alloc
@@ -1369,6 +1373,12 @@ void sched_test_timer_cancel(struct timer *timer)
     }
 }
 
+bool sched_test_timer_cancel_nosync(struct timer *timer)
+{
+    sched_test_timer_cancel(timer);
+    return true;
+}
+
 void sched_test_timer_arm_with_slack(
     struct timer *timer, u64 deadline_ns, u64 slack_ns
 )
@@ -1391,6 +1401,11 @@ void sched_test_timer_arm_with_slack(
     BUG();
 }
 
+void sched_test_timer_arm_lazy(struct timer *timer, u64 deadline_ns)
+{
+    sched_test_timer_arm_with_slack(timer, deadline_ns, 0);
+}
+
 u64 sched_test_now(void)
 {
     return s_now;

From 457ecd2e28cc1b276498f6fb55c62c9105471e94 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 5 Nov 2026 22:59:58 +0300
Subject: [PATCH 115/152] common/list: add list_is_singular()

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/common/list.h |  5 +++++
 tests/test-cases/test_list.c | 17 +++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/kernel/include/common/list.h b/kernel/include/common/list.h
index 5875fc19..ffb31efa 100644
--- a/kernel/include/common/list.h
+++ b/kernel/include/common/list.h
@@ -57,6 +57,11 @@ static inline bool list_is_empty(const struct list_link *head)
     return head->next == head;
 }
 
+static inline bool list_is_singular(const struct list_link *head)
+{
+    return !list_is_empty(head) && head->next == head->prev;
+}
+
 #define list_entry(ptr, type, member) container_of(ptr, type, member)
 
 #define list_first_entry(head, type, member) \
diff --git a/tests/test-cases/test_list.c b/tests/test-cases/test_list.c
index e9937470..ae212cad 100644
--- a/tests/test-cases/test_list.c
+++ b/tests/test-cases/test_list.c
@@ -280,3 +280,20 @@ TEST_CASE(list_for_each_entry_safe_remove_all)
     ASSERT_EQ(count, 3);
     ASSERT(list_is_empty(&head));
 }
+
+TEST_CASE(list_is_singular_with_exactly_one_entry)
+{
+    LIST_HEAD(head);
+    struct list_link a, b;
+
+    ASSERT_FALSE(list_is_singular(&head));
+
+    list_insert_prev(&head, &a);
+    ASSERT(list_is_singular(&head));
+
+    list_insert_prev(&head, &b);
+    ASSERT_FALSE(list_is_singular(&head));
+
+    list_remove(&a);
+    ASSERT(list_is_singular(&head));
+}

From 3939ac32ab23d0307afb5514725a61873aca1256 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 6 Nov 2026 14:37:07 +0300
Subject: [PATCH 116/152] common/list: add list_is_first()

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/common/list.h |  7 +++++++
 tests/test-cases/test_list.c | 14 ++++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/kernel/include/common/list.h b/kernel/include/common/list.h
index ffb31efa..923dddd3 100644
--- a/kernel/include/common/list.h
+++ b/kernel/include/common/list.h
@@ -62,6 +62,13 @@ static inline bool list_is_singular(const struct list_link *head)
     return !list_is_empty(head) && head->next == head->prev;
 }
 
+static inline bool list_is_first(
+    const struct list_link *head, const struct list_link *link
+)
+{
+    return head->next == link;
+}
+
 #define list_entry(ptr, type, member) container_of(ptr, type, member)
 
 #define list_first_entry(head, type, member) \
diff --git a/tests/test-cases/test_list.c b/tests/test-cases/test_list.c
index ae212cad..5d2e63b9 100644
--- a/tests/test-cases/test_list.c
+++ b/tests/test-cases/test_list.c
@@ -297,3 +297,17 @@ TEST_CASE(list_is_singular_with_exactly_one_entry)
     list_remove(&a);
     ASSERT(list_is_singular(&head));
 }
+
+TEST_CASE(list_is_first_follows_the_first_entry)
+{
+    LIST_HEAD(head);
+    struct list_link a, b;
+
+    list_insert_prev(&head, &a);
+    list_insert_prev(&head, &b);
+    ASSERT(list_is_first(&head, &a));
+    ASSERT_FALSE(list_is_first(&head, &b));
+
+    list_remove(&a);
+    ASSERT(list_is_first(&head, &b));
+}

From 041da36a7c650b955d62119b4d8db5f74f51fde2 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:20:18 +0300
Subject: [PATCH 117/152] sched: make round-robin preemptive via a slice timer

A task that never sleeps or yields keeps every other task of its
priority off the CPU. Give round-robin tasks a 10 ms slice, armed only
while another task of the same priority is runnable. Being preempted by
a higher priority does not refill a slice.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/sched/rr.h |   2 +
 kernel/sched/rr.c                | 119 ++++++++++++++--
 tests/test-cases/test_sched.c    | 238 +++++++++++++++++++++++++++++++
 3 files changed, 351 insertions(+), 8 deletions(-)

diff --git a/kernel/include/detail/sched/rr.h b/kernel/include/detail/sched/rr.h
index 9906dd86..334f2897 100644
--- a/kernel/include/detail/sched/rr.h
+++ b/kernel/include/detail/sched/rr.h
@@ -5,4 +5,6 @@
 
 struct rr_entity {
     struct list_link link;
+    u64 slice_left_ns;
+    u64 running_since_ns;
 };
diff --git a/kernel/sched/rr.c b/kernel/sched/rr.c
index 41fdde7d..8ecd6454 100644
--- a/kernel/sched/rr.c
+++ b/kernel/sched/rr.c
@@ -2,6 +2,12 @@
 #include <private/sched/rr.h>
 
 #include <common/bit.h>
+#include <common/overflow.h>
+
+#include <time/time.h>
+#include <time/units.h>
+
+#define RR_SLICE_NS (10 * NS_PER_MS)
 
 void rr_rq_init(struct rr_rq *rr)
 {
@@ -13,12 +19,80 @@ void rr_rq_init(struct rr_rq *rr)
     rr->non_empty_queues_mask = 0;
 }
 
+/*
+ * A task only competes with others of its own priority, and without
+ * any there is nothing to rotate to, so no deadline is armed for it.
+ */
+static bool rr_is_alone(struct rq *rq, struct task *task)
+{
+    return list_is_singular(&rq->rr.queues[task->prio]);
+}
+
+static void rr_charge(struct task *task, u64 now_ns)
+{
+    u64 ran_ns;
+
+    ran_ns = now_ns - task->rr.running_since_ns;
+    task->rr.running_since_ns = now_ns;
+
+    task->rr.slice_left_ns = saturating_sub(task->rr.slice_left_ns, ran_ns);
+}
+
+static void rr_move_to_tail(struct rq *rq, struct task *task)
+{
+    struct rr_rq *rr = &rq->rr;
+
+    list_remove(&task->rr.link);
+    list_insert_prev(&rr->queues[task->prio], &task->rr.link);
+
+    task->rr.slice_left_ns = RR_SLICE_NS;
+}
+
+/*
+ * Called before 'peer' is queued. If 'peer' is the first task to share the
+ * priority of rq->curr, arm the slice of rq->curr, or call resched_curr() if
+ * that slice is used up already.
+ */
+static void rr_maybe_arm_slice(struct rq *rq, struct task *peer)
+{
+    struct task *curr = rq->curr;
+    u64 now_ns;
+
+    if (curr->sched_class != &SCHED_CLASS_INSTANCE(rr))
+        return;
+    if (curr->prio != peer->prio)
+        return;
+
+    // It already has a peer, so its slice is armed already
+    if (!rr_is_alone(rq, curr))
+        return;
+
+    now_ns = ns_since_boot();
+    rr_charge(curr, now_ns);
+
+    /*
+     * The running task now has a peer. Whatever it has run so far counts,
+     * so that it's rotated out right away if it has been running alone for a
+     * long time.
+     */
+    if (curr->rr.slice_left_ns == 0) {
+        resched_curr(rq);
+        return;
+    }
+
+    rq_arm_slice(rq, now_ns + curr->rr.slice_left_ns);
+}
+
 static void rr_enqueue(struct rq *rq, struct task *task)
 {
     struct rr_rq *rr = &rq->rr;
 
+    rr_maybe_arm_slice(rq, task);
+
     list_insert_prev(&rr->queues[task->prio], &task->rr.link);
     rr->non_empty_queues_mask |= BIT_U32(task->prio);
+
+    task->rr.slice_left_ns = RR_SLICE_NS;
 }
 
 static void rr_dequeue(struct rq *rq, struct task *task)
@@ -45,11 +119,11 @@ static struct task *rr_pick(struct rq *rq)
 
 static void rr_yield(struct rq *rq)
 {
-    struct task *curr = rq->curr;
-    struct rr_rq *rr = &rq->rr;
-
-    list_remove(&curr->rr.link);
-    list_insert_prev(&rr->queues[curr->prio], &curr->rr.link);
+    /*
+     * The task gives up the rest of its slice, so the rr_switch_out() that
+     * follows moves it to the tail of its queue and gives it a new slice.
+     */
+    rq->curr->rr.slice_left_ns = 0;
 }
 
 static void rr_wakeup_preempt(struct rq *rq, struct task *task)
@@ -60,17 +134,46 @@ static void rr_wakeup_preempt(struct rq *rq, struct task *task)
 
 static void rr_switch_in(struct rq *rq)
 {
-    UNREFERENCED_PARAMETER(rq);
+    struct task *curr = rq->curr;
+    u64 now_ns;
+
+    now_ns = ns_since_boot();
+    curr->rr.running_since_ns = now_ns;
+
+    if (!rr_is_alone(rq, curr))
+        rq_arm_slice(rq, now_ns + curr->rr.slice_left_ns);
 }
 
 static void rr_switch_out(struct rq *rq)
 {
-    UNREFERENCED_PARAMETER(rq);
+    struct task *curr = rq->curr;
+
+    rq_cancel_slice(rq);
+    rr_charge(curr, ns_since_boot());
+
+    /*
+     * A task that blocked is no longer queued, and gets a new slice once it
+     * wakes up. A task that is still queued moves to the tail of its queue
+     * with a new slice if it used up its slice. Otherwise, e.g. when it was
+     * preempted by a higher priority, it keeps whatever remains of its slice
+     * and stays at the head of the queue.
+     */
+    if (curr->on_rq && curr->rr.slice_left_ns == 0)
+        rr_move_to_tail(rq, curr);
 }
 
 static void rr_slice_expired(struct rq *rq)
 {
-    UNREFERENCED_PARAMETER(rq);
+    struct task *curr = rq->curr;
+
+    rr_charge(curr, ns_since_boot());
+
+    if (rr_is_alone(rq, curr)) {
+        curr->rr.slice_left_ns = RR_SLICE_NS;
+        return;
+    }
+
+    resched_curr(rq);
 }
 
 DEFINE_SCHED_CLASS(rr) = {
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 427bca7a..7a10e9e1 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -1739,3 +1739,241 @@ TEST_CASE(completion_timeout_does_not_wait_when_already_done)
 
     switching_test_end();
 }
+
+#define SLICE_NS (10 * 1000 * 1000ull)
+
+// A timer interrupt taken by the running task, with the preemption check
+static void take_timer_interrupt_at(u64 now)
+{
+    struct registers regs = { .irq_state = true };
+    irq_state_t irq_state;
+
+    irq_state = local_irq_state_save();
+    advance_time_to(now);
+    preempt_schedule_irq(&regs);
+    local_irq_state_restore(irq_state);
+}
+
+struct spinner {
+    int id;
+    bool *stop;
+};
+
+/*
+ * Stands in for a task that computes without ever giving up the CPU. The
+ * test drives time from inside it.
+ */
+static void spin_and_log(void *user)
+{
+    struct spinner *spinner = user;
+
+    while (!*spinner->stop) {
+        log_id(spinner->id);
+
+        take_timer_interrupt_at(s_now + SLICE_NS / 2);
+
+        if (s_log_length >= 12)
+            *spinner->stop = true;
+    }
+}
+
+TEST_CASE(slice_rotates_tasks_that_never_yield)
+{
+    bool stop = false;
+    struct spinner first = { .id = 1, .stop = &stop };
+    struct spinner second = { .id = 2, .stop = &stop };
+    int i;
+
+    timed_test_begin();
+    spawn(spin_and_log, &first, TASK_PRIO_DEFAULT);
+    spawn(spin_and_log, &second, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 12);
+    for (i = 0; i < 12; i++)
+        ASSERT_EQ(s_log[i], (i / 2) % 2 + 1);
+
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+    ASSERT_EQ(count_pending_timers(), 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(slice_is_not_armed_for_a_task_alone_at_its_priority)
+{
+    bool stop = false;
+    struct spinner alone = { .id = 1, .stop = &stop };
+    struct runner lower = { .id = 2, .rounds = 1 };
+
+    timed_test_begin();
+    spawn(spin_and_log, &alone, TASK_PRIO_DEFAULT);
+    spawn(run_to_completion, &lower, TASK_PRIO_DEFAULT + 1);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 13);
+    ASSERT_EQ(count_ids(1), 12);
+    ASSERT_EQ(s_log[12], 2);
+
+    switching_test_end();
+}
+
+struct slice_user {
+    int id;
+    int rounds;
+    struct runner intruder;
+};
+
+static void get_preempted_every_round(void *user)
+{
+    struct slice_user *slice_user = user;
+    int i;
+
+    for (i = 0; i < slice_user->rounds; i++) {
+        log_id(slice_user->id);
+
+        spawn(run_to_completion, &slice_user->intruder, TASK_PRIO_HIGHEST);
+        take_timer_interrupt_at(s_now + SLICE_NS / 4);
+    }
+}
+
+TEST_CASE(slice_is_not_refilled_by_being_preempted)
+{
+    struct slice_user greedy = {
+        .id = 1, .rounds = 6, .intruder = { .id = 9, .rounds = 1 },
+    };
+    struct runner patient = { .id = 2, .rounds = 1 };
+    int i, patient_at = -1;
+
+    timed_test_begin();
+    spawn(get_preempted_every_round, &greedy, TASK_PRIO_DEFAULT);
+    spawn(run_to_completion, &patient, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    for (i = 0; i < s_log_length; i++) {
+        if (s_log[i] == 2)
+            patient_at = i;
+    }
+
+    ASSERT_EQ(count_ids(1), 6);
+    ASSERT_EQ(count_ids(9), 6);
+    ASSERT_EQ(patient_at, 8);
+
+    switching_test_end();
+}
+
+struct slice_probe {
+    int id;
+    u64 slice_left_ns;
+    int timers_while_alone;
+    int timers_with_a_peer;
+    struct runner peer;
+};
+
+static void yield_late_in_the_slice(void *user)
+{
+    struct slice_probe *probe = user;
+
+    advance_time_to(s_now + SLICE_NS - 1);
+    yield();
+
+    probe->slice_left_ns = current->rr.slice_left_ns;
+    log_id(probe->id);
+}
+
+TEST_CASE(slice_is_refilled_by_a_yield)
+{
+    struct slice_probe probe = { .id = 1 };
+    struct runner other = { .id = 2, .rounds = 1 };
+
+    timed_test_begin();
+    spawn(yield_late_in_the_slice, &probe, TASK_PRIO_DEFAULT);
+    spawn(run_to_completion, &other, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 2);
+    ASSERT_EQ(s_log[0], 2);
+    ASSERT_EQ(s_log[1], 1);
+    ASSERT_EQ(probe.slice_left_ns, SLICE_NS);
+
+    switching_test_end();
+}
+
+static void run_alone_then_get_a_peer(void *user)
+{
+    struct slice_probe *probe = user;
+
+    log_id(probe->id);
+    advance_time_to(s_now + SLICE_NS / 2);
+    probe->timers_while_alone = count_pending_timers();
+
+    spawn(run_to_completion, &probe->peer, TASK_PRIO_DEFAULT);
+    probe->timers_with_a_peer = count_pending_timers();
+    log_id(probe->id);
+
+    take_timer_interrupt_at(s_now + SLICE_NS / 2 - 1);
+    log_id(probe->id);
+
+    take_timer_interrupt_at(s_now + 1);
+    log_id(probe->id);
+}
+
+TEST_CASE(slice_starts_counting_when_a_peer_shows_up)
+{
+    struct slice_probe probe = {
+        .id = 1, .peer = { .id = 2, .rounds = 1 },
+    };
+
+    timed_test_begin();
+    spawn(run_alone_then_get_a_peer, &probe, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(probe.timers_while_alone, 0);
+    ASSERT_EQ(probe.timers_with_a_peer, 1);
+
+    ASSERT_EQ(s_log_length, 5);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 1);
+    ASSERT_EQ(s_log[2], 1);
+    ASSERT_EQ(s_log[3], 2);
+    ASSERT_EQ(s_log[4], 1);
+
+    switching_test_end();
+}
+
+static void use_up_the_slice_alone_then_get_a_peer(void *user)
+{
+    struct slice_probe *probe = user;
+
+    log_id(probe->id);
+    advance_time_to(s_now + 3 * SLICE_NS);
+
+    spawn(run_to_completion, &probe->peer, TASK_PRIO_DEFAULT);
+    probe->timers_with_a_peer = count_pending_timers();
+    log_id(probe->id);
+}
+
+TEST_CASE(slice_used_up_alone_lets_a_peer_run_right_away)
+{
+    struct slice_probe probe = {
+        .id = 1, .peer = { .id = 2, .rounds = 1 },
+    };
+
+    timed_test_begin();
+    spawn(use_up_the_slice_alone_then_get_a_peer, &probe, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(probe.timers_with_a_peer, 0);
+
+    ASSERT_EQ(s_log_length, 3);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 2);
+    ASSERT_EQ(s_log[2], 1);
+
+    switching_test_end();
+}

From d619da82719d5e641f2f5508659b2e63784ea42a Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 6 Nov 2026 02:07:35 +0300
Subject: [PATCH 118/152] common/atomic: add
 atomic_cmpxchg_{relaxed,acquire,release}()

pvclock.c and page_table.h passed the relaxed and the release orders
to atomic_cmpxchg_explicit() themselves, and now use the new helpers
instead.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/drivers/time/pvclock.c      | 3 +--
 kernel/include/common/atomic.h     | 6 ++++++
 kernel/include/memory/page_table.h | 4 +---
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/kernel/drivers/time/pvclock.c b/kernel/drivers/time/pvclock.c
index 2aaf1272..876c9abd 100644
--- a/kernel/drivers/time/pvclock.c
+++ b/kernel/drivers/time/pvclock.c
@@ -73,8 +73,7 @@ static u64 pvclock_read_stablize(u64 this_read)
          */
         if (this_read <= last_read)
             return last_read;
-    } while (!atomic_cmpxchg_explicit(&s_last_read_ns, &last_read, this_read,
-                                      MO_RELAXED, MO_RELAXED));
+    } while (!atomic_cmpxchg_relaxed(&s_last_read_ns, &last_read, this_read));
     return this_read;
 }
 
diff --git a/kernel/include/common/atomic.h b/kernel/include/common/atomic.h
index ec82d8e6..0b342a31 100644
--- a/kernel/include/common/atomic.h
+++ b/kernel/include/common/atomic.h
@@ -65,3 +65,9 @@ enum memory_order : u32 {
 
 #define atomic_cmpxchg_acq_rel(ptr, expected, desired) \
     atomic_cmpxchg_explicit(ptr, expected, desired, MO_ACQ_REL, MO_ACQUIRE)
+#define atomic_cmpxchg_relaxed(ptr, expected, desired) \
+    atomic_cmpxchg_explicit(ptr, expected, desired, MO_RELAXED, MO_RELAXED)
+#define atomic_cmpxchg_acquire(ptr, expected, desired) \
+    atomic_cmpxchg_explicit(ptr, expected, desired, MO_ACQUIRE, MO_RELAXED)
+#define atomic_cmpxchg_release(ptr, expected, desired) \
+    atomic_cmpxchg_explicit(ptr, expected, desired, MO_RELEASE, MO_RELAXED)
diff --git a/kernel/include/memory/page_table.h b/kernel/include/memory/page_table.h
index a56ca3ed..8ec59111 100644
--- a/kernel/include/memory/page_table.h
+++ b/kernel/include/memory/page_table.h
@@ -290,9 +290,7 @@ static inline pt_entry_word pt_entry_cmpxchg_populate(
     pt_entry_word expected = 0;
     bool success;
 
-    success = atomic_cmpxchg_explicit(
-        entry, &expected, value, MO_RELEASE, MO_RELAXED
-    );
+    success = atomic_cmpxchg_release(entry, &expected, value);
     if (success)
         return 0;
 

From 540d240767640fcfb07218b31f1e9b9faa81a279 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:24:22 +0300
Subject: [PATCH 119/152] mutex: implement for real

This used to be a stub based on spinlocks, but now that we have all the
primitives needed to implement it we can actually do it for real.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/locking/mutex.h |   7 +-
 kernel/include/locking/mutex.h        |  12 +-
 kernel/locking/mutex.c                | 166 ++++++++++++++++++++++++--
 tests/test-cases/test_sched.c         | 114 ++++++++++++++++++
 4 files changed, 285 insertions(+), 14 deletions(-)

diff --git a/kernel/include/detail/locking/mutex.h b/kernel/include/detail/locking/mutex.h
index 3620ae9f..606639c1 100644
--- a/kernel/include/detail/locking/mutex.h
+++ b/kernel/include/detail/locking/mutex.h
@@ -1,7 +1,12 @@
 #pragma once
 
+#include <common/types.h>
+#include <common/list.h>
+
 #include <locking/spinlock.h>
 
 struct mutex {
-    struct spinlock lock;
+    ptr_t owner;
+    struct spinlock wait_lock;
+    struct list_link waiters;
 };
diff --git a/kernel/include/locking/mutex.h b/kernel/include/locking/mutex.h
index e1d35016..d02954d4 100644
--- a/kernel/include/locking/mutex.h
+++ b/kernel/include/locking/mutex.h
@@ -1,5 +1,8 @@
 #pragma once
 
+#include <common/types.h>
+#include <common/list.h>
+
 #include <locking/spinlock.h>
 
 #include <detail/locking/mutex.h>
@@ -8,14 +11,13 @@
  * A mutual exclusion lock for process context only. Holders are
  * allowed to block, so it must never be acquired from interrupt
  * context.
- *
- * Sleeping does not exist yet, so the current implementation is a
- * spinlock that must be free when taken, but every caller must
- * already honor the contract as if it could sleep.
  */
 struct mutex;
 
-#define MUTEX_INIT(name) (struct mutex) { SPINLOCK_INIT((name).lock) }
+#define MUTEX_INIT(name) (struct mutex) {         \
+    .wait_lock = SPINLOCK_INIT((name).wait_lock), \
+    .waiters = LIST_INIT((name).waiters),         \
+}
 #define DEFINE_MUTEX(name) struct mutex name = MUTEX_INIT(name)
 
 void mutex_init(struct mutex*);
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 7d09a1f2..6b8c456e 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -2,29 +2,179 @@
 #include <preempt.h>
 #include <bug.h>
 
+#include <common/atomic.h>
+#include <common/bit.h>
+
+#include <sched/task.h>
+#include <sched/sched.h>
+#include <private/sched/sched.h>
+
+/*
+ * The owner word holds a pointer to the owning task, with these flags stored
+ * in its low bits.
+ */
+enum mutex_flags : ptr_t {
+    // The wait list is not empty, so mutex_unlock() has to wake up a waiter
+    MUTEX_FLAG_HAS_WAITERS = BIT_OF_TYPE(ptr_t, 0),
+};
+
+#define MUTEX_FLAGS_MASK MUTEX_FLAG_HAS_WAITERS
+
+BUILD_BUG_ON_WITH_MSG(
+    alignof(struct task) <= MUTEX_FLAGS_MASK,
+    "The mutex flags must fit in the low bits of a task pointer"
+);
+
+struct mutex_waiter {
+    struct list_link link;
+    struct task *task;
+};
+
 void mutex_init(struct mutex *mutex)
 {
     *mutex = MUTEX_INIT(*mutex);
 }
 
+static bool try_lock(struct mutex *mutex)
+{
+    ptr_t owner, task, flags, me;
+
+    me = (ptr_t)current;
+    owner = atomic_load_relaxed(&mutex->owner);
+
+    for (;;) {
+        task = owner & ~MUTEX_FLAGS_MASK;
+        flags = owner & MUTEX_FLAGS_MASK;
+
+        if (task != 0)
+            return false;
+
+        if (atomic_cmpxchg_acquire(&mutex->owner, &owner, me | flags))
+            return true;
+    }
+}
+
+static bool try_lock_fast(struct mutex *mutex)
+{
+    ptr_t expected = 0;
+
+    return atomic_cmpxchg_acquire(&mutex->owner, &expected, (ptr_t)current);
+}
+
+static bool try_unlock_fast(struct mutex *mutex)
+{
+    ptr_t expected;
+
+    expected = (ptr_t)current;
+    return atomic_cmpxchg_release(&mutex->owner, &expected, 0);
+}
+
+static void lock_slow(struct mutex *mutex)
+{
+    struct mutex_waiter waiter;
+
+    waiter.task = current;
+    spin_lock(&mutex->wait_lock);
+
+    if (try_lock(mutex)) {
+        spin_unlock(&mutex->wait_lock);
+        return;
+    }
+
+    if (list_is_empty(&mutex->waiters))
+        atomic_fetch_or(&mutex->owner, MUTEX_FLAG_HAS_WAITERS, MO_RELAXED);
+    list_insert_prev(&mutex->waiters, &waiter.link);
+
+    for (;;) {
+        /*
+         * We set our state before trying to take the lock, so that an unlock
+         * that happens after the attempt finds us on the wait list and makes us
+         * runnable again. Otherwise the schedule() below could sleep forever.
+         */
+        set_current_state(TASK_STATE_UNINTERRUPTIBLE);
+
+        if (try_lock(mutex))
+            break;
+
+        spin_unlock(&mutex->wait_lock);
+        schedule();
+        spin_lock(&mutex->wait_lock);
+    }
+
+    set_current_state(TASK_STATE_RUNNING);
+
+    list_remove(&waiter.link);
+    if (list_is_empty(&mutex->waiters))
+        atomic_fetch_and(&mutex->owner, ~MUTEX_FLAG_HAS_WAITERS, MO_RELAXED);
+
+    spin_unlock(&mutex->wait_lock);
+}
+
 void mutex_lock(struct mutex *mutex)
 {
-    BUG_ON(in_hard_irq());
+    might_sleep();
+
+    if (likely(try_lock_fast(mutex)))
+        return;
 
-    /*
-     * With a single CPU and no way to sleep, a busy lock can only be
-     * held by the caller itself, so spinning on it would hang forever.
-     */
-    BUG_ON(!spin_try_lock(&mutex->lock));
+    lock_slow(mutex);
 }
 
 bool mutex_try_lock(struct mutex *mutex)
 {
     BUG_ON(in_hard_irq());
-    return spin_try_lock(&mutex->lock);
+
+    if (likely(try_lock_fast(mutex)))
+        return true;
+
+    return try_lock(mutex);
+}
+
+static void unlock_slow(struct mutex *mutex)
+{
+    struct mutex_waiter *waiter;
+    struct task *next = nullptr;
+    ptr_t owner, flags;
+
+    owner = atomic_load_relaxed(&mutex->owner);
+
+    for (;;) {
+        BUG_ON((owner & ~MUTEX_FLAGS_MASK) != (ptr_t)current);
+
+        flags = owner & MUTEX_FLAGS_MASK;
+        if (atomic_cmpxchg_release(&mutex->owner, &owner, flags))
+            break;
+    }
+
+    spin_lock(&mutex->wait_lock);
+
+    if (!list_is_empty(&mutex->waiters)) {
+        waiter = list_first_entry(&mutex->waiters, struct mutex_waiter, link);
+        next = waiter->task;
+
+        /*
+         * The waiter may already be running if an earlier unlock woke it up
+         * and another task took the mutex before it did. Once the wait lock
+         * is dropped, it can then take the mutex, return from mutex_lock()
+         * and exit before we get to wake it up. Holding a reference keeps its
+         * task valid until wake_up_task() is done with it.
+         */
+        task_ref(next);
+    }
+
+    spin_unlock(&mutex->wait_lock);
+
+    if (next == nullptr)
+        return;
+
+    wake_up_task(next, TASK_STATE_UNINTERRUPTIBLE);
+    task_unref(next);
 }
 
 void mutex_unlock(struct mutex *mutex)
 {
-    spin_unlock(&mutex->lock);
+    if (likely(try_unlock_fast(mutex)))
+        return;
+
+    unlock_slow(mutex);
 }
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 7a10e9e1..6b303903 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -44,6 +44,8 @@ u64 sched_test_now(void);
 #include <kernel-source/sched/wait_queue.c>
 #include <kernel-source/sched/completion.c>
 
+#include <locking/mutex.h>
+
 #include <private/arch/task.h>
 #include <private/arch/preempt.h>
 
@@ -1977,3 +1979,115 @@ TEST_CASE(slice_used_up_alone_lets_a_peer_run_right_away)
 
     switching_test_end();
 }
+
+struct mutex_user {
+    int id;
+    struct mutex *mutex;
+    int rounds;
+    bool yield_while_holding;
+};
+
+static void lock_log_unlock(void *user)
+{
+    struct mutex_user *mutex_user = user;
+    int i;
+
+    for (i = 0; i < mutex_user->rounds; i++) {
+        mutex_lock(mutex_user->mutex);
+        log_id(mutex_user->id);
+
+        if (mutex_user->yield_while_holding)
+            yield();
+
+        mutex_unlock(mutex_user->mutex);
+    }
+}
+
+TEST_CASE(mutex_contended_lock_sleeps_until_the_unlock)
+{
+    DEFINE_MUTEX(mutex);
+    struct mutex_user holder = {
+        .id = 1, .mutex = &mutex, .rounds = 1, .yield_while_holding = true,
+    };
+    struct mutex_user waiter = { .id = 2, .mutex = &mutex, .rounds = 1 };
+    struct rq *rq;
+
+    rq = switching_test_begin();
+    spawn(lock_log_unlock, &holder, TASK_PRIO_DEFAULT);
+    spawn(lock_log_unlock, &waiter, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 2);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 2);
+
+    ASSERT_EQ(mutex.owner, 0);
+    ASSERT(list_is_empty(&mutex.waiters));
+    ASSERT_EQ(rq->nr_running, 0);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}
+
+TEST_CASE(mutex_waiters_get_the_lock_in_arrival_order)
+{
+    DEFINE_MUTEX(mutex);
+    struct mutex_user holder = {
+        .id = 1, .mutex = &mutex, .rounds = 1, .yield_while_holding = true,
+    };
+    struct mutex_user waiters[3];
+    int i;
+
+    switching_test_begin();
+    spawn(lock_log_unlock, &holder, TASK_PRIO_DEFAULT);
+
+    for (i = 0; i < 3; i++) {
+        waiters[i] = (struct mutex_user) {
+            .id = 2 + i, .mutex = &mutex, .rounds = 1,
+        };
+        spawn(lock_log_unlock, &waiters[i], TASK_PRIO_DEFAULT);
+    }
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 4);
+    for (i = 0; i < 4; i++)
+        ASSERT_EQ(s_log[i], 1 + i);
+
+    ASSERT_EQ(mutex.owner, 0);
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}
+
+static void try_lock_and_log(void *user)
+{
+    struct mutex_user *mutex_user = user;
+
+    if (mutex_try_lock(mutex_user->mutex)) {
+        log_id(mutex_user->id);
+        mutex_unlock(mutex_user->mutex);
+    }
+}
+
+TEST_CASE(mutex_try_lock_fails_while_another_task_holds_it)
+{
+    DEFINE_MUTEX(mutex);
+    struct mutex_user holder = {
+        .id = 1, .mutex = &mutex, .rounds = 1, .yield_while_holding = true,
+    };
+    struct mutex_user other = { .id = 2, .mutex = &mutex };
+
+    switching_test_begin();
+    spawn(lock_log_unlock, &holder, TASK_PRIO_DEFAULT);
+    spawn(try_lock_and_log, &other, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 1);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(mutex.owner, 0);
+
+    switching_test_end();
+}

From 8ab03efd6db5b13291e824107834d9572e19b401 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 4 Oct 2026 16:24:22 +0300
Subject: [PATCH 120/152] mutex: hand the lock off to a waiter that lost it

A woken waiter competes with running tasks for the lock and can lose it
every time it wakes. The first waiter that loses asks for a handoff, and
the next unlock passes the lock directly to it.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/locking/mutex.c        | 78 ++++++++++++++++++++++++++++++-----
 tests/test-cases/test_sched.c | 27 ++++++++++++
 2 files changed, 95 insertions(+), 10 deletions(-)

diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 6b8c456e..c3cf5371 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -16,9 +16,25 @@
 enum mutex_flags : ptr_t {
     // The wait list is not empty, so mutex_unlock() has to wake up a waiter
     MUTEX_FLAG_HAS_WAITERS = BIT_OF_TYPE(ptr_t, 0),
+
+    /*
+     * The first waiter failed to take the lock because another task got it
+     * first. The next unlock hands the lock directly to that waiter instead of
+     * releasing it.
+     */
+    MUTEX_FLAG_HANDOFF_REQUESTED = BIT_OF_TYPE(ptr_t, 1),
+
+    /*
+     * The lock was handed to the task in the owner word, but that task hasn't
+     * taken it yet. Nobody else may take the lock in the meantime.
+     */
+    MUTEX_FLAG_PICKUP_PENDING = BIT_OF_TYPE(ptr_t, 2),
 };
 
-#define MUTEX_FLAGS_MASK MUTEX_FLAG_HAS_WAITERS
+#define MUTEX_FLAGS_MASK (                                  \
+    MUTEX_FLAG_HAS_WAITERS | MUTEX_FLAG_HANDOFF_REQUESTED | \
+    MUTEX_FLAG_PICKUP_PENDING                               \
+)
 
 BUILD_BUG_ON_WITH_MSG(
     alignof(struct task) <= MUTEX_FLAGS_MASK,
@@ -35,7 +51,7 @@ void mutex_init(struct mutex *mutex)
     *mutex = MUTEX_INIT(*mutex);
 }
 
-static bool try_lock(struct mutex *mutex)
+static bool try_lock(struct mutex *mutex, bool request_handoff)
 {
     ptr_t owner, task, flags, me;
 
@@ -46,11 +62,21 @@ static bool try_lock(struct mutex *mutex)
         task = owner & ~MUTEX_FLAGS_MASK;
         flags = owner & MUTEX_FLAGS_MASK;
 
-        if (task != 0)
+        if (task == 0) {
+            task = me;
+        } else if (flags & MUTEX_FLAG_PICKUP_PENDING) {
+            if (task != me)
+                return false;
+
+            flags &= ~MUTEX_FLAG_PICKUP_PENDING;
+        } else if (request_handoff && !(flags & MUTEX_FLAG_HANDOFF_REQUESTED)) {
+            flags |= MUTEX_FLAG_HANDOFF_REQUESTED;
+        } else {
             return false;
+        }
 
-        if (atomic_cmpxchg_acquire(&mutex->owner, &owner, me | flags))
-            return true;
+        if (atomic_cmpxchg_acquire(&mutex->owner, &owner, task | flags))
+            return task == me;
     }
 }
 
@@ -72,11 +98,12 @@ static bool try_unlock_fast(struct mutex *mutex)
 static void lock_slow(struct mutex *mutex)
 {
     struct mutex_waiter waiter;
+    bool is_first = false;
 
     waiter.task = current;
     spin_lock(&mutex->wait_lock);
 
-    if (try_lock(mutex)) {
+    if (try_lock(mutex, false)) {
         spin_unlock(&mutex->wait_lock);
         return;
     }
@@ -93,19 +120,26 @@ static void lock_slow(struct mutex *mutex)
          */
         set_current_state(TASK_STATE_UNINTERRUPTIBLE);
 
-        if (try_lock(mutex))
+        if (try_lock(mutex, is_first))
             break;
 
         spin_unlock(&mutex->wait_lock);
         schedule();
         spin_lock(&mutex->wait_lock);
+
+        is_first = list_is_first(&mutex->waiters, &waiter.link);
     }
 
     set_current_state(TASK_STATE_RUNNING);
 
     list_remove(&waiter.link);
-    if (list_is_empty(&mutex->waiters))
-        atomic_fetch_and(&mutex->owner, ~MUTEX_FLAG_HAS_WAITERS, MO_RELAXED);
+    if (list_is_empty(&mutex->waiters)) {
+        atomic_fetch_and(
+            &mutex->owner,
+            ~(MUTEX_FLAG_HAS_WAITERS | MUTEX_FLAG_HANDOFF_REQUESTED),
+            MO_RELAXED
+        );
+    }
 
     spin_unlock(&mutex->wait_lock);
 }
@@ -127,7 +161,24 @@ bool mutex_try_lock(struct mutex *mutex)
     if (likely(try_lock_fast(mutex)))
         return true;
 
-    return try_lock(mutex);
+    return try_lock(mutex, false);
+}
+
+static void hand_off_to(struct mutex *mutex, struct task *next)
+{
+    ptr_t owner, new_owner;
+
+    owner = atomic_load_relaxed(&mutex->owner);
+
+    new_owner = (ptr_t)next | MUTEX_FLAG_PICKUP_PENDING;
+    new_owner |= owner & MUTEX_FLAG_HAS_WAITERS;
+
+    /*
+     * While we own the mutex, every other write to the owner word happens
+     * under the wait lock, which we hold. Nothing can change the owner word
+     * between the load and this store.
+     */
+    atomic_store_release(&mutex->owner, new_owner);
 }
 
 static void unlock_slow(struct mutex *mutex)
@@ -140,6 +191,10 @@ static void unlock_slow(struct mutex *mutex)
 
     for (;;) {
         BUG_ON((owner & ~MUTEX_FLAGS_MASK) != (ptr_t)current);
+        BUG_ON(owner & MUTEX_FLAG_PICKUP_PENDING);
+
+        if (owner & MUTEX_FLAG_HANDOFF_REQUESTED)
+            break;
 
         flags = owner & MUTEX_FLAGS_MASK;
         if (atomic_cmpxchg_release(&mutex->owner, &owner, flags))
@@ -162,6 +217,9 @@ static void unlock_slow(struct mutex *mutex)
         task_ref(next);
     }
 
+    if (owner & MUTEX_FLAG_HANDOFF_REQUESTED)
+        hand_off_to(mutex, next);
+
     spin_unlock(&mutex->wait_lock);
 
     if (next == nullptr)
diff --git a/tests/test-cases/test_sched.c b/tests/test-cases/test_sched.c
index 6b303903..a0516ef5 100644
--- a/tests/test-cases/test_sched.c
+++ b/tests/test-cases/test_sched.c
@@ -2061,6 +2061,33 @@ TEST_CASE(mutex_waiters_get_the_lock_in_arrival_order)
     switching_test_end();
 }
 
+TEST_CASE(mutex_is_handed_to_a_waiter_that_lost_it_once)
+{
+    DEFINE_MUTEX(mutex);
+    struct mutex_user greedy = {
+        .id = 1, .mutex = &mutex, .rounds = 3, .yield_while_holding = true,
+    };
+    struct mutex_user waiter = { .id = 2, .mutex = &mutex, .rounds = 1 };
+
+    switching_test_begin();
+    spawn(lock_log_unlock, &greedy, TASK_PRIO_DEFAULT);
+    spawn(lock_log_unlock, &waiter, TASK_PRIO_DEFAULT);
+
+    schedule_idle();
+
+    ASSERT_EQ(s_log_length, 4);
+    ASSERT_EQ(s_log[0], 1);
+    ASSERT_EQ(s_log[1], 1);
+    ASSERT_EQ(s_log[2], 2);
+    ASSERT_EQ(s_log[3], 1);
+
+    ASSERT_EQ(mutex.owner, 0);
+    ASSERT(list_is_empty(&mutex.waiters));
+    ASSERT_EQ(g_sched_test_live_tasks, 0);
+
+    switching_test_end();
+}
+
 static void try_lock_and_log(void *user)
 {
     struct mutex_user *mutex_user = user;

From 8391401f3574f1a17853a5ef722985e267cee469 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 13:55:43 +0300
Subject: [PATCH 121/152] build: define ULTRA_MODULE_NAME for every kernel
 source

Either the name of the module passed to add_ultra_module(), or the
basename of a regular kernel source file. This will be used for module
parameters.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/module.cmake         |  1 +
 kernel/target_helpers.cmake | 11 +++++++++++
 2 files changed, 12 insertions(+)

diff --git a/kernel/module.cmake b/kernel/module.cmake
index 1baf49e9..64753e24 100644
--- a/kernel/module.cmake
+++ b/kernel/module.cmake
@@ -46,6 +46,7 @@ function(add_ultra_module)
         ${MODULE_OBJECT_TARGET}
         PRIVATE
         $<TARGET_PROPERTY:${ULTRA_KERNEL_IFACE},INTERFACE_COMPILE_DEFINITIONS>
+        ULTRA_MODULE_NAME="${MODULE_NAME}"
         ${MODULE_PRIVATE_DEFINITIONS}
     )
     target_include_directories(
diff --git a/kernel/target_helpers.cmake b/kernel/target_helpers.cmake
index a2ee3ced..9f3a1c83 100644
--- a/kernel/target_helpers.cmake
+++ b/kernel/target_helpers.cmake
@@ -54,6 +54,17 @@ function(ultra_sources)
         PRIVATE
         ${ARGN}
     )
+
+    foreach (SOURCE ${ARGN})
+        get_filename_component(SOURCE_NAME ${SOURCE} NAME_WE)
+        set_property(
+            SOURCE ${SOURCE}
+            TARGET_DIRECTORY ${ULTRA_KERNEL_OBJECTS}
+            APPEND
+            PROPERTY COMPILE_DEFINITIONS
+            ULTRA_MODULE_NAME="${SOURCE_NAME}"
+        )
+    endforeach ()
 endfunction()
 
 function(ultra_sources_if CONFIG)

From e2d9320e2d0979530096b1bd9f1dc98ce9ccb139 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 13:58:09 +0300
Subject: [PATCH 122/152] tests/test_harness: count printed messages

A test may need to check whether the code under test logged something.
vprint() now counts the messages each test thread prints.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 tests/kernel_helpers.cpp | 4 ++++
 tests/test_harness.h     | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/tests/kernel_helpers.cpp b/tests/kernel_helpers.cpp
index 8f31015c..cfc3cccc 100644
--- a/tests/kernel_helpers.cpp
+++ b/tests/kernel_helpers.cpp
@@ -34,11 +34,15 @@ void for_each_memory_map_range(
     UNREFERENCED_PARAMETER(user);
 }
 
+__thread size_t g_test_num_prints;
+
 void vprint(const char *msg, va_list vlist)
 {
     // TODO: save to a buffer and only show if a test fails
     UNREFERENCED_PARAMETER(msg);
     UNREFERENCED_PARAMETER(vlist);
+
+    g_test_num_prints++;
 }
 
 void print(const char *msg, ...)
diff --git a/tests/test_harness.h b/tests/test_harness.h
index be22248a..2a25cff7 100644
--- a/tests/test_harness.h
+++ b/tests/test_harness.h
@@ -13,6 +13,8 @@ struct test_case {
 extern "C" {
 #endif
 
+extern __thread size_t g_test_num_prints;
+
 void add_test_case(struct test_case *test, const char *file);
 void teardown_callback_register(void (*callback)(void), const char *file);
 

From fb9cd6a788746420043ab7f61b65d8f77113557e Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 13:58:09 +0300
Subject: [PATCH 123/152] param: don't warn for unknown dotted keys

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/param.c                    |  8 +++++++-
 tests/test-cases/test_parameter.c | 29 +++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/kernel/param.c b/kernel/param.c
index 2368b989..894982e8 100644
--- a/kernel/param.c
+++ b/kernel/param.c
@@ -384,7 +384,13 @@ struct string cmdline_parse_tables(
 
         p = find_param_in_tables(key, tables, num_tables);
         if (p == nullptr) {
-            warn_unknown_param(key, value);
+            /*
+             * A key with a dot in it belongs to whatever owns the part before
+             * the dot, e.g. a module that isn't loaded yet, so we don't warn
+             * about it.
+             */
+            if (str_find_one(key, '.', 0) == -1)
+                warn_unknown_param(key, value);
             goto do_next;
         }
 
diff --git a/tests/test-cases/test_parameter.c b/tests/test-cases/test_parameter.c
index b34811f7..064c568c 100644
--- a/tests/test-cases/test_parameter.c
+++ b/tests/test-cases/test_parameter.c
@@ -192,6 +192,35 @@ TEST_CASE(string_capacity)
     CMDLINE_PARSE_CHECKED();
 }
 
+TEST_CASE(unknown_keys_warn)
+{
+    size_t prints;
+
+    CMDLINE_TEST(
+        "foo qux=1 quux",
+        DEFINE_PARAM(bool, foo, false, true),
+    );
+
+    prints = g_test_num_prints;
+    CMDLINE_PARSE_CHECKED();
+    ASSERT_EQ(g_test_num_prints, prints + 2);
+}
+
+TEST_CASE(unknown_dotted_keys_are_skipped_silently)
+{
+    size_t prints;
+
+    CMDLINE_TEST(
+        "foo other.setting=1 other.flag foo.bar=2 bar",
+        DEFINE_PARAM(bool, foo, false, true),
+        DEFINE_PARAM(bool, bar, false, true),
+    );
+
+    prints = g_test_num_prints;
+    CMDLINE_PARSE_CHECKED();
+    ASSERT_EQ(g_test_num_prints, prints);
+}
+
 TEST_CASE(dashes_are_underscores)
 {
     CMDLINE_TEST(

From d0fc24b0189f1221478bd47181e6569a6841b516 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 6 Nov 2026 02:32:33 +0300
Subject: [PATCH 124/152] common/string_container: parenthesize MAKE_STR()

The braces of a compound literal don't protect its commas from the
preprocessor. A NULL_STR() or STR_CONSTEXPR() handed from one macro to
another was split into several arguments, the parentheses keep it one.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/common/string_container.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel/include/common/string_container.h b/kernel/include/common/string_container.h
index b03e3a08..d5ae7388 100644
--- a/kernel/include/common/string_container.h
+++ b/kernel/include/common/string_container.h
@@ -15,7 +15,7 @@ struct string {
     size_t size;
 };
 
-#define MAKE_STR(ptr, size) (struct string) { { (ptr) }, (size) }
+#define MAKE_STR(ptr, size) ((struct string) { { (ptr) }, (size) })
 #define NULL_STR() MAKE_STR(nullptr, 0)
 #define STR_CONSTEXPR(str) MAKE_STR(str, sizeof((str)) - 1)
 #define STR_RUNTIME(str) MAKE_STR(str, (str) ? strlen((str)) : 0)

From 50dc34511f2e3b7324adc23574b341b2afe7c0f5 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 6 Nov 2026 02:55:22 +0300
Subject: [PATCH 125/152] param: move the command line parsing API into
 private/param.h

The kernel parses its command line in entry.c and nowhere else, so
cmdline_parse(), cmdline_parse_tables() and struct param_table are not
something the author of a parameter uses.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/param.h         | 17 -----------------
 kernel/include/private/param.h | 17 +++++++++++++++++
 kernel/param.c                 |  1 +
 tests/CMakeLists.txt           |  1 +
 4 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/kernel/include/param.h b/kernel/include/param.h
index 94788fde..51804d7d 100644
--- a/kernel/include/param.h
+++ b/kernel/include/param.h
@@ -276,23 +276,6 @@ struct suboption {
 #define action_suboption(name, fn) \
     { PARAM_NAME(name), (fn), { nullptr, 0 }, SUBOPTION_ALLOWS_EMPTY_VALUE }
 
-struct param_table {
-    struct param *params;
-    size_t count;
-};
-
-/*
- * Parses the given command line against the given parameter tables, searched
- * in order, and returns the string (if any) after --. Unknown parameters and
- * bad values are logged. cmdline_parse is the single table form.
- */
-struct string cmdline_parse_tables(
-    struct string cmdline, const struct param_table *tables, size_t num_tables
-);
-struct string cmdline_parse(
-    struct string cmdline, struct param *params, size_t num_params
-);
-
 /*
  * Parses the sub-options of a parameter value, a list of key[=value] entries
  * separated by commas or the given separator, against the given table. An
diff --git a/kernel/include/private/param.h b/kernel/include/private/param.h
index b24538e2..6f44d3dc 100644
--- a/kernel/include/private/param.h
+++ b/kernel/include/private/param.h
@@ -5,3 +5,20 @@
 
 DECLARE_TYPED_SECTION_MARKERS(struct param, PARAMETERS_SECTION);
 DECLARE_TYPED_SECTION_MARKERS(struct param, FREE_AFTER_INIT_PARAMETERS_SECTION);
+
+struct param_table {
+    struct param *params;
+    size_t count;
+};
+
+/*
+ * Parses the given command line against the given parameter tables, searched
+ * in order, and returns the string (if any) after --. Unknown parameters and
+ * bad values are logged. cmdline_parse is the single table form.
+ */
+struct string cmdline_parse_tables(
+    struct string cmdline, const struct param_table *tables, size_t num_tables
+);
+struct string cmdline_parse(
+    struct string cmdline, struct param *params, size_t num_params
+);
diff --git a/kernel/param.c b/kernel/param.c
index 894982e8..811a384d 100644
--- a/kernel/param.c
+++ b/kernel/param.c
@@ -5,6 +5,7 @@
 #include <common/ctype.h>
 
 #include <param.h>
+#include <private/param.h>
 
 #define PARAM_SET_OPS_TEMPLATE(type)                              \
     error_t param_set_##type(                                     \
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index ae76062a..b473523b 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -100,6 +100,7 @@ KERNEL_FILE(
 KERNEL_FILE(INCLUDE_FILE "detail/log_ring.h")
 KERNEL_FILE(INCLUDE_FILE "smbios_structures.h")
 KERNEL_FILE(SOURCE_FILE "param.c"  INCLUDE_FILE "param.h")
+KERNEL_FILE(INCLUDE_FILE "private/param.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "helpers.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "types.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "attributes.h")

From 1ad90ce8b6731bcc882ee281fa0612c9fff9af9e Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 6 Nov 2026 02:47:01 +0300
Subject: [PATCH 126/152] param: move the macro internals into detail/param.h

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/param.h | 80 +++++++++++++++++++++++++++++++++
 kernel/include/param.h        | 83 ++---------------------------------
 tests/CMakeLists.txt          |  1 +
 3 files changed, 85 insertions(+), 79 deletions(-)
 create mode 100644 kernel/include/detail/param.h

diff --git a/kernel/include/detail/param.h b/kernel/include/detail/param.h
new file mode 100644
index 00000000..9f30ccf1
--- /dev/null
+++ b/kernel/include/detail/param.h
@@ -0,0 +1,80 @@
+#pragma once
+
+#include <common/helpers.h>
+#include <common/string_container.h>
+
+#include <free_after_init.h>
+
+/*
+ * Dispatch on the address so that a char array is seen as such along with
+ * its size, a plain char pointer intentionally matches nothing.
+ */
+#define PARAM_TYPE_OPS(value) _Generic(&(value), \
+    char (*)[sizeof(value)]: g_param_string_ops, \
+    i8*: g_param_i8_ops,                         \
+    u8*: g_param_u8_ops,                         \
+    i16*: g_param_i16_ops,                       \
+    u16*: g_param_u16_ops,                       \
+    i32*: g_param_i32_ops,                       \
+    u32*: g_param_u32_ops,                       \
+    i64*: g_param_i64_ops,                       \
+    u64*: g_param_u64_ops,                       \
+    bool*: g_param_bool_ops                      \
+)
+
+#define PARAM_CAPACITY(value) _Generic(&(value), \
+    char (*)[sizeof(value)]: sizeof(value),      \
+    default: 0                                   \
+)
+
+#define SUBOPTION_VARIANT_TABLE(...) \
+    CONCAT(SUBOPTION_VARIANT_TABLE_, GET_NUM_ARGS(__VA_ARGS__))(__VA_ARGS__)
+#define SUBOPTION_VARIANT_TABLE_0() nullptr, 0
+#define SUBOPTION_VARIANT_TABLE_1(table)                 \
+    (table), ARRAY_SIZE(table) + EMBED_STATIC_ASSERT(    \
+        !ARE_SAME_TYPE(table, &(table)[0]),              \
+        "a table given without a count must be an array" \
+    )
+#define SUBOPTION_VARIANT_TABLE_2(table, count) (table), (count)
+
+// The name of a parameter is that of its variable minus a g_ or s_ prefix
+#define PARAM_NAME(name)                                        \
+    __builtin_choose_expr(                                      \
+        __builtin_strncmp(#name, "g_", 2) == 0 ||               \
+        __builtin_strncmp(#name, "s_", 2) == 0,                 \
+        (struct string) { { &(#name)[2] }, sizeof(#name) - 3 }, \
+        STR_CONSTEXPR(#name)                                    \
+    )
+
+#define PARAM_ENTRY(name, value, ops, flags) \
+    { PARAM_NAME(name), &(ops), { &(value), PARAM_CAPACITY(value) }, (flags) }
+
+#define PARAM_ACTION_OPS(name, fn, placement)                        \
+    static const struct param_ops placement s_param_##name##_ops = { \
+        .allows_empty_value = true,                                  \
+        .set = (fn),                                                 \
+    }
+
+#define PARAM_ACTION_ENTRY(name, flags) \
+    { PARAM_NAME(name), &s_param_##name##_ops, { nullptr, 0 }, (flags) }
+
+#define PARAM_TYPE_SET(value) _Generic(&(value), \
+    char (*)[sizeof(value)]: param_set_string,   \
+    i8*: param_set_i8,                           \
+    u8*: param_set_u8,                           \
+    i16*: param_set_i16,                         \
+    u16*: param_set_u16,                         \
+    i32*: param_set_i32,                         \
+    u32*: param_set_u32,                         \
+    i64*: param_set_i64,                         \
+    u64*: param_set_u64,                         \
+    bool*: param_set_bool                        \
+)
+
+#define PARAM_TYPE_SUBOPTION_FLAGS(value) _Generic(&(value), \
+    bool*: SUBOPTION_ALLOWS_EMPTY_VALUE,                     \
+    default: SUBOPTION_FLAG_NONE                             \
+)
+
+#define SUBOPTION_ENTRY(name, value, set, flags) \
+    { PARAM_NAME(name), (set), { &(value), PARAM_CAPACITY(value) }, (flags) }
diff --git a/kernel/include/param.h b/kernel/include/param.h
index 51804d7d..775a741f 100644
--- a/kernel/include/param.h
+++ b/kernel/include/param.h
@@ -7,7 +7,8 @@
 
 #include <linker.h>
 #include <common/bit.h>
-#include <free_after_init.h>
+
+#include <detail/param.h>
 
 enum param_flags : u32 {
     /*
@@ -94,47 +95,6 @@ PARAMETER_OPS_DECL(u64);
 PARAMETER_OPS_DECL(bool)
 PARAMETER_OPS_DECL(string)
 
-/*
- * Dispatch on the address so that a char array is seen as such along with
- * its size, a plain char pointer intentionally matches nothing.
- */
-#define PARAM_TYPE_OPS(value) _Generic(&(value), \
-    char (*)[sizeof(value)]: g_param_string_ops, \
-    i8*: g_param_i8_ops,                         \
-    u8*: g_param_u8_ops,                         \
-    i16*: g_param_i16_ops,                       \
-    u16*: g_param_u16_ops,                       \
-    i32*: g_param_i32_ops,                       \
-    u32*: g_param_u32_ops,                       \
-    i64*: g_param_i64_ops,                       \
-    u64*: g_param_u64_ops,                       \
-    bool*: g_param_bool_ops                      \
-)
-
-#define PARAM_CAPACITY(value) _Generic(&(value), \
-    char (*)[sizeof(value)]: sizeof(value),      \
-    default: 0                                   \
-)
-
-#define SUBOPTION_VARIANT_TABLE(...) \
-    CONCAT(SUBOPTION_VARIANT_TABLE_, GET_NUM_ARGS(__VA_ARGS__))(__VA_ARGS__)
-#define SUBOPTION_VARIANT_TABLE_0() nullptr, 0
-#define SUBOPTION_VARIANT_TABLE_1(table)                 \
-    (table), ARRAY_SIZE(table) + EMBED_STATIC_ASSERT(    \
-        !ARE_SAME_TYPE(table, &(table)[0]),              \
-        "a table given without a count must be an array" \
-    )
-#define SUBOPTION_VARIANT_TABLE_2(table, count) (table), (count)
-
-// The name of a parameter is that of its variable minus a g_ or s_ prefix
-#define PARAM_NAME(name)                                        \
-    __builtin_choose_expr(                                      \
-        __builtin_strncmp(#name, "g_", 2) == 0 ||               \
-        __builtin_strncmp(#name, "s_", 2) == 0,                 \
-        (struct string) { { &(#name)[2] }, sizeof(#name) - 3 }, \
-        STR_CONSTEXPR(#name)                                    \
-    )
-
 /*
  * Parameters are set from the kernel command line as soon as it is available
  * at boot and can be read back if their ops provide a get callback. See enum
@@ -146,9 +106,6 @@ PARAMETER_OPS_DECL(string)
  * custom_parameter is the fully explicit form, the shorthands deduce the
  * name from the variable, the ops from its type, or both.
  */
-#define PARAM_ENTRY(name, value, ops, flags) \
-    { PARAM_NAME(name), &(ops), { &(value), PARAM_CAPACITY(value) }, (flags) }
-
 #define custom_parameter(name, value, ops, flags)               \
     SECTION_VAR(PARAMETERS_SECTION, static const, struct param) \
     s_param_##name = PARAM_ENTRY(name, value, ops, flags)
@@ -190,25 +147,14 @@ PARAMETER_OPS_DECL(string)
  * the parameter is given, including an empty one, and the parameter is never
  * exposed to readers. fn has the signature of the set callback.
  */
-#define PARAM_ACTION_OPS(name, fn, placement)                        \
-    static const struct param_ops placement s_param_##name##_ops = { \
-        .allows_empty_value = true,                                  \
-        .set = (fn),                                                 \
-    }
-#define param_action_ops(name, fn) PARAM_ACTION_OPS(name, fn, )
-#define init_param_action_ops(name, fn) PARAM_ACTION_OPS(name, fn, INIT_RODATA)
-
-#define PARAM_ACTION_ENTRY(name, flags) \
-    { PARAM_NAME(name), &s_param_##name##_ops, { nullptr, 0 }, (flags) }
-
 #define action_parameter_with_flags(name, fn, flags)            \
-    param_action_ops(name, fn);                                 \
+    PARAM_ACTION_OPS(name, fn, );                               \
     SECTION_VAR(PARAMETERS_SECTION, static const, struct param) \
     s_param_##name = PARAM_ACTION_ENTRY(name, flags)
 #define action_parameter(name, fn) action_parameter_with_flags(name, fn, 0)
 
 #define init_action_parameter(name, fn)                                \
-    init_param_action_ops(name, fn);                                   \
+    PARAM_ACTION_OPS(name, fn, INIT_RODATA);                           \
     SECTION_VAR(                                                       \
         FREE_AFTER_INIT_PARAMETERS_SECTION, static const, struct param \
     )                                                                  \
@@ -235,27 +181,6 @@ struct suboption {
     enum suboption_flags flags;
 };
 
-#define PARAM_TYPE_SET(value) _Generic(&(value), \
-    char (*)[sizeof(value)]: param_set_string,   \
-    i8*: param_set_i8,                           \
-    u8*: param_set_u8,                           \
-    i16*: param_set_i16,                         \
-    u16*: param_set_u16,                         \
-    i32*: param_set_i32,                         \
-    u32*: param_set_u32,                         \
-    i64*: param_set_i64,                         \
-    u64*: param_set_u64,                         \
-    bool*: param_set_bool                        \
-)
-
-#define PARAM_TYPE_SUBOPTION_FLAGS(value) _Generic(&(value), \
-    bool*: SUBOPTION_ALLOWS_EMPTY_VALUE,                     \
-    default: SUBOPTION_FLAG_NONE                             \
-)
-
-#define SUBOPTION_ENTRY(name, value, set, flags) \
-    { PARAM_NAME(name), (set), { &(value), PARAM_CAPACITY(value) }, (flags) }
-
 #define custom_suboption(name, value, set, flags) \
     SUBOPTION_ENTRY(name, value, set, flags)
 
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index b473523b..deb1ab8f 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -101,6 +101,7 @@ KERNEL_FILE(INCLUDE_FILE "detail/log_ring.h")
 KERNEL_FILE(INCLUDE_FILE "smbios_structures.h")
 KERNEL_FILE(SOURCE_FILE "param.c"  INCLUDE_FILE "param.h")
 KERNEL_FILE(INCLUDE_FILE "private/param.h")
+KERNEL_FILE(INCLUDE_FILE "detail/param.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "helpers.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "types.h")
 KERNEL_FILE(INCLUDE_PATH "common" INCLUDE_FILE "attributes.h")

From b804a621197c25a0f8f4a2a0e932a941ce5dd34d Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:01:25 +0300
Subject: [PATCH 127/152] param: add module parameters

Modules need parameters that do not clash with each other. The key is
module.name, so the parser needs no new table and plain parameters are
unaffected. Init parameters get no module variant: they belong to the
boot of the kernel itself and would mean nothing to a module loaded
after it. In a runtime module an init parameter is an unused entry, so
a driver that has one still builds either way.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/param.h     | 41 +++++++++++++++++--
 kernel/include/param.h            | 65 +++++++++++++++++++++----------
 kernel/param.c                    | 19 ++++++++-
 tests/test-cases/test_parameter.c | 62 ++++++++++++++++++++++++++++-
 4 files changed, 160 insertions(+), 27 deletions(-)

diff --git a/kernel/include/detail/param.h b/kernel/include/detail/param.h
index 9f30ccf1..b1f31e29 100644
--- a/kernel/include/detail/param.h
+++ b/kernel/include/detail/param.h
@@ -1,8 +1,10 @@
 #pragma once
 
 #include <common/helpers.h>
+#include <common/attributes.h>
 #include <common/string_container.h>
 
+#include <linker.h>
 #include <free_after_init.h>
 
 /*
@@ -46,8 +48,32 @@
         STR_CONSTEXPR(#name)                                    \
     )
 
-#define PARAM_ENTRY(name, value, ops, flags) \
-    { PARAM_NAME(name), &(ops), { &(value), PARAM_CAPACITY(value) }, (flags) }
+#define PARAM_NO_MODULE_NAME NULL_STR()
+#define PARAM_THIS_MODULE_NAME STR_CONSTEXPR(ULTRA_MODULE_NAME)
+
+#define PARAM_ENTRY(module_name, name, value, ops, flags) { \
+    PARAM_NAME(name), &(ops), (module_name),                \
+    { &(value), PARAM_CAPACITY(value) }, (flags)            \
+}
+
+#define DO_CUSTOM_PARAMETER(module_name, name, value, ops, flags)      \
+    SECTION_VAR(PARAMETERS_SECTION, static const, struct param)        \
+    s_param_##name = PARAM_ENTRY(module_name, name, value, ops, flags)
+
+#ifdef ULTRA_RUNTIME_MODULE
+
+#define INIT_PARAM_VAR(name)                                         \
+    static const struct param INIT_RODATA UNUSED_DECL s_param_##name
+
+#else
+
+#define INIT_PARAM_VAR(name)                                           \
+    SECTION_VAR(                                                       \
+        FREE_AFTER_INIT_PARAMETERS_SECTION, static const, struct param \
+    )                                                                  \
+    s_param_##name
+
+#endif
 
 #define PARAM_ACTION_OPS(name, fn, placement)                        \
     static const struct param_ops placement s_param_##name##_ops = { \
@@ -55,8 +81,15 @@
         .set = (fn),                                                 \
     }
 
-#define PARAM_ACTION_ENTRY(name, flags) \
-    { PARAM_NAME(name), &s_param_##name##_ops, { nullptr, 0 }, (flags) }
+#define PARAM_ACTION_ENTRY(module_name, name, flags) {      \
+    PARAM_NAME(name), &s_param_##name##_ops, (module_name), \
+    { nullptr, 0 }, (flags)                                 \
+}
+
+#define DO_ACTION_PARAMETER(module_name, name, fn, flags)         \
+    PARAM_ACTION_OPS(name, fn, );                                 \
+    SECTION_VAR(PARAMETERS_SECTION, static const, struct param)   \
+    s_param_##name = PARAM_ACTION_ENTRY(module_name, name, flags)
 
 #define PARAM_TYPE_SET(value) _Generic(&(value), \
     char (*)[sizeof(value)]: param_set_string,   \
diff --git a/kernel/include/param.h b/kernel/include/param.h
index 775a741f..4f16cd86 100644
--- a/kernel/include/param.h
+++ b/kernel/include/param.h
@@ -5,7 +5,6 @@
 #include <common/string_container.h>
 #include <common/error.h>
 
-#include <linker.h>
 #include <common/bit.h>
 
 #include <detail/param.h>
@@ -58,7 +57,11 @@ struct param {
     struct string name;
     const struct param_ops *ops;
 
-    // TODO: reference to the module defining this parameter
+    /*
+     * The name of the module a parameter belongs to (empty if it doesn't belong
+     * to one). The command line key of a module parameter is module_name.name.
+     */
+    struct string module_name;
 
     struct param_value value;
     u32 flags;
@@ -106,20 +109,26 @@ PARAMETER_OPS_DECL(string)
  * custom_parameter is the fully explicit form, the shorthands deduce the
  * name from the variable, the ops from its type, or both.
  */
-#define custom_parameter(name, value, ops, flags)               \
-    SECTION_VAR(PARAMETERS_SECTION, static const, struct param) \
-    s_param_##name = PARAM_ENTRY(name, value, ops, flags)
+#define custom_parameter(name, value, ops, flags)                      \
+    DO_CUSTOM_PARAMETER(PARAM_NO_MODULE_NAME, name, value, ops, flags)
+
+/*
+ * Every parameter macro other than the init ones has a module_ variant like
+ * this one, which defines a module parameter instead. Its command line key is
+ * module_name.name, where module_name is the ULTRA_MODULE_NAME of the source
+ * file that defines the parameter.
+ */
+#define module_custom_parameter(name, value, ops, flags)                 \
+    DO_CUSTOM_PARAMETER(PARAM_THIS_MODULE_NAME, name, value, ops, flags)
 
 /*
  * An init parameter is released after init together with everything it
  * references, its ops, its setter and whatever the setter uses may be INIT_*
  * as well. It is parsed at boot and neither written nor read afterwards.
  */
-#define custom_init_parameter(name, value, ops)                        \
-    SECTION_VAR(                                                       \
-        FREE_AFTER_INIT_PARAMETERS_SECTION, static const, struct param \
-    )                                                                  \
-    s_param_##name = PARAM_ENTRY(name, value, ops, 0)
+#define custom_init_parameter(name, value, ops)                \
+    INIT_PARAM_VAR(name) =                                     \
+        PARAM_ENTRY(PARAM_NO_MODULE_NAME, name, value, ops, 0)
 
 #define renamed_init_parameter(name, var) \
     custom_init_parameter(name, var, PARAM_TYPE_OPS(var))
@@ -129,36 +138,52 @@ PARAMETER_OPS_DECL(string)
 
 #define renamed_parameter_with_ops(name, var, ops) \
     custom_parameter(name, var, ops, 0)
+#define module_renamed_parameter_with_ops(name, var, ops) \
+    module_custom_parameter(name, var, ops, 0)
 
 #define renamed_parameter_with_flags(name, var, flags)      \
     custom_parameter(name, var, PARAM_TYPE_OPS(var), flags)
 #define renamed_parameter(name, var) renamed_parameter_with_flags(name, var, 0)
 
+#define module_renamed_parameter_with_flags(name, var, flags)      \
+    module_custom_parameter(name, var, PARAM_TYPE_OPS(var), flags)
+#define module_renamed_parameter(name, var)           \
+    module_renamed_parameter_with_flags(name, var, 0)
+
 #define parameter_with_ops_and_flags(var, ops, flags) \
     custom_parameter(var, var, ops, flags)
 #define parameter_with_ops(var, ops) parameter_with_ops_and_flags(var, ops, 0)
 
+#define module_parameter_with_ops_and_flags(var, ops, flags) \
+    module_custom_parameter(var, var, ops, flags)
+#define module_parameter_with_ops(var, ops)          \
+    module_parameter_with_ops_and_flags(var, ops, 0)
+
 #define parameter_with_flags(var, flags)          \
     renamed_parameter_with_flags(var, var, flags)
 #define parameter(var) parameter_with_flags(var, 0)
 
+#define module_parameter_with_flags(var, flags)          \
+    module_renamed_parameter_with_flags(var, var, flags)
+#define module_parameter(var) module_parameter_with_flags(var, 0)
+
 /*
  * Action parameters have no stored value, fn is called with whatever value
  * the parameter is given, including an empty one, and the parameter is never
  * exposed to readers. fn has the signature of the set callback.
  */
-#define action_parameter_with_flags(name, fn, flags)            \
-    PARAM_ACTION_OPS(name, fn, );                               \
-    SECTION_VAR(PARAMETERS_SECTION, static const, struct param) \
-    s_param_##name = PARAM_ACTION_ENTRY(name, flags)
+#define action_parameter_with_flags(name, fn, flags)           \
+    DO_ACTION_PARAMETER(PARAM_NO_MODULE_NAME, name, fn, flags)
 #define action_parameter(name, fn) action_parameter_with_flags(name, fn, 0)
 
-#define init_action_parameter(name, fn)                                \
-    PARAM_ACTION_OPS(name, fn, INIT_RODATA);                           \
-    SECTION_VAR(                                                       \
-        FREE_AFTER_INIT_PARAMETERS_SECTION, static const, struct param \
-    )                                                                  \
-    s_param_##name = PARAM_ACTION_ENTRY(name, 0)
+#define module_action_parameter_with_flags(name, fn, flags)      \
+    DO_ACTION_PARAMETER(PARAM_THIS_MODULE_NAME, name, fn, flags)
+#define module_action_parameter(name, fn)           \
+    module_action_parameter_with_flags(name, fn, 0)
+
+#define init_action_parameter(name, fn)                                      \
+    PARAM_ACTION_OPS(name, fn, INIT_RODATA);                                 \
+    INIT_PARAM_VAR(name) = PARAM_ACTION_ENTRY(PARAM_NO_MODULE_NAME, name, 0)
 
 enum suboption_flags : u32 {
     SUBOPTION_FLAG_NONE = 0,
diff --git a/kernel/param.c b/kernel/param.c
index 811a384d..69cce582 100644
--- a/kernel/param.c
+++ b/kernel/param.c
@@ -120,6 +120,23 @@ static bool cmdline_name_compare(char lhs, char rhs)
     return dash_to_underscore(lhs) == dash_to_underscore(rhs);
 }
 
+static bool param_matches(const struct param *p, struct string key)
+{
+    struct string module_name = p->module_name, name;
+
+    if (str_empty(module_name))
+        return str_equals_with_cb(p->name, key, cmdline_name_compare);
+
+    if (key.size <= module_name.size || key.text[module_name.size] != '.')
+        return false;
+
+    name = str_substring(key, module_name.size + 1, key.size);
+    key = str_substring(key, 0, module_name.size);
+
+    return str_equals_with_cb(module_name, key, cmdline_name_compare) &&
+           str_equals_with_cb(p->name, name, cmdline_name_compare);
+}
+
 static struct param *find_param(
     struct string name, struct param *params, size_t num_params
 )
@@ -130,7 +147,7 @@ static struct param *find_param(
     for (i = 0; i < num_params; i++) {
         p = &params[i];
 
-        if (str_equals_with_cb(p->name, name, cmdline_name_compare))
+        if (param_matches(p, name))
             return p;
     }
 
diff --git a/tests/test-cases/test_parameter.c b/tests/test-cases/test_parameter.c
index 064c568c..ef9a8225 100644
--- a/tests/test-cases/test_parameter.c
+++ b/tests/test-cases/test_parameter.c
@@ -46,6 +46,18 @@ struct test_param {
         },                                             \
     }
 
+#define DEFINE_MODULE_PARAM(type, mod, param_name, init, expected) \
+    {                                                              \
+        .value = { .as_##type = init, },                           \
+        .expect = { .as_##type = expected, },                      \
+        .stored_type = st_##type,                                  \
+        .param = {                                                 \
+            .name = STR(#param_name),                              \
+            .module_name = STR(#mod),                              \
+            .ops = &g_param_##type##_ops,                          \
+        },                                                         \
+    }
+
 #define CMDLINE_TEST(cmdline, ...)                        \
     struct test_param params[] = {                        \
         __VA_ARGS__                                       \
@@ -221,6 +233,48 @@ TEST_CASE(unknown_dotted_keys_are_skipped_silently)
     ASSERT_EQ(g_test_num_prints, prints);
 }
 
+TEST_CASE(module_parameters_are_keyed_by_module_and_name)
+{
+    size_t prints;
+
+    CMDLINE_TEST(
+        "demo.foo=5 bar demo.flag",
+        DEFINE_MODULE_PARAM(u8, demo, foo, 0, 5),
+        DEFINE_MODULE_PARAM(bool, demo, flag, false, true),
+        DEFINE_PARAM(bool, bar, false, true),
+    );
+
+    prints = g_test_num_prints;
+    CMDLINE_PARSE_CHECKED();
+    ASSERT_EQ(g_test_num_prints, prints);
+}
+
+TEST_CASE(module_parameters_need_their_prefix)
+{
+    size_t prints;
+
+    CMDLINE_TEST(
+        "foo=5 other.foo=6 demo.bar=1",
+        DEFINE_MODULE_PARAM(u8, demo, foo, 0, 0),
+        DEFINE_PARAM(bool, bar, false, false),
+    );
+
+    prints = g_test_num_prints;
+    CMDLINE_PARSE_CHECKED();
+    ASSERT_EQ(g_test_num_prints, prints + 1);
+}
+
+TEST_CASE(module_parameters_dashes_are_underscores)
+{
+    CMDLINE_TEST(
+        "my-demo.some_flag my_demo.other-flag=1",
+        DEFINE_MODULE_PARAM(bool, my_demo, some_flag, false, true),
+        DEFINE_MODULE_PARAM(bool, my_demo, other_flag, false, true),
+    );
+
+    CMDLINE_PARSE_CHECKED();
+}
+
 TEST_CASE(dashes_are_underscores)
 {
     CMDLINE_TEST(
@@ -499,8 +553,12 @@ TEST_CASE(suboptions_actions)
 TEST_CASE(tables_are_searched_in_order)
 {
     bool a = false, b = false;
-    struct param first[] = { PARAM_ENTRY(a, a, g_param_bool_ops, 0) };
-    struct param second[] = { PARAM_ENTRY(b, b, g_param_bool_ops, 0) };
+    struct param first[] = {
+        PARAM_ENTRY(PARAM_NO_MODULE_NAME, a, a, g_param_bool_ops, 0),
+    };
+    struct param second[] = {
+        PARAM_ENTRY(PARAM_NO_MODULE_NAME, b, b, g_param_bool_ops, 0),
+    };
     struct param_table tables[] = {
         { first, ARRAY_SIZE(first) }, { second, ARRAY_SIZE(second) },
     };

From 42845766215db2c22435aeb6f53140017fcd29b8 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:04:19 +0300
Subject: [PATCH 128/152] build: add --cmdline for extra kernel arguments

The image's command line was fixed, so trying a parameter meant editing
the script. --cmdline extends a baremetal job's command line as well,
which makes --baremetal-cmdline redundant, so build.py drops it.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 build.py                       | 28 ++++++++++++++++++++++------
 scripts/baremetal_ci/client.py | 17 ++++++++++-------
 2 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/build.py b/build.py
index 69b5cc7a..2a6d79cf 100755
--- a/build.py
+++ b/build.py
@@ -235,7 +235,7 @@ def build_ultra(
 
 
 QEMU_HYPER_OPTIONS = """
-cmdline = "log-level=debug earlycon=e9,colored"
+cmdline = "log-level=debug earlycon=e9,colored{cmdline_extra}"
 
 # Hardcode a resolution that's good enough for local testing,
 # we don't need a huge QEMU window.
@@ -250,7 +250,15 @@ def build_ultra(
 }
 
 
-def make_hyper_config(arch: str, target: str) -> str:
+# The QEMU image gets a fixed command line, extra arguments are appended
+def image_target_options(target: str, extra_cmdline: str) -> str:
+    if extra_cmdline:
+        extra_cmdline = " " + extra_cmdline
+
+    return IMAGE_TARGETS[target].replace("{cmdline_extra}", extra_cmdline)
+
+
+def make_hyper_config(arch: str, target: str, extra_cmdline: str) -> str:
     return \
 f"""
 default-entry = ultra-{arch}
@@ -266,7 +274,7 @@ def make_hyper_config(arch: str, target: str) -> str:
 page-table:
     levels = 5
     constraint = maximum
-{IMAGE_TARGETS[target]}
+{image_target_options(target, extra_cmdline)}
 [ultra-aarch64]
 protocol = ultra
 higher-half-exclusive = true
@@ -290,7 +298,8 @@ def get_kernel_path(execution_node: str, build_dir: str) -> str:
 def make_hyper_image(
     br_type: str, fs_type: str, arch: str, target: str, build_dir: str,
     hyper_installer: Optional[str], hyper_iso_br: Optional[str],
-    hyper_uefi_binaries: List[str], image_path: str, force_regenerate: bool
+    hyper_uefi_binaries: List[str], image_path: str, force_regenerate: bool,
+    extra_cmdline: str
 ) -> Optional[ultr.DiskImage]:
     kernel_path = get_kernel_path(arch, build_dir)
     image_root_path = os.path.join(build_dir, "image-root")
@@ -309,7 +318,7 @@ def make_hyper_image(
 
     return ultr.DiskImage(
         image_root_path, br_type, fs_type,
-        hyper_config=make_hyper_config(arch, target),
+        hyper_config=make_hyper_config(arch, target, extra_cmdline),
         hyper_uefi_binary_paths=hyper_uefi_binaries,
         hyper_iso_br_path=hyper_iso_br,
         hyper_installer_path=hyper_installer,
@@ -525,6 +534,9 @@ def main() -> None:
                        help="Produce a bootable image after building, "
                             "for QEMU or for real hardware with "
                             "'baremetal' (default: qemu)")
+    image.add_argument("--cmdline", type=str, default="",
+                       help="Extra kernel command line arguments (implies "
+                            "--make-image unless this is a baremetal run)")
     image.add_argument("--image-type", choices=["iso", "raw"], default="iso",
                        help="Image type to produce (with --make-image)")
     image.add_argument("--hyper-installer", type=str,
@@ -594,6 +606,10 @@ def main() -> None:
     if rc is not None:
         sys.exit(rc)
 
+    # A changed command line only reaches QEMU through a new image
+    if args.cmdline and args.machines is None and args.make_image is None:
+        args.make_image = "qemu"
+
     config_layers = cl.build_layers(args.config, args.arch)
     config_file = config_layers[-1]
     default_preset = cl.preset_path(cl.DEFAULT_PRESET)
@@ -688,7 +704,7 @@ def main() -> None:
         make_hyper_image(
             "MBR", fs_type, args.arch, image_target, build_dir,
             hyper_installer, hyper_iso_br, hyper_uefi_binary_paths,
-            image_path, args.make_image is not None
+            image_path, args.make_image is not None, args.cmdline
         )
 
     if should_run:
diff --git a/scripts/baremetal_ci/client.py b/scripts/baremetal_ci/client.py
index e69160b3..502b2482 100755
--- a/scripts/baremetal_ci/client.py
+++ b/scripts/baremetal_ci/client.py
@@ -141,12 +141,14 @@ def git_meta() -> Dict[str, str]:
 
 
 def add_run_args(target: argparse._ActionsContainer, prefix: str = "",
-                 run_for: bool = True, log_dir: bool = True) -> None:
-    target.add_argument(f"--{prefix}cmdline", dest="cmdline",
-                        metavar="ARGS",
-                        help="Extra kernel command line, appended "
-                             "after the machine's console line and "
-                             "the server's base")
+                 cmdline: bool = True, run_for: bool = True,
+                 log_dir: bool = True) -> None:
+    if cmdline:
+        target.add_argument(f"--{prefix}cmdline", dest="cmdline",
+                            metavar="ARGS",
+                            help="Extra kernel command line, appended "
+                                 "after the machine's console line and "
+                                 "the server's base")
     target.add_argument(f"--{prefix}timeout", dest="timeout",
                         type=float, metavar="SECONDS",
                         help="Give up on a run after this long, the "
@@ -181,7 +183,8 @@ def add_build_args(group: argparse._ActionsContainer) -> None:
                        metavar=("HOST", "CONFIG"),
                        help="Install or update the CI server on HOST "
                             "with the server CONFIG")
-    add_run_args(group, prefix="baremetal-", run_for=False, log_dir=False)
+    add_run_args(group, prefix="baremetal-", cmdline=False, run_for=False,
+                 log_dir=False)
 
 
 def require_server(args: argparse.Namespace,

From 44879b73b9902f4fef32c64099447342db610b04 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Mon, 5 Oct 2026 00:34:14 +0300
Subject: [PATCH 129/152] sched: add sched_nr_switches()

Let code outside the scheduler read how many context switches a CPU
has done without reaching into its runqueue.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/sched/sched.h | 3 +++
 kernel/sched/core.c          | 8 +++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/kernel/include/sched/sched.h b/kernel/include/sched/sched.h
index 63847731..5b7dc84e 100644
--- a/kernel/include/sched/sched.h
+++ b/kernel/include/sched/sched.h
@@ -10,3 +10,6 @@ void might_sleep(void);
 
 // Sleep until at least 'ns' nanoseconds have passed
 void sleep_ns(u64 ns);
+
+// How many context switches 'cpu' has done since boot
+u64 sched_nr_switches(cpu_t cpu);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 8fb67c28..5305a753 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -205,7 +205,8 @@ static void do_schedule(bool preempted)
         return;
     }
 
-    rq->nr_switches++;
+    // Read without the runqueue lock by sched_nr_switches()
+    atomic_store_relaxed(&rq->nr_switches, rq->nr_switches + 1);
     atomic_store_relaxed(&next->on_cpu, true);
 
     prev = arch_switch_to(prev, next);
@@ -421,6 +422,11 @@ void task_exit(void)
     BUG();
 }
 
+u64 sched_nr_switches(cpu_t cpu)
+{
+    return atomic_load_relaxed(&cpu_rq(cpu)->nr_switches);
+}
+
 static void rq_init(struct rq *rq, cpu_t cpu)
 {
     struct task *idle;

From 1d34fafd3f99aa818fd60f97bf7da1ae19ed12c2 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Mon, 5 Oct 2026 00:34:14 +0300
Subject: [PATCH 130/152] demo: add a fireworks scheduler stress demo

Nothing exercises the scheduler with many threads. Every rocket and
particle is a thread that sleeps for a frame between draws, which keeps
thousands of threads waking 60 times a second, and the log gets switch
rate and wakeup latency every two seconds.

Exclusive with the framebuffer console and built with --config
fireworks. fireworks.runtime-seconds=<n> makes it stop and print its
totals.

The demo is a tristate module, so it can be loaded at runtime once
the kernel has a module loader.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 configs/fireworks.config   |   4 +
 kernel/CMakeLists.txt      |   1 +
 kernel/Kconfig             |   1 +
 kernel/demo/CMakeLists.txt |   6 +
 kernel/demo/Kconfig        |  18 ++
 kernel/demo/fireworks.c    | 503 +++++++++++++++++++++++++++++++++++++
 6 files changed, 533 insertions(+)
 create mode 100644 configs/fireworks.config
 create mode 100644 kernel/demo/CMakeLists.txt
 create mode 100644 kernel/demo/Kconfig
 create mode 100644 kernel/demo/fireworks.c

diff --git a/configs/fireworks.config b/configs/fireworks.config
new file mode 100644
index 00000000..1135cf34
--- /dev/null
+++ b/configs/fireworks.config
@@ -0,0 +1,4 @@
+CONFIG_DEBUG_MM=y
+CONFIG_UBSAN=y
+# CONFIG_FLANTERM_CONSOLE is not set
+CONFIG_FIREWORKS_DEMO=y
diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt
index 3e0aaaa5..908185bd 100644
--- a/kernel/CMakeLists.txt
+++ b/kernel/CMakeLists.txt
@@ -174,6 +174,7 @@ add_subdirectory(memory)
 add_subdirectory(sched)
 add_subdirectory(time)
 add_subdirectory(drivers)
+add_subdirectory(demo)
 
 set(ULTRA_ALL_OBJECTS "")
 get_property(ULTRA_OBJECT_TARGETS_FINAL GLOBAL PROPERTY ULTRA_OBJECT_TARGETS)
diff --git a/kernel/Kconfig b/kernel/Kconfig
index 104f64f3..9590c192 100644
--- a/kernel/Kconfig
+++ b/kernel/Kconfig
@@ -96,3 +96,4 @@ endmenu
 rsource "arch/Kconfig"
 rsource "irq/Kconfig"
 rsource "drivers/Kconfig"
+rsource "demo/Kconfig"
diff --git a/kernel/demo/CMakeLists.txt b/kernel/demo/CMakeLists.txt
new file mode 100644
index 00000000..3ca80868
--- /dev/null
+++ b/kernel/demo/CMakeLists.txt
@@ -0,0 +1,6 @@
+add_ultra_module(
+    NAME fireworks
+    CONFIG FIREWORKS_DEMO
+    SOURCES
+        fireworks.c
+)
diff --git a/kernel/demo/Kconfig b/kernel/demo/Kconfig
new file mode 100644
index 00000000..58e0c05f
--- /dev/null
+++ b/kernel/demo/Kconfig
@@ -0,0 +1,18 @@
+menu "Demos"
+
+config FIREWORKS_DEMO
+    tristate "Fireworks scheduler stress demo"
+    depends on !FLANTERM_CONSOLE
+    help
+        Draw fireworks on the framebuffer once the kernel is up, with
+        one thread per rocket and per particle. Every thread plots its
+        pixel, sleeps for a frame and moves on, so a volley keeps a
+        few thousand threads sleeping and waking 60 times a second.
+        Statistics go to the log every couple of seconds. It runs until
+        the machine is turned off, or for fireworks.runtime-seconds=<n>
+        from the kernel command line.
+
+        The demo owns the framebuffer, which is why it cannot be
+        combined with the framebuffer log console.
+
+endmenu
diff --git a/kernel/demo/fireworks.c b/kernel/demo/fireworks.c
new file mode 100644
index 00000000..123f7a61
--- /dev/null
+++ b/kernel/demo/fireworks.c
@@ -0,0 +1,503 @@
+#define MSG_FMT(msg) "fireworks: " msg
+
+#include <common/types.h>
+#include <common/atomic.h>
+#include <common/bit.h>
+#include <common/minmax.h>
+
+#include <boot/boot.h>
+#include <memory/io.h>
+#include <memory/alloc.h>
+#include <time/time.h>
+#include <time/units.h>
+#include <sched/sched.h>
+#include <sched/kthread.h>
+
+#include <log.h>
+#include <param.h>
+#include <smp.h>
+#include <module.h>
+#include <free_after_init.h>
+
+#define FP_ONE ((i64)1 << 16)
+
+#define FRAMES_PER_SEC 60
+#define SIGNED_MS_PER_SEC ((i64)MS_PER_SEC)
+#define STATS_PERIOD_MS 2000
+#define DRAIN_POLL_PERIOD_MS 100
+
+// A dot is the screen height divided by this, but at least one pixel
+#define DOTS_PER_SCREEN_HEIGHT 384
+
+/*
+ * Speeds are in percent of the screen width or height per second, and gravity
+ * is in percent of the screen height per second squared.
+ */
+#define GRAVITY_PCT 5
+
+#define ROCKETS_PER_VOLLEY_MIN 1
+#define ROCKETS_PER_VOLLEY_MAX 20
+#define VOLLEY_INTERVAL_MIN_MS 2000
+#define VOLLEY_INTERVAL_MAX_MS 4000
+
+#define ROCKET_SIDEWAYS_SPEED_MAX_PCT 40
+#define ROCKET_RISE_SPEED_MIN_PCT 50
+#define ROCKET_RISE_SPEED_MAX_PCT 100
+#define ROCKET_LIFETIME_MIN_MS 500
+#define ROCKET_LIFETIME_MAX_MS 1000
+
+#define PARTICLES_PER_ROCKET_MIN 100
+#define PARTICLES_PER_ROCKET_MAX 200
+#define PARTICLE_SPEED_MIN_PCT 12
+#define PARTICLE_SPEED_MAX_PCT 25
+#define PARTICLE_LIFETIME_MIN_MS 2000
+#define PARTICLE_LIFETIME_MAX_MS 3000
+
+struct color {
+    u8 r, g, b;
+};
+
+#define BACKGROUND_COLOR ((struct color) { 0, 0, 0 })
+
+struct screen {
+    u8 *pixels;
+    u32 width, height, pitch;
+    u32 dot_size;
+    u8 r_shift, g_shift, b_shift;
+};
+
+/*
+ * A moving dot on the screen. Its position is in pixels and its velocity in
+ * pixels per second, both stored as 48.16 fixed point numbers.
+ */
+struct body {
+    i64 x, y;
+    i64 vx, vy;
+    struct color color;
+};
+
+struct particle_seed {
+    i64 x, y;
+    i64 speed;
+};
+
+static io_window s_fb_window;
+static struct screen s_screen;
+static i64 s_gravity;
+
+static u64 s_rand_state;
+
+static u32 s_runtime_seconds;
+module_parameter(s_runtime_seconds);
+
+static struct task *s_stats_task;
+
+static u32 s_num_alive;
+static u64 s_num_spawned;
+static u64 s_num_failed;
+static u64 s_worst_lateness_ns;
+
+// SplitMix64 by Steele, Lea and Flood (OOPSLA 2014)
+static u32 rand_u32(void)
+{
+    u64 z;
+
+    z = atomic_fetch_add(&s_rand_state, 0x9E3779B97F4A7C15ull, MO_RELAXED);
+    z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
+    z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
+
+    return (z ^ (z >> 31)) >> 32;
+}
+
+static u32 rand_below(u32 limit)
+{
+    return rand_u32() % limit;
+}
+
+static i64 rand_between(i64 first, i64 last)
+{
+    return first + rand_below(last - first + 1);
+}
+
+// A random bright color, with every channel at least half of its maximum
+static struct color rand_color(void)
+{
+    struct color color;
+
+    color.r = rand_between(UNSIGNED_HALF_MAX(u8), UNSIGNED_MAX(u8));
+    color.g = rand_between(UNSIGNED_HALF_MAX(u8), UNSIGNED_MAX(u8));
+    color.b = rand_between(UNSIGNED_HALF_MAX(u8), UNSIGNED_MAX(u8));
+
+    return color;
+}
+
+// 'pct' percent of 'length' pixels, as a 48.16 fixed point number
+static i64 percent_of(u32 length, u32 pct)
+{
+    return (i64)length * pct * FP_ONE / 100;
+}
+
+static u32 pixel_value(struct color color)
+{
+    struct screen *screen = &s_screen;
+    u32 value;
+
+    value = (u32)color.r << screen->r_shift;
+    value |= (u32)color.g << screen->g_shift;
+    value |= (u32)color.b << screen->b_shift;
+
+    return value;
+}
+
+static u32 *screen_row(u32 y)
+{
+    return (u32*)(s_screen.pixels + y * s_screen.pitch);
+}
+
+static void plot(i64 x, i64 y, struct color color)
+{
+    struct screen *screen = &s_screen;
+    u32 *row, value, dx, dy;
+
+    if (x < 0 || y < 0)
+        return;
+    if (x + screen->dot_size > screen->width)
+        return;
+    if (y + screen->dot_size > screen->height)
+        return;
+
+    value = pixel_value(color);
+
+    for (dy = 0; dy < screen->dot_size; dy++) {
+        row = screen_row(y + dy);
+
+        for (dx = 0; dx < screen->dot_size; dx++)
+            row[x + dx] = value;
+    }
+}
+
+static void note_lateness(u64 lateness_ns)
+{
+    u64 worst;
+
+    worst = atomic_load_relaxed(&s_worst_lateness_ns);
+
+    while (lateness_ns > worst) {
+        if (atomic_cmpxchg_relaxed(&s_worst_lateness_ns, &worst, lateness_ns))
+            break;
+    }
+}
+
+static void sleep_for_a_frame(u32 ms)
+{
+    u64 before, slept_ns, wanted_ns;
+
+    wanted_ns = MS_TO_NS(ms);
+    before = ns_since_boot();
+    sleep_ns(wanted_ns);
+    slept_ns = ns_since_boot() - before;
+
+    note_lateness(slept_ns - wanted_ns);
+}
+
+static void body_fly(struct body *body, u32 lifetime_ms)
+{
+    u32 frame_ms, elapsed_ms = 0, frame = 0;
+    i64 x, y;
+
+    while (elapsed_ms < lifetime_ms) {
+        x = body->x / FP_ONE;
+        y = body->y / FP_ONE;
+
+        /*
+         * Frame number 'frame' ends frame / FRAMES_PER_SEC seconds after the
+         * first frame started, rounded down to a whole millisecond.
+         */
+        frame++;
+        frame_ms = frame * MS_PER_SEC / FRAMES_PER_SEC - elapsed_ms;
+
+        plot(x, y, body->color);
+        sleep_for_a_frame(frame_ms);
+        plot(x, y, BACKGROUND_COLOR);
+
+        body->x += body->vx * frame_ms / SIGNED_MS_PER_SEC;
+        body->y += body->vy * frame_ms / SIGNED_MS_PER_SEC;
+        body->vy += s_gravity * frame_ms / SIGNED_MS_PER_SEC;
+
+        elapsed_ms += frame_ms;
+    }
+}
+
+static void particle_thread(void *user)
+{
+    struct particle_seed *seed = user;
+    struct body body;
+    i64 dx, dy;
+
+    /*
+     * Pick a random point inside the unit circle. Its direction and its
+     * distance from the center become the direction and the speed of the
+     * particle.
+     */
+    do {
+        dx = rand_between(-FP_ONE, FP_ONE);
+        dy = rand_between(-FP_ONE, FP_ONE);
+    } while (dx * dx + dy * dy > FP_ONE * FP_ONE);
+
+    body.x = seed->x;
+    body.y = seed->y;
+    body.vx = dx * seed->speed / FP_ONE;
+    body.vy = dy * seed->speed / FP_ONE;
+    body.color = rand_color();
+
+    free(seed);
+
+    body_fly(
+        &body,
+        rand_between(PARTICLE_LIFETIME_MIN_MS, PARTICLE_LIFETIME_MAX_MS)
+    );
+    atomic_fetch_sub(&s_num_alive, 1, MO_RELAXED);
+}
+
+static bool spawn(task_fn_t fn, void *user, const char *name)
+{
+    error_t ret;
+
+    atomic_fetch_add(&s_num_alive, 1, MO_RELAXED);
+
+    ret = kthread_run(fn, user, "%s", name);
+    if (is_error(ret)) {
+        atomic_fetch_sub(&s_num_alive, 1, MO_RELAXED);
+        atomic_fetch_add(&s_num_failed, 1, MO_RELAXED);
+        return false;
+    }
+
+    atomic_fetch_add(&s_num_spawned, 1, MO_RELAXED);
+    return true;
+}
+
+static void explode(struct body *rocket)
+{
+    struct particle_seed *seed;
+    u32 i, num_particles;
+    i64 speed;
+
+    num_particles = rand_between(
+        PARTICLES_PER_ROCKET_MIN, PARTICLES_PER_ROCKET_MAX
+    );
+    speed = rand_between(
+        percent_of(s_screen.height, PARTICLE_SPEED_MIN_PCT),
+        percent_of(s_screen.height, PARTICLE_SPEED_MAX_PCT)
+    );
+
+    for (i = 0; i < num_particles; i++) {
+        seed = alloc(sizeof(*seed), ALLOC_GENERIC);
+        if (seed == nullptr) {
+            atomic_fetch_add(&s_num_failed, 1, MO_RELAXED);
+            continue;
+        }
+
+        seed->x = rocket->x;
+        seed->y = rocket->y;
+        seed->speed = speed;
+
+        if (!spawn(particle_thread, seed, "particle"))
+            free(seed);
+    }
+}
+
+static void rocket_thread(void *user)
+{
+    struct screen *screen = &s_screen;
+    struct body body;
+    i64 max_vx;
+
+    UNREFERENCED_PARAMETER(user);
+
+    max_vx = percent_of(screen->width, ROCKET_SIDEWAYS_SPEED_MAX_PCT);
+
+    body.x = (i64)screen->width / 2 * FP_ONE;
+    body.y = ((i64)screen->height - 1) * FP_ONE;
+    body.vx = rand_between(-max_vx, max_vx);
+    body.vy = -rand_between(
+        percent_of(screen->height, ROCKET_RISE_SPEED_MIN_PCT),
+        percent_of(screen->height, ROCKET_RISE_SPEED_MAX_PCT)
+    );
+    body.color = rand_color();
+
+    body_fly(
+        &body, rand_between(ROCKET_LIFETIME_MIN_MS, ROCKET_LIFETIME_MAX_MS)
+    );
+    explode(&body);
+
+    atomic_fetch_sub(&s_num_alive, 1, MO_RELAXED);
+}
+
+static u64 count_switches(void)
+{
+    u64 total = 0;
+    cpu_t cpu;
+
+    for_each_present_cpu(cpu)
+        total += sched_nr_switches(cpu);
+
+    return total;
+}
+
+static void stats_thread(void *user)
+{
+    u64 switches, prev_switches, worst_ns;
+
+    UNREFERENCED_PARAMETER(user);
+
+    prev_switches = count_switches();
+
+    while (!kthread_should_stop()) {
+        sleep_ns(MS_TO_NS(STATS_PERIOD_MS));
+
+        switches = count_switches();
+        worst_ns = atomic_xchg(&s_worst_lateness_ns, 0, MO_RELAXED);
+
+        pr_info(
+            "%u threads, %llu spawned, %llu failed, %llu switches/s, "
+            "worst wakeup %llu us late\n",
+            atomic_load_relaxed(&s_num_alive),
+            atomic_load_relaxed(&s_num_spawned),
+            atomic_load_relaxed(&s_num_failed),
+            (switches - prev_switches) * MS_PER_SEC / STATS_PERIOD_MS,
+            NS_TO_US(worst_ns)
+        );
+
+        prev_switches = switches;
+    }
+}
+
+static void launcher_thread(void *user)
+{
+    u32 i, num_rockets;
+    u64 end_ns = 0, interval_ms;
+
+    UNREFERENCED_PARAMETER(user);
+
+    if (s_runtime_seconds != 0)
+        end_ns = ns_from_now(SEC_TO_NS(s_runtime_seconds));
+
+    for (;;) {
+        num_rockets = rand_between(
+            ROCKETS_PER_VOLLEY_MIN, ROCKETS_PER_VOLLEY_MAX
+        );
+
+        for (i = 0; i < num_rockets; i++)
+            spawn(rocket_thread, nullptr, "rocket");
+
+        interval_ms = rand_between(
+            VOLLEY_INTERVAL_MIN_MS, VOLLEY_INTERVAL_MAX_MS
+        );
+        sleep_ns(MS_TO_NS(interval_ms));
+
+        if (end_ns != 0 && ns_since_boot() >= end_ns)
+            break;
+    }
+
+    while (atomic_load_relaxed(&s_num_alive) != 0)
+        sleep_ns(MS_TO_NS(DRAIN_POLL_PERIOD_MS));
+
+    kthread_stop(s_stats_task);
+
+    pr_info(
+        "done after %u s: %llu spawned, %llu failed\n", s_runtime_seconds,
+        atomic_load_relaxed(&s_num_spawned),
+        atomic_load_relaxed(&s_num_failed)
+    );
+}
+
+static void INIT_CODE screen_clear(void)
+{
+    struct screen *screen = &s_screen;
+    u32 *row, value, x, y;
+
+    value = pixel_value(BACKGROUND_COLOR);
+
+    for (y = 0; y < screen->height; y++) {
+        row = screen_row(y);
+
+        for (x = 0; x < screen->width; x++)
+            row[x] = value;
+    }
+}
+
+static error_t INIT_CODE screen_init(void)
+{
+    struct screen *screen = &s_screen;
+    struct ultra_framebuffer *fb;
+    error_t ret;
+
+    if (!g_boot_ctx.fb) {
+        pr_notice("no framebuffer available\n");
+        return ENODEV;
+    }
+
+    fb = &g_boot_ctx.fb->fb;
+
+    switch (fb->format) {
+    case ULTRA_FB_FORMAT_RGBX8888:
+        screen->r_shift = 24;
+        screen->g_shift = 16;
+        screen->b_shift = 8;
+        break;
+    case ULTRA_FB_FORMAT_XRGB8888:
+        screen->r_shift = 16;
+        screen->g_shift = 8;
+        screen->b_shift = 0;
+        break;
+    default:
+        pr_warn(
+            "unsupported framebuffer format %u (%u bpp)\n",
+            fb->format, fb->bpp
+        );
+        return ENOSYS;
+    }
+
+    ret = io_window_map_wc(
+        fb->physical_address, fb->pitch * fb->height, &s_fb_window
+    );
+    if (is_error(ret))
+        return ret;
+
+    screen->pixels = io_window_raw_ptr(&s_fb_window);
+    screen->width = fb->width;
+    screen->height = fb->height;
+    screen->pitch = fb->pitch;
+    screen->dot_size = MAX(fb->height / DOTS_PER_SCREEN_HEIGHT, 1u);
+
+    screen_clear();
+    return EOK;
+}
+
+static error_t INIT_CODE fireworks_init(void)
+{
+    error_t ret;
+
+    ret = screen_init();
+    if (is_error(ret))
+        return ret == ENODEV ? EOK : ret;
+
+    s_gravity = percent_of(s_screen.height, GRAVITY_PCT);
+    s_rand_state = ns_since_boot();
+
+    s_stats_task = kthread_create(stats_thread, nullptr, "fireworks-stats");
+    if (s_stats_task == nullptr)
+        return ENOMEM;
+
+    pr_info("%ux%u\n", s_screen.width, s_screen.height);
+
+    ret = kthread_run(launcher_thread, nullptr, "fireworks");
+    if (is_error(ret)) {
+        kthread_stop(s_stats_task);
+        return ret;
+    }
+
+    kthread_start(s_stats_task);
+    return EOK;
+}
+MODULE_INIT(fireworks_init);

From be2efebf0d84dfbbcedfc87211e46729ddcab59c Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 8 Nov 2026 15:50:53 +0300
Subject: [PATCH 131/152] kthread: add kthread_wait_for_exit()

Only kthread_stop() could wait for a kernel thread to exit, and it also
asks the thread to stop. Add kthread_wait_for_exit() for a thread that
ends on its own, and use it in kthread_stop().

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/sched/kthread.h | 7 +++++++
 kernel/sched/kthread.c         | 7 ++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/kernel/include/sched/kthread.h b/kernel/include/sched/kthread.h
index f016b074..37c3e67f 100644
--- a/kernel/include/sched/kthread.h
+++ b/kernel/include/sched/kthread.h
@@ -42,6 +42,13 @@ error_t kthread_run(task_fn_t fn, void *user, const char *name_fmt, ...);
 NORETURN
 void kthread_exit(void);
 
+/*
+ * Wait until a kernel thread that was started has exited. The caller must
+ * hold a reference to the task, e.g. the one from kthread_create(), and
+ * still owns it afterwards.
+ */
+void kthread_wait_for_exit(struct task *task);
+
 /*
  * Ask a kernel thread to stop and wait until it has exited. The thread
  * has to cooperate by checking kthread_should_stop() whenever it is
diff --git a/kernel/sched/kthread.c b/kernel/sched/kthread.c
index df0815c7..b01ccd6f 100644
--- a/kernel/sched/kthread.c
+++ b/kernel/sched/kthread.c
@@ -97,11 +97,16 @@ void kthread_did_exit(struct task *task)
     complete_all(&task->kthread.exited);
 }
 
+void kthread_wait_for_exit(struct task *task)
+{
+    wait_for_completion(&task->kthread.exited);
+}
+
 void kthread_stop(struct task *task)
 {
     task_set_flag(task, TASK_FLAG_KTHREAD_SHOULD_STOP);
     wake_up_task(task, TASK_STATE_NEW | TASK_STATE_UNINTERRUPTIBLE);
-    wait_for_completion(&task->kthread.exited);
+    kthread_wait_for_exit(task);
 
     task_unref(task);
 }

From d7e07e2a97d40092962cfdd94bcbe5bf58beef21 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 10 Nov 2026 22:12:33 +0300
Subject: [PATCH 132/152] common/helpers: add CLASSIFY_TYPE()

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/common/helpers.h | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/kernel/include/common/helpers.h b/kernel/include/common/helpers.h
index fc06045a..60507648 100644
--- a/kernel/include/common/helpers.h
+++ b/kernel/include/common/helpers.h
@@ -30,6 +30,13 @@
 #define IS_POWER_OF_TWO_OR_ZERO(x) (((x) & ((x) - 1)) == 0)
 #define IS_POWER_OF_TWO(x) ((x) != 0 && IS_POWER_OF_TWO_OR_ZERO(x))
 
+#define CLASSIFY_TYPE(x) __builtin_classify_type(x)
+
+#define TYPE_CLASS_INTEGER 1
+#define TYPE_CLASS_POINTER 5
+#define TYPE_CLASS_STRUCT 12
+#define TYPE_CLASS_UNION 13
+
 #define DO_CONTAINER_OF(ptr, ptr_name, type, member) ({                    \
     char *ptr_name = (char*)(ptr);                                         \
     BUILD_BUG_ON(!ARE_SAME_TYPE(*(ptr), ((type*)sizeof(type))->member) &&  \

From c30dbd058cd5654d57665978a7bcab00da4ddd53 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:15:29 +0300
Subject: [PATCH 133/152] ktest: introduce the in-kernel test runner

A framework for testing kernel features that require the real kernel
environment.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 configs/ktest.config                   |   3 +
 kernel/CMakeLists.txt                  |   1 +
 kernel/Kconfig                         |   1 +
 kernel/include/detail/ktest/ktest.h    | 150 ++++++++++
 kernel/include/detail/sched/task.h     |   6 +
 kernel/include/ktest/ktest.h           |  52 ++++
 kernel/include/linker.h                |   1 +
 kernel/include/linker_common.ld.h      |  11 +-
 kernel/include/private/ktest/task.h    |  28 ++
 kernel/include/private/sched/kthread.h |   3 +
 kernel/ktest/CMakeLists.txt            |   9 +
 kernel/ktest/Kconfig                   |  23 ++
 kernel/ktest/ktest.c                   | 387 +++++++++++++++++++++++++
 kernel/ktest/selftest.c                |  50 ++++
 kernel/sched/kthread.c                 |   8 +
 kernel/sched/task.c                    |   4 +
 tests/CMakeLists.txt                   |   1 +
 17 files changed, 733 insertions(+), 5 deletions(-)
 create mode 100644 configs/ktest.config
 create mode 100644 kernel/include/detail/ktest/ktest.h
 create mode 100644 kernel/include/ktest/ktest.h
 create mode 100644 kernel/include/private/ktest/task.h
 create mode 100644 kernel/ktest/CMakeLists.txt
 create mode 100644 kernel/ktest/Kconfig
 create mode 100644 kernel/ktest/ktest.c
 create mode 100644 kernel/ktest/selftest.c

diff --git a/configs/ktest.config b/configs/ktest.config
new file mode 100644
index 00000000..0e0cffc6
--- /dev/null
+++ b/configs/ktest.config
@@ -0,0 +1,3 @@
+CONFIG_DEBUG_MM=y
+CONFIG_UBSAN=y
+CONFIG_KTEST=y
diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt
index 908185bd..2026d96c 100644
--- a/kernel/CMakeLists.txt
+++ b/kernel/CMakeLists.txt
@@ -174,6 +174,7 @@ add_subdirectory(memory)
 add_subdirectory(sched)
 add_subdirectory(time)
 add_subdirectory(drivers)
+add_subdirectory(ktest)
 add_subdirectory(demo)
 
 set(ULTRA_ALL_OBJECTS "")
diff --git a/kernel/Kconfig b/kernel/Kconfig
index 9590c192..93e594ec 100644
--- a/kernel/Kconfig
+++ b/kernel/Kconfig
@@ -96,4 +96,5 @@ endmenu
 rsource "arch/Kconfig"
 rsource "irq/Kconfig"
 rsource "drivers/Kconfig"
+rsource "ktest/Kconfig"
 rsource "demo/Kconfig"
diff --git a/kernel/include/detail/ktest/ktest.h b/kernel/include/detail/ktest/ktest.h
new file mode 100644
index 00000000..25a48e14
--- /dev/null
+++ b/kernel/include/detail/ktest/ktest.h
@@ -0,0 +1,150 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/attributes.h>
+#include <common/helpers.h>
+#include <common/string_container.h>
+
+/*
+ * A test that needs a running kernel. Each case runs in a thread of its own and
+ * may sleep, take locks and start threads.
+ */
+struct ktest_case {
+    // The module name of the source file that defines the case
+    struct string suite;
+    struct string name;
+    void (*fn)(void);
+};
+
+PRINTF_DECL(3, 4)
+void ktest_fail(const char *file, u32 line, const char *fmt, ...);
+
+NORETURN
+void ktest_abort(void);
+
+enum ktest_value_type : u8 {
+    KTEST_VALUE_INTEGER,
+    KTEST_VALUE_POINTER,
+    KTEST_VALUE_STRING,
+};
+
+struct ktest_value {
+    enum ktest_value_type type;
+    bool is_signed;
+
+    union {
+        i64 as_signed;
+        u64 as_unsigned;
+        const void *as_pointer;
+        struct string as_string;
+    };
+};
+
+static inline struct ktest_value ktest_value_signed(i64 value)
+{
+    return (struct ktest_value) {
+        .type = KTEST_VALUE_INTEGER,
+        .is_signed = true,
+        .as_signed = value,
+    };
+}
+
+static inline struct ktest_value ktest_value_unsigned(u64 value)
+{
+    return (struct ktest_value) {
+        .type = KTEST_VALUE_INTEGER,
+        .as_unsigned = value,
+    };
+}
+
+static inline struct ktest_value ktest_value_pointer(const void *value)
+{
+    return (struct ktest_value) {
+        .type = KTEST_VALUE_POINTER,
+        .as_pointer = value,
+    };
+}
+
+static inline struct ktest_value ktest_value_string(struct string value)
+{
+    return (struct ktest_value) {
+        .type = KTEST_VALUE_STRING,
+        .as_string = value,
+    };
+}
+
+int ktest_compare(const struct ktest_value *lhs, const struct ktest_value *rhs);
+
+void ktest_fail_cmp(
+    const char *file, u32 line, const char *expr, const struct ktest_value *lhs,
+    const struct ktest_value *rhs
+);
+
+#define KTEST_VALUE_CTOR(x) _Generic((x),                              \
+    bool: ktest_value_unsigned,                                        \
+    char: CHOOSE_EXPR(                                                 \
+        IS_SIGNED_TYPE(char), ktest_value_signed, ktest_value_unsigned \
+    ),                                                                 \
+    signed char: ktest_value_signed,                                   \
+    short: ktest_value_signed,                                         \
+    int: ktest_value_signed,                                           \
+    long: ktest_value_signed,                                          \
+    long long: ktest_value_signed,                                     \
+    unsigned char: ktest_value_unsigned,                               \
+    unsigned short: ktest_value_unsigned,                              \
+    unsigned int: ktest_value_unsigned,                                \
+    unsigned long: ktest_value_unsigned,                               \
+    unsigned long long: ktest_value_unsigned,                          \
+    struct string: ktest_value_string,                                 \
+    default: ktest_value_pointer                                       \
+)
+
+#define KTEST_VALUE_TYPE_OF(x) _Generic((x),     \
+    struct string: KTEST_VALUE_STRING,           \
+    typeof(nullptr): KTEST_VALUE_POINTER,        \
+    default: CHOOSE_EXPR(                        \
+        CLASSIFY_TYPE(x) == TYPE_CLASS_POINTER,  \
+        KTEST_VALUE_POINTER, KTEST_VALUE_INTEGER \
+    )                                            \
+)
+
+#define KTEST_IS_C_STRING(x)                                        \
+    _Generic((x), char *: true, const char *: true, default: false)
+
+#define KTEST_CHECK_OPERANDS(lhs, op, rhs)                              \
+    STATIC_ASSERT(                                                      \
+        !KTEST_IS_C_STRING(lhs) && !KTEST_IS_C_STRING(rhs),             \
+        "C strings must be passed as STR()"                             \
+    );                                                                  \
+    STATIC_ASSERT(                                                      \
+        KTEST_VALUE_TYPE_OF(lhs) == KTEST_VALUE_TYPE_OF(rhs),           \
+        "operands must both be integers, both pointers or both strings" \
+    );                                                                  \
+    STATIC_ASSERT(                                                      \
+        KTEST_VALUE_TYPE_OF(lhs) != KTEST_VALUE_STRING ||               \
+        __builtin_strcmp(#op, "==") == 0 ||                             \
+        __builtin_strcmp(#op, "!=") == 0,                               \
+        "strings only support == and !="                                \
+    )
+
+#define KTEST_CMP(lhs, op, rhs, lhs_text, rhs_text, on_failure) do {       \
+    struct ktest_value ktest_lhs, ktest_rhs;                               \
+                                                                           \
+    KTEST_CHECK_OPERANDS(lhs, op, rhs);                                    \
+                                                                           \
+    ktest_lhs = KTEST_VALUE_CTOR(lhs)(lhs);                                \
+    ktest_rhs = KTEST_VALUE_CTOR(rhs)(rhs);                                \
+                                                                           \
+    if (unlikely(!(ktest_compare(&ktest_lhs, &ktest_rhs) op 0))) {         \
+        ktest_fail_cmp(                                                    \
+            __FILE__, __LINE__, lhs_text " " #op " " rhs_text, &ktest_lhs, \
+            &ktest_rhs                                                     \
+        );                                                                 \
+        on_failure;                                                        \
+    }                                                                      \
+} while (0)
+
+#define KTEST_CMP_EXPECT(lhs, op, rhs, lhs_text, rhs_text) \
+    KTEST_CMP(lhs, op, rhs, lhs_text, rhs_text, )
+#define KTEST_CMP_ASSERT(lhs, op, rhs, lhs_text, rhs_text)     \
+    KTEST_CMP(lhs, op, rhs, lhs_text, rhs_text, ktest_abort())
diff --git a/kernel/include/detail/sched/task.h b/kernel/include/detail/sched/task.h
index 62bd1af9..4ea3f567 100644
--- a/kernel/include/detail/sched/task.h
+++ b/kernel/include/detail/sched/task.h
@@ -20,6 +20,7 @@
 #endif
 
 struct sched_class;
+struct ktest_run;
 
 #define TASK_NAME_MAX 32
 
@@ -62,6 +63,11 @@ struct task {
         struct kthread kthread;
     };
 
+#if IS_ENABLED(KTEST)
+    // The test run this thread reports to, inherited by the threads it creates
+    struct ktest_run *ktest_run;
+#endif
+
     struct arch_task arch;
 };
 
diff --git a/kernel/include/ktest/ktest.h b/kernel/include/ktest/ktest.h
new file mode 100644
index 00000000..3aa7d0ab
--- /dev/null
+++ b/kernel/include/ktest/ktest.h
@@ -0,0 +1,52 @@
+#pragma once
+
+#include <common/types.h>
+#include <common/attributes.h>
+#include <common/helpers.h>
+
+#include <linker.h>
+
+#include <detail/ktest/ktest.h>
+
+#define KTEST_CASE(fn_name)                                           \
+    static void fn_name(void);                                        \
+    SECTION_VAR(KTEST_CASES_SECTION, static const, struct ktest_case) \
+    s_ktest_case_##fn_name = {                                        \
+        STR(ULTRA_MODULE_NAME), STR(#fn_name), fn_name,               \
+    };                                                                \
+    static void fn_name(void)
+
+// End the case and report it as skipped, printing 'reason' with the result
+NORETURN
+void ktest_skip(const char *reason);
+
+/*
+ * KTEST_EXPECT_*() records a failure and lets the case continue, while
+ * KTEST_ASSERT_*() records it and ends the case. An assert that fails in a
+ * thread started by the case ends only that thread.
+ */
+#define KTEST_EXPECT(cond) do {                               \
+    if (unlikely(!(cond)))                                    \
+        ktest_fail(__FILE__, __LINE__, "expected %s", #cond); \
+} while (0)
+
+#define KTEST_EXPECT_EQ(lhs, rhs) KTEST_CMP_EXPECT(lhs, ==, rhs, #lhs, #rhs)
+#define KTEST_EXPECT_NE(lhs, rhs) KTEST_CMP_EXPECT(lhs, !=, rhs, #lhs, #rhs)
+#define KTEST_EXPECT_LT(lhs, rhs) KTEST_CMP_EXPECT(lhs, <, rhs, #lhs, #rhs)
+#define KTEST_EXPECT_LE(lhs, rhs) KTEST_CMP_EXPECT(lhs, <=, rhs, #lhs, #rhs)
+#define KTEST_EXPECT_GT(lhs, rhs) KTEST_CMP_EXPECT(lhs, >, rhs, #lhs, #rhs)
+#define KTEST_EXPECT_GE(lhs, rhs) KTEST_CMP_EXPECT(lhs, >=, rhs, #lhs, #rhs)
+
+#define KTEST_ASSERT(cond) do {                               \
+    if (unlikely(!(cond))) {                                  \
+        ktest_fail(__FILE__, __LINE__, "expected %s", #cond); \
+        ktest_abort();                                        \
+    }                                                         \
+} while (0)
+
+#define KTEST_ASSERT_EQ(lhs, rhs) KTEST_CMP_ASSERT(lhs, ==, rhs, #lhs, #rhs)
+#define KTEST_ASSERT_NE(lhs, rhs) KTEST_CMP_ASSERT(lhs, !=, rhs, #lhs, #rhs)
+#define KTEST_ASSERT_LT(lhs, rhs) KTEST_CMP_ASSERT(lhs, <, rhs, #lhs, #rhs)
+#define KTEST_ASSERT_LE(lhs, rhs) KTEST_CMP_ASSERT(lhs, <=, rhs, #lhs, #rhs)
+#define KTEST_ASSERT_GT(lhs, rhs) KTEST_CMP_ASSERT(lhs, >, rhs, #lhs, #rhs)
+#define KTEST_ASSERT_GE(lhs, rhs) KTEST_CMP_ASSERT(lhs, >=, rhs, #lhs, #rhs)
diff --git a/kernel/include/linker.h b/kernel/include/linker.h
index 8ebc0992..23e8702f 100644
--- a/kernel/include/linker.h
+++ b/kernel/include/linker.h
@@ -35,6 +35,7 @@
 #define ABORTABLE_INSTRUCTIONS_SECTION abortable_instructions
 #define BUG_TABLE_SECTION bug_table
 #define PARAMETERS_SECTION parameters
+#define KTEST_CASES_SECTION ktest_cases
 #define PER_CPU_SECTION per_cpu
 
 #define FREE_AFTER_INIT_SECTION free_after_init
diff --git a/kernel/include/linker_common.ld.h b/kernel/include/linker_common.ld.h
index 23a6e4ad..d4bbd02d 100644
--- a/kernel/include/linker_common.ld.h
+++ b/kernel/include/linker_common.ld.h
@@ -81,11 +81,12 @@
     SCHED_CLASSES                                    \
     SECTION_MARKER_END(SCHED_CLASSES_SECTION) = .;
 
-#define RODATA                                           \
-    *(.rodata .rodata.*)                                 \
-    *(.INIT_DATA_REFERENCE_RODATA_SECTION)               \
-    SCHED_CLASS_SECTIONS                                 \
-    NATURALLY_ALIGNED_MARKED_SECTION(PARAMETERS_SECTION)
+#define RODATA                                            \
+    *(.rodata .rodata.*)                                  \
+    *(.INIT_DATA_REFERENCE_RODATA_SECTION)                \
+    SCHED_CLASS_SECTIONS                                  \
+    NATURALLY_ALIGNED_MARKED_SECTION(PARAMETERS_SECTION)  \
+    NATURALLY_ALIGNED_MARKED_SECTION(KTEST_CASES_SECTION)
 
 #define RODATA_FREE_AFTER_INIT_BEGIN \
     FREE_AFTER_INIT_BEGIN(FREE_AFTER_INIT_RODATA_SECTION)
diff --git a/kernel/include/private/ktest/task.h b/kernel/include/private/ktest/task.h
new file mode 100644
index 00000000..5fdabe68
--- /dev/null
+++ b/kernel/include/private/ktest/task.h
@@ -0,0 +1,28 @@
+#pragma once
+
+#include <common/helpers.h>
+
+#include <config.h>
+
+struct task;
+
+#if IS_ENABLED(KTEST)
+/*
+ * Called for every new kernel thread, which then reports to the same run as
+ * the thread that created it and holds a reference to that run.
+ */
+void ktest_inherit_run(struct task *task);
+
+// Called when a kernel thread is freed, drops the reference to its run
+void ktest_release_run(struct task *task);
+#else
+static inline void ktest_inherit_run(struct task *task)
+{
+    UNREFERENCED_PARAMETER(task);
+}
+
+static inline void ktest_release_run(struct task *task)
+{
+    UNREFERENCED_PARAMETER(task);
+}
+#endif
diff --git a/kernel/include/private/sched/kthread.h b/kernel/include/private/sched/kthread.h
index da7aaa17..4859ddc0 100644
--- a/kernel/include/private/sched/kthread.h
+++ b/kernel/include/private/sched/kthread.h
@@ -4,3 +4,6 @@ struct task;
 
 // Called by task_exit() for a kthread once it is marked dead
 void kthread_did_exit(struct task *task);
+
+// Called by task_unref() for a kthread once its last reference is gone
+void kthread_release(struct task *task);
diff --git a/kernel/ktest/CMakeLists.txt b/kernel/ktest/CMakeLists.txt
new file mode 100644
index 00000000..56693402
--- /dev/null
+++ b/kernel/ktest/CMakeLists.txt
@@ -0,0 +1,9 @@
+ultra_sources_if(
+    CONFIG_KTEST
+    ktest.c
+)
+
+ultra_sources_if(
+    CONFIG_KTEST_SELFTEST
+    selftest.c
+)
diff --git a/kernel/ktest/Kconfig b/kernel/ktest/Kconfig
new file mode 100644
index 00000000..8012a5ee
--- /dev/null
+++ b/kernel/ktest/Kconfig
@@ -0,0 +1,23 @@
+menuconfig KTEST
+    bool "In-kernel integration tests"
+    help
+        A runner for tests that need the real kernel around them, e.g.
+        interrupts, timers and threads on real hardware. Each test runs
+        in its own thread and the results are printed in KTAP format,
+        ending in a summary line.
+
+        Nothing runs unless the kernel command line has ktest.run=all.
+
+if KTEST
+
+config KTEST_ALL_TESTS
+    bool "Build every test"
+    default y
+    help
+        Turn off to pick the tests to build one by one.
+
+config KTEST_SELFTEST
+    bool "Tests of the test runner itself" if !KTEST_ALL_TESTS
+    default KTEST_ALL_TESTS
+
+endif
diff --git a/kernel/ktest/ktest.c b/kernel/ktest/ktest.c
new file mode 100644
index 00000000..9b2cebde
--- /dev/null
+++ b/kernel/ktest/ktest.c
@@ -0,0 +1,387 @@
+#define MSG_FMT(msg) "ktest: " msg
+
+#include <ktest/ktest.h>
+#include <private/ktest/task.h>
+#include <detail/sched/task.h>
+
+#include <common/string_container.h>
+#include <common/format.h>
+#include <common/string.h>
+#include <common/atomic.h>
+
+#include <sched/kthread.h>
+#include <log.h>
+#include <param.h>
+#include <init_level.h>
+#include <free_after_init.h>
+#include <memory/alloc.h>
+#include <refcount.h>
+#include <time/time.h>
+#include <time/units.h>
+
+DECLARE_TYPED_SECTION_MARKERS(const struct ktest_case, KTEST_CASES_SECTION);
+
+static char s_run[128];
+module_parameter(s_run);
+
+struct ktest_run {
+    struct refcount32 refcount;
+    const struct ktest_case *test;
+    u32 num_failures;
+    bool skipped;
+    const char *skip_reason;
+};
+
+static void run_ref(struct ktest_run *run)
+{
+    refcount_inc(&run->refcount);
+}
+
+static void run_unref(struct ktest_run *run)
+{
+    if (refcount_dec_and_test(&run->refcount))
+        free(run);
+}
+
+void ktest_inherit_run(struct task *task)
+{
+    struct ktest_run *run = current->ktest_run;
+
+    task->ktest_run = run;
+    if (run != nullptr)
+        run_ref(run);
+}
+
+void ktest_release_run(struct task *task)
+{
+    if (task->ktest_run != nullptr)
+        run_unref(task->ktest_run);
+}
+
+void ktest_fail(const char *file, u32 line, const char *fmt, ...)
+{
+    struct ktest_run *run = current->ktest_run;
+    struct nested_printf message;
+    va_list vlist;
+
+    if (run == nullptr)
+        return;
+
+    va_start(vlist, fmt);
+    message.fmt = fmt;
+    message.vlist = &vlist;
+
+    atomic_fetch_add(&run->num_failures, 1, MO_RELAXED);
+    print(LOG_NOTICE "# %s:%u: %pV\n", file, line, &message);
+    va_end(vlist);
+}
+
+#define MAX_KTEST_VALUE_DESCRIPTION_LENGTH 32
+
+#define DESCRIBE_PRINTF(out_desc_buf, ...)                                  \
+    str_printf(out_desc_buf, MAX_KTEST_VALUE_DESCRIPTION_LENGTH, __VA_ARGS__)
+
+#define KTEST_CMP_FMT(value_fmt) "expected %s, got " value_fmt " and " value_fmt
+
+struct ktest_value_type_ops {
+    /*
+     * A three-way comparison of 'lhs' and 'rhs'. A value type without an
+     * order returns zero for equal values and any other number otherwise.
+     * The build only allows == and != on such a type.
+     */
+    int (*compare)(
+        const struct ktest_value *lhs, const struct ktest_value *rhs
+    );
+
+    /*
+     * Return a string representation of 'value', possibly using 'out_desc_buf'
+     * for storage if needed for this type.
+     */
+    struct string (*describe)(
+        const struct ktest_value *value,
+        char out_desc_buf[static MAX_KTEST_VALUE_DESCRIPTION_LENGTH]
+    );
+    bool quoted;
+};
+
+static int compare_u64(u64 lhs, u64 rhs)
+{
+    if (lhs == rhs)
+        return 0;
+
+    return lhs < rhs ? -1 : 1;
+}
+
+static bool is_negative(const struct ktest_value *value)
+{
+    return value->is_signed && value->as_signed < 0;
+}
+
+static int compare_integers(
+    const struct ktest_value *lhs, const struct ktest_value *rhs
+)
+{
+    bool lhs_negative, rhs_negative;
+
+    lhs_negative = is_negative(lhs);
+    rhs_negative = is_negative(rhs);
+
+    if (lhs_negative != rhs_negative)
+        return lhs_negative ? -1 : 1;
+
+    // Two values of the same sign are ordered by their bit patterns
+    return compare_u64(lhs->as_unsigned, rhs->as_unsigned);
+}
+
+static struct string describe_integer(
+    const struct ktest_value *value,
+    char out_desc_buf[static MAX_KTEST_VALUE_DESCRIPTION_LENGTH]
+)
+{
+    if (value->is_signed)
+        return DESCRIBE_PRINTF(out_desc_buf, "%lld", value->as_signed);
+
+    return DESCRIBE_PRINTF(out_desc_buf, "%llu", value->as_unsigned);
+}
+
+static int compare_pointers(
+    const struct ktest_value *lhs, const struct ktest_value *rhs
+)
+{
+    return compare_u64((ptr_t)lhs->as_pointer, (ptr_t)rhs->as_pointer);
+}
+
+static struct string describe_pointer(
+    const struct ktest_value *value,
+    char out_desc_buf[static MAX_KTEST_VALUE_DESCRIPTION_LENGTH]
+)
+{
+    return DESCRIBE_PRINTF(out_desc_buf, "%p", value->as_pointer);
+}
+
+static int compare_strings(
+    const struct ktest_value *lhs, const struct ktest_value *rhs
+)
+{
+    return str_equals(lhs->as_string, rhs->as_string) ? 0 : 1;
+}
+
+static struct string describe_string(
+    const struct ktest_value *value,
+    char out_desc_buf[static MAX_KTEST_VALUE_DESCRIPTION_LENGTH]
+)
+{
+    UNREFERENCED_PARAMETER(out_desc_buf);
+    return value->as_string;
+}
+
+static const struct ktest_value_type_ops s_value_type_ops[] = {
+    [KTEST_VALUE_INTEGER] = {
+        .compare = compare_integers,
+        .describe = describe_integer,
+    },
+    [KTEST_VALUE_POINTER] = {
+        .compare = compare_pointers,
+        .describe = describe_pointer,
+    },
+    [KTEST_VALUE_STRING] = {
+        .compare = compare_strings,
+        .describe = describe_string,
+        .quoted = true,
+    },
+};
+
+int ktest_compare(const struct ktest_value *lhs, const struct ktest_value *rhs)
+{
+    return s_value_type_ops[lhs->type].compare(lhs, rhs);
+}
+
+void ktest_fail_cmp(
+    const char *file, u32 line, const char *expr, const struct ktest_value *lhs,
+    const struct ktest_value *rhs
+)
+{
+    char lhs_buf[MAX_KTEST_VALUE_DESCRIPTION_LENGTH];
+    char rhs_buf[MAX_KTEST_VALUE_DESCRIPTION_LENGTH];
+    const struct ktest_value_type_ops *ops;
+    struct string lhs_desc, rhs_desc;
+
+    ops = &s_value_type_ops[lhs->type];
+    lhs_desc = ops->describe(lhs, lhs_buf);
+    rhs_desc = ops->describe(rhs, rhs_buf);
+
+    ktest_fail(
+        file, line,
+        ops->quoted ? KTEST_CMP_FMT("\"%pS\"") : KTEST_CMP_FMT("%pS"), expr,
+        &lhs_desc, &rhs_desc
+    );
+}
+
+void ktest_abort(void)
+{
+    kthread_exit();
+}
+
+void ktest_skip(const char *reason)
+{
+    struct ktest_run *run = current->ktest_run;
+
+    run->skipped = true;
+    run->skip_reason = reason;
+    kthread_exit();
+}
+
+static void case_thread(void *user)
+{
+    struct ktest_run *run = user;
+
+    run->test->fn();
+}
+
+static struct task *create_case_thread(struct ktest_run *run)
+{
+    struct task *thread;
+
+    thread = kthread_create(case_thread, run, "ktest-%pS", &run->test->name);
+    if (unlikely(thread == nullptr))
+        return nullptr;
+
+    thread->ktest_run = run;
+    run_ref(run);
+    return thread;
+}
+
+enum ktest_result : u8 {
+    KTEST_PASSED,
+    KTEST_FAILED,
+    KTEST_SKIPPED,
+};
+
+// Run the case in a separate thread, returns true if the thread ran and exited
+static bool run_case_thread(struct ktest_run *run)
+{
+    struct task *thread;
+
+    thread = create_case_thread(run);
+    if (unlikely(thread == nullptr)) {
+        print(LOG_NOTICE "# no memory for the thread\n");
+        return false;
+    }
+
+    kthread_start(thread);
+    kthread_wait_for_exit(thread);
+    task_unref(thread);
+
+    return true;
+}
+
+static enum ktest_result case_result(const struct ktest_run *run, bool exited)
+{
+    if (!exited)
+        return KTEST_FAILED;
+
+    if (run->skipped)
+        return KTEST_SKIPPED;
+
+    if (run->num_failures != 0)
+        return KTEST_FAILED;
+
+    return KTEST_PASSED;
+}
+
+static void print_result(
+    const struct ktest_case *test, u32 index, enum ktest_result result,
+    const char *skip_reason
+)
+{
+    if (result == KTEST_SKIPPED) {
+        print(
+            LOG_NOTICE "ok %u %pS.%pS # SKIP %s\n", index, &test->suite,
+            &test->name, skip_reason
+        );
+        return;
+    }
+
+    print(
+        LOG_NOTICE "%s %u %pS.%pS\n", result == KTEST_PASSED ? "ok" : "not ok",
+        index, &test->suite, &test->name
+    );
+}
+
+static enum ktest_result run_case(const struct ktest_case *test, u32 index)
+{
+    struct ktest_run *run;
+    u64 start_ns, duration_ms;
+    enum ktest_result result;
+    bool exited;
+
+    run = alloc(sizeof(*run), ALLOC_GENERIC_ZEROED);
+    if (unlikely(run == nullptr)) {
+        print(LOG_NOTICE "# out of memory\n");
+        print_result(test, index, KTEST_FAILED, nullptr);
+        return KTEST_FAILED;
+    }
+
+    refcount_set(&run->refcount, 1);
+    run->test = test;
+
+    start_ns = ns_since_boot();
+    exited = run_case_thread(run);
+    duration_ms = NS_TO_MS(ns_since_boot() - start_ns);
+
+    result = case_result(run, exited);
+
+    print(
+        LOG_NOTICE "# %pS.%pS: %llu ms\n", &test->suite, &test->name,
+        duration_ms
+    );
+    print_result(test, index, result, run->skip_reason);
+    run_unref(run);
+
+    return result;
+}
+
+static void runner_thread(void *user)
+{
+    const struct ktest_case *test;
+    u32 num_passed = 0, num_failed = 0, num_skipped = 0;
+    u32 index = 0;
+
+    UNREFERENCED_PARAMETER(user);
+
+    if (strcmp(s_run, "all") != 0) {
+        pr_notice("nothing matched \"%s\"\n", s_run);
+        return;
+    }
+
+    print(LOG_NOTICE "KTAP version 1\n");
+    print(LOG_NOTICE "1..%zu\n", SECTION_ARRAY_SIZE(KTEST_CASES_SECTION));
+
+    for_each_section_array_item(test, KTEST_CASES_SECTION) {
+        switch (run_case(test, ++index)) {
+        case KTEST_PASSED:
+            num_passed++;
+            break;
+        case KTEST_FAILED:
+            num_failed++;
+            break;
+        case KTEST_SKIPPED:
+            num_skipped++;
+            break;
+        }
+    }
+
+    pr_notice(
+        "%u passed, %u failed, %u skipped\n",
+        num_passed, num_failed, num_skipped
+    );
+}
+
+static error_t INIT_CODE ktest_init(void)
+{
+    if (s_run[0] == '\0')
+        return EOK;
+
+    return kthread_run(runner_thread, nullptr, "ktest");
+}
+INIT_CALL_POST(GENERIC_MODULES_INITIALIZED, ktest_init);
diff --git a/kernel/ktest/selftest.c b/kernel/ktest/selftest.c
new file mode 100644
index 00000000..e14436ff
--- /dev/null
+++ b/kernel/ktest/selftest.c
@@ -0,0 +1,50 @@
+#include <ktest/ktest.h>
+
+#include <sched/kthread.h>
+#include <sched/completion.h>
+
+KTEST_CASE(expectations_hold)
+{
+    struct string name;
+    int x = 3;
+
+    name = STR("ktest");
+
+    KTEST_EXPECT(x == 3);
+    KTEST_EXPECT_EQ(x + 1, 4);
+    KTEST_EXPECT_NE(x, 0);
+    KTEST_EXPECT_LT(x, 4);
+    KTEST_EXPECT_LE(x, 3);
+    KTEST_EXPECT_LE(-x, 0);
+    KTEST_EXPECT_GT(x, 2);
+    KTEST_EXPECT_GE(x, 3);
+    KTEST_EXPECT_LT(-x, 1u);
+    KTEST_EXPECT_NE(&x, nullptr);
+    KTEST_EXPECT_EQ(name, STR("ktest"));
+    KTEST_EXPECT_NE(name, STR("test"));
+
+    KTEST_ASSERT(x > 0);
+    KTEST_ASSERT_EQ(x * 2, 6);
+    KTEST_ASSERT_EQ(name, STR("ktest"));
+}
+
+KTEST_CASE(skipping_is_reported)
+{
+    ktest_skip("always skipped!");
+}
+
+static void expect_from_a_helper(void *user)
+{
+    struct completion *done = user;
+
+    KTEST_EXPECT(true);
+    complete(done);
+}
+
+KTEST_CASE(helper_threads_may_expect)
+{
+    DEFINE_COMPLETION(done);
+
+    KTEST_ASSERT(!is_error(kthread_run(expect_from_a_helper, &done, "helper")));
+    wait_for_completion(&done);
+}
diff --git a/kernel/sched/kthread.c b/kernel/sched/kthread.c
index b01ccd6f..23137877 100644
--- a/kernel/sched/kthread.c
+++ b/kernel/sched/kthread.c
@@ -11,6 +11,7 @@
 #include <private/sched/rr.h>
 #include <private/sched/kthread.h>
 #include <private/arch/task.h>
+#include <private/ktest/task.h>
 
 static void kthread_entry(void *user)
 {
@@ -45,6 +46,8 @@ struct task *kthread_vcreate(
 
     completion_init(&task->kthread.exited);
 
+    ktest_inherit_run(task);
+
     arch_task_frame_init(task, kthread_entry, nullptr);
 
     return task;
@@ -97,6 +100,11 @@ void kthread_did_exit(struct task *task)
     complete_all(&task->kthread.exited);
 }
 
+void kthread_release(struct task *task)
+{
+    ktest_release_run(task);
+}
+
 void kthread_wait_for_exit(struct task *task)
 {
     wait_for_completion(&task->kthread.exited);
diff --git a/kernel/sched/task.c b/kernel/sched/task.c
index 9b0c4dd5..689aaeb9 100644
--- a/kernel/sched/task.c
+++ b/kernel/sched/task.c
@@ -1,6 +1,7 @@
 #include <private/sched/task.h>
 #include <private/arch/task.h>
 #include <private/sched/core.h>
+#include <private/sched/kthread.h>
 
 #include <common/format.h>
 
@@ -107,6 +108,9 @@ void task_unref(struct task *task)
     if (!refcount_dec_and_test(&task->refcount))
         return;
 
+    if (task_is_kthread(task))
+        kthread_release(task);
+
     vrelease(task->stack);
     free(task);
 }
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index deb1ab8f..f21c0f54 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -223,6 +223,7 @@ KERNEL_FILE(
 )
 KERNEL_FILE(INCLUDE_PATH "detail/sched" INCLUDE_FILE "task.h")
 KERNEL_FILE(INCLUDE_PATH "private/sched" INCLUDE_FILE "task.h")
+KERNEL_FILE(INCLUDE_PATH "private/ktest" INCLUDE_FILE "task.h")
 KERNEL_FILE(INCLUDE_PATH "private/arch" INCLUDE_FILE "task.h")
 KERNEL_FILE(INCLUDE_PATH "sched" INCLUDE_FILE "sched.h")
 KERNEL_FILE(

From 67b14c4f811b6995671e3c6b8f25290a41b99d90 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 8 Nov 2026 15:50:53 +0300
Subject: [PATCH 134/152] kthread: add kthread_wait_for_exit_timeout()

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/sched/kthread.h | 7 +++++++
 kernel/sched/kthread.c         | 5 +++++
 2 files changed, 12 insertions(+)

diff --git a/kernel/include/sched/kthread.h b/kernel/include/sched/kthread.h
index 37c3e67f..13607345 100644
--- a/kernel/include/sched/kthread.h
+++ b/kernel/include/sched/kthread.h
@@ -49,6 +49,13 @@ void kthread_exit(void);
  */
 void kthread_wait_for_exit(struct task *task);
 
+/*
+ * Same as kthread_wait_for_exit(), except the caller stops waiting once
+ * 'timeout_ns' have passed. Returns true if the thread has exited, false if
+ * it still hadn't when the timeout ran out.
+ */
+bool kthread_wait_for_exit_timeout(struct task *task, u64 timeout_ns);
+
 /*
  * Ask a kernel thread to stop and wait until it has exited. The thread
  * has to cooperate by checking kthread_should_stop() whenever it is
diff --git a/kernel/sched/kthread.c b/kernel/sched/kthread.c
index 23137877..8bab81e4 100644
--- a/kernel/sched/kthread.c
+++ b/kernel/sched/kthread.c
@@ -110,6 +110,11 @@ void kthread_wait_for_exit(struct task *task)
     wait_for_completion(&task->kthread.exited);
 }
 
+bool kthread_wait_for_exit_timeout(struct task *task, u64 timeout_ns)
+{
+    return wait_for_completion_timeout(&task->kthread.exited, timeout_ns);
+}
+
 void kthread_stop(struct task *task)
 {
     task_set_flag(task, TASK_FLAG_KTHREAD_SHOULD_STOP);

From 7bfcc4025bf1968141960032d9d99bcfc1f099c4 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:15:29 +0300
Subject: [PATCH 135/152] ktest: add per-case timeouts

A case that hangs would stall the whole run. Each case now gets 30 s
unless KTEST_CASE_WITH_TIMEOUT() says otherwise, after which it fails
and its thread is left running. A result that arrives after its case
finished is reported as late and not counted.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/ktest/ktest.h |  1 +
 kernel/include/ktest/ktest.h        | 12 +++++++--
 kernel/ktest/Kconfig                |  4 +--
 kernel/ktest/ktest.c                | 34 ++++++++++++++++++++++---
 kernel/ktest/selftest.c             | 39 +++++++++++++++++++++++++++++
 5 files changed, 82 insertions(+), 8 deletions(-)

diff --git a/kernel/include/detail/ktest/ktest.h b/kernel/include/detail/ktest/ktest.h
index 25a48e14..e6a253db 100644
--- a/kernel/include/detail/ktest/ktest.h
+++ b/kernel/include/detail/ktest/ktest.h
@@ -14,6 +14,7 @@ struct ktest_case {
     struct string suite;
     struct string name;
     void (*fn)(void);
+    u32 timeout_ms;
 };
 
 PRINTF_DECL(3, 4)
diff --git a/kernel/include/ktest/ktest.h b/kernel/include/ktest/ktest.h
index 3aa7d0ab..9351d5cd 100644
--- a/kernel/include/ktest/ktest.h
+++ b/kernel/include/ktest/ktest.h
@@ -8,13 +8,21 @@
 
 #include <detail/ktest/ktest.h>
 
-#define KTEST_CASE(fn_name)                                           \
+#define KTEST_DEFAULT_TIMEOUT_MS 30000
+
+/*
+ * Define a test case that fails if it hasn't finished within 'ms' milliseconds.
+ * The thread is left abandoned in this case.
+ */
+#define KTEST_CASE_WITH_TIMEOUT(fn_name, ms)                          \
     static void fn_name(void);                                        \
     SECTION_VAR(KTEST_CASES_SECTION, static const, struct ktest_case) \
     s_ktest_case_##fn_name = {                                        \
-        STR(ULTRA_MODULE_NAME), STR(#fn_name), fn_name,               \
+        STR(ULTRA_MODULE_NAME), STR(#fn_name), fn_name, (ms),         \
     };                                                                \
     static void fn_name(void)
+#define KTEST_CASE(fn_name)                                    \
+    KTEST_CASE_WITH_TIMEOUT(fn_name, KTEST_DEFAULT_TIMEOUT_MS)
 
 // End the case and report it as skipped, printing 'reason' with the result
 NORETURN
diff --git a/kernel/ktest/Kconfig b/kernel/ktest/Kconfig
index 8012a5ee..ebcb4340 100644
--- a/kernel/ktest/Kconfig
+++ b/kernel/ktest/Kconfig
@@ -3,8 +3,8 @@ menuconfig KTEST
     help
         A runner for tests that need the real kernel around them, e.g.
         interrupts, timers and threads on real hardware. Each test runs
-        in its own thread and the results are printed in KTAP format,
-        ending in a summary line.
+        in its own thread with a timeout and the results are printed
+        in KTAP format, ending in a summary line.
 
         Nothing runs unless the kernel command line has ktest.run=all.
 
diff --git a/kernel/ktest/ktest.c b/kernel/ktest/ktest.c
index 9b2cebde..99eb528e 100644
--- a/kernel/ktest/ktest.c
+++ b/kernel/ktest/ktest.c
@@ -30,6 +30,7 @@ struct ktest_run {
     u32 num_failures;
     bool skipped;
     const char *skip_reason;
+    bool finished;
 };
 
 static void run_ref(struct ktest_run *run)
@@ -71,8 +72,16 @@ void ktest_fail(const char *file, u32 line, const char *fmt, ...)
     message.fmt = fmt;
     message.vlist = &vlist;
 
-    atomic_fetch_add(&run->num_failures, 1, MO_RELAXED);
-    print(LOG_NOTICE "# %s:%u: %pV\n", file, line, &message);
+    if (atomic_load_relaxed(&run->finished)) {
+        print(
+            LOG_NOTICE "# %s:%u: %pV (late, %pS.%pS is already finished)\n",
+            file, line, &message, &run->test->suite, &run->test->name
+        );
+    } else {
+        atomic_fetch_add(&run->num_failures, 1, MO_RELAXED);
+        print(LOG_NOTICE "# %s:%u: %pV\n", file, line, &message);
+    }
+
     va_end(vlist);
 }
 
@@ -261,6 +270,7 @@ enum ktest_result : u8 {
 static bool run_case_thread(struct ktest_run *run)
 {
     struct task *thread;
+    bool exited;
 
     thread = create_case_thread(run);
     if (unlikely(thread == nullptr)) {
@@ -269,10 +279,19 @@ static bool run_case_thread(struct ktest_run *run)
     }
 
     kthread_start(thread);
-    kthread_wait_for_exit(thread);
+    exited = kthread_wait_for_exit_timeout(
+        thread, MS_TO_NS(run->test->timeout_ms)
+    );
     task_unref(thread);
 
-    return true;
+    if (!exited) {
+        print(
+            LOG_NOTICE "# timed out after %u ms, thread left running\n",
+            run->test->timeout_ms
+        );
+    }
+
+    return exited;
 }
 
 static enum ktest_result case_result(const struct ktest_run *run, bool exited)
@@ -329,6 +348,13 @@ static enum ktest_result run_case(const struct ktest_case *test, u32 index)
     exited = run_case_thread(run);
     duration_ms = NS_TO_MS(ns_since_boot() - start_ns);
 
+    /*
+     * The case fails if the main thread hasn't exited within the timeout,
+     * but the threads that it started may still be running. If they report
+     * a result after this point it's simply discarded as "late" and is not
+     * counted.
+     */
+    atomic_store_relaxed(&run->finished, true);
     result = case_result(run, exited);
 
     print(
diff --git a/kernel/ktest/selftest.c b/kernel/ktest/selftest.c
index e14436ff..3c883920 100644
--- a/kernel/ktest/selftest.c
+++ b/kernel/ktest/selftest.c
@@ -1,7 +1,12 @@
 #include <ktest/ktest.h>
 
+#include <common/helpers.h>
+
 #include <sched/kthread.h>
+#include <sched/sched.h>
 #include <sched/completion.h>
+#include <time/time.h>
+#include <time/units.h>
 
 KTEST_CASE(expectations_hold)
 {
@@ -33,6 +38,17 @@ KTEST_CASE(skipping_is_reported)
     ktest_skip("always skipped!");
 }
 
+KTEST_CASE_WITH_TIMEOUT(cases_may_sleep, 1000)
+{
+    u64 before, after;
+
+    before = ns_since_boot();
+    sleep_ns(20 * NS_PER_MS);
+    after = ns_since_boot();
+
+    KTEST_EXPECT_GE(after - before, 20 * NS_PER_MS);
+}
+
 static void expect_from_a_helper(void *user)
 {
     struct completion *done = user;
@@ -48,3 +64,26 @@ KTEST_CASE(helper_threads_may_expect)
     KTEST_ASSERT(!is_error(kthread_run(expect_from_a_helper, &done, "helper")));
     wait_for_completion(&done);
 }
+
+static void expect_late(void *user)
+{
+    UNREFERENCED_PARAMETER(user);
+
+    sleep_ns(50 * NS_PER_MS);
+    KTEST_EXPECT(false);
+}
+
+KTEST_CASE(late_results_are_not_counted)
+{
+    KTEST_ASSERT(!is_error(kthread_run(expect_late, nullptr, "late")));
+}
+
+/*
+ * The failure that expect_late() records arrives while this case is running,
+ * after late_results_are_not_counted has finished. It must not fail this case
+ * because it belongs to the run of the case that started the thread.
+ */
+KTEST_CASE_WITH_TIMEOUT(late_results_dont_fail_the_running_case, 1000)
+{
+    sleep_ns(100 * NS_PER_MS);
+}

From f6a4c16c1675c9316aee75886338d7e36a87c15d Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:15:29 +0300
Subject: [PATCH 136/152] ktest: allow selecting suites and cases with
 ktest.run=

Running every case is slow when working on one subsystem. ktest.run=
now takes a comma-separated list of suites and suite.case names, "all"
still runs everything.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/private/ktest/helpers.h |  7 +++++
 kernel/ktest/CMakeLists.txt            |  1 +
 kernel/ktest/Kconfig                   |  3 +-
 kernel/ktest/helpers.c                 | 37 +++++++++++++++++++++++
 kernel/ktest/ktest.c                   | 18 +++++++++--
 tests/CMakeLists.txt                   |  4 +++
 tests/test-cases/files.cmake           |  1 +
 tests/test-cases/test_ktest.c          | 41 ++++++++++++++++++++++++++
 8 files changed, 108 insertions(+), 4 deletions(-)
 create mode 100644 kernel/include/private/ktest/helpers.h
 create mode 100644 kernel/ktest/helpers.c
 create mode 100644 tests/test-cases/test_ktest.c

diff --git a/kernel/include/private/ktest/helpers.h b/kernel/include/private/ktest/helpers.h
new file mode 100644
index 00000000..73f8fc1e
--- /dev/null
+++ b/kernel/include/private/ktest/helpers.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include <common/string_container.h>
+
+bool ktest_case_is_selected(
+    struct string suite, struct string name, struct string selection
+);
diff --git a/kernel/ktest/CMakeLists.txt b/kernel/ktest/CMakeLists.txt
index 56693402..ca202669 100644
--- a/kernel/ktest/CMakeLists.txt
+++ b/kernel/ktest/CMakeLists.txt
@@ -1,5 +1,6 @@
 ultra_sources_if(
     CONFIG_KTEST
+    helpers.c
     ktest.c
 )
 
diff --git a/kernel/ktest/Kconfig b/kernel/ktest/Kconfig
index ebcb4340..7d9a1d4a 100644
--- a/kernel/ktest/Kconfig
+++ b/kernel/ktest/Kconfig
@@ -6,7 +6,8 @@ menuconfig KTEST
         in its own thread with a timeout and the results are printed
         in KTAP format, ending in a summary line.
 
-        Nothing runs unless the kernel command line has ktest.run=all.
+        Nothing runs unless the kernel command line selects tests with
+        ktest.run=all or ktest.run=suite,suite.case.
 
 if KTEST
 
diff --git a/kernel/ktest/helpers.c b/kernel/ktest/helpers.c
new file mode 100644
index 00000000..82453c09
--- /dev/null
+++ b/kernel/ktest/helpers.c
@@ -0,0 +1,37 @@
+#include <private/ktest/helpers.h>
+
+bool ktest_case_is_selected(
+    struct string suite, struct string name, struct string selection
+)
+{
+    struct string token, token_suite, token_name;
+    ssize_t dot;
+
+    /*
+     * 'selection' is a comma separated list of the following:
+     *     "all" - runs all registered tests
+     *     "suite" - runs all cases of 'suite'
+     *     "suite.case" - runs a specific test 'case' of 'suite'
+     */
+    while (str_pop_token(&selection, ',', &token)) {
+        if (str_equals(token, STR("all")))
+            return true;
+
+        dot = str_find_one(token, '.', 0);
+        if (dot < 0) {
+            if (str_equals(token, suite))
+                return true;
+
+            continue;
+        }
+
+        token_suite = str_substring(token, 0, dot);
+        token_name = str_substring(token, dot + 1, token.size);
+
+        if (str_equals(token_suite, suite) &&
+            str_equals(token_name, name))
+            return true;
+    }
+
+    return false;
+}
diff --git a/kernel/ktest/ktest.c b/kernel/ktest/ktest.c
index 99eb528e..baa0a8fd 100644
--- a/kernel/ktest/ktest.c
+++ b/kernel/ktest/ktest.c
@@ -1,6 +1,7 @@
 #define MSG_FMT(msg) "ktest: " msg
 
 #include <ktest/ktest.h>
+#include <private/ktest/helpers.h>
 #include <private/ktest/task.h>
 #include <detail/sched/task.h>
 
@@ -370,20 +371,31 @@ static enum ktest_result run_case(const struct ktest_case *test, u32 index)
 static void runner_thread(void *user)
 {
     const struct ktest_case *test;
-    u32 num_passed = 0, num_failed = 0, num_skipped = 0;
+    struct string selection;
+    u32 num_selected = 0, num_passed = 0, num_failed = 0, num_skipped = 0;
     u32 index = 0;
 
     UNREFERENCED_PARAMETER(user);
 
-    if (strcmp(s_run, "all") != 0) {
+    selection = MAKE_STR(s_run, strlen(s_run));
+
+    for_each_section_array_item(test, KTEST_CASES_SECTION) {
+        if (ktest_case_is_selected(test->suite, test->name, selection))
+            num_selected++;
+    }
+
+    if (num_selected == 0) {
         pr_notice("nothing matched \"%s\"\n", s_run);
         return;
     }
 
     print(LOG_NOTICE "KTAP version 1\n");
-    print(LOG_NOTICE "1..%zu\n", SECTION_ARRAY_SIZE(KTEST_CASES_SECTION));
+    print(LOG_NOTICE "1..%u\n", num_selected);
 
     for_each_section_array_item(test, KTEST_CASES_SECTION) {
+        if (!ktest_case_is_selected(test->suite, test->name, selection))
+            continue;
+
         switch (run_case(test, ++index)) {
         case KTEST_PASSED:
             num_passed++;
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index f21c0f54..ac2e06d2 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -181,6 +181,10 @@ KERNEL_FILE(
     SOURCE_PATH "time" SOURCE_FILE "units.c"
     INCLUDE_PATH "time" INCLUDE_FILE "units.h"
 )
+KERNEL_FILE(
+    SOURCE_PATH "ktest" SOURCE_FILE "helpers.c"
+    INCLUDE_PATH "private/ktest" INCLUDE_FILE "helpers.h"
+)
 KERNEL_FILE(
     SOURCE_PATH "time" SOURCE_FILE "timer.c"
     INCLUDE_PATH "time" INCLUDE_FILE "timer.h"
diff --git a/tests/test-cases/files.cmake b/tests/test-cases/files.cmake
index 87df2075..5a3e9dc6 100644
--- a/tests/test-cases/files.cmake
+++ b/tests/test-cases/files.cmake
@@ -13,6 +13,7 @@ add_test_cases(
     test_slist.c
     test_time.c
     test_timer.c
+    test_ktest.c
     test_rbtree.c
     test_cpu_mask.c
     test_cpu_mask_out_of_line.c
diff --git a/tests/test-cases/test_ktest.c b/tests/test-cases/test_ktest.c
new file mode 100644
index 00000000..729818e2
--- /dev/null
+++ b/tests/test-cases/test_ktest.c
@@ -0,0 +1,41 @@
+#include <kernel-source/ktest/helpers.c>
+
+#include <test_harness.h>
+
+static bool selected(const char *suite, const char *name, const char *selection)
+{
+    return ktest_case_is_selected(
+        STR_RUNTIME(suite), STR_RUNTIME(name), STR_RUNTIME(selection)
+    );
+}
+
+TEST_CASE(ktest_select_all)
+{
+    ASSERT(selected("sched", "slice", "all"));
+    ASSERT(selected("sched", "slice", "mutex,all"));
+    ASSERT(!selected("sched", "slice", ""));
+}
+
+TEST_CASE(ktest_select_by_suite)
+{
+    ASSERT(selected("sched", "slice", "sched"));
+    ASSERT(selected("sched", "slice", "mutex,sched"));
+    ASSERT(!selected("sched", "slice", "mutex"));
+    ASSERT(!selected("sched", "slice", "sche"));
+    ASSERT(!selected("sched", "slice", "scheduler"));
+}
+
+TEST_CASE(ktest_select_by_case)
+{
+    ASSERT(selected("sched", "slice", "sched.slice"));
+    ASSERT(selected("sched", "slice", "mutex.handoff,sched.slice"));
+    ASSERT(!selected("sched", "slice", "sched.sleep"));
+    ASSERT(!selected("sched", "slice", "mutex.slice"));
+    ASSERT(!selected("sched", "slice", "sched."));
+}
+
+TEST_CASE(ktest_select_ignores_empty_tokens)
+{
+    ASSERT(selected("sched", "slice", ",sched,"));
+    ASSERT(!selected("sched", "slice", ","));
+}

From 079d07f0657aef0dc70ba52bed9be6008a1849c2 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:15:29 +0300
Subject: [PATCH 137/152] ktest: add KTEST_RUN_THREADS()

A helper for starting a number of threads that all run the same function
with an index per-thread.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/include/detail/ktest/ktest.h |  5 +++
 kernel/include/ktest/ktest.h        |  9 +++++
 kernel/ktest/ktest.c                | 54 +++++++++++++++++++++++++++++
 kernel/ktest/selftest.c             | 18 ++++++++++
 4 files changed, 86 insertions(+)

diff --git a/kernel/include/detail/ktest/ktest.h b/kernel/include/detail/ktest/ktest.h
index e6a253db..b71bbcf0 100644
--- a/kernel/include/detail/ktest/ktest.h
+++ b/kernel/include/detail/ktest/ktest.h
@@ -23,6 +23,11 @@ void ktest_fail(const char *file, u32 line, const char *fmt, ...);
 NORETURN
 void ktest_abort(void);
 
+void ktest_run_threads(
+    const char *file, u32 line, void (*fn)(void *user, u32 index), void *user,
+    u32 num, const char *name
+);
+
 enum ktest_value_type : u8 {
     KTEST_VALUE_INTEGER,
     KTEST_VALUE_POINTER,
diff --git a/kernel/include/ktest/ktest.h b/kernel/include/ktest/ktest.h
index 9351d5cd..aa0d59a4 100644
--- a/kernel/include/ktest/ktest.h
+++ b/kernel/include/ktest/ktest.h
@@ -58,3 +58,12 @@ void ktest_skip(const char *reason);
 #define KTEST_ASSERT_LE(lhs, rhs) KTEST_CMP_ASSERT(lhs, <=, rhs, #lhs, #rhs)
 #define KTEST_ASSERT_GT(lhs, rhs) KTEST_CMP_ASSERT(lhs, >, rhs, #lhs, #rhs)
 #define KTEST_ASSERT_GE(lhs, rhs) KTEST_CMP_ASSERT(lhs, >=, rhs, #lhs, #rhs)
+
+/*
+ * Run fn(user, i) for every i in [0, num), each in its own thread, and return
+ * once all of them have exited. If a thread can't be started, no more threads
+ * are started, and once the threads that did start have exited, the failure
+ * is handled like a failed KTEST_ASSERT*().
+ */
+#define KTEST_RUN_THREADS(fn, user, num, name)                         \
+    ktest_run_threads(__FILE__, __LINE__, (fn), (user), (num), (name))
diff --git a/kernel/ktest/ktest.c b/kernel/ktest/ktest.c
index baa0a8fd..3dcd711e 100644
--- a/kernel/ktest/ktest.c
+++ b/kernel/ktest/ktest.c
@@ -241,6 +241,60 @@ void ktest_skip(const char *reason)
     kthread_exit();
 }
 
+struct ktest_thread {
+    void (*fn)(void *user, u32 index);
+    void *user;
+    u32 index;
+    struct task *task;
+};
+
+static void thread_entry(void *user)
+{
+    struct ktest_thread *thread = user;
+
+    thread->fn(thread->user, thread->index);
+}
+
+void ktest_run_threads(
+    const char *file, u32 line, void (*fn)(void *user, u32 index), void *user,
+    u32 num, const char *name
+)
+{
+    struct ktest_thread *threads;
+    struct task *task;
+    u32 i, num_started = 0;
+
+    threads = alloc(num * sizeof(*threads), ALLOC_GENERIC);
+    if (unlikely(threads == nullptr)) {
+        ktest_fail(file, line, "no memory for %u threads", num);
+        ktest_abort();
+    }
+
+    for (i = 0; i < num; i++) {
+        threads[i] = (struct ktest_thread) { fn, user, i, nullptr };
+
+        task = kthread_create(thread_entry, &threads[i], "%s-%u", name, i);
+        if (unlikely(task == nullptr)) {
+            ktest_fail(file, line, "no memory for %s-%u", name, i);
+            break;
+        }
+
+        threads[i].task = task;
+        kthread_start(task);
+        num_started++;
+    }
+
+    for (i = 0; i < num_started; i++) {
+        kthread_wait_for_exit(threads[i].task);
+        task_unref(threads[i].task);
+    }
+
+    free(threads);
+
+    if (unlikely(num_started != num))
+        ktest_abort();
+}
+
 static void case_thread(void *user)
 {
     struct ktest_run *run = user;
diff --git a/kernel/ktest/selftest.c b/kernel/ktest/selftest.c
index 3c883920..4786b912 100644
--- a/kernel/ktest/selftest.c
+++ b/kernel/ktest/selftest.c
@@ -87,3 +87,21 @@ KTEST_CASE_WITH_TIMEOUT(late_results_dont_fail_the_running_case, 1000)
 {
     sleep_ns(100 * NS_PER_MS);
 }
+
+static void mark_index(void *user, u32 index)
+{
+    bool *ran = user;
+
+    ran[index] = true;
+}
+
+KTEST_CASE(threads_have_all_returned_when_the_case_goes_on)
+{
+    bool ran[4] = { 0 };
+    u32 i;
+
+    KTEST_RUN_THREADS(mark_index, ran, ARRAY_SIZE(ran), "marker");
+
+    for (i = 0; i < ARRAY_SIZE(ran); i++)
+        KTEST_EXPECT(ran[i]);
+}

From 38cdf0d0a954bf87eff12facfe09fb5b0e2dc6cc Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:42:49 +0300
Subject: [PATCH 138/152] scripts/baremetal_ci: add criteria for judging a
 ktest run

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 scripts/baremetal_ci/client.py           | 18 ++++++++++++------
 scripts/baremetal_ci/criteria-ktest.json | 11 +++++++++++
 2 files changed, 23 insertions(+), 6 deletions(-)
 create mode 100644 scripts/baremetal_ci/criteria-ktest.json

diff --git a/scripts/baremetal_ci/client.py b/scripts/baremetal_ci/client.py
index 502b2482..924563b1 100755
--- a/scripts/baremetal_ci/client.py
+++ b/scripts/baremetal_ci/client.py
@@ -115,9 +115,10 @@ def state(self) -> Dict[str, Any]:
         return self._request("/api/state")
 
 
-def load_default_criteria() -> Dict[str, Any]:
+def load_criteria(name: Optional[str] = None) -> Dict[str, Any]:
+    file_name = f"criteria-{name}.json" if name else "criteria.json"
     path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                        "criteria.json")
+                        file_name)
     with open(path) as f:
         criteria: Dict[str, Any] = json.load(f)
     return criteria
@@ -149,6 +150,9 @@ def add_run_args(target: argparse._ActionsContainer, prefix: str = "",
                             help="Extra kernel command line, appended "
                                  "after the machine's console line and "
                                  "the server's base")
+    target.add_argument(f"--{prefix}criteria", dest="criteria",
+                        metavar="NAME",
+                        help="Judge the run by criteria-NAME.json")
     target.add_argument(f"--{prefix}timeout", dest="timeout",
                         type=float, metavar="SECONDS",
                         help="Give up on a run after this long, the "
@@ -223,14 +227,16 @@ def run_from_args(args: argparse.Namespace, arch: str, kernel_path: str,
 def payload_from_args(args: argparse.Namespace,
                       kernel_path: str) -> Dict[str, Any]:
     return build_payload(kernel_path, args.run_for is None, args.machines,
-                         args.run_for, args.timeout, args.cmdline)
+                         args.run_for, args.timeout, args.cmdline,
+                         args.criteria)
 
 
 def build_payload(kernel_path: str, ci: bool,
                   machines: Optional[str],
                   run_for: Optional[float],
                   timeout: Optional[float],
-                  cmdline: Optional[str]) -> Dict[str, Any]:
+                  cmdline: Optional[str],
+                  criteria_name: Optional[str] = None) -> Dict[str, Any]:
     with open(kernel_path, "rb") as f:
         kernel = f.read()
 
@@ -246,12 +252,12 @@ def build_payload(kernel_path: str, ci: bool,
             m.strip() for m in machines.split(",") if m.strip()
         ]
     if ci:
-        criteria = load_default_criteria()
+        criteria = load_criteria(criteria_name)
         payload["success"] = criteria["success"]
         payload["failures"] = criteria.get("failures", [])
     elif run_for is not None:
         payload["run_for"] = run_for
-        marker = load_default_criteria().get("kernel_start")
+        marker = load_criteria(criteria_name).get("kernel_start")
         if marker:
             payload["run_for_after"] = marker
     if timeout is not None:
diff --git a/scripts/baremetal_ci/criteria-ktest.json b/scripts/baremetal_ci/criteria-ktest.json
new file mode 100644
index 00000000..9571ddfd
--- /dev/null
+++ b/scripts/baremetal_ci/criteria-ktest.json
@@ -0,0 +1,11 @@
+{
+  "success": "ktest: \\d+ passed, 0 failed",
+  "failures": [
+    "Kernel panic:",
+    "BUG!",
+    "Oops!",
+    "ktest: \\d+ passed, [1-9]\\d* failed",
+    "ktest: nothing matched"
+  ],
+  "kernel_start": "Starting ultra kernel"
+}

From 1593b5fbf71a25c09f5f294a7df5374e5be569e6 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:28:14 +0300
Subject: [PATCH 139/152] build: let qemu_watch() stop at a pattern

A run that waits for a result has to end once the kernel printed it.
qemu_watch() can now stop at the first match of a pattern, which it
looks for with the color escapes of the console removed, and returns
the match after a short drain of the remaining output.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 build.py | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/build.py b/build.py
index 2a6d79cf..6f493b9d 100755
--- a/build.py
+++ b/build.py
@@ -360,12 +360,27 @@ def hyper_get_iso_br() -> str:
     return ret
 
 
-def qemu_watch_run_for(qp: subprocess.Popen, run_for: float) -> None:
+QEMU_DRAIN_SECONDS = 1
+
+# The colored console wraps parts of a line in escape sequences
+ANSI_ESCAPE = re.compile(rb"\x1b\[[0-9;]*m")
+
+
+# Streams the kernel output, returns the first match of 'until' or None
+# when QEMU exited or the wait ran out. With 'until' the run is bounded by
+# 'run_for' seconds of total output, otherwise by 'run_for' seconds since
+# the first output byte.
+def qemu_watch(
+    qp: subprocess.Popen, run_for: float,
+    until: Optional[re.Pattern] = None
+) -> Optional[re.Match]:
     assert qp.stdout is not None
     sel = selectors.DefaultSelector()
     sel.register(qp.stdout, selectors.EVENT_READ)
     deadline = None
     silent_deadline = time.monotonic() + run_for + QEMU_SILENT_BOOT_SECONDS
+    tail = b""
+    match = None
 
     while qp.poll() is None:
         if deadline is not None:
@@ -388,7 +403,16 @@ def qemu_watch_run_for(qp: subprocess.Popen, run_for: float) -> None:
         sys.stdout.buffer.flush()
         if deadline is None:
             deadline = time.monotonic() + run_for
+        if until is not None and match is None:
+            tail = (tail + data)[-4096:]
+            match = until.search(ANSI_ESCAPE.sub(b"", tail))
+            if match is not None:
+                deadline = time.monotonic() + QEMU_DRAIN_SECONDS
+    if match is not None:
+        sys.stdout.buffer.write(b"\n")
+        sys.stdout.buffer.flush()
     qp.wait()
+    return match
 
 
 def run_qemu(
@@ -452,7 +476,7 @@ def run_qemu(
     if run_for is not None and not debug:
         qp = subprocess.Popen(args, stdout=subprocess.PIPE)
         try:
-            qemu_watch_run_for(qp, run_for)
+            qemu_watch(qp, run_for)
         except KeyboardInterrupt:
             qp.terminate()
             qp.wait()

From fbb9fc0a704f44a6f1602b2e6d13a3e8e0802aa2 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 12 Nov 2026 01:08:10 +0300
Subject: [PATCH 140/152] build: retry failed downloads of the hyper binaries

Use download() from build_utils instead, which the toolchain build
already uses, it retries a failed download and only moves the file into
place once it is complete.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 build.py            | 4 ++--
 scripts/build_utils | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/build.py b/build.py
index 6f493b9d..5454c722 100755
--- a/build.py
+++ b/build.py
@@ -7,7 +7,6 @@
 import selectors
 import shutil
 import time
-import urllib.request
 import signal
 import sys
 from contextlib import contextmanager
@@ -24,6 +23,7 @@
     import scripts.build_utils.package_manager as pm
     import scripts.build_utils.toolchain_builder as tb
     import scripts.build_utils.toolchain_args as ta
+    import scripts.build_utils.download as dl
     import scripts.image_utils.ultra as ultr
     import scripts.image_utils.uefi as uefi
     import scripts.image_utils.path_guesser as pg
@@ -339,7 +339,7 @@ def hyper_get_binary(name: str, optional: bool = False) -> Optional[str]:
         base_url += hyper_version
         file_url = f"{base_url}/{name}"
         try:
-            urllib.request.urlretrieve(file_url, binary_path)
+            dl.download([file_url], binary_path)
         except Exception as ex:
             print(f"Failed to retrieve {name}:", ex)
             return None
diff --git a/scripts/build_utils b/scripts/build_utils
index e4920dbc..3f9222b2 160000
--- a/scripts/build_utils
+++ b/scripts/build_utils
@@ -1 +1 @@
-Subproject commit e4920dbcc2a847a28e78ac1c1c3e81c0c83ee150
+Subproject commit 3f9222b292eb0e6c983445cd7e335eb4c4819de8

From b3b235577c44c87140374e416c50f6bfbb91af57 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 12 Nov 2026 00:23:58 +0300
Subject: [PATCH 141/152] build: add --boot-test to boot the kernel and report
 the verdict

A kernel that builds can still crash early in boot. --boot-test boots it
headless and exits 0 once it idles, 1 on a panic, a BUG or an oops, and
2 when it never got that far. The output is judged by criteria.json,
like a plain baremetal run. --baremetal-criteria becomes --criteria
and picks the criteria for both.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 build.py                       | 56 ++++++++++++++++++++++++++++++++--
 scripts/baremetal_ci/client.py | 15 ++++-----
 2 files changed, 62 insertions(+), 9 deletions(-)

diff --git a/build.py b/build.py
index 5454c722..67413317 100755
--- a/build.py
+++ b/build.py
@@ -360,6 +360,7 @@ def hyper_get_iso_br() -> str:
     return ret
 
 
+QEMU_VERDICT_TIMEOUT_SECONDS = 120
 QEMU_DRAIN_SECONDS = 1
 
 # The colored console wraps parts of a line in escape sequences
@@ -415,10 +416,19 @@ def qemu_watch(
     return match
 
 
+# A QEMU run is judged by the same criteria as a baremetal run
+def verdict_pattern(criteria_name: Optional[str]) -> re.Pattern:
+    criteria = bmc.load_criteria(criteria_name)
+    failures = "|".join(criteria["failures"])
+    return re.compile(
+        f"(?P<success>{criteria['success']})|(?P<failure>{failures})".encode()
+    )
+
+
 def run_qemu(
     arch: str, image_path: str, image_type: str, debug: bool, uefi_boot: bool,
     uefi_firmware: str, kvm: bool, la57: bool, headless: bool, dry: bool,
-    run_for: Optional[float] = None
+    run_for: Optional[float] = None, verdict: Optional[re.Pattern] = None
 ) -> Optional[subprocess.Popen]:
     extra_args = []
     force_uefi = False
@@ -473,6 +483,27 @@ def run_qemu(
         print(" ".join(args))
         return None
 
+    if verdict is not None:
+        qp = subprocess.Popen(args, stdout=subprocess.PIPE)
+        try:
+            match = qemu_watch(qp, QEMU_VERDICT_TIMEOUT_SECONDS, verdict)
+        except KeyboardInterrupt:
+            qp.terminate()
+            qp.wait()
+            sys.exit(130)
+
+        if match is None:
+            print("Run did not finish", file=sys.stderr)
+            sys.exit(2)
+
+        failure = match.group("failure")
+        if failure is not None:
+            print(f"Run failed: {failure.decode(errors='replace')}",
+                  file=sys.stderr)
+            sys.exit(1)
+
+        sys.exit(0)
+
     if run_for is not None and not debug:
         qp = subprocess.Popen(args, stdout=subprocess.PIPE)
         try:
@@ -572,6 +603,18 @@ def main() -> None:
                        help="Paths to the hyper UEFI binaries "
                             "(BOOT{X64,AA64}.EFI)")
 
+    boot_tests = parser.add_argument_group(
+        "Boot tests",
+        "Boot the kernel headless and exit with the verdict, in QEMU unless "
+        "--baremetal is given"
+    )
+    boot_tests.add_argument("--boot-test", action="store_true",
+                            help="Check that the kernel boots until it idles "
+                                 "(the default for --baremetal)")
+    boot_tests.add_argument("--criteria", metavar="NAME",
+                            help="Judge the run by criteria-NAME.json "
+                                 "instead of criteria.json")
+
     qemu = parser.add_argument_group("QEMU")
     qemu.add_argument("--run", action="store_true",
                       help="Run in QEMU after building")
@@ -630,6 +673,11 @@ def main() -> None:
     if rc is not None:
         sys.exit(rc)
 
+    if args.criteria is not None:
+        args.boot_test = True
+    if args.boot_test:
+        args.headless = True
+
     # A changed command line only reaches QEMU through a new image
     if args.cmdline and args.machines is None and args.make_image is None:
         args.make_image = "qemu"
@@ -739,9 +787,13 @@ def main() -> None:
         if args.la57 and args.arch != "x86_64":
             raise RuntimeError("--la57 is only supported on x86_64")
 
+        verdict: Optional[re.Pattern] = None
+        if args.boot_test:
+            verdict = verdict_pattern(args.criteria)
+
         qp = run_qemu(args.arch, image_path, args.image_type, is_debug,
                       uefi_boot, args.uefi_firmware_path, args.kvm, args.la57,
-                      args.headless, args.dry, args.run_for)
+                      args.headless, args.dry, args.run_for, verdict)
 
     if args.debug:
         assert qp
diff --git a/scripts/baremetal_ci/client.py b/scripts/baremetal_ci/client.py
index 924563b1..5632824e 100755
--- a/scripts/baremetal_ci/client.py
+++ b/scripts/baremetal_ci/client.py
@@ -142,17 +142,18 @@ def git_meta() -> Dict[str, str]:
 
 
 def add_run_args(target: argparse._ActionsContainer, prefix: str = "",
-                 cmdline: bool = True, run_for: bool = True,
-                 log_dir: bool = True) -> None:
+                 cmdline: bool = True, criteria: bool = True,
+                 run_for: bool = True, log_dir: bool = True) -> None:
     if cmdline:
         target.add_argument(f"--{prefix}cmdline", dest="cmdline",
                             metavar="ARGS",
                             help="Extra kernel command line, appended "
                                  "after the machine's console line and "
                                  "the server's base")
-    target.add_argument(f"--{prefix}criteria", dest="criteria",
-                        metavar="NAME",
-                        help="Judge the run by criteria-NAME.json")
+    if criteria:
+        target.add_argument(f"--{prefix}criteria", dest="criteria",
+                            metavar="NAME",
+                            help="Judge the run by criteria-NAME.json")
     target.add_argument(f"--{prefix}timeout", dest="timeout",
                         type=float, metavar="SECONDS",
                         help="Give up on a run after this long, the "
@@ -187,8 +188,8 @@ def add_build_args(group: argparse._ActionsContainer) -> None:
                        metavar=("HOST", "CONFIG"),
                        help="Install or update the CI server on HOST "
                             "with the server CONFIG")
-    add_run_args(group, prefix="baremetal-", cmdline=False, run_for=False,
-                 log_dir=False)
+    add_run_args(group, prefix="baremetal-", cmdline=False, criteria=False,
+                 run_for=False, log_dir=False)
 
 
 def require_server(args: argparse.Namespace,

From d8efcd0e9d020bc543d702ee9f5d60f00e0c0957 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Thu, 12 Nov 2026 00:23:58 +0300
Subject: [PATCH 142/152] build: add --ktest to run the in-kernel tests and
 report the verdict

This boots the tests headless and exits with 0 when everything passed,
1 on a failure or a crash and 2 when the run never finished. The output
is judged by criteria-ktest.json, the same criteria as a baremetal run.
--ktest always builds the ktest preset and uses its criteria, so it
can't be combined with --config or --criteria.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 build.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/build.py b/build.py
index 67413317..29fd3c1b 100755
--- a/build.py
+++ b/build.py
@@ -611,6 +611,10 @@ def main() -> None:
     boot_tests.add_argument("--boot-test", action="store_true",
                             help="Check that the kernel boots until it idles "
                                  "(the default for --baremetal)")
+    boot_tests.add_argument("--ktest", nargs="?", const="all",
+                            metavar="TESTS",
+                            help="Run the in-kernel tests selected by TESTS "
+                                 "(default: all)")
     boot_tests.add_argument("--criteria", metavar="NAME",
                             help="Judge the run by criteria-NAME.json "
                                  "instead of criteria.json")
@@ -673,6 +677,18 @@ def main() -> None:
     if rc is not None:
         sys.exit(rc)
 
+    if args.ktest is not None:
+        if args.config is not None:
+            sys.exit("--config cannot be combined with --ktest")
+        if args.criteria is not None:
+            sys.exit("--criteria cannot be combined with --ktest")
+        args.config = "ktest"
+        args.criteria = "ktest"
+
+        if args.cmdline:
+            args.cmdline += " "
+        args.cmdline += f"ktest.run={args.ktest}"
+
     if args.criteria is not None:
         args.boot_test = True
     if args.boot_test:

From adcd4f2e4675cd249d0913df4fc8dd03a511a2e6 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 11 Nov 2026 02:31:08 +0300
Subject: [PATCH 143/152] build: add --optimization-level and --lto to override
 the config

A preset fixes the optimization level and whether LTO is used, so
building a preset with another combination meant editing its .config by
hand. Add --optimization-level and --lto/--no-lto to override them.

The options only apply to the build that is given them. They win over
the preset and over a value changed by hand in .config, and build.py
records the values they replace in the build directory, so that a later
build without them restores those values.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 build.py                 |  15 +++++-
 scripts/config_layers.py | 101 +++++++++++++++++++++++++++++++++++----
 2 files changed, 104 insertions(+), 12 deletions(-)

diff --git a/build.py b/build.py
index 29fd3c1b..c26da1eb 100755
--- a/build.py
+++ b/build.py
@@ -10,7 +10,7 @@
 import signal
 import sys
 from contextlib import contextmanager
-from typing import Optional, List, Callable, Iterator, Tuple
+from typing import Optional, Dict, List, Callable, Iterator, Tuple
 
 try:
     import scripts.kconfiglib.kconfiglib as kc
@@ -567,6 +567,11 @@ def main() -> None:
     build.add_argument("--reset-config", action="store_true",
                        help="Recreate the build directory's config from "
                             "--config, discarding the changes made to it")
+    build.add_argument("--optimization-level", type=int, choices=range(4),
+                       help="Override the optimization level of the config")
+    build.add_argument("--lto", action=argparse.BooleanOptionalAction,
+                       help="Override whether the config uses link-time "
+                            "optimization")
     build.add_argument("--menuconfig", action="store_true",
                        help="Run menuconfig to edit the current config file")
     build.add_argument("--guiconfig", action="store_true",
@@ -713,8 +718,14 @@ def main() -> None:
     build_dir = pg.project_root_relative(build_dir_name)
     os.makedirs(build_dir, exist_ok=True)
 
+    overrides: Dict[str, str] = {}
+    if args.optimization_level is not None:
+        overrides["OPTIMIZATION_LEVEL"] = str(args.optimization_level)
+    if args.lto is not None:
+        overrides["LTO"] = "y" if args.lto else "n"
+
     args.config = os.path.join(build_dir, ".config")
-    cl.prepare_config(args.config, config_layers, args.toolchain,
+    cl.prepare_config(args.config, config_layers, overrides, args.toolchain,
                       args.arch, args.reset_config)
 
     if args.menuconfig or args.guiconfig:
diff --git a/scripts/config_layers.py b/scripts/config_layers.py
index 0a973553..2fe735bb 100644
--- a/scripts/config_layers.py
+++ b/scripts/config_layers.py
@@ -1,4 +1,5 @@
 import hashlib
+import json
 import os
 import sys
 from typing import Dict, List, Optional, Set
@@ -23,6 +24,8 @@
 MERGE_BASE = ".config.merge-base"
 MERGE_BASE_LAYER = "# layer: "
 
+OVERRIDES = ".config.overrides"
+
 
 def root_kconfig() -> kc.Kconfig:
     old_dir = os.getcwd()
@@ -176,9 +179,77 @@ def stamp_path(stamp: str) -> str:
     return stamp.split(" ", 1)[1]
 
 
+def read_overrides(path: str) -> Dict[str, Dict[str, str]]:
+    if not os.path.isfile(path):
+        return {}
+
+    with open(path) as f:
+        return json.load(f)
+
+
+def overrides_in_place(
+    record: Dict[str, Dict[str, str]], overrides: Dict[str, str],
+    cur_values: Dict[str, str]
+) -> bool:
+    for name, entry in record.items():
+        if overrides.get(name) != entry["value"]:
+            return False
+
+    return all(cur_values.get(name) == value
+               for name, value in overrides.items())
+
+
+def restore_overridden(
+    record: Dict[str, Dict[str, str]], overrides: Dict[str, str],
+    cur_values: Dict[str, str]
+) -> List[str]:
+    messages = []
+
+    for name, entry in record.items():
+        # The value was changed by hand after it was overridden, keep it
+        if cur_values.get(name) != entry["value"]:
+            continue
+
+        cur_values[name] = entry["replaced"]
+        if name not in overrides:
+            messages.append(f"restoring {name} {entry['value']} => "
+                            f"{entry['replaced']}, no longer overridden "
+                            "from the command line")
+
+    return messages
+
+
+def apply_overrides(
+    kconfig: kc.Kconfig, overrides: Dict[str, str], path: str
+) -> List[str]:
+    values = config_values(kconfig)
+    record = {}
+    messages = []
+
+    for name, value in overrides.items():
+        kconfig.syms[name].set_value(value)
+        if values[name] == value:
+            continue
+
+        record[name] = {"value": value, "replaced": values[name]}
+        messages.append(f"overriding {name} {values[name]} => {value} "
+                        "from the command line")
+
+    if not record:
+        if os.path.isfile(path):
+            os.remove(path)
+
+        return messages
+
+    with open(path, "w") as f:
+        json.dump(record, f, indent=4)
+
+    return messages
+
+
 def merge_config(
-    config: str, merge_base: str, layered: kc.Kconfig, layers: List[str],
-    toolchain: str, arch: str
+    config: str, merge_base: str, overrides_path: str, layered: kc.Kconfig,
+    layers: List[str], overrides: Dict[str, str], toolchain: str, arch: str
 ) -> None:
     where = display_path(config)
     stamps = layer_stamps(layers)
@@ -197,15 +268,17 @@ def merge_config(
         reader.load_config(merge_base)
         base_values = config_values(reader)
 
+    reader.load_config(config)
+    cur_values = config_values(reader)
+    record = read_overrides(overrides_path)
+
     # Leave .config alone when nothing changed, rewriting it would
     # make cmake reconfigure on every build
-    if old_stamps == stamps and base_values == new_values:
+    if (old_stamps == stamps and base_values == new_values and
+            overrides_in_place(record, overrides, cur_values)):
         return
 
-    reader.load_config(config)
-    cur_values = config_values(reader)
-
-    messages = []
+    messages = restore_overridden(record, overrides, cur_values)
     if old_paths is None and paths:
         messages.append(f"now made from {', '.join(paths)}")
 
@@ -276,21 +349,29 @@ def merge_config(
         messages.append(f"applied {applied} {changes} from "
                         f"{', '.join(changed) or 'Kconfig'}")
 
-    for message in messages + conflicts + hidden:
+    overridden = apply_overrides(layered, overrides, overrides_path)
+    for message in messages + conflicts + hidden + overridden:
         print(f"{where}: {message}")
 
     layered.write_config(config)
 
 
 def prepare_config(
-    config: str, layers: List[str], toolchain: str, arch: str, reset: bool
+    config: str, layers: List[str], overrides: Dict[str, str],
+    toolchain: str, arch: str, reset: bool
 ) -> None:
     merge_base = os.path.join(os.path.dirname(config), MERGE_BASE)
+    overrides_path = os.path.join(os.path.dirname(config), OVERRIDES)
     layered = load_layers(layers, toolchain, arch)
 
     if not reset and os.path.isfile(config):
-        merge_config(config, merge_base, layered, layers, toolchain, arch)
+        merge_config(config, merge_base, overrides_path, layered, layers,
+                     overrides, toolchain, arch)
         return
 
     write_merge_base(layered, layer_stamps(layers), merge_base)
+
+    for message in apply_overrides(layered, overrides, overrides_path):
+        print(f"{display_path(config)}: {message}")
+
     layered.write_config(config)

From 613a27e63c85a043381ce5098af9b7778673549b Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 8 Nov 2026 15:50:53 +0300
Subject: [PATCH 144/152] ci: build the GCC toolchain in a job of its own

On a cache miss, both GCC legs of the build matrix built the toolchain
at the same time. Build it once in a toolchain job that the build job
waits for, with the toolchain steps in a composite action that both
jobs use.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .github/actions/toolchain/action.yml | 46 ++++++++++++++++++++++++++++
 .github/workflows/main.yml           | 42 ++++++++++++-------------
 2 files changed, 65 insertions(+), 23 deletions(-)
 create mode 100644 .github/actions/toolchain/action.yml

diff --git a/.github/actions/toolchain/action.yml b/.github/actions/toolchain/action.yml
new file mode 100644
index 00000000..9787c5ac
--- /dev/null
+++ b/.github/actions/toolchain/action.yml
@@ -0,0 +1,46 @@
+name: Platform toolchain
+description: Build the platform toolchain, with GCC cached between runs
+
+inputs:
+  arch:
+    description: The architecture to build for
+    required: true
+  toolchain:
+    description: gcc or clang
+    required: true
+  lookup-only:
+    description: Don't restore a cached toolchain, only build a missing one
+    default: "false"
+
+runs:
+  using: composite
+  steps:
+    - if: ${{ inputs.toolchain == 'gcc' }}
+      id: version
+      name: Get the toolchain version
+      shell: bash
+      run: |
+        python3 -c 'import scripts.build_utils.toolchain_builder as tb
+        print(f"key=gcc-{tb.GCC_VERSION}-binutils-{tb.BINUTILS_VERSION}")' \
+          >> "$GITHUB_OUTPUT"
+
+    - if: ${{ inputs.toolchain == 'gcc' }}
+      id: cache
+      name: Restore the toolchain cache
+      uses: actions/cache/restore@v4.2.2
+      with:
+        path: "${{ github.workspace }}/toolchain/tools-${{ inputs.toolchain }}-${{ inputs.arch }}"
+        key: "${{ runner.os }}-${{ runner.arch }}-${{ inputs.arch }}-${{ steps.version.outputs.key }}"
+        lookup-only: ${{ inputs.lookup-only }}
+
+    - if: ${{ inputs.lookup-only != 'true' || steps.cache.outputs.cache-hit != 'true' }}
+      name: Build the platform toolchain
+      shell: bash
+      run: "${{ github.workspace }}/build.py --arch ${{ inputs.arch }} --toolchain ${{ inputs.toolchain }} --no-tune-native --toolchain-only"
+
+    - if: ${{ inputs.toolchain == 'gcc' && steps.cache.outputs.cache-hit != 'true' }}
+      name: Save the toolchain cache
+      uses: actions/cache/save@v4.2.2
+      with:
+        path: "${{ github.workspace }}/toolchain/tools-${{ inputs.toolchain }}-${{ inputs.arch }}"
+        key: ${{ steps.cache.outputs.cache-primary-key }}
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 323a8cd3..e056be50 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -45,7 +45,23 @@ jobs:
       - name: Check the coding conventions
         run: "${{ github.workspace }}/build.py --toolchain clang --lint"
 
+  toolchain:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Build the GCC toolchain unless it is cached
+        uses: ./.github/actions/toolchain
+        with:
+          arch: x86_64
+          toolchain: gcc
+          lookup-only: true
+
   build:
+    needs: toolchain
     runs-on: ${{ matrix.os }}
 
     strategy:
@@ -66,31 +82,11 @@ jobs:
         with:
           submodules: true
 
-      - if: ${{ matrix.toolchain == 'gcc' }}
-        id: toolchain-version
-        name: Get the toolchain version
-        run: |
-          python3 -c 'import scripts.build_utils.toolchain_builder as tb
-          print(f"key=gcc-{tb.GCC_VERSION}-binutils-{tb.BINUTILS_VERSION}")' \
-            >> "$GITHUB_OUTPUT"
-
-      - if: ${{ matrix.toolchain == 'gcc' }}
-        id: toolchain-cache
-        name: Restore the toolchain cache
-        uses: actions/cache/restore@v4.2.2
-        with:
-          path: "${{ github.workspace }}/toolchain/tools-${{ matrix.toolchain }}-${{ matrix.arch }}"
-          key: "${{ runner.os }}-${{ runner.arch }}-${{ matrix.arch }}-${{ steps.toolchain-version.outputs.key }}"
-
       - name: Build the platform toolchain
-        run: "${{ github.workspace }}/build.py --arch ${{ matrix.arch }} --toolchain ${{ matrix.toolchain }} --no-tune-native --toolchain-only"
-
-      - if: ${{ matrix.toolchain == 'gcc' && steps.toolchain-cache.outputs.cache-hit != 'true' }}
-        name: Save the toolchain cache
-        uses: actions/cache/save@v4.2.2
+        uses: ./.github/actions/toolchain
         with:
-          path: "${{ github.workspace }}/toolchain/tools-${{ matrix.toolchain }}-${{ matrix.arch }}"
-          key: ${{ steps.toolchain-cache.outputs.cache-primary-key }}
+          arch: ${{ matrix.arch }}
+          toolchain: ${{ matrix.toolchain }}
 
       - name: Build the kernel
         run: "${{ github.workspace }}/build.py --arch ${{ matrix.arch }} --config ${{ matrix.preset }} --toolchain ${{ matrix.toolchain }} --no-tune-native"

From ab192447ff7518c1c1dbb8c7525de1bdae8a06a3 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:28:14 +0300
Subject: [PATCH 145/152] ci: run the in-kernel tests

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .github/workflows/main.yml | 44 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index e056be50..01a268da 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -91,6 +91,50 @@ jobs:
       - name: Build the kernel
         run: "${{ github.workspace }}/build.py --arch ${{ matrix.arch }} --config ${{ matrix.preset }} --toolchain ${{ matrix.toolchain }} --no-tune-native"
 
+  ktest:
+    needs: toolchain
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - os: ubuntu-latest
+            arch: x86_64
+            toolchain: clang
+            accel: kvm
+            overrides: --optimization-level 3 --lto
+          - os: ubuntu-latest
+            arch: x86_64
+            toolchain: gcc
+            accel: tcg
+            overrides: --optimization-level 2
+          - os: macos-latest
+            arch: x86_64
+            toolchain: clang
+            accel: tcg
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Build the platform toolchain
+        uses: ./.github/actions/toolchain
+        with:
+          arch: ${{ matrix.arch }}
+          toolchain: ${{ matrix.toolchain }}
+
+      - if: ${{ matrix.accel == 'kvm' }}
+        name: Give the runner access to KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      - name: Run the in-kernel tests
+        run: "${{ github.workspace }}/build.py --arch ${{ matrix.arch }} --toolchain ${{ matrix.toolchain }} --no-tune-native --ktest ${{ matrix.accel == 'kvm' && '--kvm' || '' }} ${{ matrix.overrides }}"
+
   unit-tests:
     runs-on: ubuntu-latest
 

From a990f329233d455b56206521c0ef3d1afd687d84 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Wed, 11 Nov 2026 23:45:03 +0300
Subject: [PATCH 146/152] ci: boot every kernel that the build job builds

A successful build doesn't mean that the kernel boots, undefined
behavior can still crash it early. Boot every kernel the job builds
with --boot-test, and rename the job to build-and-boot.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .github/workflows/main.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 01a268da..9610fa92 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -60,7 +60,7 @@ jobs:
           toolchain: gcc
           lookup-only: true
 
-  build:
+  build-and-boot:
     needs: toolchain
     runs-on: ${{ matrix.os }}
 
@@ -88,8 +88,8 @@ jobs:
           arch: ${{ matrix.arch }}
           toolchain: ${{ matrix.toolchain }}
 
-      - name: Build the kernel
-        run: "${{ github.workspace }}/build.py --arch ${{ matrix.arch }} --config ${{ matrix.preset }} --toolchain ${{ matrix.toolchain }} --no-tune-native"
+      - name: Build and boot the kernel
+        run: "${{ github.workspace }}/build.py --arch ${{ matrix.arch }} --config ${{ matrix.preset }} --toolchain ${{ matrix.toolchain }} --no-tune-native --boot-test"
 
   ktest:
     needs: toolchain

From 2d78bca9528da25d8576302c8fe458c2888ed874 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 10 Nov 2026 22:59:41 +0300
Subject: [PATCH 147/152] ci: lint every config preset

The AST checks of the linter only see the code the linted build
compiles, and the default preset leaves out everything behind an option
it doesn't enable. Lint every preset in configs/ instead.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 .github/workflows/main.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 9610fa92..0877c6cd 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -42,8 +42,12 @@ jobs:
       - name: Build the platform toolchain
         run: "${{ github.workspace }}/build.py --toolchain clang --toolchain-only"
 
-      - name: Check the coding conventions
-        run: "${{ github.workspace }}/build.py --toolchain clang --lint"
+      - name: Check the coding conventions of every preset
+        run: |
+          for preset in configs/*.config; do
+            "${{ github.workspace }}/build.py" --toolchain clang \
+                --config "$(basename "$preset" .config)" --lint
+          done
 
   toolchain:
     runs-on: ubuntu-latest

From 162c973dcab3b749b0798e3dd6229c8a2644c112 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:39:18 +0300
Subject: [PATCH 148/152] sched: add the scheduler integration tests

Cover timed sleeps, equal priority spinners sharing the CPU, wakeup
preemption by a higher priority but not an equal one, and kthread_stop()
on a sleeping and on a never started thread.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/ktest/Kconfig        |   2 +
 kernel/sched/CMakeLists.txt |   5 ++
 kernel/sched/tests/Kconfig  |   3 +
 kernel/sched/tests/sched.c  | 168 ++++++++++++++++++++++++++++++++++++
 4 files changed, 178 insertions(+)
 create mode 100644 kernel/sched/tests/Kconfig
 create mode 100644 kernel/sched/tests/sched.c

diff --git a/kernel/ktest/Kconfig b/kernel/ktest/Kconfig
index 7d9a1d4a..1a98cdfe 100644
--- a/kernel/ktest/Kconfig
+++ b/kernel/ktest/Kconfig
@@ -21,4 +21,6 @@ config KTEST_SELFTEST
     bool "Tests of the test runner itself" if !KTEST_ALL_TESTS
     default KTEST_ALL_TESTS
 
+rsource "../sched/tests/Kconfig"
+
 endif
diff --git a/kernel/sched/CMakeLists.txt b/kernel/sched/CMakeLists.txt
index 1a2200cf..f2050b01 100644
--- a/kernel/sched/CMakeLists.txt
+++ b/kernel/sched/CMakeLists.txt
@@ -7,3 +7,8 @@ ultra_sources(
     wait_queue.c
     completion.c
 )
+
+ultra_sources_if(
+    CONFIG_KTEST_SCHED
+    tests/sched.c
+)
diff --git a/kernel/sched/tests/Kconfig b/kernel/sched/tests/Kconfig
new file mode 100644
index 00000000..e25cdbd5
--- /dev/null
+++ b/kernel/sched/tests/Kconfig
@@ -0,0 +1,3 @@
+config KTEST_SCHED
+    bool "Scheduler tests" if !KTEST_ALL_TESTS
+    default KTEST_ALL_TESTS
diff --git a/kernel/sched/tests/sched.c b/kernel/sched/tests/sched.c
new file mode 100644
index 00000000..d7b6076e
--- /dev/null
+++ b/kernel/sched/tests/sched.c
@@ -0,0 +1,168 @@
+#include <ktest/ktest.h>
+
+#include <private/sched/task.h>
+#include <private/sched/sched.h>
+
+#include <common/atomic.h>
+
+#include <sched/kthread.h>
+#include <sched/sched.h>
+#include <sched/wait_queue.h>
+#include <time/time.h>
+#include <time/units.h>
+
+KTEST_CASE(sleep_returns_no_earlier_than_asked)
+{
+    static const u64 s_durations_ns[] = {
+        NS_PER_MS, 5 * NS_PER_MS, 20 * NS_PER_MS,
+    };
+    u64 before, after;
+    size_t i;
+
+    for (i = 0; i < ARRAY_SIZE(s_durations_ns); i++) {
+        before = ns_since_boot();
+        sleep_ns(s_durations_ns[i]);
+        after = ns_since_boot();
+
+        KTEST_EXPECT_GE(after - before, s_durations_ns[i]);
+    }
+}
+
+#define NUM_SPINNERS 3
+#define SPIN_LIMIT_NS NS_PER_SEC
+
+struct spinners {
+    u32 last;
+    u32 num_saw_another;
+    bool saw_another[NUM_SPINNERS];
+};
+
+static void spin(void *user, u32 index)
+{
+    struct spinners *s = user;
+    bool saw_another = false;
+    u64 deadline;
+    u32 me, last;
+
+    me = index + 1;
+    deadline = ns_from_now(SPIN_LIMIT_NS);
+
+    for (;;) {
+        if (atomic_load_relaxed(&s->num_saw_another) == NUM_SPINNERS)
+            break;
+        if (ns_since_boot() > deadline)
+            break;
+
+        last = atomic_xchg(&s->last, me, MO_RELAXED);
+        if (last == me || last == 0)
+            continue;
+        if (saw_another)
+            continue;
+
+        saw_another = true;
+        atomic_fetch_add(&s->num_saw_another, 1, MO_RELAXED);
+    }
+
+    s->saw_another[index] = saw_another;
+}
+
+KTEST_CASE(equal_priority_spinners_share_the_cpu)
+{
+    struct spinners s = { 0 };
+    u32 i;
+
+    KTEST_RUN_THREADS(spin, &s, NUM_SPINNERS, "spinner");
+
+    for (i = 0; i < NUM_SPINNERS; i++)
+        KTEST_EXPECT(s.saw_another[i]);
+}
+
+static void set_flag(void *user)
+{
+    atomic_store_release((bool*)user, true);
+}
+
+static struct task *create_flag_setter(bool *flag, u8 prio)
+{
+    struct task *task;
+
+    task = kthread_create(set_flag, flag, "flag-setter");
+    if (task != nullptr)
+        task->prio = prio;
+
+    return task;
+}
+
+KTEST_CASE(higher_priority_wakeup_preempts_right_away)
+{
+    struct task *task;
+    bool flag = false;
+
+    task = create_flag_setter(&flag, TASK_PRIO_HIGHEST);
+    KTEST_ASSERT(task != nullptr);
+
+    kthread_start(task);
+    KTEST_EXPECT(atomic_load_acquire(&flag));
+
+    task_unref(task);
+}
+
+KTEST_CASE(equal_priority_wakeup_does_not_preempt)
+{
+    struct task *task;
+    bool flag = false;
+
+    sleep_ns(NS_PER_MS);
+
+    task = create_flag_setter(&flag, TASK_PRIO_DEFAULT);
+    KTEST_ASSERT(task != nullptr);
+
+    kthread_start(task);
+    KTEST_EXPECT(!atomic_load_acquire(&flag));
+
+    // yield() queues us behind the flag setter, so the setter runs first
+    yield();
+    KTEST_EXPECT(atomic_load_acquire(&flag));
+
+    task_unref(task);
+}
+
+static void sleep_until_stopped(void *user)
+{
+    DEFINE_WAIT_QUEUE(wq);
+    bool *ran = user;
+
+    wait_event(&wq, kthread_should_stop());
+    *ran = true;
+}
+
+KTEST_CASE(kthread_stop_wakes_a_sleeping_thread)
+{
+    struct task *task;
+    bool ran = false;
+
+    task = kthread_create(sleep_until_stopped, &ran, "sleeper");
+    KTEST_ASSERT(task != nullptr);
+    kthread_start(task);
+
+    // Let it run until it sleeps, so that kthread_stop() has to wake it up
+    while (task_state(task) != TASK_STATE_UNINTERRUPTIBLE)
+        yield();
+
+    kthread_stop(task);
+    KTEST_EXPECT(ran);
+}
+
+KTEST_CASE(kthread_stop_before_start_never_runs_it)
+{
+    struct task *task;
+    bool ran = false;
+
+    task = kthread_create(sleep_until_stopped, &ran, "never-started");
+    KTEST_ASSERT(task != nullptr);
+
+    kthread_stop(task);
+    yield();
+
+    KTEST_EXPECT(!ran);
+}

From e34ac9b17bf73160e1f2d84f18cdf0e556aa51e0 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:39:18 +0300
Subject: [PATCH 149/152] time: add the timer and clock integration tests

Cover hundreds of timers expiring at the same time, cancellation, a
callback rearming its own timer and the clock never stepping back
between two readers.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/ktest/Kconfig       |   1 +
 kernel/time/CMakeLists.txt |   5 ++
 kernel/time/tests/Kconfig  |   3 +
 kernel/time/tests/time.c   | 179 +++++++++++++++++++++++++++++++++++++
 4 files changed, 188 insertions(+)
 create mode 100644 kernel/time/tests/Kconfig
 create mode 100644 kernel/time/tests/time.c

diff --git a/kernel/ktest/Kconfig b/kernel/ktest/Kconfig
index 1a98cdfe..427d0176 100644
--- a/kernel/ktest/Kconfig
+++ b/kernel/ktest/Kconfig
@@ -22,5 +22,6 @@ config KTEST_SELFTEST
     default KTEST_ALL_TESTS
 
 rsource "../sched/tests/Kconfig"
+rsource "../time/tests/Kconfig"
 
 endif
diff --git a/kernel/time/CMakeLists.txt b/kernel/time/CMakeLists.txt
index b839a910..fe326f79 100644
--- a/kernel/time/CMakeLists.txt
+++ b/kernel/time/CMakeLists.txt
@@ -5,3 +5,8 @@ ultra_sources(
     timer.c
     units.c
 )
+
+ultra_sources_if(
+    CONFIG_KTEST_TIME
+    tests/time.c
+)
diff --git a/kernel/time/tests/Kconfig b/kernel/time/tests/Kconfig
new file mode 100644
index 00000000..6101fa6c
--- /dev/null
+++ b/kernel/time/tests/Kconfig
@@ -0,0 +1,3 @@
+config KTEST_TIME
+    bool "Timer and clock tests" if !KTEST_ALL_TESTS
+    default KTEST_ALL_TESTS
diff --git a/kernel/time/tests/time.c b/kernel/time/tests/time.c
new file mode 100644
index 00000000..ffdc68ed
--- /dev/null
+++ b/kernel/time/tests/time.c
@@ -0,0 +1,179 @@
+#include <ktest/ktest.h>
+
+#include <common/atomic.h>
+
+#include <sched/sched.h>
+#include <sched/completion.h>
+#include <time/time.h>
+#include <time/timer.h>
+#include <time/units.h>
+
+#define NUM_MASS_TIMERS 500
+
+struct counted_timer {
+    struct timer timer;
+    u32 num_fired;
+};
+
+static u32 s_num_mass_fired;
+static struct completion s_all_mass_fired;
+
+static enum timer_result count_mass_firing(struct timer *timer)
+{
+    struct counted_timer *counted;
+
+    counted = container_of(timer, struct counted_timer, timer);
+    counted->num_fired++;
+
+    if (atomic_add_fetch(&s_num_mass_fired, 1, MO_RELAXED) == NUM_MASS_TIMERS)
+        complete(&s_all_mass_fired);
+
+    return TIMER_RESULT_DONE;
+}
+
+KTEST_CASE(timers_due_together_all_fire_once)
+{
+    static struct counted_timer s_timers[NUM_MASS_TIMERS];
+    u64 deadline_ns;
+    size_t i;
+
+    atomic_store_relaxed(&s_num_mass_fired, 0);
+    completion_init(&s_all_mass_fired);
+    deadline_ns = ns_from_now(20 * NS_PER_MS);
+
+    for (i = 0; i < NUM_MASS_TIMERS; i++) {
+        s_timers[i].num_fired = 0;
+        timer_init(&s_timers[i].timer, count_mass_firing);
+        timer_arm(&s_timers[i].timer, deadline_ns);
+    }
+
+    wait_for_completion(&s_all_mass_fired);
+
+    for (i = 0; i < NUM_MASS_TIMERS; i++)
+        KTEST_EXPECT_EQ(s_timers[i].num_fired, 1);
+}
+
+static u32 s_num_fired;
+
+static enum timer_result count_firing(struct timer *timer)
+{
+    UNREFERENCED_PARAMETER(timer);
+    atomic_fetch_add(&s_num_fired, 1, MO_RELAXED);
+
+    return TIMER_RESULT_DONE;
+}
+
+KTEST_CASE(cancelled_timer_does_not_fire)
+{
+    DEFINE_TIMER(queued, count_firing);
+    DEFINE_TIMER(never_armed, count_firing);
+
+    atomic_store_relaxed(&s_num_fired, 0);
+
+    timer_arm(&queued, ns_from_now(10 * NS_PER_MS));
+    timer_cancel(&queued);
+    KTEST_EXPECT(timer_cancel_nosync(&never_armed));
+
+    sleep_ns(30 * NS_PER_MS);
+
+    KTEST_EXPECT_EQ(atomic_load_relaxed(&s_num_fired), 0);
+}
+
+struct periodic {
+    struct timer timer;
+    u32 runs;
+    u32 wanted;
+    u64 period_ns;
+    struct completion done;
+};
+
+static enum timer_result rearm(struct timer *timer)
+{
+    struct periodic *p;
+
+    p = container_of(timer, struct periodic, timer);
+
+    if (++p->runs == p->wanted) {
+        complete(&p->done);
+        return TIMER_RESULT_DONE;
+    }
+
+    timer_advance(timer, p->period_ns);
+    return TIMER_RESULT_REARM;
+}
+
+KTEST_CASE(timer_may_rearm_itself_from_its_callback)
+{
+    struct periodic p = {
+        .wanted = 10,
+        .period_ns = 2 * NS_PER_MS,
+        .done = COMPLETION_INIT(p.done),
+    };
+    u64 before, after;
+
+    timer_init(&p.timer, rearm);
+
+    before = ns_since_boot();
+    timer_arm(&p.timer, before + p.period_ns);
+    wait_for_completion(&p.done);
+    after = ns_since_boot();
+
+    timer_cancel(&p.timer);
+
+    KTEST_EXPECT_EQ(p.runs, p.wanted);
+    KTEST_EXPECT_GE(after - before, p.wanted * p.period_ns);
+}
+
+#define NUM_CLOCK_READERS 2
+#define NUM_CLOCK_READS 200000
+
+struct clock_readers {
+    u64 latest;
+    u32 steps_back;
+};
+
+static void publish_latest(u64 *latest, u64 now)
+{
+    u64 old;
+
+    old = atomic_load_relaxed(latest);
+
+    while (old < now) {
+        if (atomic_cmpxchg_acq_rel(latest, &old, now))
+            break;
+    }
+}
+
+static void read_the_clock(void *user, u32 index)
+{
+    struct clock_readers *readers = user;
+    u64 seen, now;
+    u32 i;
+
+    UNREFERENCED_PARAMETER(index);
+
+    for (i = 0; i < NUM_CLOCK_READS; i++) {
+        /*
+         * The latest time that any reader has seen so far. The time we read
+         * next must not be earlier than that, no matter which reader published
+         * it.
+         */
+        seen = atomic_load_acquire(&readers->latest);
+        now = ns_since_boot();
+
+        if (unlikely(now < seen))
+            atomic_fetch_add(&readers->steps_back, 1, MO_RELAXED);
+
+        publish_latest(&readers->latest, now);
+    }
+}
+
+KTEST_CASE(clock_never_steps_back)
+{
+    struct clock_readers readers = { 0 };
+
+    KTEST_RUN_THREADS(
+        read_the_clock, &readers, NUM_CLOCK_READERS, "clock-reader"
+    );
+    KTEST_EXPECT_EQ(readers.steps_back, 0);
+}

From d80d16a8dda3f95e72e6b543d2dfc39dc9607328 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Tue, 6 Oct 2026 14:39:18 +0300
Subject: [PATCH 150/152] mutex: add the mutex integration tests

Cover mutual exclusion between contending threads, a waiter not starved
by a thread that keeps retaking the lock, and try-lock failing only
while the lock is held.

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/ktest/Kconfig          |   1 +
 kernel/locking/CMakeLists.txt |   5 ++
 kernel/locking/tests/Kconfig  |   3 +
 kernel/locking/tests/mutex.c  | 149 ++++++++++++++++++++++++++++++++++
 4 files changed, 158 insertions(+)
 create mode 100644 kernel/locking/tests/Kconfig
 create mode 100644 kernel/locking/tests/mutex.c

diff --git a/kernel/ktest/Kconfig b/kernel/ktest/Kconfig
index 427d0176..862d84d6 100644
--- a/kernel/ktest/Kconfig
+++ b/kernel/ktest/Kconfig
@@ -23,5 +23,6 @@ config KTEST_SELFTEST
 
 rsource "../sched/tests/Kconfig"
 rsource "../time/tests/Kconfig"
+rsource "../locking/tests/Kconfig"
 
 endif
diff --git a/kernel/locking/CMakeLists.txt b/kernel/locking/CMakeLists.txt
index cfb50c51..f1ec8aee 100644
--- a/kernel/locking/CMakeLists.txt
+++ b/kernel/locking/CMakeLists.txt
@@ -2,3 +2,8 @@ ultra_sources(
     spinlock.c
     mutex.c
 )
+
+ultra_sources_if(
+    CONFIG_KTEST_MUTEX
+    tests/mutex.c
+)
diff --git a/kernel/locking/tests/Kconfig b/kernel/locking/tests/Kconfig
new file mode 100644
index 00000000..fef2e38c
--- /dev/null
+++ b/kernel/locking/tests/Kconfig
@@ -0,0 +1,3 @@
+config KTEST_MUTEX
+    bool "Mutex tests" if !KTEST_ALL_TESTS
+    default KTEST_ALL_TESTS
diff --git a/kernel/locking/tests/mutex.c b/kernel/locking/tests/mutex.c
new file mode 100644
index 00000000..f6859be9
--- /dev/null
+++ b/kernel/locking/tests/mutex.c
@@ -0,0 +1,149 @@
+#include <ktest/ktest.h>
+
+#include <common/atomic.h>
+
+#include <locking/mutex.h>
+#include <sched/kthread.h>
+#include <sched/sched.h>
+#include <sched/completion.h>
+#include <private/sched/sched.h>
+#include <time/units.h>
+
+#define NUM_CONTENDERS 4
+#define ITERATIONS_PER_CONTENDER 200
+
+struct contention {
+    struct mutex mutex;
+    u32 inside;
+    u32 violations;
+    u64 total;
+};
+
+static void contend(void *user, u32 index)
+{
+    struct contention *c = user;
+    u32 i;
+
+    UNREFERENCED_PARAMETER(index);
+
+    for (i = 0; i < ITERATIONS_PER_CONTENDER; i++) {
+        mutex_lock(&c->mutex);
+
+        if (unlikely(atomic_fetch_add(&c->inside, 1, MO_RELAXED) != 0))
+            atomic_fetch_add(&c->violations, 1, MO_RELAXED);
+
+        if (i % 50 == 0)
+            sleep_ns(NS_PER_MS);
+
+        c->total++;
+
+        atomic_fetch_sub(&c->inside, 1, MO_RELAXED);
+        mutex_unlock(&c->mutex);
+    }
+}
+
+KTEST_CASE(contended_lock_excludes)
+{
+    struct contention c = { 0 };
+
+    mutex_init(&c.mutex);
+
+    KTEST_RUN_THREADS(contend, &c, NUM_CONTENDERS, "contender");
+
+    KTEST_EXPECT_EQ(c.violations, 0);
+    KTEST_EXPECT_EQ(c.total, NUM_CONTENDERS * ITERATIONS_PER_CONTENDER);
+    KTEST_EXPECT(mutex_try_lock(&c.mutex));
+    mutex_unlock(&c.mutex);
+}
+
+#define GREEDY_ITERATIONS 20000
+
+struct handoff {
+    struct mutex mutex;
+    u32 greedy_count;
+    struct completion done;
+};
+
+static void hammer(void *user)
+{
+    struct handoff *h = user;
+    u32 i;
+
+    for (i = 0; i < GREEDY_ITERATIONS; i++) {
+        mutex_lock(&h->mutex);
+        atomic_store_relaxed(&h->greedy_count, i + 1);
+        yield();
+        mutex_unlock(&h->mutex);
+    }
+
+    complete(&h->done);
+}
+
+KTEST_CASE(waiter_is_not_starved_by_a_hammering_holder)
+{
+    struct handoff h = { .done = COMPLETION_INIT(h.done) };
+    u32 before, after;
+
+    mutex_init(&h.mutex);
+
+    KTEST_ASSERT(!is_error(kthread_run(hammer, &h, "hammer")));
+    sleep_ns(5 * NS_PER_MS);
+
+    before = atomic_load_relaxed(&h.greedy_count);
+    mutex_lock(&h.mutex);
+    after = atomic_load_relaxed(&h.greedy_count);
+    mutex_unlock(&h.mutex);
+
+    wait_for_completion(&h.done);
+
+    KTEST_EXPECT_GE(after, 1);
+
+    /*
+     * We may fail to take the lock once while the hammer holds it across its
+     * yield. That failed attempt makes the next unlock hand the lock to us
+     * instead of letting the hammer take it again.
+     */
+    KTEST_EXPECT_LE(after - before, 2);
+}
+
+struct holder {
+    struct mutex mutex;
+    struct completion locked;
+    struct completion release;
+    struct completion done;
+};
+
+static void hold_until_released(void *user)
+{
+    struct holder *h = user;
+
+    mutex_lock(&h->mutex);
+    complete(&h->locked);
+
+    wait_for_completion(&h->release);
+    mutex_unlock(&h->mutex);
+
+    complete(&h->done);
+}
+
+KTEST_CASE(try_lock_fails_while_another_thread_holds_it)
+{
+    struct holder h = {
+        .locked = COMPLETION_INIT(h.locked),
+        .release = COMPLETION_INIT(h.release),
+        .done = COMPLETION_INIT(h.done),
+    };
+
+    mutex_init(&h.mutex);
+
+    KTEST_ASSERT(!is_error(kthread_run(hold_until_released, &h, "holder")));
+    wait_for_completion(&h.locked);
+
+    KTEST_EXPECT(!mutex_try_lock(&h.mutex));
+
+    complete(&h.release);
+    wait_for_completion(&h.done);
+
+    KTEST_EXPECT(mutex_try_lock(&h.mutex));
+    mutex_unlock(&h.mutex);
+}

From 35e4c8e347075780735d26425a6462c26e7e8c1e Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Sun, 8 Nov 2026 16:45:49 +0300
Subject: [PATCH 151/152] common/string_container: add str_ends_with()

Add the counterpart of str_starts_with().

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/common/string_container.c         | 11 +++++++++++
 kernel/include/common/string_container.h |  2 ++
 tests/test-cases/test_string.c           |  9 +++++++++
 3 files changed, 22 insertions(+)

diff --git a/kernel/common/string_container.c b/kernel/common/string_container.c
index 8dd974eb..417d3064 100644
--- a/kernel/common/string_container.c
+++ b/kernel/common/string_container.c
@@ -70,6 +70,17 @@ bool str_starts_with_with_cb(
     return true;
 }
 
+bool str_ends_with(struct string str, struct string suffix)
+{
+    struct string tail;
+
+    if (suffix.size > str.size)
+        return false;
+
+    tail = str_substring(str, str.size - suffix.size, str.size);
+    return str_equals(tail, suffix);
+}
+
 ssize_t str_find_with_cb(
     struct string str, bool (*is_match)(struct string str), size_t offset
 )
diff --git a/kernel/include/common/string_container.h b/kernel/include/common/string_container.h
index d5ae7388..92664898 100644
--- a/kernel/include/common/string_container.h
+++ b/kernel/include/common/string_container.h
@@ -52,6 +52,8 @@ static inline bool str_starts_with_caseless(
     return str_starts_with_with_cb(str, prefix, chars_caseless_compare);
 }
 
+bool str_ends_with(struct string str, struct string suffix);
+
 ssize_t str_find_with_cb(
     struct string str, bool (*is_match)(struct string str), size_t starting_at
 );
diff --git a/tests/test-cases/test_string.c b/tests/test-cases/test_string.c
index b4172a94..3a7d05be 100644
--- a/tests/test-cases/test_string.c
+++ b/tests/test-cases/test_string.c
@@ -85,3 +85,12 @@ TEST_CASE(starts_with_caseless)
     ASSERT_FALSE(str_starts_with_caseless(STR("ASU"), STR("ASUS")));
     ASSERT_FALSE(str_starts_with_caseless(STR("BSUS X"), STR("ASUS")));
 }
+
+TEST_CASE(ends_with)
+{
+    ASSERT_TRUE(str_ends_with(STR("data+0x10/0x20"), STR("+0x10/0x20")));
+    ASSERT_TRUE(str_ends_with(STR("abc"), STR("abc")));
+    ASSERT_TRUE(str_ends_with(STR("abc"), STR("")));
+    ASSERT_FALSE(str_ends_with(STR("bc"), STR("abc")));
+    ASSERT_FALSE(str_ends_with(STR("abd"), STR("bc")));
+}

From 857b886b54805597343b61ae44d99a8c46d24f42 Mon Sep 17 00:00:00 2001
From: Daniil Tatianin <99danilt@gmail.com>
Date: Fri, 23 Oct 2026 13:53:40 +0300
Subject: [PATCH 152/152] memory: add the mem_describe() integration tests

Check each describer on various memory types:
- A function and a variable in the image
- Per-CPU variable on every CPU
- Heap object and a large allocation
- Block and a struct page
- The whole chain through %pMEM

Signed-off-by: Daniil Tatianin <99danilt@gmail.com>
---
 kernel/ktest/Kconfig           |   1 +
 kernel/memory/CMakeLists.txt   |   5 +
 kernel/memory/tests/Kconfig    |   3 +
 kernel/memory/tests/describe.c | 188 +++++++++++++++++++++++++++++++++
 4 files changed, 197 insertions(+)
 create mode 100644 kernel/memory/tests/Kconfig
 create mode 100644 kernel/memory/tests/describe.c

diff --git a/kernel/ktest/Kconfig b/kernel/ktest/Kconfig
index 862d84d6..b821ec55 100644
--- a/kernel/ktest/Kconfig
+++ b/kernel/ktest/Kconfig
@@ -24,5 +24,6 @@ config KTEST_SELFTEST
 rsource "../sched/tests/Kconfig"
 rsource "../time/tests/Kconfig"
 rsource "../locking/tests/Kconfig"
+rsource "../memory/tests/Kconfig"
 
 endif
diff --git a/kernel/memory/CMakeLists.txt b/kernel/memory/CMakeLists.txt
index 2748f15d..7776ae6d 100644
--- a/kernel/memory/CMakeLists.txt
+++ b/kernel/memory/CMakeLists.txt
@@ -9,3 +9,8 @@ ultra_sources(
     valloc.c
     describe.c
 )
+
+ultra_sources_if(
+    CONFIG_KTEST_MEMORY_DESCRIBE
+    tests/describe.c
+)
diff --git a/kernel/memory/tests/Kconfig b/kernel/memory/tests/Kconfig
new file mode 100644
index 00000000..a1e09dec
--- /dev/null
+++ b/kernel/memory/tests/Kconfig
@@ -0,0 +1,3 @@
+config KTEST_MEMORY_DESCRIBE
+    bool "Memory description tests" if !KTEST_ALL_TESTS
+    default KTEST_ALL_TESTS
diff --git a/kernel/memory/tests/describe.c b/kernel/memory/tests/describe.c
new file mode 100644
index 00000000..c66f0a67
--- /dev/null
+++ b/kernel/memory/tests/describe.c
@@ -0,0 +1,188 @@
+#include <ktest/ktest.h>
+
+#include <common/format.h>
+#include <common/string_container.h>
+
+#include <memory/alloc.h>
+#include <memory/buddy.h>
+#include <memory/describe.h>
+#include <memory/page.h>
+
+#include <per_cpu.h>
+#include <smp.h>
+#include <symbols.h>
+
+#define LARGE_ALLOCATION_SIZE (4 * PAGE_SIZE)
+#define LARGE_ALLOCATION_ORDER 2u
+
+static char s_got[MAX_MEM_DESCRIPTION_LENGTH];
+static char s_expected[MAX_MEM_DESCRIPTION_LENGTH];
+
+/*
+ * Expect 'got' to equal the string that str_printf() makes from 'fmt' and the
+ * arguments after it.
+ */
+#define EXPECT_FORMATTED(got, fmt, ...) do {               \
+    struct string expect_formatted_str;                    \
+                                                           \
+    expect_formatted_str = str_printf(                     \
+        s_expected, sizeof(s_expected), fmt, ##__VA_ARGS__ \
+    );                                                     \
+    KTEST_EXPECT_EQ(got, expect_formatted_str);            \
+} while (0)
+
+// EXPECT_FORMATTED() for the description of 'addr', a pointer or a ptr_t
+#define EXPECT_DESCRIBED(addr, fmt, ...)                             \
+    EXPECT_FORMATTED(                                                \
+        mem_describe((const void*)(addr), s_got), fmt, ##__VA_ARGS__ \
+    )
+
+// EXPECT_FORMATTED() for 'arg' printed with the format 'spec'
+#define EXPECT_PRINTED(spec, arg, fmt, ...)                             \
+    EXPECT_FORMATTED(                                                   \
+        str_printf(s_got, sizeof(s_got), spec, arg), fmt, ##__VA_ARGS__ \
+    )
+
+struct described {
+    u64 first;
+    u64 second;
+};
+
+/*
+ * LTO renames static variables that exist in other translation units, e.g. x
+ * becomes x.lto_priv.0 with GCC or x.1 with clang. The cases below expect
+ * the exact names of these two, so both are named after this test to hopefully
+ * be unique.
+ */
+static u64 s_describe_ktest_data[4];
+static DEFINE_PER_CPU(struct described, s_describe_ktest_per_cpu);
+
+KTEST_CASE(image_addresses_are_named_by_symbol)
+{
+    char name_buf[MAX_SYMBOL_LENGTH];
+    struct string got, name;
+    void *past_end, *object;
+    size_t symbol_size;
+    ptr_t pc, mem_describe_addr = (ptr_t)mem_describe;
+
+    name = symbol_lookup_code_by_address(
+        mem_describe_addr, name_buf, nullptr, &symbol_size
+    );
+    KTEST_ASSERT_EQ(name, STR("mem_describe"));
+
+    EXPECT_DESCRIBED(
+        mem_describe_addr + 1, "mem_describe+0x1/0x%zx", symbol_size
+    );
+    EXPECT_PRINTED("%pSM", &mem_describe_addr, "mem_describe");
+
+    pc = mem_describe_addr + 1;
+    EXPECT_PRINTED("%pSM", &pc, "mem_describe+1/%zu", symbol_size);
+
+    pc = mem_describe_addr + symbol_size;
+    EXPECT_PRINTED(
+        "%pRSM", &pc, "mem_describe+%zu/%zu", symbol_size, symbol_size
+    );
+
+    EXPECT_DESCRIBED(
+        &s_describe_ktest_data[2], "s_describe_ktest_data+0x10/0x20"
+    );
+
+    // The address right after the end of the array isn't part of it
+    past_end = &s_describe_ktest_data[ARRAY_SIZE(s_describe_ktest_data)];
+    got = mem_describe(past_end, s_got);
+    KTEST_EXPECT(!str_starts_with(got, STR("s_describe_ktest_data")));
+
+    object = alloc(16, ALLOC_GENERIC);
+    KTEST_ASSERT(object != nullptr);
+    EXPECT_DESCRIBED(object, "kheap builtin_16 object %p +0x0", object);
+    free(object);
+}
+
+KTEST_CASE(per_cpu_addresses_are_rebased_onto_the_variable)
+{
+    struct described *described;
+    cpu_t cpu;
+
+    for_each_present_cpu(cpu) {
+        described = per_cpu_ptr(&s_describe_ktest_per_cpu, cpu);
+        EXPECT_DESCRIBED(
+            &described->second,
+            "s_describe_ktest_per_cpu+0x%zx/0x%zx on CPU %u",
+            offsetof(struct described, second), sizeof(struct described), cpu
+        );
+    }
+
+    EXPECT_DESCRIBED(
+        &s_describe_ktest_per_cpu, "s_describe_ktest_per_cpu+0x0/0x%zx",
+        sizeof(struct described)
+    );
+}
+
+KTEST_CASE(heap_addresses_name_the_object)
+{
+    struct page_block *block;
+    char *small, *large;
+
+    small = alloc(100, ALLOC_GENERIC);
+    KTEST_ASSERT(small != nullptr);
+
+    EXPECT_DESCRIBED(small + 8, "kheap builtin_128 object %p +0x8", small);
+    free(small);
+
+    large = alloc(LARGE_ALLOCATION_SIZE, ALLOC_GENERIC);
+    KTEST_ASSERT(large != nullptr);
+
+    EXPECT_DESCRIBED(
+        large + 0x10, "kheap large allocation %p order %u +0x10", large,
+        LARGE_ALLOCATION_ORDER
+    );
+    free(large);
+
+    block = alloc_page(ALLOC_GENERIC);
+    KTEST_ASSERT(block != nullptr);
+    EXPECT_DESCRIBED(
+        block_to_virt(block), "generic block pfn 0x%zx order 0 +0x0",
+        page_to_pfn(block_to_page(block))
+    );
+    block_unref(block);
+}
+
+KTEST_CASE(direct_map_addresses_name_the_block)
+{
+    struct page_block *block;
+    char *inside, *tail_page;
+    pfn_t pfn;
+
+    block = alloc_block(1, ALLOC_GENERIC);
+    KTEST_ASSERT(block != nullptr);
+    pfn = page_to_pfn(block_to_page(block));
+
+    inside = (char*)block_to_virt(block) + PAGE_SIZE + 0x20;
+    EXPECT_DESCRIBED(
+        inside, "generic block pfn 0x%zx order 1 +0x%zx", pfn, PAGE_SIZE + 0x20
+    );
+
+    tail_page = (char*)&block_to_page(block)[1] + 8;
+    EXPECT_DESCRIBED(
+        tail_page,
+        "struct page of pfn 0x%zx +0x8, generic block pfn 0x%zx order 1",
+        pfn + 1, pfn
+    );
+    block_unref(block);
+
+    EXPECT_DESCRIBED(0x1234, "unknown memory");
+}
+
+KTEST_CASE(mem_describe_backs_the_format)
+{
+    char *object;
+
+    object = alloc(100, ALLOC_GENERIC);
+    KTEST_ASSERT(object != nullptr);
+
+    EXPECT_DESCRIBED(object + 8, "%pMEM", object + 8);
+    free(object);
+
+    EXPECT_PRINTED("%pMEM", (void*)0x1234, "unknown memory");
+    EXPECT_DESCRIBED(nullptr, "unknown memory");
+}