From 24f0d0c8c8877c165c7fdca13ee29b2c64c93585 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Tue, 29 Sep 2026 22:40:15 +0300 Subject: [PATCH 01/37] build: don't let LTO drop the functions the compiler calls itself GCC's LTO plugin resolves which IR symbols are referenced before code generation, but the memset() and memcpy() calls GCC emits for struct initializers only appear during code generation. With LTO on, memset() is dropped as unreferenced and the link fails. Add ultra_sources_no_lto() and build both string.c files with it, so these functions always reach the linker as machine code. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- kernel/arch/x86_64/CMakeLists.txt | 3 ++- kernel/common/CMakeLists.txt | 3 ++- kernel/target_helpers.cmake | 11 +++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/kernel/arch/x86_64/CMakeLists.txt b/kernel/arch/x86_64/CMakeLists.txt index a8d5de0c..e6e41e13 100644 --- a/kernel/arch/x86_64/CMakeLists.txt +++ b/kernel/arch/x86_64/CMakeLists.txt @@ -1,7 +1,8 @@ include(CheckCompilerFlag) +ultra_sources_no_lto(string.c) + ultra_sources( - string.c apic.c vector_alloc.c lapic_domain.c diff --git a/kernel/common/CMakeLists.txt b/kernel/common/CMakeLists.txt index 195dfc61..d1ac8a34 100644 --- a/kernel/common/CMakeLists.txt +++ b/kernel/common/CMakeLists.txt @@ -4,7 +4,8 @@ ultra_sources( ctype.c error.c format.c - string.c string_container.c rb_tree.c ) + +ultra_sources_no_lto(string.c) diff --git a/kernel/target_helpers.cmake b/kernel/target_helpers.cmake index 419f30a4..a2ee3ced 100644 --- a/kernel/target_helpers.cmake +++ b/kernel/target_helpers.cmake @@ -82,3 +82,14 @@ function(ultra_sources_no_ubsan) -fno-sanitize=undefined ) endfunction() + +function(ultra_sources_no_lto) + ultra_sources(${ARGN}) + set_property( + SOURCE ${ARGN} + TARGET_DIRECTORY ${ULTRA_KERNEL_OBJECTS} + APPEND + PROPERTY COMPILE_OPTIONS + -fno-lto + ) +endfunction() From efd98e5601dbd48224b85041ef333b61a85ac535 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Tue, 29 Sep 2026 22:40:15 +0300 Subject: [PATCH 02/37] common/string: add strncmp() At -O2 and above GCC lowers __builtin_strcmp() to strncmp() when it knows a bound on one of the strings, so any optimized build that compares against a fixed size array fails to link without it. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- kernel/common/string.c | 16 ++++++++++++++++ kernel/include/common/string.h | 1 + 2 files changed, 17 insertions(+) diff --git a/kernel/common/string.c b/kernel/common/string.c index a95907d4..d28fb49c 100644 --- a/kernel/common/string.c +++ b/kernel/common/string.c @@ -130,4 +130,20 @@ int strcmp(const char *s1, const char *s2) } #endif +#ifndef ARCH_HAS_CUSTOM_STRNCMP +#undef strncmp +int strncmp(const char *s1, const char *s2, size_t count) +{ + while (count--) { + if (*s1 != *s2 || *s1 == '\0') + return (int)(unsigned char)*s1 - (int)(unsigned char)*s2; + + ++s1; + ++s2; + } + + return 0; +} +#endif + #pragma GCC diagnostic pop diff --git a/kernel/include/common/string.h b/kernel/include/common/string.h index 555425b5..8555c440 100644 --- a/kernel/include/common/string.h +++ b/kernel/include/common/string.h @@ -10,6 +10,7 @@ #define strlen __builtin_strlen #define strstr __builtin_strstr #define strcmp __builtin_strcmp +#define strncmp __builtin_strncmp static ALWAYS_INLINE void *memzero(void *dest, size_t count) { From 47a9f4c999880aed5bd5224a41f3ad226b63f460 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Tue, 29 Sep 2026 22:46:43 +0300 Subject: [PATCH 03/37] tests/arg_parser: include arg_parser.h uses uint64_t and int64_t but relies on another standard header to pull in , which GCC 16's libstdc++ no longer does, so the unit tests fail to build. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- tests/arg_parser.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/arg_parser.h b/tests/arg_parser.h index 80fbf07b..25b82cfe 100644 --- a/tests/arg_parser.h +++ b/tests/arg_parser.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include From 83126f8ffeb5f87fd36f6887c6ab2e845dfabc65 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Wed, 30 Sep 2026 00:41:54 +0300 Subject: [PATCH 04/37] toolchain: pass brew's lld to clang with --ld-path clang deprecated giving -fuse-ld= a path in favor of --ld-path=, so use that. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- toolchain/toolchain_clang.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchain/toolchain_clang.cmake b/toolchain/toolchain_clang.cmake index bfc1e333..aeb59b3f 100644 --- a/toolchain/toolchain_clang.cmake +++ b/toolchain/toolchain_clang.cmake @@ -6,7 +6,7 @@ if (APPLE) OUTPUT_VARIABLE BREW_LLD_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE ) - set(CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=${BREW_LLD_PREFIX}/bin/ld.lld") + set(CMAKE_EXE_LINKER_FLAGS_INIT "--ld-path=${BREW_LLD_PREFIX}/bin/ld.lld") execute_process( COMMAND brew --prefix llvm From 86e573f63996f4ba2e09c7faebad3b5043d9e5c3 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Tue, 29 Sep 2026 11:16:39 +0300 Subject: [PATCH 05/37] build: add a baremetal target to --make-image It's annoying to have to manually edit the config every time you want to make an image for real hardware, so just add a target option instead. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- build.py | 57 ++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/build.py b/build.py index 73cc1c4e..360c1ac9 100755 --- a/build.py +++ b/build.py @@ -223,7 +223,23 @@ def rebuild_toolchain() -> None: ) -def make_hyper_config(arch: str) -> str: +QEMU_HYPER_OPTIONS = """ +cmdline = "log-level=debug earlycon=e9,colored" + +# Hardcode a resolution that's good enough for local testing, +# we don't need a huge QEMU window. +video-mode: + width = 1024 + height = 768 +""" + +IMAGE_TARGETS = { + "qemu": QEMU_HYPER_OPTIONS, + "baremetal": "", +} + + +def make_hyper_config(arch: str, target: str) -> str: return \ f""" default-entry = ultra-{arch} @@ -239,15 +255,7 @@ def make_hyper_config(arch: str) -> str: page-table: levels = 5 constraint = maximum - -cmdline = "log-level=debug earlycon=e9,colored" - -# Hardcode a resolution that's good enough for local testing, -# we don't need a huge QEMU window. -video-mode: - width = 1024 - height = 768 - +{IMAGE_TARGETS[target]} [ultra-aarch64] protocol = ultra higher-half-exclusive = true @@ -269,7 +277,7 @@ def get_kernel_path(execution_node: str, build_dir: str) -> str: def make_hyper_image( - br_type: str, fs_type: str, arch: str, build_dir: str, + br_type: str, fs_type: str, arch: str, target: str, build_dir: str, hyper_installer: Optional[str], hyper_iso_br: Optional[str], hyper_uefi_binaries: List[str], image_path: str, force_regenerate: bool ) -> Optional[ultr.DiskImage]: @@ -290,7 +298,7 @@ def make_hyper_image( return ultr.DiskImage( image_root_path, br_type, fs_type, - hyper_config=make_hyper_config(arch), + hyper_config=make_hyper_config(arch, target), hyper_uefi_binary_paths=hyper_uefi_binaries, hyper_iso_br_path=hyper_iso_br, hyper_installer_path=hyper_installer, @@ -536,8 +544,11 @@ def main() -> None: help="Build the toolchain and exit") image = parser.add_argument_group("Image") - image.add_argument("--make-image", action="store_true", - help="Produce a bootable image after building") + image.add_argument("--make-image", nargs="?", const="qemu", + choices=IMAGE_TARGETS.keys(), metavar="TARGET", + help="Produce a bootable image after building, " + "for QEMU or for real hardware with " + "'baremetal' (default: qemu)") image.add_argument("--image-type", choices=["iso", "raw"], default="iso", help="Image type to produce (with --make-image)") image.add_argument("--hyper-installer", type=str, @@ -667,6 +678,11 @@ def main() -> None: if args.run_for is not None and is_debug: sys.exit("--run-for cannot be combined with a debug run") + image_target = args.make_image or "qemu" + if should_run and image_target != "qemu": + sys.exit(f"--make-image {image_target} cannot be combined " + "with a QEMU run") + if should_run or args.make_image: hyper_installer = args.hyper_installer hyper_iso_br = args.hyper_iso_loader @@ -674,10 +690,15 @@ def main() -> None: if args.image_type == "iso": fs_type = "ISO9660" - image_name = "image.iso" + image_ext = "iso" else: fs_type = "FAT32" - image_name = "image.raw" + image_ext = "raw" + + image_name = "image" + if image_target != "qemu": + image_name += f"-{image_target}" + image_name += f".{image_ext}" if not hyper_installer: hyper_installer = hyper_get_installer() @@ -696,9 +717,9 @@ def main() -> None: image_path = os.path.join(build_dir, image_name) make_hyper_image( - "MBR", fs_type, args.arch, build_dir, + "MBR", fs_type, args.arch, image_target, build_dir, hyper_installer, hyper_iso_br, hyper_uefi_binary_paths, - image_path, args.make_image + image_path, args.make_image is not None ) if should_run: From 83864746ac21caf6af8090b33914b76cd78d977a Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Tue, 29 Sep 2026 13:15:12 +0300 Subject: [PATCH 06/37] build: move the config making into scripts/config_layers.py build.py is getting huge as it is, and we're going to need this code to be reusable to implement a lint check, so factor it out. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- build.py | 57 ++++---------------------------------- scripts/config_layers.py | 59 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 52 deletions(-) create mode 100644 scripts/config_layers.py diff --git a/build.py b/build.py index 360c1ac9..421eebfe 100755 --- a/build.py +++ b/build.py @@ -20,6 +20,7 @@ sys.modules["kconfiglib"] = kc import scripts.build_utils.wsl_wrap as ww + import scripts.config_layers as cl import scripts.build_utils.package_manager as pm import scripts.build_utils.toolchain_builder as tb import scripts.build_utils.toolchain_args as ta @@ -73,16 +74,6 @@ # How long --run-for waits for the first output before giving up QEMU_SILENT_BOOT_SECONDS = 30 -ARCH_TO_CONFIG_KEY = { - "x86_64": "ARCH_X86_64", - "aarch64": "ARCH_AARCH64", -} - -TOOLCHAIN_TO_CONFIG_KEY = { - "clang": "TOOLCHAIN_CLANG", - "gcc": "TOOLCHAIN_GCC", -} - @contextmanager def enter_work_dir(path: str) -> Iterator[None]: @@ -473,44 +464,6 @@ def run_unit_tests(args: argparse.Namespace, this_os: str) -> int: return subprocess.run([binary]).returncode -def root_kconfig() -> kc.Kconfig: - with enter_work_dir(pg.project_root()): - kconfig = kc.Kconfig(pg.project_root_relative("Kconfig")) - - return kconfig - - -def make_config_from_preset( - preset: str, out_path: str, toolchain: str, arch: str -) -> str: - kconfig = root_kconfig() - kconfig.load_config(preset) - - preset_arch = kconfig.syms["ARCH_STRING"].str_value - if arch != "auto" and arch != preset_arch: - sys.exit(f"--arch {arch} conflicts with {preset}, " - f"which is for {preset_arch}") - - toolchain_sym = kconfig.syms[TOOLCHAIN_TO_CONFIG_KEY[toolchain]] - preset_toolchain = toolchain_sym.choice.user_selection - - if preset_toolchain is not None and preset_toolchain is not toolchain_sym: - print(f"Ignoring {preset_toolchain.name} from {preset}, " - f"building with --toolchain {toolchain}") - - toolchain_sym.set_value("y") - kconfig.write_config(out_path) - - return preset_arch - - -def make_default_config(out_path: str, toolchain: str, arch: str) -> None: - kconfig = root_kconfig() - kconfig.syms[TOOLCHAIN_TO_CONFIG_KEY[toolchain]].set_value("y") - kconfig.syms[ARCH_TO_CONFIG_KEY[arch]].set_value("y") - kconfig.write_config(out_path) - - def main() -> None: ww.relaunch_in_wsl_if_windows() pg.set_project_root(os.path.dirname(os.path.abspath(__file__))) @@ -628,8 +581,8 @@ def main() -> None: os.makedirs(build_dir, exist_ok=True) config = os.path.join(build_dir, ".config") - args.arch = make_config_from_preset(args.config, config, - args.toolchain, args.arch) + args.arch = cl.make_config_from_preset(args.config, config, + args.toolchain, args.arch) args.config = config else: if args.arch == "auto": @@ -642,7 +595,7 @@ def main() -> None: args.config = os.path.join(build_dir, ".config") if not os.path.isfile(args.config): - make_default_config(args.config, args.toolchain, args.arch) + cl.make_default_config(args.config, args.toolchain, args.arch) if args.menuconfig or args.guiconfig: os.environ["KCONFIG_CONFIG"] = args.config @@ -653,7 +606,7 @@ def main() -> None: module = mc if args.menuconfig else gc with enter_work_dir(pg.project_root()): - module.menuconfig(root_kconfig()) + module.menuconfig(cl.root_kconfig()) sys.exit(0) if args.toolchain_only: diff --git a/scripts/config_layers.py b/scripts/config_layers.py new file mode 100644 index 00000000..20556f37 --- /dev/null +++ b/scripts/config_layers.py @@ -0,0 +1,59 @@ +import os +import sys + +import scripts.kconfiglib.kconfiglib as kc + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +ARCH_TO_CONFIG_KEY = { + "x86_64": "ARCH_X86_64", + "aarch64": "ARCH_AARCH64", +} + +TOOLCHAIN_TO_CONFIG_KEY = { + "clang": "TOOLCHAIN_CLANG", + "gcc": "TOOLCHAIN_GCC", +} + + +def root_kconfig() -> kc.Kconfig: + old_dir = os.getcwd() + os.chdir(ROOT) + try: + kconfig = kc.Kconfig(os.path.join(ROOT, "Kconfig")) + finally: + os.chdir(old_dir) + + return kconfig + + +def make_config_from_preset( + preset: str, out_path: str, toolchain: str, arch: str +) -> str: + kconfig = root_kconfig() + kconfig.load_config(preset) + + preset_arch = kconfig.syms["ARCH_STRING"].str_value + if arch != "auto" and arch != preset_arch: + sys.exit(f"--arch {arch} conflicts with {preset}, " + f"which is for {preset_arch}") + + toolchain_sym = kconfig.syms[TOOLCHAIN_TO_CONFIG_KEY[toolchain]] + preset_toolchain = toolchain_sym.choice.user_selection + + if preset_toolchain is not None and preset_toolchain is not toolchain_sym: + print(f"Ignoring {preset_toolchain.name} from {preset}, " + f"building with --toolchain {toolchain}") + + toolchain_sym.set_value("y") + kconfig.write_config(out_path) + + return preset_arch + + +def make_default_config(out_path: str, toolchain: str, arch: str) -> None: + kconfig = root_kconfig() + kconfig.syms[TOOLCHAIN_TO_CONFIG_KEY[toolchain]].set_value("y") + kconfig.syms[ARCH_TO_CONFIG_KEY[arch]].set_value("y") + kconfig.write_config(out_path) From cfab682d903229b48b271a9d1dadd17cab8cc683 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Tue, 29 Sep 2026 11:55:53 +0300 Subject: [PATCH 07/37] build: make --arch apply to --config builds A preset has to select its arch today, and --arch only checks it for a conflict, so one preset cannot serve several architectures. Take the arch from --arch, x86_64 by default, and apply it on top of the preset like the toolchain. A preset that selects a different arch is an error. The arch becomes part of a preset build's directory name. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- build.py | 37 +++++++++++++---------------------- scripts/config_layers.py | 42 +++++++++++++++++++++------------------- 2 files changed, 35 insertions(+), 44 deletions(-) diff --git a/build.py b/build.py index 421eebfe..59378992 100755 --- a/build.py +++ b/build.py @@ -473,11 +473,9 @@ def main() -> None: ) build = parser.add_argument_group("Build") - build.add_argument("--arch", default="auto", - choices=["auto", "x86_64", "aarch64"], - help="CPU architecture to build the kernel for " - "(auto implies x86_64 or the config setting if " - "--config is specified)") + build.add_argument("--arch", default="x86_64", + choices=["x86_64", "aarch64"], + help="CPU architecture to build the kernel for") build.add_argument("--config", help="Configuration file to use for this build") build.add_argument("--menuconfig", action="store_true", @@ -567,6 +565,9 @@ def main() -> None: if rc is not None: sys.exit(rc) + config_layers = [] + build_dir_name = f"build-{args.toolchain}-{args.arch}" + if args.config: if not os.path.isfile(args.config): raise RuntimeError(f"Invalid --config path: {args.config}") @@ -575,27 +576,15 @@ def main() -> None: if preset_name.startswith("."): preset_name = "user-config" - build_dir = pg.project_root_relative( - f"build-{args.toolchain}-{preset_name}" - ) - os.makedirs(build_dir, exist_ok=True) - - config = os.path.join(build_dir, ".config") - args.arch = cl.make_config_from_preset(args.config, config, - args.toolchain, args.arch) - args.config = config - else: - if args.arch == "auto": - args.arch = "x86_64" + build_dir_name += f"-{preset_name}" + config_layers.append(args.config) - build_dir = pg.project_root_relative( - f"build-{args.toolchain}-{args.arch}" - ) - os.makedirs(build_dir, exist_ok=True) + build_dir = pg.project_root_relative(build_dir_name) + os.makedirs(build_dir, exist_ok=True) - args.config = os.path.join(build_dir, ".config") - if not os.path.isfile(args.config): - cl.make_default_config(args.config, args.toolchain, args.arch) + args.config = os.path.join(build_dir, ".config") + if config_layers or not os.path.isfile(args.config): + cl.make_config(config_layers, args.config, args.toolchain, args.arch) if args.menuconfig or args.guiconfig: os.environ["KCONFIG_CONFIG"] = args.config diff --git a/scripts/config_layers.py b/scripts/config_layers.py index 20556f37..677999b2 100644 --- a/scripts/config_layers.py +++ b/scripts/config_layers.py @@ -1,5 +1,6 @@ import os import sys +from typing import List import scripts.kconfiglib.kconfiglib as kc @@ -28,32 +29,33 @@ def root_kconfig() -> kc.Kconfig: return kconfig -def make_config_from_preset( - preset: str, out_path: str, toolchain: str, arch: str -) -> str: +def make_config( + layers: List[str], out_path: str, toolchain: str, arch: str +) -> None: kconfig = root_kconfig() - kconfig.load_config(preset) - - preset_arch = kconfig.syms["ARCH_STRING"].str_value - if arch != "auto" and arch != preset_arch: - sys.exit(f"--arch {arch} conflicts with {preset}, " - f"which is for {preset_arch}") + kconfig.warn_assign_undef = True + kconfig.warn_assign_override = False + kconfig.warn_assign_redun = False + arch_sym = kconfig.syms[ARCH_TO_CONFIG_KEY[arch]] toolchain_sym = kconfig.syms[TOOLCHAIN_TO_CONFIG_KEY[toolchain]] - preset_toolchain = toolchain_sym.choice.user_selection - if preset_toolchain is not None and preset_toolchain is not toolchain_sym: - print(f"Ignoring {preset_toolchain.name} from {preset}, " - f"building with --toolchain {toolchain}") + for layer in layers: + kconfig.load_config(layer, replace=False) - toolchain_sym.set_value("y") - kconfig.write_config(out_path) + layer_arch = arch_sym.choice.user_selection + if layer_arch is not None and layer_arch is not arch_sym: + sys.exit(f"{layer} selects {layer_arch.name}, which conflicts " + f"with --arch {arch}") - return preset_arch + layer_tc = toolchain_sym.choice.user_selection + if layer_tc is not None and layer_tc is not toolchain_sym: + print(f"Ignoring {layer_tc.name} from {layer}, " + f"building with --toolchain {toolchain}") + arch_sym.choice.unset_value() + toolchain_sym.choice.unset_value() -def make_default_config(out_path: str, toolchain: str, arch: str) -> None: - kconfig = root_kconfig() - kconfig.syms[TOOLCHAIN_TO_CONFIG_KEY[toolchain]].set_value("y") - kconfig.syms[ARCH_TO_CONFIG_KEY[arch]].set_value("y") + arch_sym.set_value("y") + toolchain_sym.set_value("y") kconfig.write_config(out_path) From 9f2e9a1cb83449178b33c14b3d2f42a5afac757c Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Tue, 29 Sep 2026 11:56:23 +0300 Subject: [PATCH 08/37] build: stop --config from overwriting the build directory's config --config recreates the build directory's .config from the preset on every run, so anything changed through --menuconfig is gone on the next build. Only create .config when it does not exist yet. When the preset changes later, apply just the options it changed and keep the ones changed through menuconfig. To tell the two apart, the build directory keeps what the preset produced last time in .config.merge-base. If both sides changed the same option, the menuconfig value wins and a warning is printed, as it is when an option stops having an effect because something it depends on was turned off. --reset-config recreates .config from the preset. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- build.py | 7 +- scripts/config_layers.py | 200 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 200 insertions(+), 7 deletions(-) diff --git a/build.py b/build.py index 59378992..c8f07b35 100755 --- a/build.py +++ b/build.py @@ -478,6 +478,9 @@ def main() -> None: help="CPU architecture to build the kernel for") build.add_argument("--config", help="Configuration file to use for this build") + build.add_argument("--reset-config", action="store_true", + help="Recreate the build directory's config from " + "--config, discarding the changes made to it") build.add_argument("--menuconfig", action="store_true", help="Run menuconfig to edit the current config file") build.add_argument("--guiconfig", action="store_true", @@ -583,8 +586,8 @@ def main() -> None: os.makedirs(build_dir, exist_ok=True) args.config = os.path.join(build_dir, ".config") - if config_layers or not os.path.isfile(args.config): - cl.make_config(config_layers, args.config, args.toolchain, args.arch) + cl.prepare_config(args.config, config_layers, args.toolchain, + args.arch, args.reset_config) if args.menuconfig or args.guiconfig: os.environ["KCONFIG_CONFIG"] = args.config diff --git a/scripts/config_layers.py b/scripts/config_layers.py index 677999b2..5ad9a3f9 100644 --- a/scripts/config_layers.py +++ b/scripts/config_layers.py @@ -1,6 +1,7 @@ +import hashlib import os import sys -from typing import List +from typing import Dict, List, Optional, Set import scripts.kconfiglib.kconfiglib as kc @@ -17,6 +18,9 @@ "gcc": "TOOLCHAIN_GCC", } +MERGE_BASE = ".config.merge-base" +MERGE_BASE_LAYER = "# layer: " + def root_kconfig() -> kc.Kconfig: old_dir = os.getcwd() @@ -29,9 +33,9 @@ def root_kconfig() -> kc.Kconfig: return kconfig -def make_config( - layers: List[str], out_path: str, toolchain: str, arch: str -) -> None: +def load_layers( + layers: List[str], toolchain: str, arch: str +) -> kc.Kconfig: kconfig = root_kconfig() kconfig.warn_assign_undef = True kconfig.warn_assign_override = False @@ -58,4 +62,190 @@ def make_config( arch_sym.set_value("y") toolchain_sym.set_value("y") - kconfig.write_config(out_path) + + return kconfig + + +def display_path(path: str) -> str: + path = os.path.abspath(path) + if path.startswith(ROOT + os.sep): + return os.path.relpath(path, ROOT) + + return path + + +def config_values(kconfig: kc.Kconfig) -> Dict[str, str]: + reserved = ( + kconfig.syms[ARCH_TO_CONFIG_KEY["x86_64"]].choice, + kconfig.syms[TOOLCHAIN_TO_CONFIG_KEY["clang"]].choice, + ) + + return { + sym.name: sym.str_value for sym in kconfig.unique_defined_syms + if sym.config_string and sym.choice not in reserved and + any(node.prompt for node in sym.nodes) + } + + +def layer_setting(layers: List[str], name: str) -> str: + for layer in reversed(layers): + with open(layer) as f: + for line in f.read().splitlines(): + if (line.startswith(f"CONFIG_{name}=") or + line == f"# CONFIG_{name} is not set"): + return display_path(layer) + + return "the config files" + + +def layer_stamps(layers: List[str]) -> List[str]: + stamps = [] + + for layer in layers: + with open(layer, "rb") as f: + digest = hashlib.sha256(f.read()).hexdigest() + + stamps.append(f"{digest} {display_path(layer)}") + + return stamps + + +def read_merge_base_stamps(path: str) -> Optional[List[str]]: + if not os.path.isfile(path): + return None + + with open(path) as f: + return [ + line[len(MERGE_BASE_LAYER):].rstrip("\n") for line in f + if line.startswith(MERGE_BASE_LAYER) + ] + + +def write_merge_base(kconfig: kc.Kconfig, stamps: List[str], + path: str) -> None: + header = "".join(f"{MERGE_BASE_LAYER}{stamp}\n" for stamp in stamps) + kconfig.write_config(path, header=header, save_old=False) + + +def stamp_path(stamp: str) -> str: + return stamp.split(" ", 1)[1] + + +def merge_config( + config: str, merge_base: str, layered: kc.Kconfig, layers: List[str], + toolchain: str, arch: str +) -> None: + where = display_path(config) + stamps = layer_stamps(layers) + paths = [stamp_path(stamp) for stamp in stamps] + new_values = config_values(layered) + + reader = root_kconfig() + reader.warn_to_stderr = False + + old_stamps = read_merge_base_stamps(merge_base) + if old_stamps is None: + old_paths = None + base_values = config_values(load_layers([], toolchain, arch)) + else: + old_paths = [stamp_path(stamp) for stamp in old_stamps] + reader.load_config(merge_base) + base_values = config_values(reader) + + # Leave .config alone when nothing changed, rewriting it would + # make cmake reconfigure on every build + if old_stamps == stamps and base_values == new_values: + return + + reader.load_config(config) + cur_values = config_values(reader) + + messages = [] + if old_paths is None and paths: + messages.append(f"now made from {', '.join(paths)}") + + if old_paths is not None and old_paths != paths: + added = [path for path in paths if path not in old_paths] + removed = [path for path in old_paths if path not in paths] + messages.append(f"now made from {', '.join(added) or 'nothing'} " + f"instead of {', '.join(removed) or 'nothing'}") + + conflicts = [] + wanted: Dict[str, str] = {} + kept: Set[str] = set() + names = sorted(set(base_values) | set(new_values) | set(cur_values)) + + for name in names: + base = base_values.get(name) + new = new_values.get(name) + cur = cur_values.get(name) + + if cur == base or (cur is None and new != base): + if new is not None: + wanted[name] = new + continue + + if cur is None: + continue + + wanted[name] = cur + kept.add(name) + if new in (base, cur, None): + continue + + conflicts.append(f"keeping {name}={cur}, " + f"{layer_setting(layers, name)} now gives {new}") + + write_merge_base(layered, stamps, merge_base) + + for name in kept: + if wanted[name] != new_values.get(name): + layered.syms[name].set_value(wanted[name]) + + hidden = [] + resolved = config_values(layered) + applied = len([ + name for name in names + if name not in kept and resolved.get(name) != cur_values.get(name) + ]) + + for name, value in wanted.items(): + if resolved.get(name) == value: + continue + + depends = kc.expr_str(layered.syms[name].direct_dep) + if name in kept: + hidden.append(f"your {name}={value} no longer has an effect, " + f"it depends on {depends}") + continue + + hidden.append(f"{name}={value} from {layer_setting(layers, name)} " + f"has no effect, it depends on {depends}") + + if applied: + changed = [ + stamp_path(stamp) for stamp in stamps + if stamp not in (old_stamps or []) + ] + changes = "change" if applied == 1 else "changes" + messages.append(f"applied {applied} {changes} from " + f"{', '.join(changed) or 'Kconfig'}") + + for message in messages + conflicts + hidden: + print(f"{where}: {message}") + + layered.write_config(config) + + +def prepare_config( + config: str, layers: List[str], toolchain: str, arch: str, reset: bool +) -> None: + merge_base = os.path.join(os.path.dirname(config), MERGE_BASE) + layered = load_layers(layers, toolchain, arch) + + if not reset and os.path.isfile(config): + merge_config(config, merge_base, layered, layers, toolchain, arch) + return + + write_merge_base(layered, layer_stamps(layers), merge_base) + layered.write_config(config) From cc067c99a2fb39dc4ee945f4db22d5ca4eb02331 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Wed, 30 Sep 2026 13:20:32 +0300 Subject: [PATCH 09/37] build: introduce per-arch base configs The base config lists the "recommended" options to enable for a chosen platform. Other architectures like aarch64 may add extra platforms in the future alognside "base", but for an architecture that is mostly one platform aka the IBM PC for x86_64, base will likely remain the only platform we support. Automatically load configs/arch//base.config, when the arch has one, for builds without --config. A --config file build is still loaded on its own without the base pre-applied. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- build.py | 4 ++++ configs/arch/x86_64/base.config | 2 ++ scripts/config_layers.py | 8 ++++++++ 3 files changed, 14 insertions(+) create mode 100644 configs/arch/x86_64/base.config diff --git a/build.py b/build.py index c8f07b35..c670ff89 100755 --- a/build.py +++ b/build.py @@ -569,6 +569,10 @@ def main() -> None: sys.exit(rc) config_layers = [] + base_config = cl.arch_base_config(args.arch) + if args.config is None and base_config is not None: + config_layers.append(base_config) + build_dir_name = f"build-{args.toolchain}-{args.arch}" if args.config: diff --git a/configs/arch/x86_64/base.config b/configs/arch/x86_64/base.config new file mode 100644 index 00000000..7d38e833 --- /dev/null +++ b/configs/arch/x86_64/base.config @@ -0,0 +1,2 @@ +CONFIG_PARAVIRT=y +CONFIG_FLANTERM_CONSOLE=y diff --git a/scripts/config_layers.py b/scripts/config_layers.py index 5ad9a3f9..c87e5e00 100644 --- a/scripts/config_layers.py +++ b/scripts/config_layers.py @@ -66,6 +66,14 @@ def load_layers( return kconfig +def arch_base_config(arch: str) -> Optional[str]: + path = os.path.join(ROOT, "configs", "arch", arch, "base.config") + if not os.path.isfile(path): + return None + + return path + + def display_path(path: str) -> str: path = os.path.abspath(path) if path.startswith(ROOT + os.sep): From 0aab1e75761ad7d07fae06061f220f0c6ddc815f Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Wed, 30 Sep 2026 13:20:32 +0300 Subject: [PATCH 10/37] build: add a debug-lite config preset and build it by default A build without --config gets only the Kconfig defaults, which leave every debug check off, while day to day development wants the cheap ones on. Add configs/debug-lite.config with the memory management assertions and the undefined behavior sanitizer, and build it when --config is not given. --config now also takes the name of a preset in configs/. Presets are loaded on top of the arch's base config. The default preset keeps the plain build-- directory. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- build.py | 25 ++++++++++--------------- configs/debug-lite.config | 2 ++ scripts/config_layers.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 15 deletions(-) create mode 100644 configs/debug-lite.config diff --git a/build.py b/build.py index c670ff89..f7205ebf 100755 --- a/build.py +++ b/build.py @@ -477,7 +477,8 @@ def main() -> None: choices=["x86_64", "aarch64"], help="CPU architecture to build the kernel for") build.add_argument("--config", - help="Configuration file to use for this build") + help="Preset from configs/ or a config file to build " + f"(default: {cl.DEFAULT_PRESET})") build.add_argument("--reset-config", action="store_true", help="Recreate the build directory's config from " "--config, discarding the changes made to it") @@ -568,23 +569,17 @@ def main() -> None: if rc is not None: sys.exit(rc) - config_layers = [] - base_config = cl.arch_base_config(args.arch) - if args.config is None and base_config is not None: - config_layers.append(base_config) - + config_layers = cl.build_layers(args.config, args.arch) + config_file = config_layers[-1] + default_preset = cl.preset_path(cl.DEFAULT_PRESET) build_dir_name = f"build-{args.toolchain}-{args.arch}" - if args.config: - if not os.path.isfile(args.config): - raise RuntimeError(f"Invalid --config path: {args.config}") - - preset_name = os.path.splitext(os.path.basename(args.config))[0] - if preset_name.startswith("."): - preset_name = "user-config" + if os.path.abspath(config_file) != os.path.abspath(default_preset): + config_name = os.path.splitext(os.path.basename(config_file))[0] + if config_name.startswith("."): + config_name = "user-config" - build_dir_name += f"-{preset_name}" - config_layers.append(args.config) + build_dir_name += f"-{config_name}" build_dir = pg.project_root_relative(build_dir_name) os.makedirs(build_dir, exist_ok=True) diff --git a/configs/debug-lite.config b/configs/debug-lite.config new file mode 100644 index 00000000..3559c0fa --- /dev/null +++ b/configs/debug-lite.config @@ -0,0 +1,2 @@ +CONFIG_DEBUG_MM=y +CONFIG_UBSAN=y diff --git a/scripts/config_layers.py b/scripts/config_layers.py index c87e5e00..0a973553 100644 --- a/scripts/config_layers.py +++ b/scripts/config_layers.py @@ -13,6 +13,8 @@ "aarch64": "ARCH_AARCH64", } +DEFAULT_PRESET = "debug-lite" + TOOLCHAIN_TO_CONFIG_KEY = { "clang": "TOOLCHAIN_CLANG", "gcc": "TOOLCHAIN_GCC", @@ -66,6 +68,27 @@ def load_layers( return kconfig +def preset_path(preset: str) -> str: + return os.path.join(ROOT, "configs", f"{preset}.config") + + +def preset_config(preset: str) -> str: + if os.path.isfile(preset): + return preset + + path = preset_path(preset) + if os.path.isfile(path): + return path + + presets = sorted( + os.path.splitext(name)[0] + for name in os.listdir(os.path.join(ROOT, "configs")) + if name.endswith(".config") + ) + sys.exit(f"--config {preset} is neither a file nor a preset, " + f"presets: {', '.join(presets)}") + + def arch_base_config(arch: str) -> Optional[str]: path = os.path.join(ROOT, "configs", "arch", arch, "base.config") if not os.path.isfile(path): @@ -74,6 +97,20 @@ def arch_base_config(arch: str) -> Optional[str]: return path +def build_layers(config: Optional[str], arch: str) -> List[str]: + path = preset_path(DEFAULT_PRESET) + if config is not None: + path = preset_config(config) + + base = arch_base_config(arch) + is_preset = (os.path.dirname(os.path.abspath(path)) == + os.path.join(ROOT, "configs")) + if base is None or not is_preset: + return [path] + + return [base, path] + + def display_path(path: str) -> str: path = os.path.abspath(path) if path.startswith(ROOT + os.sep): From 3e8379c5dafa850fec833cc111a5938585d71581 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Wed, 30 Sep 2026 12:50:00 +0300 Subject: [PATCH 11/37] scripts: add a lint check for the files in configs/ Config files go stale without anyone noticing: a renamed symbol is dropped with no warning, and a changed Kconfig default can turn a line into a no-op or change what it means. Add scripts/check_configs.py and run it as the "configs" lint check. Each line must name an existing symbol, must not pick the arch or the toolchain, and must change the resulting config for at least one arch and toolchain the file is built with. Presets must not set symbols that only exist for one arch, those belong in the arch's base config. The check resolves many configs on one Kconfig object, which needs the kconfiglib fix for unset_values() leaving symbols marked as assigned, so pull it in here. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- scripts/check_configs.py | 162 +++++++++++++++++++++++++++++++++++++++ scripts/kconfiglib | 2 +- scripts/lint.py | 13 +++- 3 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 scripts/check_configs.py diff --git a/scripts/check_configs.py b/scripts/check_configs.py new file mode 100644 index 00000000..7e715c3b --- /dev/null +++ b/scripts/check_configs.py @@ -0,0 +1,162 @@ +import os +import re +import tempfile +from typing import List, Tuple + +import scripts.config_layers as cl +import scripts.kconfiglib.kconfiglib as kc + + +Problem = Tuple[str, int, str] +ConfigBuild = Tuple[List[str], str, str] + +ASSIGNMENT_RE = re.compile(r"^(?:CONFIG_(\w+)=|# CONFIG_(\w+) is not set$)") +WARNING_RE = re.compile(r"^.*:(\d+): warning: (.*)$") + + +def resolve( + kconfig: kc.Kconfig, layers: List[str], arch: str, toolchain: str +) -> str: + kconfig.unset_values() + for layer in layers: + kconfig.load_config(layer, replace=False) + + kconfig.syms[cl.ARCH_TO_CONFIG_KEY[arch]].set_value("y") + kconfig.syms[cl.TOOLCHAIN_TO_CONFIG_KEY[toolchain]].set_value("y") + + return "".join(sym.config_string for sym in kconfig.unique_defined_syms) + + +def is_arch_kconfig(path: str) -> bool: + parts = path.split("/") + return parts[:2] == ["kernel", "arch"] and len(parts) > 3 + + +def builds_for(arch: str, below: List[str]) -> List[ConfigBuild]: + return [(below, arch, tc) for tc in cl.TOOLCHAIN_TO_CONFIG_KEY] + + +def line_matters( + kconfig: kc.Kconfig, path: str, lines: List[str], i: int, + builds: List[ConfigBuild], scratch: str +) -> bool: + without = os.path.join(scratch, "without.config") + with open(without, "w") as f: + f.write("".join(f"{x}\n" for x in lines[:i] + lines[i + 1:])) + + return any( + resolve(kconfig, below + [path], arch, tc) != + resolve(kconfig, below + [without], arch, tc) + for below, arch, tc in builds + ) + + +def check_file( + kconfig: kc.Kconfig, path: str, is_preset: bool, + builds: List[ConfigBuild], scratch: str +) -> List[Problem]: + problems = [] + where = os.path.relpath(path, cl.ROOT) + reserved = ( + kconfig.syms[cl.ARCH_TO_CONFIG_KEY["x86_64"]].choice, + kconfig.syms[cl.TOOLCHAIN_TO_CONFIG_KEY["clang"]].choice, + ) + + kconfig.warnings = [] + kconfig.load_config(path) + + for warning in kconfig.warnings: + match = WARNING_RE.match(warning) + if match is None: + problems.append((where, 1, warning)) + continue + + problems.append((where, int(match.group(1)), match.group(2))) + + with open(path) as f: + lines = f.read().splitlines() + + for i, line in enumerate(lines): + match = ASSIGNMENT_RE.match(line) + if match is None: + continue + + name = match.group(1) or match.group(2) + sym = kconfig.syms.get(name) + if sym is None or not sym.nodes: + continue + + if sym.choice in reserved: + problems.append((where, i + 1, f"{name} is chosen by --arch or " + "--toolchain, not by config " + "files")) + continue + + if is_preset and all(is_arch_kconfig(n.filename) for n in sym.nodes): + problems.append((where, i + 1, f"{name} only exists for one " + "arch, set it in configs/arch/" + "/ instead")) + continue + + if line_matters(kconfig, path, lines, i, builds, scratch): + continue + + problems.append((where, i + 1, f"'{line}' changes nothing for any " + "arch and toolchain")) + + return problems + + +def stray_arch_files() -> List[Problem]: + problems = [] + arch_root = os.path.join(cl.ROOT, "configs", "arch") + + for arch in sorted(os.listdir(arch_root)): + for name in sorted(os.listdir(os.path.join(arch_root, arch))): + where = f"configs/arch/{arch}/{name}" + + if arch not in cl.ARCH_TO_CONFIG_KEY: + problems.append((where, 1, f"{arch} is not a known arch")) + continue + + if name != "base.config": + problems.append((where, 1, "only base.config is currently " + "supported in an arch directory")) + + return problems + + +def check_all(kconfig: kc.Kconfig, scratch: str) -> List[Problem]: + problems = stray_arch_files() + preset_builds: List[ConfigBuild] = [] + + for arch in cl.ARCH_TO_CONFIG_KEY: + base = cl.arch_base_config(arch) + if base is None: + preset_builds.extend(builds_for(arch, [])) + continue + + preset_builds.extend(builds_for(arch, [base])) + problems.extend(check_file(kconfig, base, False, + builds_for(arch, []), scratch)) + + configs_dir = os.path.join(cl.ROOT, "configs") + for name in sorted(os.listdir(configs_dir)): + if not name.endswith(".config"): + continue + + problems.extend(check_file(kconfig, os.path.join(configs_dir, name), + True, preset_builds, scratch)) + + return problems + + +def check() -> List[Problem]: + kconfig = cl.root_kconfig() + kconfig.warn_to_stderr = False + kconfig.warn_assign_undef = True + kconfig.warn_assign_override = False + kconfig.warn_assign_redun = False + + with tempfile.TemporaryDirectory() as scratch: + return check_all(kconfig, scratch) diff --git a/scripts/kconfiglib b/scripts/kconfiglib index 3232cf8b..f849d0db 160000 --- a/scripts/kconfiglib +++ b/scripts/kconfiglib @@ -1 +1 @@ -Subproject commit 3232cf8ba43aca0da94467c1456dc8796c7622db +Subproject commit f849d0db56fd7956e803517d4d0b1fd41aff0ca9 diff --git a/scripts/lint.py b/scripts/lint.py index f8f5cb31..9d0b7471 100755 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -24,6 +24,8 @@ class Context(NamedTuple): Check = Callable[[Context], List[Finding]] +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + THIRD_PARTY = ("/uacpi/", "/flanterm/", "/boot/ultra_protocol.h") GENERATED_SOURCES = ("kernel_symbols", "build_banner") SOURCE_SUFFIXES = (".c", ".h", ".S", ".cpp", ".hpp") @@ -310,8 +312,15 @@ def check_variable_prefixes(ctx: Context) -> List[Finding]: return list(seen) +def check_configs(ctx: Context) -> List[Finding]: + from scripts.check_configs import check + + return [Finding(path, line, message) for path, line, message in check()] + + CHECKS: Dict[str, Check] = { "comments": check_comments, + "configs": check_configs, "file-end": check_file_end, "includes": check_includes, "null": check_null, @@ -321,8 +330,7 @@ def check_variable_prefixes(ctx: Context) -> List[Finding]: def run(build_dir: str, checks: Optional[List[str]] = None) -> int: - root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ctx = Context(root, os.path.abspath(build_dir)) + ctx = Context(ROOT, os.path.abspath(build_dir)) findings: List[Finding] = [] for name in checks or sorted(CHECKS): @@ -352,6 +360,7 @@ def main() -> None: help="Only run this check, may be repeated") args = parser.parse_args() + sys.path.insert(0, ROOT) sys.exit(run(args.build_dir, args.check)) From d478d0168a07fe4a3a610fbc2b3c478c50cbafe5 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Tue, 29 Sep 2026 22:47:35 +0300 Subject: [PATCH 12/37] build: add a release config preset A config with -O3 and LTO. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- configs/release.config | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 configs/release.config diff --git a/configs/release.config b/configs/release.config new file mode 100644 index 00000000..2cedb6f1 --- /dev/null +++ b/configs/release.config @@ -0,0 +1,2 @@ +CONFIG_LTO=y +CONFIG_OPTIMIZATION_LEVEL=3 From b6786e08343493523ce7ab7abbdd6db4bd01711f Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Wed, 30 Sep 2026 01:32:11 +0300 Subject: [PATCH 13/37] build: make --lint configure without building --lint only needs compile_commands.json, so there's no reason to build the entire kernel to run it. Split configuring out of cmake_build() and make --lint configure, then check and stop. An existing build directory is configured again, so a source added since the last build is checked too. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- build.py | 65 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/build.py b/build.py index f7205ebf..4e7891ed 100755 --- a/build.py +++ b/build.py @@ -171,28 +171,36 @@ def cmake_native_build_args(build_dir: str) -> List[str]: return ["--", "-Oline"] -def cmake_build( +def cmake_configure( args: argparse.Namespace, build_dir: str, extra_args: List[str] = [], - reconfigure_cb: Optional[Callable[[], None]] = None + reconfigure_cb: Optional[Callable[[], None]] = None, + refresh: bool = False ) -> None: cmake_cache = os.path.join(build_dir, "CMakeCache.txt") - rerun_cmake = args.reconfigure or not os.path.isfile(cmake_cache) + has_cache = os.path.isfile(cmake_cache) - if rerun_cmake: - if reconfigure_cb is not None: - reconfigure_cb() - os.makedirs(build_dir, exist_ok=True) + if has_cache and refresh and not args.reconfigure: + subprocess.run(["cmake", "."], check=True, cwd=build_dir) + return - generator_args = [] - if not os.path.isfile(cmake_cache) and shutil.which("ninja"): - generator_args = ["-G", "Ninja"] - - subprocess.run(["cmake", "..", *generator_args, *extra_args], - check=True, cwd=build_dir) - else: + if has_cache and not args.reconfigure: print("Not rerunning cmake since build directory already exists " "(--reconfigure)") + return + + if reconfigure_cb is not None: + reconfigure_cb() + os.makedirs(build_dir, exist_ok=True) + + generator_args = [] + if not has_cache and shutil.which("ninja"): + generator_args = ["-G", "Ninja"] + + subprocess.run(["cmake", "..", *generator_args, *extra_args], + check=True, cwd=build_dir) + +def cmake_run_build(build_dir: str) -> None: subprocess.run( ["cmake", "--build", ".", "-j", str(os.cpu_count()), *cmake_native_build_args(build_dir)], @@ -200,20 +208,32 @@ def cmake_build( ) -def build_ultra( - args: argparse.Namespace, build_dir: str +def cmake_build(args: argparse.Namespace, build_dir: str) -> None: + cmake_configure(args, build_dir) + cmake_run_build(build_dir) + + +def configure_ultra( + args: argparse.Namespace, build_dir: str, refresh: bool = False ) -> None: def rebuild_toolchain() -> None: # Only rerun toolchain builder if reconfigure is not artificial if not args.reconfigure: build_toolchain(args) - cmake_build( + cmake_configure( args, build_dir, [f"-DCONFIG_FILE={args.config}"], - rebuild_toolchain + rebuild_toolchain, refresh ) +def build_ultra( + args: argparse.Namespace, build_dir: str +) -> None: + configure_ultra(args, build_dir) + cmake_run_build(build_dir) + + QEMU_HYPER_OPTIONS = """ cmdline = "log-level=debug earlycon=e9,colored" @@ -453,6 +473,7 @@ def run_lint(args: argparse.Namespace, build_dir: str) -> int: if args.toolchain != "clang": sys.exit("--lint only works with --toolchain clang") + configure_ultra(args, build_dir, refresh=True) return lint.run(build_dir) @@ -552,7 +573,7 @@ def main() -> None: tests.add_argument("--unit-tests", action="store_true", help="Run the userspace test suite") tests.add_argument("--lint", action="store_true", - help="Check the coding conventions after building," + help="Check the coding conventions without building," " needs the clang toolchain") args = parser.parse_args() @@ -604,12 +625,12 @@ def main() -> None: build_toolchain(args) sys.exit(0) - if not args.no_build: - build_ultra(args, build_dir) - if args.lint: sys.exit(run_lint(args, build_dir)) + if not args.no_build: + build_ultra(args, build_dir) + rc = bmc.run_from_args(args, args.arch, get_kernel_path(args.arch, build_dir), build_dir) if rc is not None: From da24567658882e729602665b88f27ca505a500f5 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Tue, 29 Sep 2026 22:53:44 +0300 Subject: [PATCH 14/37] ci: run the unit tests and the lint in jobs of their own The unit tests only need the host compiler but run in every build job, once per toolchain, and the lint step hides behind a condition that picks one of those jobs. Move each into its own job: the unit tests once per OS, the lint once on ubuntu with clang. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- .github/workflows/main.yml | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3b43a558..6def2d55 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,6 +31,20 @@ jobs: mypy *.py mypy scripts/ + lint-kernel: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Build the platform toolchain + run: "${{ github.workspace }}/build.py --toolchain clang --toolchain-only" + + - name: Check the coding conventions + run: "${{ github.workspace }}/build.py --config ${{ github.workspace }}/.github/configs/x86_64-ci.config --toolchain clang --lint" + build: runs-on: ${{ matrix.os }} @@ -80,9 +94,18 @@ jobs: - name: Build the kernel run: "${{ github.workspace }}/build.py --config ${{ github.workspace }}/.github/configs/${{ matrix.arch }}-ci.config --toolchain ${{ matrix.toolchain }} --no-tune-native" + unit-tests: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest, macos-latest] + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Run the userspace unit tests run: "${{ github.workspace }}/build.py --unit-tests" - - - if: ${{ matrix.toolchain == 'clang' && matrix.os == 'ubuntu-latest' }} - name: Check the coding conventions - run: "${{ github.workspace }}/build.py --config ${{ github.workspace }}/.github/configs/${{ matrix.arch }}-ci.config --toolchain ${{ matrix.toolchain }} --no-build --lint" From 511daf4fc401546a8a351bf42d238841519777cf Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Tue, 29 Sep 2026 22:53:44 +0300 Subject: [PATCH 15/37] ci: build the config presets instead of a CI-only config Now that we have clean config presets, use those for CI instead of hardcoding a separate config for it. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- .github/configs/x86_64-ci.config | 8 -------- .github/workflows/main.yml | 5 +++-- 2 files changed, 3 insertions(+), 10 deletions(-) delete mode 100644 .github/configs/x86_64-ci.config diff --git a/.github/configs/x86_64-ci.config b/.github/configs/x86_64-ci.config deleted file mode 100644 index a270d8a1..00000000 --- a/.github/configs/x86_64-ci.config +++ /dev/null @@ -1,8 +0,0 @@ -CONFIG_ARCH_X86_64=y -CONFIG_VERBOSE_INIT_LEVELS=y -CONFIG_DEBUG_MM=y -CONFIG_PARAVIRT=y -CONFIG_KVM_GUEST=y -CONFIG_BHYVE_GUEST=y -CONFIG_EMULATOR_GUEST=y -CONFIG_FLANTERM_CONSOLE=y diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6def2d55..3418745c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,7 +43,7 @@ jobs: run: "${{ github.workspace }}/build.py --toolchain clang --toolchain-only" - name: Check the coding conventions - run: "${{ github.workspace }}/build.py --config ${{ github.workspace }}/.github/configs/x86_64-ci.config --toolchain clang --lint" + run: "${{ github.workspace }}/build.py --toolchain clang --lint" build: runs-on: ${{ matrix.os }} @@ -54,6 +54,7 @@ jobs: os: [ubuntu-latest, macos-latest] arch: [x86_64] toolchain: [gcc, clang] + preset: [debug-lite, release] exclude: # FIXME: the GCC build somehow ends up being corrupted with # libexec/cc1 containing garbage data. @@ -92,7 +93,7 @@ jobs: key: ${{ steps.toolchain-cache.outputs.cache-primary-key }} - name: Build the kernel - run: "${{ github.workspace }}/build.py --config ${{ github.workspace }}/.github/configs/${{ matrix.arch }}-ci.config --toolchain ${{ matrix.toolchain }} --no-tune-native" + run: "${{ github.workspace }}/build.py --arch ${{ matrix.arch }} --config ${{ matrix.preset }} --toolchain ${{ matrix.toolchain }} --no-tune-native" unit-tests: runs-on: ${{ matrix.os }} From 37b36e0627c13269dec2e880c46e4d293a51e86a Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Wed, 30 Sep 2026 16:12:42 +0300 Subject: [PATCH 16/37] build: validate the arch bitness and endianness in CMake An arch selects its bitness and endianness, and Kconfig cannot stop it from selecting both, since choice members cannot be selected. Make CMake reject both, next to its checks for neither. The bitness check now tests ARCH_WIDTH, leaving ARCH_UNKNOWN_BIT_WIDTH unused, and nothing ever read ARCH_UNKNOWN_ENDIAN, so drop both. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- kernel/CMakeLists.txt | 21 ++++++++++++++++----- kernel/arch/Kconfig | 8 -------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 1ff2318a..8ff85988 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -33,21 +33,32 @@ set( ${ULTRA_KERNEL_BASE} ) -if (CONFIG_ARCH_UNKNOWN_BIT_WIDTH) - message( +if (CONFIG_ARCH_64BIT AND CONFIG_ARCH_32BIT) + message( + FATAL_ERROR + "Please fix your architecture's Kconfig to specify only one bitness" + ) +elseif (NOT CONFIG_ARCH_WIDTH) + message( FATAL_ERROR - "Please fix your achitecture's Kconfig to specify bitness" + "Please fix your architecture's Kconfig to specify bitness" ) endif () -if (CONFIG_ARCH_LITTLE_ENDIAN) +if (CONFIG_ARCH_LITTLE_ENDIAN AND CONFIG_ARCH_BIG_ENDIAN) + message( + FATAL_ERROR + "Please fix your architecture's Kconfig to specify only one " + "endianness" + ) +elseif (CONFIG_ARCH_LITTLE_ENDIAN) ultra_compile_definitions(ULTRA_ARCH_LITTLE_ENDIAN) elseif (CONFIG_ARCH_BIG_ENDIAN) ultra_compile_definitions(ULTRA_ARCH_BIG_ENDIAN) else () message( FATAL_ERROR - "Please fix your achitecture's Kconfig to specify endianness" + "Please fix your architecture's Kconfig to specify endianness" ) endif () diff --git a/kernel/arch/Kconfig b/kernel/arch/Kconfig index 870f819c..3e43c831 100644 --- a/kernel/arch/Kconfig +++ b/kernel/arch/Kconfig @@ -5,10 +5,6 @@ config ARCH_LITTLE_ENDIAN config ARCH_BIG_ENDIAN bool -config ARCH_UNKNOWN_ENDIAN - bool - default y if !ARCH_LITTLE_ENDIAN && !ARCH_BIG_ENDIAN - # Arch bitness settings config ARCH_64BIT bool @@ -16,10 +12,6 @@ config ARCH_64BIT config ARCH_32BIT bool -config ARCH_UNKNOWN_BIT_WIDTH - bool - default y if !ARCH_64BIT && !ARCH_32BIT - config ARCH_WIDTH int default 8 if ARCH_64BIT From 72ca457bd763e6a1bace0099f80624d7dbdf7127 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Wed, 30 Sep 2026 15:39:15 +0300 Subject: [PATCH 17/37] memory: move MAX_PHYS_BITS into arch/memory.h It isn't a constant but a value memory.c sets at boot, like everything else in arch/memory.h. smbios.c got PAGE_SIZE through memory/io.h and now includes arch/constants.h itself. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- kernel/arch/x86_64/include/arch/constants.h | 7 ------- kernel/arch/x86_64/include/arch/memory.h | 3 +++ kernel/drivers/smbios.c | 1 + kernel/include/memory/io.h | 2 +- tests/synthetic-include/arch/constants.h | 7 ------- tests/synthetic-include/arch/memory.h | 7 +++++++ tests/synthetic-include/memory/io.h | 2 +- 7 files changed, 13 insertions(+), 16 deletions(-) diff --git a/kernel/arch/x86_64/include/arch/constants.h b/kernel/arch/x86_64/include/arch/constants.h index a601e5c3..e66a0b9c 100644 --- a/kernel/arch/x86_64/include/arch/constants.h +++ b/kernel/arch/x86_64/include/arch/constants.h @@ -1,12 +1,5 @@ #pragma once -#ifndef __ASSEMBLER__ -#include - -extern u8 g_max_phys_bits; -#define MAX_PHYS_BITS g_max_phys_bits -#endif - #define PAGE_SIZE 4096 #define PAGE_SHIFT 12 diff --git a/kernel/arch/x86_64/include/arch/memory.h b/kernel/arch/x86_64/include/arch/memory.h index 38dc239e..1813ec6b 100644 --- a/kernel/arch/x86_64/include/arch/memory.h +++ b/kernel/arch/x86_64/include/arch/memory.h @@ -9,3 +9,6 @@ extern virt_addr_t g_memory_map_base, g_memory_map_end; extern virt_addr_t g_valloc_base, g_valloc_end; #define VALLOC_BASE g_valloc_base #define VALLOC_END g_valloc_end + +extern u8 g_max_phys_bits; +#define MAX_PHYS_BITS g_max_phys_bits diff --git a/kernel/drivers/smbios.c b/kernel/drivers/smbios.c index 4eb50cdb..51a870da 100644 --- a/kernel/drivers/smbios.c +++ b/kernel/drivers/smbios.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include diff --git a/kernel/include/memory/io.h b/kernel/include/memory/io.h index ffde1329..8c010624 100644 --- a/kernel/include/memory/io.h +++ b/kernel/include/memory/io.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include #define MAX_PHYS_ADDR BIT_OF_TYPE(phys_addr_t, MAX_PHYS_BITS) diff --git a/tests/synthetic-include/arch/constants.h b/tests/synthetic-include/arch/constants.h index f2ed3c34..11d7416b 100644 --- a/tests/synthetic-include/arch/constants.h +++ b/tests/synthetic-include/arch/constants.h @@ -10,10 +10,3 @@ #define PAGE_SIZE 4096 #define PAGE_SHIFT 12 - -/* - * The harness physical backing store is tiny, so this only has to be a - * plausible ceiling for the range checks that consult it. 52 is what x86_64 - * reports with 5-level paging, matching the backend in arch/page_table.h. - */ -#define MAX_PHYS_BITS 52 diff --git a/tests/synthetic-include/arch/memory.h b/tests/synthetic-include/arch/memory.h index f750291b..53a09430 100644 --- a/tests/synthetic-include/arch/memory.h +++ b/tests/synthetic-include/arch/memory.h @@ -15,3 +15,10 @@ extern phys_addr_t g_memory_map_base, g_memory_map_end; extern phys_addr_t g_valloc_base, g_valloc_end; #define VALLOC_BASE g_valloc_base #define VALLOC_END g_valloc_end + +/* + * The harness physical backing store is tiny, so this only has to be a + * plausible ceiling for the range checks that consult it. 52 is what x86_64 + * reports with 5-level paging, matching the backend in arch/page_table.h. + */ +#define MAX_PHYS_BITS 52 diff --git a/tests/synthetic-include/memory/io.h b/tests/synthetic-include/memory/io.h index 4373d18c..d7ef67fa 100644 --- a/tests/synthetic-include/memory/io.h +++ b/tests/synthetic-include/memory/io.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include From 9d2c40a7861cd6d8a28ffcfb2c9e200482213646 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin <99danilt@gmail.com> Date: Wed, 30 Sep 2026 15:21:10 +0300 Subject: [PATCH 18/37] arch: move the page and cache line sizes into Kconfig Every arch hardcoded the same constants in its own arch/constants.h. Make the page size a Kconfig choice limited to the sizes an arch selects with ARCH_HAS_PAGE_SIZE_*, add CACHE_LINE_SHIFT defaulting to 6, which an arch can override through ARCH_CACHE_LINE_SHIFT, and derive the constants from both in a generic that replaces arch/constants.h. x2apic.c used nothing from arch/constants.h and just drops the include. Signed-off-by: Daniil Tatianin <99danilt@gmail.com> --- kernel/CMakeLists.txt | 7 ++++++ kernel/arch/Kconfig | 25 +++++++++++++++++++ kernel/arch/aarch64/Kconfig | 1 + kernel/arch/aarch64/include/arch/constants.h | 6 ----- kernel/arch/x86_64/Kconfig | 1 + kernel/arch/x86_64/include/arch/constants.h | 6 ----- kernel/arch/x86_64/include/arch/page_table.h | 2 +- kernel/arch/x86_64/ioapic.c | 2 +- kernel/arch/x86_64/x2apic.c | 1 - kernel/arch/x86_64/xapic.c | 2 +- kernel/drivers/acpi/acpi.c | 3 +-- kernel/drivers/smbios.c | 2 +- kernel/include/arch_constants.h | 9 +++++++ kernel/include/common/align.h | 2 +- kernel/include/linker_common.ld.h | 2 +- kernel/include/memory/buddy.h | 2 +- kernel/include/memory/page.h | 2 +- kernel/log.c | 3 +-- tests/synthetic-include/arch/page_table.h | 2 +- .../{arch/constants.h => arch_constants.h} | 2 +- tests/test_harness.cpp | 2 +- 21 files changed, 56 insertions(+), 28 deletions(-) delete mode 100644 kernel/arch/aarch64/include/arch/constants.h delete mode 100644 kernel/arch/x86_64/include/arch/constants.h create mode 100644 kernel/include/arch_constants.h rename tests/synthetic-include/{arch/constants.h => arch_constants.h} (87%) diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 8ff85988..c692431f 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -62,6 +62,13 @@ else () ) endif () +if (NOT CONFIG_PAGE_SHIFT) + message( + FATAL_ERROR + "Please fix your architecture's Kconfig to select a page size" + ) +endif () + ultra_compile_definitions( ULTRA_ARCH_WIDTH=${CONFIG_ARCH_WIDTH} ULTRA_ARCH_PHYS_ADDR_WIDTH=${CONFIG_ARCH_PHYS_ADDR_WIDTH} diff --git a/kernel/arch/Kconfig b/kernel/arch/Kconfig index 3e43c831..0a5a6b8a 100644 --- a/kernel/arch/Kconfig +++ b/kernel/arch/Kconfig @@ -28,6 +28,31 @@ config ARCH_PHYS_ADDR_WIDTH config ARCH_HAS_UBSAN bool +config ARCH_HAS_PAGE_SIZE_4K + bool + +choice + prompt "Page size" + + config PAGE_SIZE_4K + bool "4KiB" + depends on ARCH_HAS_PAGE_SIZE_4K +endchoice + +config PAGE_SHIFT + int + default 12 if PAGE_SIZE_4K + +# Declared here so the symbol always exists as an int. An arch with +# a different cache line size gives it a default of its own. +config ARCH_CACHE_LINE_SHIFT + int + +config CACHE_LINE_SHIFT + int + default ARCH_CACHE_LINE_SHIFT if ARCH_CACHE_LINE_SHIFT > 0 + default 6 + if ARCH_X86_64 rsource "x86_64/Kconfig" endif diff --git a/kernel/arch/aarch64/Kconfig b/kernel/arch/aarch64/Kconfig index 16f49cbf..11e32e02 100644 --- a/kernel/arch/aarch64/Kconfig +++ b/kernel/arch/aarch64/Kconfig @@ -15,3 +15,4 @@ config ARCH_AARCH64_BASE # Big endian mode is not supported select ARCH_LITTLE_ENDIAN select ARCH_64BIT + select ARCH_HAS_PAGE_SIZE_4K diff --git a/kernel/arch/aarch64/include/arch/constants.h b/kernel/arch/aarch64/include/arch/constants.h deleted file mode 100644 index e66a0b9c..00000000 --- a/kernel/arch/aarch64/include/arch/constants.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once - -#define PAGE_SIZE 4096 -#define PAGE_SHIFT 12 - -#define CACHE_LINE_SIZE 64 diff --git a/kernel/arch/x86_64/Kconfig b/kernel/arch/x86_64/Kconfig index 721018ea..5079f6e1 100644 --- a/kernel/arch/x86_64/Kconfig +++ b/kernel/arch/x86_64/Kconfig @@ -14,6 +14,7 @@ config ARCH_X86_64_BASE bool select ARCH_LITTLE_ENDIAN select ARCH_64BIT + select ARCH_HAS_PAGE_SIZE_4K select ARCH_HAS_UBSAN select SMBIOS select ACPI diff --git a/kernel/arch/x86_64/include/arch/constants.h b/kernel/arch/x86_64/include/arch/constants.h deleted file mode 100644 index e66a0b9c..00000000 --- a/kernel/arch/x86_64/include/arch/constants.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once - -#define PAGE_SIZE 4096 -#define PAGE_SHIFT 12 - -#define CACHE_LINE_SIZE 64 diff --git a/kernel/arch/x86_64/include/arch/page_table.h b/kernel/arch/x86_64/include/arch/page_table.h index e0790ad8..4002f2fd 100644 --- a/kernel/arch/x86_64/include/arch/page_table.h +++ b/kernel/arch/x86_64/include/arch/page_table.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include diff --git a/kernel/arch/x86_64/ioapic.c b/kernel/arch/x86_64/ioapic.c index a62e5957..2e7fbccb 100644 --- a/kernel/arch/x86_64/ioapic.c +++ b/kernel/arch/x86_64/ioapic.c @@ -4,12 +4,12 @@ #include #include #include -#include #include #include #include +#include #include #include #include diff --git a/kernel/arch/x86_64/x2apic.c b/kernel/arch/x86_64/x2apic.c index 82103c76..2278640a 100644 --- a/kernel/arch/x86_64/x2apic.c +++ b/kernel/arch/x86_64/x2apic.c @@ -2,7 +2,6 @@ #include #include -#include #include diff --git a/kernel/arch/x86_64/xapic.c b/kernel/arch/x86_64/xapic.c index b726a455..3aacb912 100644 --- a/kernel/arch/x86_64/xapic.c +++ b/kernel/arch/x86_64/xapic.c @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -13,6 +12,7 @@ #include #include +#include #include static u64 s_xapic_phys_base; diff --git a/kernel/drivers/acpi/acpi.c b/kernel/drivers/acpi/acpi.c index 2cd1a52d..36967847 100644 --- a/kernel/drivers/acpi/acpi.c +++ b/kernel/drivers/acpi/acpi.c @@ -1,8 +1,7 @@ #include #include -#include - +#include #include #include diff --git a/kernel/drivers/smbios.c b/kernel/drivers/smbios.c index 51a870da..052404cd 100644 --- a/kernel/drivers/smbios.c +++ b/kernel/drivers/smbios.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include diff --git a/kernel/include/arch_constants.h b/kernel/include/arch_constants.h new file mode 100644 index 00000000..0ea38eeb --- /dev/null +++ b/kernel/include/arch_constants.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +#define PAGE_SHIFT CONFIG_PAGE_SHIFT +#define PAGE_SIZE (1 << PAGE_SHIFT) + +#define CACHE_LINE_SHIFT CONFIG_CACHE_LINE_SHIFT +#define CACHE_LINE_SIZE (1 << CACHE_LINE_SHIFT) diff --git a/kernel/include/common/align.h b/kernel/include/common/align.h index 5cfc3c81..e96f54ca 100644 --- a/kernel/include/common/align.h +++ b/kernel/include/common/align.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #if HAS_BUILTIN(__builtin_align_up) #define ALIGN_UP(x, val) __builtin_align_up(x, val) diff --git a/kernel/include/linker_common.ld.h b/kernel/include/linker_common.ld.h index 649d5e5b..f20c2ac6 100644 --- a/kernel/include/linker_common.ld.h +++ b/kernel/include/linker_common.ld.h @@ -2,7 +2,7 @@ #include -#include +#include #include #define PHDR_READ (1 << 2) diff --git a/kernel/include/memory/buddy.h b/kernel/include/memory/buddy.h index 4a05e3ee..1f3b4b2d 100644 --- a/kernel/include/memory/buddy.h +++ b/kernel/include/memory/buddy.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include diff --git a/kernel/include/memory/page.h b/kernel/include/memory/page.h index bebb2a26..83de1c7c 100644 --- a/kernel/include/memory/page.h +++ b/kernel/include/memory/page.h @@ -39,7 +39,7 @@ #include #include -#include +#include #include /* diff --git a/kernel/log.c b/kernel/log.c index 023175a1..89ac1c17 100644 --- a/kernel/log.c +++ b/kernel/log.c @@ -14,11 +14,10 @@ #include #include #include +#include #include