diff --git a/crates/hal/src/arch/aarch64/boot.rs b/crates/hal/src/arch/aarch64/boot.rs index 118f07d..6d78400 100644 --- a/crates/hal/src/arch/aarch64/boot.rs +++ b/crates/hal/src/arch/aarch64/boot.rs @@ -8,8 +8,8 @@ //! 2. Zero the BSS section. //! 3. Allocate a 64 KiB stack in the BOOT_STACK symbol. //! 4. Install the exception vector table via VBAR_EL1. -//! 5. Configure TCR_EL1 / MAIR_EL1 / SCTLR_EL1 to enable the MMU with a -//! flat (identity) mapping in TTBR0_EL1. +//! 5. Configure TCR_EL1 / MAIR_EL1 / SCTLR_EL1 and install the kernel and +//! user mappings in TTBR0_EL1 before enabling the MMU. //! 6. Call `kernel_main`. //! //! # Assumptions @@ -17,7 +17,7 @@ //! - The kernel image is loaded by QEMU at 0x4008_0000 (standard ELF load //! address for `-kernel` mode on the `virt` machine). //! - QEMU places us in EL1 with the MMU off and caches off. -//! - The linker script defines `__bss_start`, `__bss_end`, `__stack_top`. +//! - The linker script defines the BSS, kernel stack, user text, and user stack symbols. #[cfg(target_arch = "aarch64")] core::arch::global_asm!( @@ -54,21 +54,22 @@ core::arch::global_asm!( " mov x0, #0xFF", " msr mair_el1, x0", " isb", - // ── TCR_EL1: T0SZ=25 (39-bit VA), TG0=4K, IRGN0/ORGN0=WB/WA ───────── - // IPS=0b001 (36-bit PA, 64 GiB), TBI0=0, AS=0 - // Value: IPS=001, TG1=10 (4K), TG0=00 (4K), T0SZ=25, T1SZ=25 + // ── TCR_EL1: T0SZ=27 (37-bit VA), TG0=4K, IRGN0/ORGN0=WB/WA ───────── + // SH0=inner-shareable, EPD1=1 (disable TTBR1 walks), IPS=000 (32-bit PA). + // A 37-bit VA with a 4 KiB granule starts translation at level 1. " ldr x0, =0x000000000080351bULL", " msr tcr_el1, x0", " isb", - // ── Build a minimal identity page table in TTBR0_EL1 ───────────────── + // ── Build the initial translation tables in TTBR0_EL1 ──────────────── // With TCR T0SZ=27 (37-bit VA) and a 4 KiB granule, the top level is - // L1 where each entry maps a 1 GiB block. We identity-map two 1 GiB - // blocks, which covers everything the QEMU `virt` machine needs: + // L1 where each entry covers a 1 GiB range. We identity-map two 1 GiB + // blocks for the QEMU `virt` machine and use page tables for userspace: // entry 0: VA/PA 0x0000_0000..0x4000_0000 — device MMIO // (GICD 0x0800_0000, GICR 0x080A_0000, PL011 0x0900_0000), // MAIR index 1 (Device-nGnRnE). - // entry 1: VA/PA 0x4000_0000..0x8000_0000 — RAM (kernel loads at - // 0x4008_0000), MAIR index 0 (Normal WB/WA). + // entry 1: VA/PA 0x4000_0000..0x8000_0000 — EL1-only RAM (kernel loads + // at 0x4008_0000), MAIR index 0 (Normal WB/WA). + // entry 3: VA 0xC000_0000..0x1_0000_0000 — L2/L3 user page tables. // Block descriptor bits: bit0=1 (valid), bit1=0 (block, not table), // AttrIndx = bits[4:2], NS=bit5, AP=0b00 bits[7:6] (EL1 RW), SH=0b11 // bits[9:8] (inner shareable), AF=bit10 (access flag) = 1. @@ -82,11 +83,41 @@ core::arch::global_asm!( " movz x2, #0x0405", // (1<<10)|(1<<2)|1 = 0x405 " orr x1, x1, x2", " str x1, [x0]", - // entry 1 → normal block at PA 0x4000_0000 + // entry 1 -> normal block at PA 0x4000_0000, EL1-only identity mapping. + // PTE 0x0701 uses AP=0b00 (bits[7:6]=00), so EL0 cannot access it. " movz x1, #0x4000, lsl #16", // PA 0x4000_0000 - " movz x2, #0x0701", // (1<<10)|(3<<8)|1 = 0x701 + " movz x2, #0x0701", // (1<<10)|(3<<8)|1 = 0x701 (EL1 RW/X) " orr x1, x1, x2", " str x1, [x0, #8]", + // entry 3 -> L2 table for the user VA range beginning at 0xC000_0000. + " adrp x1, __ttbr0_user_l2", + " add x1, x1, :lo12:__ttbr0_user_l2", + " mov x2, #0x3", // Valid level-1 table descriptor. + " orr x4, x1, x2", + " str x4, [x0, #24]", + // L2[0] -> L3 table covering VA 0xC000_0000..0xC020_0000. + " adrp x3, __ttbr0_user_l3", + " add x3, x3, :lo12:__ttbr0_user_l3", + " orr x4, x3, x2", + " str x4, [x1]", + // L3[0] maps the user text page read-only and executable only at EL0. + " adrp x4, __user_text_start", + " add x4, x4, :lo12:__user_text_start", + " ldr x5, =0x0020000000000FC3ULL", // PXN|nG|AF|SH|AP11|page. + " orr x4, x4, x5", + " str x4, [x3]", + // L3[1] stays invalid as a guard page. L3[2..5] map the 16 KiB user stack. + " adrp x4, __user_stack_start", + " add x4, x4, :lo12:__user_stack_start", + " ldr x5, =0x0060000000000F43ULL", // UXN|PXN|nG|AF|SH|AP01|page. + " add x3, x3, #16", + " mov x6, #4", + ".Lmap_user_stack:", + " orr x7, x4, x5", + " str x7, [x3], #8", + " add x4, x4, #1, lsl #12", + " subs x6, x6, #1", + " b.ne .Lmap_user_stack", // TTBR0_EL1 = &__ttbr0_l1 " msr ttbr0_el1, x0", " isb", @@ -122,10 +153,10 @@ core::arch::global_asm!( // The kernel runs at EL1h (SP_EL1), so hardware IRQs are delivered to // the "Current EL with SPx / IRQ" slot at offset 0x280 (index 5). That // slot vectors to `el1h_irq`, which saves a full trap frame, calls the - // Rust handler `aarch64_handle_irq`, restores, and `eret`s. Every other - // slot vectors to `el1_default`, which spins so an unexpected exception - // stops at a known PC (visible under `qemu -d int`) instead of running - // off into garbage. Each slot is padded to its 128-byte boundary. + // Rust handler `aarch64_handle_irq`, restores, and `eret`s. Apart from that + // IRQ slot and the handled lower-EL AArch64 synchronous slot, vectors use + // `el1_default`, which spins so an unexpected exception stops at a known PC + // (visible under `qemu -d int`). Each slot is padded to 128 bytes. ".balign 2048", ".global exception_vectors", "exception_vectors:", @@ -147,8 +178,8 @@ core::arch::global_asm!( ".balign 0x80", " b el1_default", // 0x380 SError ".balign 0x80", - // Lower EL using AArch64 (no userspace yet) - " b el1_default", // 0x400 Sync + // Lower EL using AArch64 (userspace) + " b el0_sync", // 0x400 Sync from EL0 (SVC / user fault) ".balign 0x80", " b el1_default", // 0x480 IRQ ".balign 0x80", @@ -213,6 +244,55 @@ core::arch::global_asm!( " ldr x30, [sp, #0xf0]", " add sp, sp, #0x120", " eret", + // ── el0_sync: synchronous exception from EL0 (SVC / user fault) ────── + // Same 288-byte trap frame as el1h_irq. The Rust handler reads ESR_EL1 + // to classify the exception (SVC #imm for a syscall); on return we + // restore ELR_EL1/SPSR_EL1 (still pointing just past the SVC, in EL0 + // state) and `eret` back to user. On EL0 entry the CPU is at EL1h, so + // SP is the kernel stack — the frame is pushed there. + "el0_sync:", + " sub sp, sp, #0x120", + " stp x0, x1, [sp, #0x00]", + " stp x2, x3, [sp, #0x10]", + " stp x4, x5, [sp, #0x20]", + " stp x6, x7, [sp, #0x30]", + " stp x8, x9, [sp, #0x40]", + " stp x10, x11, [sp, #0x50]", + " stp x12, x13, [sp, #0x60]", + " stp x14, x15, [sp, #0x70]", + " stp x16, x17, [sp, #0x80]", + " stp x18, x19, [sp, #0x90]", + " stp x20, x21, [sp, #0xa0]", + " stp x22, x23, [sp, #0xb0]", + " stp x24, x25, [sp, #0xc0]", + " stp x26, x27, [sp, #0xd0]", + " stp x28, x29, [sp, #0xe0]", + " str x30, [sp, #0xf0]", + " mrs x0, elr_el1", + " mrs x1, spsr_el1", + " stp x0, x1, [sp, #0x100]", + " bl aarch64_handle_sync_lower", + " ldp x0, x1, [sp, #0x100]", + " msr elr_el1, x0", + " msr spsr_el1, x1", + " ldp x0, x1, [sp, #0x00]", + " ldp x2, x3, [sp, #0x10]", + " ldp x4, x5, [sp, #0x20]", + " ldp x6, x7, [sp, #0x30]", + " ldp x8, x9, [sp, #0x40]", + " ldp x10, x11, [sp, #0x50]", + " ldp x12, x13, [sp, #0x60]", + " ldp x14, x15, [sp, #0x70]", + " ldp x16, x17, [sp, #0x80]", + " ldp x18, x19, [sp, #0x90]", + " ldp x20, x21, [sp, #0xa0]", + " ldp x22, x23, [sp, #0xb0]", + " ldp x24, x25, [sp, #0xc0]", + " ldp x26, x27, [sp, #0xd0]", + " ldp x28, x29, [sp, #0xe0]", + " ldr x30, [sp, #0xf0]", + " add sp, sp, #0x120", + " eret", // ── el1_default: park on an unexpected exception ───────────────────── "el1_default:", " b .", @@ -223,12 +303,17 @@ core::arch::global_asm!( " .space 65536", ".global __stack_top", "__stack_top:", - // ── TTBR0_EL1 level-1 identity page table (4 KiB, 4 KiB-aligned) ───── - // 512 × 8-byte descriptors; only entries 0 (device) and 1 (RAM) are - // populated by _start, the rest stay zero (invalid). BSS-resident so - // it is zeroed by the boot BSS-clear before use. + // ── TTBR0_EL1 translation tables (three 4 KiB-aligned pages) ────────── + // L1 entries 0 and 1 are identity blocks; L1[3] points through L2[0] to + // the L3 user mappings. BSS zeroing leaves every other entry invalid. ".section .bss.pagetable", ".align 12", "__ttbr0_l1:", " .space 4096", + ".align 12", + "__ttbr0_user_l2:", + " .space 4096", + ".align 12", + "__ttbr0_user_l3:", + " .space 4096", ); diff --git a/crates/kernel/linker-aarch64.ld b/crates/kernel/linker-aarch64.ld index f2adbe5..53f6d63 100644 --- a/crates/kernel/linker-aarch64.ld +++ b/crates/kernel/linker-aarch64.ld @@ -3,10 +3,9 @@ /* AArch64 kernel linker script. * - * QEMU -kernel loads the ELF at its physical addresses; we use a flat - * 1:1 (identity) mapping so VMA == LMA. The boot stub enables the MMU - * with TTBR0_EL1 pointing at identity-mapped page tables, so physical - * and virtual addresses are equal during early boot. + * QEMU -kernel loads the ELF at its physical addresses. The boot stub uses + * identity mappings for the kernel and MMIO while installing dedicated user + * mappings in the same TTBR0_EL1 hierarchy before enabling the MMU. * * QEMU virt machine loads ELF at 0x40080000 by default. */ @@ -21,6 +20,19 @@ SECTIONS KEEP(*(.text.boot)) } + .text.user ALIGN(4096) : { + __user_text_start = .; + KEEP(*(.text.user)) + __user_text_end = .; + . = __user_text_start + 4096; + } + + ASSERT((__user_text_start & 0xFFF) == 0, + ".text.user must start on a 4 KiB boundary") + ASSERT((__user_text_end - __user_text_start) <= 4096, + ".text.user payload exceeds one page") + ASSERT(SIZEOF(.text.user) == 4096, ".text.user must occupy exactly one page") + .text ALIGN(4096) : { __text_start = .; *(.text .text.*) @@ -40,8 +52,19 @@ SECTIONS __data_end = .; } - .bss ALIGN(4096) (NOLOAD) : { + .bss.user ALIGN(4096) (NOLOAD) : { __bss_start = .; + __user_stack_start = .; + KEEP(*(.bss.user)) + __user_stack_end = .; + } + + ASSERT((__user_stack_start & 0xFFF) == 0, + ".bss.user must start on a 4 KiB boundary") + ASSERT((__user_stack_end - __user_stack_start) == 16384, + ".bss.user must contain exactly 16 KiB") + + .bss ALIGN(4096) (NOLOAD) : { *(.bss .bss.*) *(.bss.stack) *(COMMON) diff --git a/crates/kernel/src/arch/aarch64/mod.rs b/crates/kernel/src/arch/aarch64/mod.rs index 109ce31..3da8f0f 100644 --- a/crates/kernel/src/arch/aarch64/mod.rs +++ b/crates/kernel/src/arch/aarch64/mod.rs @@ -10,12 +10,17 @@ //! //! The `context`, `kthread`, and `sched_glue` submodules are functional //! for **kernel-thread cooperative scheduling** (the [`switch_context`] -//! primitive plus a seeded-stack spawn path). The `clone`, `init_embed`, -//! and `syscall_entry` submodules remain aarch64 build stubs that mirror -//! the x86_64 public API so architecture-neutral kernel code type-checks; -//! the userspace/EL0 transition they need is not written yet. +//! primitive plus a seeded-stack spawn path). The `usermode` submodule +//! provides a smoke-only EL1->EL0 transition on a dedicated 4KiB RX code +//! page, followed by an invalid guard page and four RW/NX stack pages. Its +//! SVC #7 canary and SVC #8 sequence proves a real exception-vector `eret` +//! back to EL0 (see [`jump_to_el0`]); it performs no syscall dispatch. The +//! `clone`, `init_embed`, and `syscall_entry` submodules remain aarch64 build +//! stubs that mirror the x86_64 public API so architecture-neutral kernel +//! code type-checks. //! //! [`switch_context`]: context::switch_context +//! [`jump_to_el0`]: usermode::jump_to_el0 pub mod clone; pub mod context; @@ -25,6 +30,7 @@ pub mod irq; pub mod kthread; pub mod sched_glue; pub mod syscall_entry; +pub mod usermode; use oncrix_hal::arch::aarch64::gic::{GICD_BASE, GICR_BASE, Gicv3}; use oncrix_hal::arch::aarch64::pl011::{PL011_BASE, Pl011}; diff --git a/crates/kernel/src/arch/aarch64/usermode.rs b/crates/kernel/src/arch/aarch64/usermode.rs new file mode 100644 index 0000000..ffa4cf6 --- /dev/null +++ b/crates/kernel/src/arch/aarch64/usermode.rs @@ -0,0 +1,323 @@ +// Copyright 2026 ONCRIX Contributors +// SPDX-License-Identifier: Apache-2.0 + +//! AArch64 transition from EL1 (kernel) to EL0 (user space). +//! +//! The smoke payload has a dedicated RX page at [`EL0_ENTRY_VA`]. Its 16 KiB +//! stack occupies dedicated RW/NX pages ending at [`EL0_STACK_TOP_VA`], with +//! an unmapped guard page between code and stack. [`jump_to_el0`] seeds the +//! exception-return state, clears every general-purpose and SIMD register plus +//! the FP control/status registers, and uses `eret` to enter the payload at +//! EL0t. This one-way confidentiality scrub does not provide SIMD/FP trap or +//! context-switch preservation; that remains separate follow-up work. +//! +//! [`el0_test_entry`] verifies a stack canary, issues `svc #7`, and then issues +//! `svc #8` only after the exception vector has restored its frame and returned +//! to EL0. The two traps therefore prove both directions of the EL0/EL1 round +//! trip without involving a syscall dispatcher. + +use oncrix_hal::arch::aarch64::pl011::{PL011_BASE, Pl011}; +use oncrix_hal::serial::SerialPort; + +/// `SPSR_EL1` value for an exception return to EL0t with DAIF masked. +/// +/// - `M[4:0] = 0b00000` -> return to EL0 using `SP_EL0` (EL0t). +/// - `DAIF` (bits `[9:6]`) all set -> Debug, SError, IRQ, and FIQ masked. +/// +/// Masking is deliberate for the smoke test so the two SVCs are the only +/// exceptions the round-trip depends on. `PSTATE.I` at EL0 does *not* mask a +/// physical IRQ that targets EL1 (a higher EL), so the caller additionally +/// disarms the generic timer before descending. A real userspace launch would +/// instead clear `I` (bit 7) so the preemptive scheduler keeps ticking. +const SPSR_EL0T_MASKED: u64 = 0x3C0; + +/// Exception Class (`ESR_EL1.EC`, bits `[31:26]`) for an `SVC` instruction +/// executed in AArch64 state. +const EC_SVC_AARCH64: u64 = 0x15; + +/// SVC immediate emitted when the EL0 stack canary comparison fails. +const SVC_STACK_FAILURE: u16 = 0x0BAD; + +/// Size of the static EL0 (user) stack for the smoke test (16 KiB). +const EL0_STACK_SIZE: usize = 16 * 1024; + +/// Virtual address of the dedicated RX page containing [`el0_test_entry`]. +pub const EL0_ENTRY_VA: u64 = 0xC000_0000; + +/// Initial EL0 stack pointer above the dedicated 16 KiB RW/NX stack mapping. +/// +/// The stack occupies `0xC000_2000..0xC000_6000`; `0xC000_1000` is an +/// unmapped guard page separating it from the payload's RX page. +pub const EL0_STACK_TOP_VA: u64 = 0xC000_6000; + +/// Backing storage for the EL0 smoke-test user stack. +/// +/// The linker maps this page-aligned input section into the dedicated RW/NX +/// EL0 stack range. It is mutable because EL0 writes the canary through that +/// mapping; Rust code deliberately takes no reference to the mutable static. +#[repr(C, align(4096))] +struct El0Stack { + _bytes: [u8; EL0_STACK_SIZE], +} + +#[used] +#[unsafe(link_section = ".bss.user")] +static mut EL0_STACK: El0Stack = El0Stack { + _bytes: [0; EL0_STACK_SIZE], +}; + +/// Debug helper: write a `u64` as `0x`-prefixed hex to the PL011 console. +fn write_hex(serial: &mut Pl011, value: u64) { + let _ = serial.write_str("0x"); + let mut buf = [0u8; 16]; + let mut n = value; + for byte in buf.iter_mut().rev() { + let digit = (n & 0xF) as u8; + *byte = if digit < 10 { + b'0' + digit + } else { + b'a' + digit - 10 + }; + n >>= 4; + } + let mut start = 0; + while start < buf.len() - 1 && buf[start] == b'0' { + start += 1; + } + for &byte in &buf[start..] { + let _ = serial.write_byte(byte); + } +} + +/// Debug helper: write a `u16` as decimal to the PL011 console. +fn write_dec(serial: &mut Pl011, value: u16) { + if value == 0 { + let _ = serial.write_byte(b'0'); + return; + } + // `u16::MAX` is 65535 -> at most 5 decimal digits. + let mut buf = [0u8; 5]; + let mut n = value; + let mut i = buf.len(); + while n > 0 { + i -= 1; + buf[i] = b'0' + (n % 10) as u8; + n /= 10; + } + for &byte in &buf[i..] { + let _ = serial.write_byte(byte); + } +} + +/// Descend to EL0 (user space). +/// +/// Seeds the exception-return system registers, clears `x0..x30`, `q0..q31`, +/// `FPCR`, and `FPSR`, then executes `eret`. DAIF remains masked for this +/// deterministic smoke-test scope. After `eret`, the CPU executes `entry` at +/// EL0t using `user_sp`; later exceptions enter EL1 through the installed +/// `VBAR_EL1` vector. This is a one-way confidentiality scrub for the smoke +/// transition; SIMD/FP trap and context-switch save/restore remain follow-up. +/// +/// # Safety +/// +/// - `entry` must name a valid EL0-accessible executable mapping. +/// - `user_sp` must be 16-byte aligned and backed below by writable EL0 memory. +/// - The EL1 lower-EL synchronous vector must save and restore `x0..x30`, +/// `ELR_EL1`, and `SPSR_EL1` around [`aarch64_handle_sync_lower`]. +/// - The caller must ensure no asynchronous source is required while DAIF is +/// masked; the smoke path disarms its timer before calling this function. +pub unsafe fn jump_to_el0(entry: u64, user_sp: u64) -> ! { + let mut serial = Pl011::new(PL011_BASE); + let _ = serial.write_str("[ONCRIX/aarch64] Transitioning to EL0...\n"); + let _ = serial.write_str("[debug] entry="); + write_hex(&mut serial, entry); + let _ = serial.write_str(" sp="); + write_hex(&mut serial, user_sp); + let _ = serial.write_str("\n"); + + // SAFETY: The caller guarantees EL1 execution and valid EL0 mappings. All + // operand-dependent system-register writes complete before the GPR, SIMD, + // FPCR, and FPSR scrub. The ISB commits the return state before ERET + // consumes it. ERET selects EL0t with DAIF masked and cannot fall through. + unsafe { + core::arch::asm!( + "msr daifset, #0xf", + "msr sp_el0, {sp}", + "msr elr_el1, {entry}", + "msr spsr_el1, {spsr}", + "isb", + "movi v0.2d, #0", + "movi v1.2d, #0", + "movi v2.2d, #0", + "movi v3.2d, #0", + "movi v4.2d, #0", + "movi v5.2d, #0", + "movi v6.2d, #0", + "movi v7.2d, #0", + "movi v8.2d, #0", + "movi v9.2d, #0", + "movi v10.2d, #0", + "movi v11.2d, #0", + "movi v12.2d, #0", + "movi v13.2d, #0", + "movi v14.2d, #0", + "movi v15.2d, #0", + "movi v16.2d, #0", + "movi v17.2d, #0", + "movi v18.2d, #0", + "movi v19.2d, #0", + "movi v20.2d, #0", + "movi v21.2d, #0", + "movi v22.2d, #0", + "movi v23.2d, #0", + "movi v24.2d, #0", + "movi v25.2d, #0", + "movi v26.2d, #0", + "movi v27.2d, #0", + "movi v28.2d, #0", + "movi v29.2d, #0", + "movi v30.2d, #0", + "movi v31.2d, #0", + "msr fpcr, xzr", + "msr fpsr, xzr", + "mov x0, xzr", + "mov x1, xzr", + "mov x2, xzr", + "mov x3, xzr", + "mov x4, xzr", + "mov x5, xzr", + "mov x6, xzr", + "mov x7, xzr", + "mov x8, xzr", + "mov x9, xzr", + "mov x10, xzr", + "mov x11, xzr", + "mov x12, xzr", + "mov x13, xzr", + "mov x14, xzr", + "mov x15, xzr", + "mov x16, xzr", + "mov x17, xzr", + "mov x18, xzr", + "mov x19, xzr", + "mov x20, xzr", + "mov x21, xzr", + "mov x22, xzr", + "mov x23, xzr", + "mov x24, xzr", + "mov x25, xzr", + "mov x26, xzr", + "mov x27, xzr", + "mov x28, xzr", + "mov x29, xzr", + "mov x30, xzr", + "eret", + sp = in(reg) user_sp, + entry = in(reg) entry, + spsr = in(reg) SPSR_EL0T_MASKED, + options(noreturn), + ); + } +} + +/// Synchronous-exception handler for a *lower* Exception Level (EL0). +/// +/// The lower-EL synchronous vector calls this function with a saved trap +/// frame. SVC #7 reports the validated stack canary and SVC #8 reports that +/// execution returned to EL0 after SVC #7. Those two expected traps return to +/// the vector so its `eret` can resume EL0. A canary failure, an unexpected +/// SVC, or any other lower-EL synchronous exception reports diagnostics and +/// fail-stops at EL1 rather than repeating a faulting EL0 instruction. +#[unsafe(no_mangle)] +pub extern "C" fn aarch64_handle_sync_lower() { + let esr: u64; + // SAFETY: `mrs` from `ESR_EL1` reads the Exception Syndrome Register, + // which is always accessible at EL1 and has no side effects. It still + // holds this exception's syndrome because no further exception has been + // taken since entry (the CPU masks DAIF on exception entry). + unsafe { + core::arch::asm!("mrs {}, esr_el1", out(reg) esr, options(nomem, nostack)); + } + + let ec = (esr >> 26) & 0x3F; + let mut serial = Pl011::new(PL011_BASE); + + if ec != EC_SVC_AARCH64 { + let far: u64; + // SAFETY: FAR_EL1 is a side-effect-free EL1 system-register read. Its + // value is useful for address-related faults and harmless otherwise. + unsafe { + core::arch::asm!("mrs {}, far_el1", out(reg) far, options(nomem, nostack)); + } + let _ = serial.write_str("[ONCRIX/aarch64] lower-EL exception: EC="); + write_hex(&mut serial, ec); + let _ = serial.write_str(" ESR="); + write_hex(&mut serial, esr); + let _ = serial.write_str(" FAR_EL1="); + write_hex(&mut serial, far); + let _ = serial.write_str("\n"); + fail_stop(); + } + + // ESR_EL1.ISS[15:0] contains the immediate from the trapped SVC. + let imm = (esr & 0xFFFF) as u16; + match imm { + 7 => { + let _ = serial.write_str("[ONCRIX/aarch64] EL0 stack canary verified\n"); + } + 8 => { + let _ = serial.write_str("[ONCRIX/aarch64] EL0 round trip verified\n"); + } + SVC_STACK_FAILURE => { + let _ = serial.write_str("[ONCRIX/aarch64] EL0 stack canary FAILED\n"); + fail_stop(); + } + _ => { + let _ = serial.write_str("[ONCRIX/aarch64] unexpected EL0 SVC #"); + write_dec(&mut serial, imm); + let _ = serial.write_str("\n"); + fail_stop(); + } + } +} + +/// Parks the processor at EL1 after an unrecoverable EL0 smoke-test failure. +fn fail_stop() -> ! { + // SAFETY: A local branch-to-self intentionally prevents return to a + // faulting or failed EL0 context and does not access memory or the stack. + unsafe { + core::arch::asm!("b .", options(noreturn, nostack)); + } +} + +/// Self-contained entry point for the dedicated EL0 RX payload page. +/// +/// The naked body has no compiler-generated prologue, calls, external branch, +/// or literal pool. It writes and reloads a 64-bit canary through `SP_EL0`, +/// restores SP, and traps with SVC #7 on success. Reaching SVC #8 proves that +/// the first exception frame was restored and `eret` resumed the next EL0 +/// instruction. A mismatch uses a distinct SVC and both terminal paths park +/// with a local branch-to-self. +#[unsafe(no_mangle)] +#[unsafe(naked)] +#[unsafe(link_section = ".text.user")] +pub extern "C" fn el0_test_entry() -> ! { + core::arch::naked_asm!( + "sub sp, sp, #16", + "movz x9, #0x454c", + "movk x9, #0x4958, lsl #16", + "movk x9, #0x4352, lsl #32", + "movk x9, #0x4f4e, lsl #48", + "str x9, [sp]", + "ldr x10, [sp]", + "add sp, sp, #16", + "cmp x9, x10", + "b.ne 1f", + "svc #7", + "svc #8", + "b .", + "1:", + "svc #0xbad", + "b .", + ); +} diff --git a/crates/kernel/src/main.rs b/crates/kernel/src/main.rs index 9057b12..eb19049 100644 --- a/crates/kernel/src/main.rs +++ b/crates/kernel/src/main.rs @@ -537,7 +537,7 @@ pub extern "C" fn kernel_main() -> ! { // `CpuContext`; copy that context into the scheduler-owned // `Thread` before registering it. // 4. Yield. Control flows boot → A → B → boot (round-robin), so - // both thread bodies print before we return here and halt. + // both thread bodies print before preemption and EL0 smoke run. { use oncrix_kernel::arch::aarch64::kthread::{kthread_context, spawn_kthread}; use oncrix_kernel::arch::init::SCHEDULER; @@ -653,7 +653,47 @@ pub extern "C" fn kernel_main() -> ! { ); } - let _ = serial.write_str("[ONCRIX/aarch64] Entering halt loop.\n"); + // ─── EL0 (userspace) smoke test ─── + // + // Prove the EL1->EL0 transition on a dedicated 4KiB RX code page, + // followed by an invalid guard page and four RW/NX stack pages. + // The EL0 payload issues an SVC #7 canary, resumes after the vector's + // `eret`, then issues SVC #8 to prove a real EL0->EL1->EL0 round-trip. + // The handler is smoke-only and performs no syscall dispatch. After + // the proof EL0 idles, so this is the last action on the aarch64 path. + { + use oncrix_hal::arch::aarch64::timer::AArch64Timer; + use oncrix_hal::timer::Timer; + use oncrix_kernel::arch::aarch64::usermode::{ + EL0_ENTRY_VA, EL0_STACK_TOP_VA, jump_to_el0, + }; + + // Mask DAIF before stopping the timer so a pending IRQ cannot land + // in the shutdown window. Intentionally omit `nomem`: the mask must + // be a compiler barrier so the timer stop cannot move ahead of it. + // SAFETY: EL1 boot context; nothing after this point needs interrupts. + unsafe { + core::arch::asm!("msr daifset, #0b1111", options(nostack)); + } + + // Disarm the generic timer after masking DAIF. EL0 PSTATE cannot + // mask a physical IRQ routed to EL1, so both steps are required for + // a deterministic smoke-only round-trip. + let mut timer = AArch64Timer::new(); + let _ = timer.stop(); + + let _ = serial.write_str( + "[ONCRIX/aarch64] entering EL0 (dedicated-page smoke; no syscall dispatch)...\n", + ); + + // SAFETY: the boot tables map `EL0_ENTRY_VA` as the dedicated 4KiB + // RX code page and `EL0_STACK_TOP_VA` above four RW/NX stack pages, + // with an invalid guard page between them. The stack top is + // 16-byte aligned; `jump_to_el0` drops via `eret` and never returns. + unsafe { + jump_to_el0(EL0_ENTRY_VA, EL0_STACK_TOP_VA); + } + } } #[cfg(target_arch = "riscv64")] @@ -823,6 +863,11 @@ pub extern "C" fn kernel_main() -> ! { let _ = serial.write_str("[ONCRIX/riscv64] Entering halt loop.\n"); } + // x86_64 and riscv64 fall through to the shared halt loop. On aarch64, + // control descended to EL0 in the block above via `jump_to_el0` (`-> !`) + // and never reaches here, so the call is compiled out to avoid an + // unreachable-code warning; `halt_loop` stays live via the panic handler. + #[cfg(not(target_arch = "aarch64"))] halt_loop(); } diff --git a/scripts/run-qemu-aarch64.sh b/scripts/run-qemu-aarch64.sh index 7aa6778..97fa56f 100755 --- a/scripts/run-qemu-aarch64.sh +++ b/scripts/run-qemu-aarch64.sh @@ -8,14 +8,28 @@ # The kernel targets the GICv3 interrupt controller, so `gic-version=3` # is required (the `virt` machine defaults to GICv2, which has no # redistributor and faults `init_gic`). Serial (PL011 @ 0x0900_0000) is -# routed to stdout. Expected output: +# routed to stdout. Representative output (C/D markers may interleave): # [ONCRIX/aarch64] Kernel booting... # [ONCRIX/aarch64] PL011 UART initialized (115200 8N1) # [ONCRIX/aarch64] Heap initialized (16 MiB) # [ONCRIX/aarch64] GICv3 initialized # [ONCRIX/aarch64] Generic timer armed (10 ms) # [ONCRIX/aarch64] All early initialization complete. -# [ONCRIX/aarch64] Entering halt loop. +# [ONCRIX/aarch64] cooperative scheduler: bring-up demo start +# [ONCRIX/aarch64] cooperative scheduler: thread A ran +# [ONCRIX/aarch64] cooperative scheduler: thread B ran +# [ONCRIX/aarch64] cooperative scheduler: thread A/B ran, back on boot thread +# [ONCRIX/aarch64] IRQs unmasked; preemptive scheduler armed (threads C, D). +# [ONCRIX/aarch64] timer IRQ received (preemptive) +# [ONCRIX/aarch64] preemptive: thread C scheduled +# [ONCRIX/aarch64] preemptive: thread D scheduled +# [ONCRIX/aarch64] preemptive: C and D both ran - timer preemption verified. +# [ONCRIX/aarch64] entering EL0 (dedicated-page smoke; no syscall dispatch)... +# [ONCRIX/aarch64] Transitioning to EL0... +# [debug] entry=0xc0000000 sp=0xc0006000 +# SVC proof: #7 validates the canary; #8 runs only after returning to EL0. +# [ONCRIX/aarch64] EL0 stack canary verified +# [ONCRIX/aarch64] EL0 round trip verified set -euo pipefail