From 7cbac9fdbe0574cbe629cf5ddb0e131483f2fdb8 Mon Sep 17 00:00:00 2001 From: KN Date: Fri, 16 Jan 2026 19:49:44 +0100 Subject: [PATCH 01/23] feat: create initial draft for async system --- include/kernel/config.h | 23 +- include/kernel/cpu/ipi.h | 2 +- include/kernel/cpu/irq.h | 4 +- include/kernel/cpu/syscall.h | 3 + include/kernel/drivers/abstract/fb.h | 2 +- include/kernel/fs/dentry.h | 6 +- include/kernel/fs/devfs.h | 8 +- include/kernel/fs/filesystem.h | 2 +- include/kernel/fs/namespace.h | 2 +- include/kernel/fs/sysfs.h | 8 +- include/kernel/mem/space.h | 4 +- include/kernel/mem/vmm.h | 9 +- include/kernel/proc/process.h | 2 + include/kernel/sched/wait.h | 2 +- include/kernel/sync/async.h | 126 +++++++ include/kernel/sync/rcu.h | 5 +- include/kernel/sync/task.h | 246 +++++++++++++ include/kernel/sync/tasks.h | 25 ++ include/kernel/utils/fifo.h | 103 +++++- include/libpatchwork/element.h | 4 +- include/libpatchwork/window.h | 2 +- include/libstd/sys/async.h | 260 +++++++++++++ include/libstd/sys/bitmap.h | 4 +- include/libstd/sys/elf.h | 2 +- include/libstd/sys/list.h | 13 +- include/libstd/sys/proc.h | 2 +- src/kernel/cpu/interrupt.c | 2 + src/kernel/cpu/ipi.c | 2 +- src/kernel/cpu/irq.c | 4 +- src/kernel/drivers/abstract/fb.c | 2 +- src/kernel/fs/devfs.c | 8 +- src/kernel/fs/namespace.c | 2 +- src/kernel/fs/netfs.c | 2 +- src/kernel/fs/procfs.c | 2 +- src/kernel/fs/sysfs.c | 8 +- src/kernel/fs/tmpfs.c | 2 +- src/kernel/mem/space.c | 2 +- src/kernel/mem/vmm.c | 12 +- src/kernel/module/module.c | 4 +- src/kernel/proc/process.c | 5 +- src/kernel/sync/async.c | 348 ++++++++++++++++++ src/kernel/sync/task.c | 66 ++++ src/kernel/sync/tasks.c | 12 + src/kernel/utils/fifo.c | 108 ------ src/libpatchwork/element.c | 8 +- src/libpatchwork/internal.h | 2 +- src/libpatchwork/window.c | 2 +- src/libstd/common/print.h | 2 +- src/libstd/common/scan.h | 2 +- src/libstd/functions/elf/elf64_relocate.c | 2 +- src/libstd/user/common/syscalls.h | 17 + src/libstd/user/common/threading.h | 6 +- .../user/functions/async/async_deinit.c | 13 + src/libstd/user/functions/async/async_init.c | 13 + .../user/functions/async/async_submit.c | 13 + src/modules/drivers/ps2/ps2.h | 2 +- src/modules/fs/9p/9p.c | 2 +- src/modules/ipc/shmem/shmem.c | 2 +- 58 files changed, 1353 insertions(+), 193 deletions(-) create mode 100644 include/kernel/sync/async.h create mode 100644 include/kernel/sync/task.h create mode 100644 include/kernel/sync/tasks.h create mode 100644 include/libstd/sys/async.h create mode 100644 src/kernel/sync/async.c create mode 100644 src/kernel/sync/task.c create mode 100644 src/kernel/sync/tasks.c delete mode 100644 src/kernel/utils/fifo.c create mode 100644 src/libstd/user/functions/async/async_deinit.c create mode 100644 src/libstd/user/functions/async/async_init.c create mode 100644 src/libstd/user/functions/async/async_submit.c diff --git a/include/kernel/config.h b/include/kernel/config.h index 2806d60e7..6cd05e59f 100644 --- a/include/kernel/config.h +++ b/include/kernel/config.h @@ -190,4 +190,25 @@ * simultaneously. * */ -#define CONFIG_MAX_WAIT_QUEUES 64 \ No newline at end of file +#define CONFIG_MAX_WAIT_QUEUES 64 + +/** + * @brief Maximum asynchronous rings configuration. + * @ingroup kernel + * @def CONFIG_MAX_ASYNC_RINGS + * + * The `CONFIG_MAX_ASYNC_RINGS` constant defines the maximum amount of asynchronous rings that each process can have. + * + */ +#define CONFIG_MAX_ASYNC_RINGS 8 + +/** + * @brief Maximum async ring pages configuration. + * @ingroup kernel + * @def CONFIG_MAX_ASYNC_PAGES + * + * The `CONFIG_MAX_ASYNC_PAGES` constant defines the maximum amount of pages that can be allocated for a async rings + * buffer. + * + */ +#define CONFIG_MAX_ASYNC_PAGES 1024 \ No newline at end of file diff --git a/include/kernel/cpu/ipi.h b/include/kernel/cpu/ipi.h index bf6a0c2f1..8feb6da14 100644 --- a/include/kernel/cpu/ipi.h +++ b/include/kernel/cpu/ipi.h @@ -166,7 +166,7 @@ uint64_t ipi_chip_amount(void); * - `EBUSY`: The target CPU's IPI queue is full, some or all IPIs could not be sent. * - Other errors returned by the IPI chip's `notify` function. */ -uint64_t ipi_send(cpu_t* cpu, ipi_flags_t flags, ipi_func_t func, void* data); +uint64_t ipi_send(cpu_t* cpu, ipi_flags_t flags, ipi_func_t func, void* data); /** * @brief Wake up one or more CPUs. diff --git a/include/kernel/cpu/irq.h b/include/kernel/cpu/irq.h index c818d01a5..27c67ab4f 100644 --- a/include/kernel/cpu/irq.h +++ b/include/kernel/cpu/irq.h @@ -229,7 +229,7 @@ uint64_t irq_virt_set_affinity(irq_virt_t virt, cpu_t* cpu); * - `ENOMEM`: Memory allocation failed. * - Other errors as returned by the IRQ chip's `enable` function. */ -uint64_t irq_chip_register(irq_chip_t* chip, irq_phys_t start, irq_phys_t end, void* data); +uint64_t irq_chip_register(irq_chip_t* chip, irq_phys_t start, irq_phys_t end, void* data); /** * @brief Unregister all instances of the given IRQ chip within the specified range. @@ -265,7 +265,7 @@ uint64_t irq_chip_amount(void); * - `ENOMEM`: Memory allocation failed. * - Other errors as returned by the IRQ chip's `enable` function. */ -uint64_t irq_handler_register(irq_virt_t virt, irq_func_t func, void* data); +uint64_t irq_handler_register(irq_virt_t virt, irq_func_t func, void* data); /** * @brief Unregister an IRQ handler. diff --git a/include/kernel/cpu/syscall.h b/include/kernel/cpu/syscall.h index 633865020..6a09202a9 100644 --- a/include/kernel/cpu/syscall.h +++ b/include/kernel/cpu/syscall.h @@ -102,6 +102,9 @@ typedef enum SYS_MOUNT, SYS_UNMOUNT, SYS_ARCH_PRCTL, + SYS_ASYNC_INIT, + SYS_ASYNC_DEINIT, + SYS_ASYNC_NOTIFY, SYS_TOTAL_AMOUNT } syscall_number_t; diff --git a/include/kernel/drivers/abstract/fb.h b/include/kernel/drivers/abstract/fb.h index 9cb44c6a2..645394d57 100644 --- a/include/kernel/drivers/abstract/fb.h +++ b/include/kernel/drivers/abstract/fb.h @@ -89,7 +89,7 @@ typedef struct fb * @param private Private data for the framebuffer. * @return On success, the new framebuffer. On failure, `NULL` and `errno` is set. */ -fb_t* fb_new(const char* name, const fb_ops_t* ops, void* data); +fb_t* fb_new(const char* name, const fb_ops_t* ops, void* data); /** * @brief Frees a framebuffer. diff --git a/include/kernel/fs/dentry.h b/include/kernel/fs/dentry.h index 8e8a2eaae..d0f2c8a9a 100644 --- a/include/kernel/fs/dentry.h +++ b/include/kernel/fs/dentry.h @@ -109,9 +109,9 @@ typedef struct dir_ctx * @return `true` to continue iterating, `false` to stop. */ bool (*emit)(dir_ctx_t* ctx, const char* name, ino_t number, itype_t type); - size_t pos; ///< The current position in the directory, can be used to skip entries. - void* data; ///< Private data that the filesystem can use to conveniently pass data. - size_t index; ///< An index that the filesystem can use for its own purposes. + size_t pos; ///< The current position in the directory, can be used to skip entries. + void* data; ///< Private data that the filesystem can use to conveniently pass data. + size_t index; ///< An index that the filesystem can use for its own purposes. } dir_ctx_t; /** diff --git a/include/kernel/fs/devfs.h b/include/kernel/fs/devfs.h index bd4f521d2..d8aefc46e 100644 --- a/include/kernel/fs/devfs.h +++ b/include/kernel/fs/devfs.h @@ -39,7 +39,7 @@ void devfs_init(void); * @param private Private data to store in the inode of the new directory, can be `NULL`. * @return On success, the new devfs directory. On failure, `NULL` and `errno` is set. */ -dentry_t* devfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); +dentry_t* devfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); /** * @brief Create a new file inside a mounted devfs instance. @@ -52,7 +52,7 @@ dentry_t* devfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* i * @return On success, the new devfs file. On failure, `NULL` and `errno` is set. */ dentry_t* devfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, const file_ops_t* fileOps, - void* data); + void* data); /** * @brief Create a new symbolic link inside a mounted devfs instance. @@ -63,7 +63,7 @@ dentry_t* devfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* * @param private Private data to store in the inode of the new symbolic link, can be `NULL`. * @return On success, the new devfs symbolic link. On failure, `NULL` and `errno` is set. */ -dentry_t* devfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); +dentry_t* devfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); /** * @brief Descriptor for batch file creation. @@ -74,7 +74,7 @@ typedef struct devfs_file_desc const char* name; ///< Name of the file, `NULL` marks end of array. const inode_ops_t* inodeOps; ///< Inode operations, can be `NULL`. const file_ops_t* fileOps; ///< File operations, can be `NULL`. - void* data; ///< Private data to store in the inode of the file. + void* data; ///< Private data to store in the inode of the file. } devfs_file_desc_t; /** diff --git a/include/kernel/fs/filesystem.h b/include/kernel/fs/filesystem.h index 180aae5c5..ed1f503c7 100644 --- a/include/kernel/fs/filesystem.h +++ b/include/kernel/fs/filesystem.h @@ -65,7 +65,7 @@ typedef struct filesystem * @param private Private data for the filesystem's mount function. * @return On success, the root dentry of the mounted filesystem. On failure, `NULL` and `errno` is set. */ - dentry_t* (*mount)(filesystem_t* fs, const char* details, void* data); + dentry_t* (*mount)(filesystem_t* fs, const char* details, void* data); } filesystem_t; /**s diff --git a/include/kernel/fs/namespace.h b/include/kernel/fs/namespace.h index 3d496ed4b..b2a714464 100644 --- a/include/kernel/fs/namespace.h +++ b/include/kernel/fs/namespace.h @@ -136,7 +136,7 @@ bool namespace_rcu_traverse(namespace_t* ns, mount_t** mount, dentry_t** dentry) * - Other errors as returned by the filesystem's `mount()` operation or `mount_new()`. */ mount_t* namespace_mount(namespace_t* ns, path_t* target, filesystem_t* fs, const char* options, mode_t mode, - void* data); + void* data); /** * @brief Bind a source path to a target path in a namespace. diff --git a/include/kernel/fs/sysfs.h b/include/kernel/fs/sysfs.h index 9657516ac..b9f5a45a0 100644 --- a/include/kernel/fs/sysfs.h +++ b/include/kernel/fs/sysfs.h @@ -44,7 +44,7 @@ void sysfs_init(void); * @param private Private data to store in the inode of the new directory, can be `NULL`. * @return On success, the new sysfs directory. On failure, `NULL` and `errno` is set. */ -dentry_t* sysfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); +dentry_t* sysfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); /** * @brief Create a new file inside a mounted sysfs instance. @@ -57,7 +57,7 @@ dentry_t* sysfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* i * @return On success, the new sysfs file. On failure, `NULL` and `errno` is set. */ dentry_t* sysfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, const file_ops_t* fileOps, - void* data); + void* data); /** * @brief Create a new symbolic link inside a mounted sysfs instance. @@ -68,7 +68,7 @@ dentry_t* sysfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* * @param private Private data to store in the inode of the new symbolic link, can be `NULL`. * @return On success, the new sysfs symbolic link. On failure, `NULL` and `errno` is set. */ -dentry_t* sysfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); +dentry_t* sysfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); /** * @brief Descriptor for batch file creation. @@ -79,7 +79,7 @@ typedef struct sysfs_file_desc const char* name; ///< Name of the file, `NULL` marks end of array. const inode_ops_t* inodeOps; ///< Inode operations, can be `NULL`. const file_ops_t* fileOps; ///< File operations, can be `NULL`. - void* data; ///< Private data to store in the inode of the file. + void* data; ///< Private data to store in the inode of the file. } sysfs_file_desc_t; /** diff --git a/include/kernel/mem/space.h b/include/kernel/mem/space.h index b6780e5b0..864134a88 100644 --- a/include/kernel/mem/space.h +++ b/include/kernel/mem/space.h @@ -39,7 +39,7 @@ typedef enum /** * @brief Space callback function. */ -typedef void (*space_callback_func_t)( void* data); +typedef void (*space_callback_func_t)(void* data); /** * @brief Space callback structure. @@ -261,7 +261,7 @@ uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* vir * @param private Private data to pass to the callback function. * @return On success, returns the callback ID. On failure, returns `PML_MAX_CALLBACK`. */ -pml_callback_id_t space_alloc_callback(space_t* space, size_t pageAmount, space_callback_func_t func, void* data); +pml_callback_id_t space_alloc_callback(space_t* space, size_t pageAmount, space_callback_func_t func, void* data); /** * @brief Free a callback. diff --git a/include/kernel/mem/vmm.h b/include/kernel/mem/vmm.h index 474993a13..de9d5c381 100644 --- a/include/kernel/mem/vmm.h +++ b/include/kernel/mem/vmm.h @@ -119,8 +119,9 @@ typedef struct */ typedef enum { - VMM_ALLOC_OVERWRITE = 0 << 0, ///< If any page is already mapped, overwrite the mapping. - VMM_ALLOC_FAIL_IF_MAPPED = 1 << 0 ///< If set and any page is already mapped, fail and set `errno` to `EEXIST`. + VMM_ALLOC_OVERWRITE = 0 << 0, ///< If any page is already mapped, overwrite the mapping. + VMM_ALLOC_FAIL_IF_MAPPED = 1 << 0, ///< If set and any page is already mapped, fail and set `errno` to `EEXIST`. + VMM_ALLOC_ZERO = 1 << 1 ///< If set, atomically zero the allocated pages. } vmm_alloc_flags_t; /** @@ -195,7 +196,7 @@ void* vmm_alloc(space_t* space, void* virtAddr, size_t length, size_t alignment, * - Other values from `space_mapping_start()`. */ void* vmm_map(space_t* space, void* virtAddr, void* physAddr, size_t length, pml_flags_t flags, - space_callback_func_t func, void* data); + space_callback_func_t func, void* data); /** * @brief Maps an array of physical pages to virtual memory in a given address space. @@ -220,7 +221,7 @@ void* vmm_map(space_t* space, void* virtAddr, void* physAddr, size_t length, pml * - Other values from `space_mapping_start()`. */ void* vmm_map_pages(space_t* space, void* virtAddr, void** pages, size_t pageAmount, pml_flags_t flags, - space_callback_func_t func, void* data); + space_callback_func_t func, void* data); /** * @brief Unmaps virtual memory from a given address space. diff --git a/include/kernel/proc/process.h b/include/kernel/proc/process.h index 3c093d4d0..faf725663 100644 --- a/include/kernel/proc/process.h +++ b/include/kernel/proc/process.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -87,6 +88,7 @@ typedef struct process file_table_t fileTable; futex_ctx_t futexCtx; perf_process_ctx_t perf; + async_ctx_t async; note_handler_t noteHandler; wait_queue_t suspendQueue; wait_queue_t dyingQueue; diff --git a/include/kernel/sched/wait.h b/include/kernel/sched/wait.h index 75afdb51d..3d9f4f9aa 100644 --- a/include/kernel/sched/wait.h +++ b/include/kernel/sched/wait.h @@ -32,7 +32,7 @@ typedef struct wait wait_t; * @note Generally its preferred to use the `WAIT_BLOCK*` macros instead of directly calling the functions provided by * this subsystem. * - * @todo Replace with `epoll()` style system? + * @todo Replace with a more optimized system for async stuff. * * @{ */ diff --git a/include/kernel/sync/async.h b/include/kernel/sync/async.h new file mode 100644 index 000000000..71dea5fe6 --- /dev/null +++ b/include/kernel/sync/async.h @@ -0,0 +1,126 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +/** + * @brief Asynchronous Rings + * @defgroup kernel_sync_rings Rings + * @ingroup kernel_sync + * + * @{ + */ + +/** + * @brief Async context flags. + * @enum async_ctx_flags_t + */ +typedef enum +{ + ASYNC_CTX_NONE = 0, ///< No flags set. + ASYNC_CTX_BUSY = 1 << 0, ///< Context is currently being used. + ASYNC_CTX_MAPPED = 1 << 1, ///< Context rings are mapped. +} async_ctx_flags_t; + +/** + * @brief The kernel-side asynchronous context structure. + * @struct async_ctx_t + */ +typedef struct async_ctx +{ + async_rings_t rings; ///< Asynchronous rings information. + task_t* tasks; ///< A preallocated array of tasks, one for each CQE. + list_t freeTasks; ///< Free list of tasks. + void* userAddr; ///< Userspace address of the rings. + void* kernelAddr; ///< Kernel address of the rings. + size_t pageAmount; ///< Amount of pages mapped for the rings. + space_t* space; ///< Pointer to the owning address space. + wait_queue_t waitQueue; ///< Wait queue for completions. + _Atomic(async_ctx_flags_t) flags; +} async_ctx_t; + +/** + * @brief Initialize a async context. + * + * @param ctx Pointer to the context to initialize. + */ +void async_ctx_init(async_ctx_t* ctx); + +/** + * @brief Deinitialize a async context. + * + * @param ctx Pointer to the context to deinitialize. + */ +void async_ctx_deinit(async_ctx_t* ctx); + +/** + * @brief Notify the context of new SQEs. + * + * @param ctx Pointer to the context. + * @param amount The number of SQEs to process. + * @param wait The minimum number of CQEs to wait for. + * @return On success, the number of SQEs processed. On failure, `ERR` and `errno` is set. + */ +uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait); + +/** + * @brief Acquire a async context. + * + * @param ctx Pointer to the context to acquire. + * @return On success, `0`. On failure, `ERR`. + */ +static inline uint64_t async_ctx_acquire(async_ctx_t* ctx) +{ + async_ctx_flags_t expected = atomic_load(&ctx->flags); + if (!(expected & ASYNC_CTX_BUSY) && + atomic_compare_exchange_strong(&ctx->flags, &expected, expected | ASYNC_CTX_BUSY)) + { + return 0; + } + + return ERR; +} + +/** + * @brief Release a async context. + * + * @param ctx Pointer to the context to release. + */ +static inline void async_ctx_release(async_ctx_t* ctx) +{ + atomic_fetch_and(&ctx->flags, ~ASYNC_CTX_BUSY); +} + +/** + * @brief Push a completion queue entry (CQE) to the completion queue. + * + * @param ctx Pointer to the async context. + * @param cqe Pointer to the CQE to push. + */ +static inline void async_ctx_push_cqe(async_ctx_t* ctx, async_cqe_t* cqe) +{ + async_rings_t* rings = &ctx->rings; + + uint32_t tail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); + uint32_t head = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); + + if ((tail - head) >= rings->centries) + { + /// @todo Handle overflow properly. + panic(NULL, "Async completion queue overflow"); + } + + rings->cqueue[tail & rings->cmask] = *cqe; + atomic_store_explicit(&rings->shared->ctail, tail + 1, memory_order_release); + + wait_unblock(&ctx->waitQueue, WAIT_ALL, EOK); +} + +/** @} */ \ No newline at end of file diff --git a/include/kernel/sync/rcu.h b/include/kernel/sync/rcu.h index b5d5c71a2..aa250840b 100644 --- a/include/kernel/sync/rcu.h +++ b/include/kernel/sync/rcu.h @@ -40,10 +40,11 @@ typedef struct rcu_entry rcu_entry_t; * `rcu_entry_t` member and a callback function that will free the structure. * * To access RCU protected data, a read-side critical section must be created using `rcu_read_lock()` and - * `rcu_read_unlock()`, or the `RCU_READ_SCOPE()` macro. + * `rcu_read_unlock()`, or the `RCU_READ_SCOPE()` macro. * * @see [Wikipedia](https://en.wikipedia.org/wiki/Read-copy-update) for more information about RCU. - * @see [kernel.org](https://www.kernel.org/doc/Documentation/RCU/whatisRCU.txt) for a explanation of RCU in the Linux kernel. + * @see [kernel.org](https://www.kernel.org/doc/Documentation/RCU/whatisRCU.txt) for a explanation of RCU in the Linux + * kernel. * * @{ */ diff --git a/include/kernel/sync/task.h b/include/kernel/sync/task.h new file mode 100644 index 000000000..94ac22e86 --- /dev/null +++ b/include/kernel/sync/task.h @@ -0,0 +1,246 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +typedef struct process process_t; + +/** + * @brief Asynchronous Task Primitive + * @defgroup kernel_sync_task Task + * @ingroup kernel_sync + * + * @{ + */ + +/** + * @brief Per-CPU task queues. + * @struct task_ctx_t + */ +typedef struct task_ctx +{ + list_t timeouts; + list_t completed; + lock_t lock; +} task_ctx_t; + +/** + * @brief Task flags. + * @enum task_flags_t + */ +typedef enum +{ + TASK_DELAYED = 1 << 0, ///< The completion of the task has been delayed. + TASK_TIMEOUT = 1 << 1, ///< The task is in a timeout queue. +} task_flags_t; + +/** + * @brief Macro to define common task structure members. + * + * All tasks contain the below common members: + * - `list_entry_t entry` - List entry for tasks queues and completion queues. + * - `list_entry_t timeoutEntry` - List entry for timeout queues. + * - `task_ctx_t* ctx` - Pointer to the per-CPU task context storing this task for timeouts. + * - `process_t* process` - Pointer to the process that created the task. + * - `void* data` - Pointer to user data. + * - `void (*complete)(_type*)` - Completion callback. + * - `bool (*cancel)(_type*)` - Cancellation callback, should return `true` if the task was cancelled. + * - `void (*timeout)(_type*)` - Timeout callback. + * - `task_flags_t flags` - Task flags. + * - `errno_t err` - Error code for the task. + * - `clock_t deadline` - Deadline for the task. + * - `_resultType result` - Result of the task. + * + * @param _type The type of the task structure. + * @param _resultType The type of the task result. + */ +#define TASK_COMMON(_type, _resultType) \ + list_entry_t entry; \ + list_entry_t timeoutEntry; \ + task_ctx_t* ctx; \ + process_t* process; \ + void* data; \ + void (*complete)(_type*); \ + bool (*cancel)(_type*); \ + void (*timeout)(_type*); \ + task_flags_t flags; \ + errno_t err; \ + clock_t deadline; \ + _resultType result + +/** + * @brief Generic task structure. + * @struct task_t + * + * @warning Due to optimization done while allocating tasks in the async system, no task structure should be larger than + * this structure. + */ +typedef struct task +{ + TASK_COMMON(struct task, uint64_t); + uint64_t _padding[4]; +} task_t; + +/** + * @brief Task queue structure. + * @struct task_queue_t + */ +typedef struct +{ + list_t tasks; +} task_queue_t; + +/** + * @brief Initializes a task queue. + * + * @param queue Pointer to the task queue to initialize. + */ +static inline void task_queue_init(task_queue_t* queue) +{ + list_init(&queue->tasks); +} + +/** + * @brief Adds a task to the per-CPU timeout queue. + * + * @param task Pointer to the task to add. + */ +void task_timeout_add(task_t* task); + +/** + * @brief Removes a task from the per-CPU timeout queue. + * + * @param task Pointer to the task to remove. + */ +void task_timeout_remove(task_t* task); + +/** + * @brief Checks for task timeouts on the current CPU and handles them. + * + * @warning Must be called with interrupts disabled. + */ +void task_timeouts_check(void); + +/** + * @brief Macro to initialize a tasks common members. + * + * @param _task Pointer to the task to initialize. + */ +#define TASK_INIT(_task) \ + ({ \ + (_task)->entry = LIST_ENTRY_CREATE((_task)->entry); \ + (_task)->timeoutEntry = LIST_ENTRY_CREATE((_task)->timeoutEntry); \ + (_task)->ctx = NULL; \ + (_task)->process = NULL; \ + (_task)->data = NULL; \ + (_task)->complete = NULL; \ + (_task)->cancel = NULL; \ + (_task)->timeout = NULL; \ + (_task)->flags = 0; \ + (_task)->err = EOK; \ + (_task)->deadline = CLOCKS_NEVER; \ + (_task)->result = (typeof((_task)->result))0; \ + }) + +#define TASK_CALL(_task, _func) \ + ({ \ + typeof((_task)->result) result = _func(_task); \ + if ((_task)->err != EOK) \ + { \ + (_task)->flags &= ~TASK_DELAYED; \ + (_task)->complete(_task); \ + } \ + else if (!((_task)->flags & TASK_DELAYED)) \ + { \ + (_task)->result = result; \ + (_task)->complete(_task); \ + } \ + result; \ + }) + +#define TASK_DELAY_NO_QUEUE(_task) \ + ({ \ + uint64_t result = 0; \ + (_task)->flags |= TASK_DELAYED; \ + if ((_task)->deadline != CLOCKS_NEVER) \ + { \ + if ((_task)->timeout == NULL) \ + { \ + errno = EINVAL; \ + result = ERR; \ + } \ + else \ + { \ + (_task)->flags |= TASK_TIMEOUT; \ + task_timeout_add((task_t*)(_task)); \ + } \ + } \ + result; \ + }) + +#define TASK_DELAY(_task, _queue) \ + ({ \ + list_push_back(&(_queue)->tasks, &(_task)->entry); \ + uint64_t result = TASK_DELAY_NO_QUEUE(_task); \ + if (result == ERR) \ + { \ + list_remove(&(_task)->entry); \ + } \ + result; \ + }) + +#define TASK_NEXT(_queue, _type) \ + (list_is_empty(&(_queue)->tasks) ? NULL : CONTAINER_OF(list_first(&(_queue)->tasks), _type, entry)) + +#define TASK_ERROR(_task, _errno) \ + ({ \ + if ((_task)->flags & TASK_TIMEOUT) \ + { \ + task_timeout_remove((task_t*)(_task)); \ + } \ + list_remove(&(_task)->entry); \ + (_task)->flags &= ~TASK_DELAYED; \ + (_task)->err = (_errno); \ + (_task)->complete((_task)); \ + }) + +#define TASK_COMPLETE(_task, _result) \ + ({ \ + if ((_task)->flags & TASK_TIMEOUT) \ + { \ + task_timeout_remove((task_t*)(_task)); \ + } \ + list_remove(&(_task)->entry); \ + (_task)->flags &= ~TASK_DELAYED; \ + (_task)->result = (_result); \ + (_task)->complete((_task)); \ + }) + +#define TASK_CANCEL(_task) \ + ({ \ + uint64_t result = 0; \ + if ((_task)->cancel == NULL) \ + { \ + errno = EINVAL; \ + result = ERR; \ + } \ + else \ + { \ + (_task)->err = ECANCELED; \ + if (!((_task)->cancel(_task))) \ + { \ + (_task)->err = EOK; \ + errno = EBUSY; \ + result = ERR; \ + } \ + } \ + result; \ + }) + +/** @} */ \ No newline at end of file diff --git a/include/kernel/sync/tasks.h b/include/kernel/sync/tasks.h new file mode 100644 index 000000000..b96233699 --- /dev/null +++ b/include/kernel/sync/tasks.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +/** + * @brief Kernel Task Implementations + * @defgroup kernel_sync_tasks Tasks + * @ingroup kernel_sync + * + * @{ + */ + +/** + * @brief No-operation task structure. + * @struct task_nop_t + */ +typedef struct task_nop +{ + TASK_COMMON(struct task_nop, uint64_t); +} task_nop_t; + +bool task_nop_cancel(task_nop_t* task); +void task_nop_timeout(task_nop_t* task); + +/** @} */ \ No newline at end of file diff --git a/include/kernel/utils/fifo.h b/include/kernel/utils/fifo.h index 40823146a..014ce2ab0 100644 --- a/include/kernel/utils/fifo.h +++ b/include/kernel/utils/fifo.h @@ -51,14 +51,24 @@ typedef struct fifo * @param buffer Pointer to the buffer memory. * @param size The size of the buffer in bytes. */ -void fifo_init(fifo_t* fifo, uint8_t* buffer, size_t size); +static inline void fifo_init(fifo_t* fifo, uint8_t* buffer, size_t size) +{ + fifo->buffer = buffer; + fifo->size = size; + fifo->head = 0; + fifo->tail = 0; +} /** * @brief Reset a fifo buffer. * * @param fifo Pointer to the fifo buffer structure. */ -void fifo_reset(fifo_t* fifo); +static inline void fifo_reset(fifo_t* fifo) +{ + fifo->head = 0; + fifo->tail = 0; +} /** * @brief Return the number of bytes available for reading in a fifo buffer. @@ -66,7 +76,15 @@ void fifo_reset(fifo_t* fifo); * @param fifo Pointer to the fifo buffer structure. * @return The number of bytes used. */ -size_t fifo_bytes_readable(const fifo_t* fifo); +static inline size_t fifo_bytes_readable(const fifo_t* fifo) +{ + if (fifo->head >= fifo->tail) + { + return fifo->head - fifo->tail; + } + + return fifo->size - (fifo->tail - fifo->head); +} /** * @brief Return the number of bytes available for writing in a fifo buffer. @@ -74,7 +92,15 @@ size_t fifo_bytes_readable(const fifo_t* fifo); * @param fifo Pointer to the fifo buffer structure. * @return The number of bytes available for writing. */ -size_t fifo_bytes_writeable(const fifo_t* fifo); +static inline size_t fifo_bytes_writeable(const fifo_t* fifo) +{ + if (fifo->tail > fifo->head) + { + return fifo->tail - fifo->head - 1; + } + + return fifo->size - (fifo->head - fifo->tail) - 1; +} /** * @brief Read data from a fifo buffer at a specific offset. @@ -84,7 +110,37 @@ size_t fifo_bytes_writeable(const fifo_t* fifo); * @param count The number of bytes to read. * @return The number of bytes read. */ -size_t fifo_read(fifo_t* fifo, void* buffer, size_t count); +static inline size_t fifo_read(fifo_t* fifo, void* buffer, size_t count) +{ + size_t readable = fifo_bytes_readable(fifo); + if (readable == 0) + { + return 0; + } + + if (count > readable) + { + count = readable; + } + + size_t firstSize = fifo->size - fifo->tail; + if (firstSize > count) + { + firstSize = count; + } + + memcpy(buffer, fifo->buffer + fifo->tail, firstSize); + fifo->tail = (fifo->tail + firstSize) % fifo->size; + + size_t remaining = count - firstSize; + if (remaining > 0) + { + memcpy((uint8_t*)buffer + firstSize, fifo->buffer + fifo->tail, remaining); + fifo->tail = (fifo->tail + remaining) % fifo->size; + } + + return count; +} /** * @brief Write data to the fifo buffer. @@ -96,7 +152,32 @@ size_t fifo_read(fifo_t* fifo, void* buffer, size_t count); * @param count The number of bytes to write. * @return The number of bytes written. */ -size_t fifo_write(fifo_t* fifo, const void* buffer, size_t count); +static inline size_t fifo_write(fifo_t* fifo, const void* buffer, size_t count) +{ + size_t writeable = fifo_bytes_writeable(fifo); + if (count > writeable) + { + count = writeable; + } + + size_t firstSize = fifo->size - fifo->head; + if (firstSize > count) + { + firstSize = count; + } + + memcpy(fifo->buffer + fifo->head, buffer, firstSize); + fifo->head = (fifo->head + firstSize) % fifo->size; + + size_t remaining = count - firstSize; + if (remaining > 0) + { + memcpy(fifo->buffer + fifo->head, (uint8_t*)buffer + firstSize, remaining); + fifo->head = (fifo->head + remaining) % fifo->size; + } + + return count; +} /** * @brief Advance the head of the fifo buffer. @@ -104,7 +185,10 @@ size_t fifo_write(fifo_t* fifo, const void* buffer, size_t count); * @param fifo Pointer to the fifo buffer structure. * @param count The number of bytes to advance the head by. */ -void fifo_advance_head(fifo_t* fifo, size_t count); +static inline void fifo_advance_head(fifo_t* fifo, size_t count) +{ + fifo->head = (fifo->head + count) % fifo->size; +} /** * @brief Advance the tail of the fifo buffer. @@ -112,6 +196,9 @@ void fifo_advance_head(fifo_t* fifo, size_t count); * @param fifo Pointer to the fifo buffer structure. * @param count The number of bytes to advance the tail by. */ -void fifo_advance_tail(fifo_t* fifo, size_t count); +static inline void fifo_advance_tail(fifo_t* fifo, size_t count) +{ + fifo->tail = (fifo->tail + count) % fifo->size; +} /** @} */ diff --git a/include/libpatchwork/element.h b/include/libpatchwork/element.h index 9eea29d9c..ba6af8fe0 100644 --- a/include/libpatchwork/element.h +++ b/include/libpatchwork/element.h @@ -98,7 +98,7 @@ typedef struct * @return On success, a pointer to the new element. On failure, `NULL` and `errno` is set. */ element_t* element_new(element_t* parent, element_id_t id, const rect_t* rect, const char* text, element_flags_t flags, - procedure_t procedure, void* data); + procedure_t procedure, void* data); /** * @brief Deinitialize and free an element and all its children. @@ -126,7 +126,7 @@ element_t* element_find(element_t* elem, element_id_t id); * @param elem The element. * @param private Pointer to the private data. */ -void element_set_private(element_t* elem, void* data); +void element_set_private(element_t* elem, void* data); /** * @brief Get private data for an element. diff --git a/include/libpatchwork/window.h b/include/libpatchwork/window.h index c79e112db..6c1f89f6a 100644 --- a/include/libpatchwork/window.h +++ b/include/libpatchwork/window.h @@ -61,7 +61,7 @@ typedef enum window_flags * @return On success, the new window. On failure, returns `NULL` and `errno` is set. */ window_t* window_new(display_t* disp, const char* name, const rect_t* rect, surface_type_t type, window_flags_t flags, - procedure_t procedure, void* data); + procedure_t procedure, void* data); /** * @brief Free a window. diff --git a/include/libstd/sys/async.h b/include/libstd/sys/async.h new file mode 100644 index 000000000..a6f4e8f8c --- /dev/null +++ b/include/libstd/sys/async.h @@ -0,0 +1,260 @@ +#include +#ifndef _SYS_ASYNC_H +#define _SYS_ASYNC_H 1 + +#include +#include +#include +#include + +#if defined(__cplusplus) +extern "C" +{ +#endif + +#include "_internal/MAX_PATH.h" +#include "_internal/clock_t.h" +#include "_internal/errno_t.h" +#include "_internal/fd_t.h" + +/** + * @brief Asynchronous operations. + * @defgroup libstd_async Async + * @ingroup libstd + * + * Asynchronous operations provide the core of all IO interfaces in PatchworkOS, all implemented in an interface + * inspired by `io_uring()` from Linux. + * + * Synchronous operations are implemented on top of this API in userspace. + * + * @todo The async system is not currently implemented, this is more just a draft for now. + * + * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`. + * @see [Manpages](https://man7.org/linux/man-pages/man7/io_uring.7.html) for more information about `io_uring`. + * + * @{ + */ + +/** + * @brief Asynchronous operation codes. + * @enum async_op_t + */ +typedef enum +{ + ASYNC_OP_NOP = 0, ///< Never completes, can be used to implement a sleep equivalent. +} async_op_t; + +/** + * @brief Asynchronous sequence flags. + * @enum async_seq_flags_t + * + * Used to modify the behavior of asynchronous operations. + * + * @todo Implement `ASYNC_SEQ_LINK`. + */ +typedef enum +{ + ASYNC_SEQ_NONE = 0, + ASYNC_SEQ_LINK = 1 << 0, ///< Must be completed before the next SQE in the submission queue is started. + ASYNC_SEQ_IMMEDIATE = 1 << 1, ///< Fail if the operation cannot be completed immediately. +} async_seq_flags_t; + +/** + * @brief Asynchronous submission queue entry (SQE). + * @struct async_sqe_t + * + * @warning For operations such as `ASYNC_OP_OPEN`, it is the responsibility of userspace to ensure that any pointers + * passed to the kernel remain valid until the operation is complete. + */ +typedef struct async_sqe +{ + void* data; ///< Private data for the operation, will be returned in the completion entry. + async_op_t opcode; ///< Operation code. + async_seq_flags_t flags; ///< Sequence flags. + clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. + union { + struct + { + + } nop; + uint64_t _raw[5]; + }; +} async_sqe_t; + +#ifdef static_assert +static_assert(sizeof(async_sqe_t) == 64, "async_sqe_t is not 64 bytes"); +#endif + +/** + * @brief Macro to create an asynchronous submission queue entry (SQE). + * + * @param _id Unique identifier for the operation. + * @param _opcode Operation code. + * @param _flags Sequence flags. + * @param _timeout Timeout for the operation, `CLOCKS_NEVER` for no timeout. + * @param _data Private data for the operation. + */ +#define ASYNC_SQE_CREATE(_id, _opcode, _flags, _timeout, _data) \ + { \ + .data = (_data), \ + .id = (_id), \ + .opcode = (_opcode), \ + .flags = (_flags), \ + .timeout = (_timeout), \ + } + +/** + * @brief Asynchronous completion queue entry (CQE). + * @struct async_cqe_t + */ +typedef struct ALIGNED(64) async_cqe +{ + void* data; ///< Private data from the submission entry. + async_op_t opcode; ///< Operation code from the submission entry. + errno_t error; ///< Error code, if not equal to `EOK` an error occurred. + union { + size_t read; ///< The number of bytes read from `ASYNC_OP_READ`. + uint64_t _raw; + }; +} async_cqe_t; + +#ifdef static_assert +static_assert(sizeof(async_cqe_t) == 64, "async_cqe_t is not 64 bytes"); +#endif +/** + * @brief Shared asynchronous rings structure. + * @struct async_shared_t + * + * Used as the intermediate between userspace and the kernel. + * + */ +typedef struct ALIGNED(64) async_shared +{ + atomic_uint32_t shead; ///< Submission head index, updated by the kernel. + atomic_uint32_t ctail; ///< Completion tail index, updated by the kernel. + uint8_t _padding[64 - + sizeof(atomic_uint32_t) * 2]; ///< Padding to prevent false sharing between user space and the kernel. + atomic_uint32_t stail; ///< Submission tail index, updated by userspace. + atomic_uint32_t chead; ///< Completion head index, updated by userspace. +} async_shared_t; + +/** + * @brief Asynchronous rings structure. + * @struct async_rings_t + * + * The kernel and userspace will have their own instances of this structure. + */ +typedef struct async_rings +{ + async_shared_t* shared; ///< Pointer to the shared structure. + async_sqe_t* squeue; ///< Pointer to the submission queue. + size_t sentries; ///< Number of entries in the submission queue. + size_t smask; ///< Bitmask for submission queue (sentries - 1). + async_cqe_t* cqueue; ///< Pointer to the completion queue. + size_t centries; ///< Number of entries in the completion queue. + size_t cmask; ///< Bitmask for completion queue (centries - 1). +} async_rings_t; + +/** + * @brief Dont wait for any submissions to complete. + */ +#define ASYNC_WAIT_NONE 0x0 + +/** + * @brief Wait for at least one submission to complete. + */ +#define ASYNC_WAIT_ONE 0x1 + +/** + * @brief Wait for all submissions to complete. + */ +#define ASYNC_WAIT_ALL SIZE_MAX + +/** + * @brief System call to initialize the asynchronous rings. + * + * This system call will populate the given structure with the necessary pointers and metadata for the submission and + * completion rings. + * + * @note Since each process can only have one rings set, the `async_deinit()` system call must be used before calling + * this function again. + * + * @param rings Pointer to the structure to populate. + * @param address Desired address to allocate the rings, or `NULL` to let the kernel choose. + * @param sentries Number of entires to allocate for the submission queue, must be a power of two. + * @param centries Number of entries to allocate for the completion queue, must be a power of two. + * @return On success, `0`. On failure, `ERR` and `errno` is set. + */ +uint64_t async_init(async_rings_t* rings, void* address, size_t sentries, size_t centries); + +/** + * @brief System call to deinitialize the asynchronous rings. + * + * @return On success, `0`. On failure, `ERR` and `errno` is set. + */ +uint64_t async_deinit(void); + +/** + * @brief System call to notify the kernel of new submission queue entries (SQEs). + * + * @param amount The number of SQEs that the kernel should process. + * @param wait The minimum number of completion queue entries (CQEs) to wait for. + * @return On success, the number of SQEs successfully processed. On failure, `ERR` and `errno` is set. + */ +uint64_t async_notify(size_t amount, size_t wait); + +/** + * @brief Pushes a submission queue entry (SQE) to the submission queue. + * + * After pushing SQEs, `async_notify()` must be called to notify the kernel of the new entries. + * + * @param rings Pointer to the asynchronous rings structure. + * @param sqe Pointer to the SQE to push. + * @return `true` if the SQE was pushed, `false` if the submission queue is full. + */ +static inline bool async_push_sqe(async_rings_t* rings, async_sqe_t* sqe) +{ + uint32_t tail = atomic_load_explicit(&rings->shared->stail, memory_order_relaxed); + uint32_t head = atomic_load_explicit(&rings->shared->shead, memory_order_acquire); + + if ((tail - head) >= rings->sentries) + { + return false; + } + + rings->squeue[tail & rings->smask] = *sqe; + atomic_store_explicit(&rings->shared->stail, tail + 1, memory_order_release); + + return true; +} + +/** + * @brief Pops a completion queue entry (CQE) from the completion queue. + * + * @param rings Pointer to the asynchronous rings structure. + * @param cqe Pointer to the CQE to pop. + * @return `true` if a CQE was popped, `false` if the completion queue is empty. + */ +static inline bool async_pop_cqe(async_rings_t* rings, async_cqe_t* cqe) +{ + uint32_t head = atomic_load_explicit(&rings->shared->chead, memory_order_relaxed); + uint32_t tail = atomic_load_explicit(&rings->shared->ctail, memory_order_acquire); + + if (head == tail) + { + return false; + } + + *cqe = rings->cqueue[head & rings->cmask]; + atomic_store_explicit(&rings->shared->chead, head + 1, memory_order_release); + + return true; +} + +/** @} */ + +#if defined(__cplusplus) +} +#endif + +#endif \ No newline at end of file diff --git a/include/libstd/sys/bitmap.h b/include/libstd/sys/bitmap.h index 09e06e068..c51702fdd 100644 --- a/include/libstd/sys/bitmap.h +++ b/include/libstd/sys/bitmap.h @@ -376,8 +376,8 @@ static inline uint64_t bitmap_find_first_set(bitmap_t* map, uint64_t startIdx, u * @param alignment Alignment of the region. * @return Starting index of the found region, or `map->length` if not found. */ -static inline uint64_t bitmap_find_clear_region_and_set(bitmap_t* map, uint64_t minIdx, uintptr_t maxIdx, uint64_t length, - uint64_t alignment) +static inline uint64_t bitmap_find_clear_region_and_set(bitmap_t* map, uint64_t minIdx, uintptr_t maxIdx, + uint64_t length, uint64_t alignment) { if (length == 0 || minIdx >= maxIdx || maxIdx > map->length) { diff --git a/include/libstd/sys/elf.h b/include/libstd/sys/elf.h index 6675836a7..3fe500278 100644 --- a/include/libstd/sys/elf.h +++ b/include/libstd/sys/elf.h @@ -896,7 +896,7 @@ void elf64_load_segments(const Elf64_File* elf, Elf64_Addr base, Elf64_Off offse * @return On success, `0`. On failure, `ERR`. */ uint64_t elf64_relocate(const Elf64_File* elf, Elf64_Addr base, Elf64_Off offset, - void* (*resolve_symbol)(const char* name, void* data), void* data); + void* (*resolve_symbol)(const char* name, void* data), void* data); /** * @brief Get a string from the string table section at the given offset diff --git a/include/libstd/sys/list.h b/include/libstd/sys/list.h index f2745d75e..87b81ca51 100644 --- a/include/libstd/sys/list.h +++ b/include/libstd/sys/list.h @@ -5,9 +5,9 @@ #include "_internal/NULL.h" #include +#include #include #include -#include /** * @brief Doubly linked list header. @@ -18,7 +18,7 @@ * * Given a entry within a structure, the `CONTAINER_OF()` macro can be used to get a pointer to the structure from the * list entry pointer. - * + * * @warning If a list is protected with RCU, the `list_*_rcu()` functions must be used. * * @{ @@ -247,10 +247,10 @@ static inline void list_add_rcu(list_entry_t* prev, list_entry_t* next, list_ent assert(prev != NULL); assert(next != NULL); assert(entry != NULL); - // For RCU we allow adding an entry that is already in a list + // For RCU we allow adding an entry that is already in a list // as we cant properly remove it until all readers are done. - //assert(entry->next == entry && entry->prev == entry); - //assert(prev->next == next && next->prev == prev); + // assert(entry->next == entry && entry->prev == entry); + // assert(prev->next == next && next->prev == prev); next->prev = entry; entry->next = next; @@ -299,7 +299,8 @@ static inline void list_remove(list_entry_t* entry) /** * @brief Removes a list entry from its current list in a RCU-safe manner. * - * @warning After calling this function the entry will still be connected to the list, but iteration over the list will not find it. + * @warning After calling this function the entry will still be connected to the list, but iteration over the list will + * not find it. * * @param entry A pointer to the `list_entry_t` to remove. */ diff --git a/include/libstd/sys/proc.h b/include/libstd/sys/proc.h index de9dc0782..8cbf2b541 100644 --- a/include/libstd/sys/proc.h +++ b/include/libstd/sys/proc.h @@ -351,7 +351,7 @@ typedef enum /** * @brief System call for setting architecture specific thread data. - * + * * @param op The operation to perform. * @param addr If getting data, a pointer to store the retrieved address. If setting data, the address to set. * @return On success, `0`. On failure, `ERR` and `errno` is set. diff --git a/src/kernel/cpu/interrupt.c b/src/kernel/cpu/interrupt.c index 1ff14a766..9134e8ded 100644 --- a/src/kernel/cpu/interrupt.c +++ b/src/kernel/cpu/interrupt.c @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -220,6 +221,7 @@ void interrupt_handler(interrupt_frame_t* frame) } note_handle_pending(frame); + task_timeouts_check(); wait_check_timeouts(frame); sched_do(frame); diff --git a/src/kernel/cpu/ipi.c b/src/kernel/cpu/ipi.c index 5cdc10987..6a31058c0 100644 --- a/src/kernel/cpu/ipi.c +++ b/src/kernel/cpu/ipi.c @@ -107,7 +107,7 @@ uint64_t ipi_chip_amount(void) return registeredChip != NULL ? 1 : 0; } -static uint64_t ipi_push(cpu_t* cpu, ipi_func_t func, void* data) +static uint64_t ipi_push(cpu_t* cpu, ipi_func_t func, void* data) { if (registeredChip == NULL) { diff --git a/src/kernel/cpu/irq.c b/src/kernel/cpu/irq.c index deabc209f..2c2a30ecc 100644 --- a/src/kernel/cpu/irq.c +++ b/src/kernel/cpu/irq.c @@ -325,7 +325,7 @@ uint64_t irq_virt_set_affinity(irq_virt_t virt, cpu_t* cpu) return 0; } -uint64_t irq_chip_register(irq_chip_t* chip, irq_phys_t start, irq_phys_t end, void* data) +uint64_t irq_chip_register(irq_chip_t* chip, irq_phys_t start, irq_phys_t end, void* data) { if (chip == NULL || chip->enable == NULL || chip->disable == NULL || start >= end) { @@ -420,7 +420,7 @@ uint64_t irq_chip_amount(void) return list_size(&domains); } -uint64_t irq_handler_register(irq_virt_t virt, irq_func_t func, void* data) +uint64_t irq_handler_register(irq_virt_t virt, irq_func_t func, void* data) { if (func == NULL) { diff --git a/src/kernel/drivers/abstract/fb.c b/src/kernel/drivers/abstract/fb.c index 5a37a1d47..8da016eef 100644 --- a/src/kernel/drivers/abstract/fb.c +++ b/src/kernel/drivers/abstract/fb.c @@ -131,7 +131,7 @@ static inode_ops_t dirInodeOps = { .cleanup = fb_dir_cleanup, }; -fb_t* fb_new(const char* name, const fb_ops_t* ops, void* data) +fb_t* fb_new(const char* name, const fb_ops_t* ops, void* data) { if (name == NULL || ops == NULL) { diff --git a/src/kernel/fs/devfs.c b/src/kernel/fs/devfs.c index 986774023..ea1e0d289 100644 --- a/src/kernel/fs/devfs.c +++ b/src/kernel/fs/devfs.c @@ -32,7 +32,7 @@ static dentry_ops_t dentryOps = { .iterate = dentry_generic_iterate, }; -static dentry_t* devfs_mount(filesystem_t* fs, const char* options, void* data) +static dentry_t* devfs_mount(filesystem_t* fs, const char* options, void* data) { UNUSED(fs); UNUSED(data); @@ -83,7 +83,7 @@ void devfs_init(void) root = dentry; } -dentry_t* devfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) +dentry_t* devfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) { if (name == NULL) { @@ -123,7 +123,7 @@ dentry_t* devfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* i } dentry_t* devfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, const file_ops_t* fileOps, - void* data) + void* data) { if (name == NULL) { @@ -162,7 +162,7 @@ dentry_t* devfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* return REF(dentry); } -dentry_t* devfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) +dentry_t* devfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) { if (parent == NULL || name == NULL || inodeOps == NULL) { diff --git a/src/kernel/fs/namespace.c b/src/kernel/fs/namespace.c index b222ae2d5..4f996b51b 100644 --- a/src/kernel/fs/namespace.c +++ b/src/kernel/fs/namespace.c @@ -385,7 +385,7 @@ bool namespace_rcu_traverse(namespace_t* ns, mount_t** mount, dentry_t** dentry) } mount_t* namespace_mount(namespace_t* ns, path_t* target, filesystem_t* fs, const char* options, mode_t mode, - void* data) + void* data) { if (ns == NULL || fs == NULL) { diff --git a/src/kernel/fs/netfs.c b/src/kernel/fs/netfs.c index 1b019bba9..0c3cf31ae 100644 --- a/src/kernel/fs/netfs.c +++ b/src/kernel/fs/netfs.c @@ -763,7 +763,7 @@ static dentry_ops_t netDentryOps = { .iterate = netfs_iterate, }; -static dentry_t* netfs_mount(filesystem_t* fs, const char* options, void* data) +static dentry_t* netfs_mount(filesystem_t* fs, const char* options, void* data) { UNUSED(data); diff --git a/src/kernel/fs/procfs.c b/src/kernel/fs/procfs.c index da3b38a4b..c97b6ce9f 100644 --- a/src/kernel/fs/procfs.c +++ b/src/kernel/fs/procfs.c @@ -1237,7 +1237,7 @@ static dentry_ops_t procDentryOps = { .iterate = procfs_iterate, }; -static dentry_t* procfs_mount(filesystem_t* fs, const char* options, void* data) +static dentry_t* procfs_mount(filesystem_t* fs, const char* options, void* data) { UNUSED(data); diff --git a/src/kernel/fs/sysfs.c b/src/kernel/fs/sysfs.c index a3018ee34..7684fdc34 100644 --- a/src/kernel/fs/sysfs.c +++ b/src/kernel/fs/sysfs.c @@ -32,7 +32,7 @@ static dentry_ops_t dentryOps = { .iterate = dentry_generic_iterate, }; -static dentry_t* sysfs_mount(filesystem_t* fs, const char* options, void* data) +static dentry_t* sysfs_mount(filesystem_t* fs, const char* options, void* data) { UNUSED(fs); UNUSED(data); @@ -115,7 +115,7 @@ void sysfs_init(void) LOG_INFO("sysfs mounted to '/sys'\n"); } -dentry_t* sysfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) +dentry_t* sysfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) { if (name == NULL) { @@ -155,7 +155,7 @@ dentry_t* sysfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* i } dentry_t* sysfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, const file_ops_t* fileOps, - void* data) + void* data) { if (name == NULL) { @@ -194,7 +194,7 @@ dentry_t* sysfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* return REF(dentry); } -dentry_t* sysfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) +dentry_t* sysfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) { if (parent == NULL || name == NULL || inodeOps == NULL) { diff --git a/src/kernel/fs/tmpfs.c b/src/kernel/fs/tmpfs.c index 980aba2a1..129cc891f 100644 --- a/src/kernel/fs/tmpfs.c +++ b/src/kernel/fs/tmpfs.c @@ -276,7 +276,7 @@ static dentry_t* tmpfs_load_dir(superblock_t* superblock, dentry_t* parent, cons return REF(dentry); } -static dentry_t* tmpfs_mount(filesystem_t* fs, const char* options, void* data) +static dentry_t* tmpfs_mount(filesystem_t* fs, const char* options, void* data) { UNUSED(data); diff --git a/src/kernel/mem/space.c b/src/kernel/mem/space.c index 633901b08..652c05c36 100644 --- a/src/kernel/mem/space.c +++ b/src/kernel/mem/space.c @@ -568,7 +568,7 @@ uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* vir return 0; // We return with the lock still acquired. } -pml_callback_id_t space_alloc_callback(space_t* space, size_t pageAmount, space_callback_func_t func, void* data) +pml_callback_id_t space_alloc_callback(space_t* space, size_t pageAmount, space_callback_func_t func, void* data) { if (space == NULL) { diff --git a/src/kernel/mem/vmm.c b/src/kernel/mem/vmm.c index c128b47f0..3ab1ef22b 100644 --- a/src/kernel/mem/vmm.c +++ b/src/kernel/mem/vmm.c @@ -200,6 +200,14 @@ void* vmm_alloc(space_t* space, void* virtAddr, size_t length, size_t alignment, return space_mapping_end(space, &mapping, ENOMEM); } + if (allocFlags & VMM_ALLOC_ZERO) + { + for (uint64_t i = 0; i < batchSize; i++) + { + memset(addresses[i], 0, PAGE_SIZE); + } + } + if (page_table_map_pages(&space->pageTable, (void*)currentVirtAddr, addresses, batchSize, mapping.flags, PML_CALLBACK_NONE) == ERR) { @@ -215,7 +223,7 @@ void* vmm_alloc(space_t* space, void* virtAddr, size_t length, size_t alignment, } void* vmm_map(space_t* space, void* virtAddr, void* physAddr, size_t length, pml_flags_t flags, - space_callback_func_t func, void* data) + space_callback_func_t func, void* data) { if (physAddr == NULL || length == 0 || !(flags & PML_PRESENT)) { @@ -270,7 +278,7 @@ void* vmm_map(space_t* space, void* virtAddr, void* physAddr, size_t length, pml } void* vmm_map_pages(space_t* space, void* virtAddr, void** pages, size_t pageAmount, pml_flags_t flags, - space_callback_func_t func, void* data) + space_callback_func_t func, void* data) { if (pages == NULL || pageAmount == 0 || !(flags & PML_PRESENT)) { diff --git a/src/kernel/module/module.c b/src/kernel/module/module.c index 7c1c80b39..6b43ce3bb 100644 --- a/src/kernel/module/module.c +++ b/src/kernel/module/module.c @@ -46,7 +46,7 @@ static bool cacheValid = false; static mutex_t lock = MUTEX_CREATE(lock); -static void* module_resolve_symbol_callback(const char* name, void* data); +static void* module_resolve_symbol_callback(const char* name, void* data); #define MODULE_SYMBOL_ALLOWED(type, binding, name) \ (((type) == STT_OBJECT || (type) == STT_FUNC) && ((binding) == STB_GLOBAL) && \ @@ -909,7 +909,7 @@ static uint64_t module_load_dependency(module_load_ctx_t* ctx, const char* symbo return 0; } -static void* module_resolve_symbol_callback(const char* symbolName, void* data) +static void* module_resolve_symbol_callback(const char* symbolName, void* data) { module_load_ctx_t* ctx = data; diff --git a/src/kernel/proc/process.c b/src/kernel/proc/process.c index 9659d6dd1..fdad878a8 100644 --- a/src/kernel/proc/process.c +++ b/src/kernel/proc/process.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -109,9 +110,10 @@ static void process_free(process_t* process) UNREF(process->nspace); } space_deinit(&process->space); + futex_ctx_deinit(&process->futexCtx); + async_ctx_deinit(&process->async); wait_queue_deinit(&process->dyingQueue); wait_queue_deinit(&process->suspendQueue); - futex_ctx_deinit(&process->futexCtx); env_deinit(&process->env); rcu_call(&process->rcu, rcu_call_cache_free, process); @@ -149,6 +151,7 @@ process_t* process_new(priority_t priority, group_member_t* group, namespace_t* file_table_init(&process->fileTable); futex_ctx_init(&process->futexCtx); perf_process_ctx_init(&process->perf); + async_ctx_init(&process->async); note_handler_init(&process->noteHandler); wait_queue_init(&process->suspendQueue); wait_queue_init(&process->dyingQueue); diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c new file mode 100644 index 000000000..ffa4e9257 --- /dev/null +++ b/src/kernel/sync/async.c @@ -0,0 +1,348 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, async_rings_t* userRings, void* address, size_t sentries, size_t centries) +{ + async_rings_t* kernelRings = &ctx->rings; + + size_t pageAmount = BYTES_TO_PAGES(sizeof(async_shared_t) + (sentries * sizeof(async_sqe_t)) + (centries * sizeof(async_cqe_t))); + if (pageAmount >= CONFIG_MAX_ASYNC_PAGES) + { + errno = ENOMEM; + return ERR; + } + + void* pages[CONFIG_MAX_ASYNC_PAGES]; + if (pmm_alloc_pages(pages, pageAmount) == ERR) + { + errno = ENOMEM; + return ERR; + } + + for (size_t i = 0; i < pageAmount; i++) + { + memset(pages[i], 0, PAGE_SIZE); + } + + // PML_OWNED means that the pages will be freed when unmapped. + void* kernelAddr = vmm_map_pages(NULL, NULL, pages, pageAmount, PML_WRITE | PML_PRESENT | PML_OWNED, NULL, NULL); + if (kernelAddr == NULL) + { + pmm_free_pages(pages, pageAmount); + return ERR; + } + + void* userAddr = vmm_map_pages(space, address, pages, pageAmount, PML_WRITE | PML_PRESENT | PML_USER, NULL, NULL); + if (userAddr == NULL) + { + vmm_unmap(NULL, kernelAddr, pageAmount * PAGE_SIZE); + return ERR; + } + + task_t* tasks = malloc(sizeof(task_t) * centries); + if (tasks == NULL) + { + vmm_unmap(space, userAddr, pageAmount * PAGE_SIZE); + vmm_unmap(NULL, kernelAddr, pageAmount * PAGE_SIZE); + return ERR; + } + + for (size_t i = 0; i < centries; i++) + { + TASK_INIT(&tasks[i]); + list_push_back(&ctx->freeTasks, &tasks[i].entry); + } + + async_shared_t* shared = (async_shared_t*)kernelAddr; + atomic_init(&shared->shead, 0); + atomic_init(&shared->stail, 0); + atomic_init(&shared->ctail, 0); + atomic_init(&shared->chead, 0); + + userRings->shared = userAddr; + userRings->squeue = (async_sqe_t*)((uintptr_t)userAddr + sizeof(async_shared_t)); + userRings->sentries = sentries; + userRings->smask = sentries - 1; + userRings->cqueue = (async_cqe_t*)((uintptr_t)userAddr + sizeof(async_shared_t) + (sentries * sizeof(async_sqe_t))); + userRings->centries = centries; + userRings->cmask = centries - 1; + + kernelRings->shared = kernelAddr; + kernelRings->squeue = (async_sqe_t*)((uintptr_t)kernelAddr + sizeof(async_shared_t)); + kernelRings->sentries = sentries; + kernelRings->smask = sentries - 1; + kernelRings->cqueue = (async_cqe_t*)((uintptr_t)kernelAddr + sizeof(async_shared_t) + (sentries * sizeof(async_sqe_t))); + kernelRings->centries = centries; + kernelRings->cmask = centries - 1; + + ctx->tasks = tasks; + ctx->userAddr = userAddr; + ctx->kernelAddr = kernelAddr; + ctx->pageAmount = pageAmount; + ctx->space = space; + + atomic_fetch_or(&ctx->flags, ASYNC_CTX_MAPPED); + return 0; +} + +static inline uint64_t async_ctx_unmap(async_ctx_t* ctx) +{ + list_init(&ctx->freeTasks); + free(ctx->tasks); + ctx->tasks = NULL; + + vmm_unmap(ctx->space, ctx->userAddr, ctx->pageAmount * PAGE_SIZE); + vmm_unmap(NULL, ctx->kernelAddr, ctx->pageAmount * PAGE_SIZE); + + atomic_fetch_and(&ctx->flags, ~ASYNC_CTX_MAPPED); + return 0; +} + +static inline task_t* async_ctx_alloc_task(async_ctx_t* ctx) +{ + if (list_is_empty(&ctx->freeTasks)) + { + return NULL; + } + + return CONTAINER_OF(list_pop_back(&ctx->freeTasks), task_t, entry); +} + +static inline void async_ctx_free_task(async_ctx_t* ctx, task_t* task) +{ + list_push_back(&ctx->freeTasks, &task->entry); +} + +void async_ctx_init(async_ctx_t* ctx) +{ + if (ctx == NULL) + { + return; + } + + memset(ctx, 0, sizeof(async_ctx_t)); + list_init(&ctx->freeTasks); + wait_queue_init(&ctx->waitQueue); + atomic_init(&ctx->flags, ASYNC_CTX_NONE); +} + +void async_ctx_deinit(async_ctx_t* ctx) +{ + if (ctx == NULL) + { + return; + } + + if (async_ctx_acquire(ctx) == ERR) + { + panic(NULL, "failed to acquire async context for deinitialization"); + } + + if (atomic_load(&ctx->flags) & ASYNC_CTX_MAPPED) + { + if (async_ctx_unmap(ctx) == ERR) + { + panic(NULL, "failed to deinitialize async context"); + } + } + + async_ctx_release(ctx); + wait_queue_deinit(&ctx->waitQueue); +} + +static void async_nop_complete(task_nop_t* nop) +{ + async_cqe_t cqe; + cqe.data = nop->data; + cqe.opcode = ASYNC_OP_NOP; + cqe.error = EOK; + + process_t* process = nop->process; + async_ctx_push_cqe(&process->async, &cqe); + async_ctx_free_task(&process->async, (task_t*)nop); + UNREF(nop->process); +} + +static uint64_t async_handle_sqe(async_ctx_t* ctx, async_sqe_t* sqe) +{ + task_t* task = async_ctx_alloc_task(ctx); + if (task == NULL) + { + errno = ENOSPC; + return ERR; + } + + switch (sqe->opcode) + { + case ASYNC_OP_NOP: + { + task_nop_t* nop = (task_nop_t*)task; + nop->data = sqe->data; + nop->process = REF(process_current()); + nop->complete = async_nop_complete; + nop->cancel = task_nop_cancel; + nop->timeout = task_nop_timeout; + TASK_DELAY_NO_QUEUE(nop); + } + break; + default: + async_ctx_free_task(ctx, task); + errno = EINVAL; + return ERR; + } + + return 0; +} + +uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) +{ + if (amount == 0) + { + return 0; + } + + if (async_ctx_acquire(ctx) == ERR) + { + errno = EBUSY; + return ERR; + } + + if (!(atomic_load(&ctx->flags) & ASYNC_CTX_MAPPED)) + { + async_ctx_release(ctx); + errno = EINVAL; + return ERR; + } + + async_rings_t* rings = &ctx->rings; + size_t processed = 0; + + while (processed < amount) + { + uint32_t stail = atomic_load_explicit(&rings->shared->stail, memory_order_acquire); + uint32_t shead = atomic_load_explicit(&rings->shared->shead, memory_order_relaxed); + + if (shead == stail) + { + break; + } + + async_sqe_t sqe = rings->squeue[shead & rings->smask]; + atomic_store_explicit(&rings->shared->shead, shead + 1, memory_order_release); + + if (async_handle_sqe(ctx, &sqe) == ERR) + { + async_ctx_release(ctx); + return ERR; + } + processed++; + } + + if (wait == 0) + { + async_ctx_release(ctx); + return processed; + } + + while (true) + { + uint32_t ctail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); + uint32_t chead = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); + + if ((ctail - chead) >= wait) + { + break; + } + + if (WAIT_BLOCK(&ctx->waitQueue, false) == ERR) + { + async_ctx_release(ctx); + return processed > 0 ? processed : ERR; + } + } + + async_ctx_release(ctx); + return processed; +} + +SYSCALL_DEFINE(SYS_ASYNC_INIT, uint64_t, async_rings_t* userRings, void* address, size_t sentries, size_t centries) +{ + if (userRings == NULL || sentries == 0 || centries == 0 || !IS_POW2(sentries) || !IS_POW2(centries)) + { + errno = EINVAL; + return ERR; + } + + process_t* process = process_current(); + async_ctx_t* ctx = &process->async; + space_t* space = &process->space; + + if (async_ctx_acquire(ctx) == ERR) + { + errno = EBUSY; + return ERR; + } + + if (atomic_load(&ctx->flags) & ASYNC_CTX_MAPPED) + { + async_ctx_release(ctx); + errno = EBUSY; + return ERR; + } + + if (async_ctx_map(ctx, space, userRings, address, sentries, centries) == ERR) + { + async_ctx_release(ctx); + return ERR; + } + + async_ctx_release(ctx); + return 0; +} + +SYSCALL_DEFINE(SYS_ASYNC_DEINIT, uint64_t) +{ + process_t* process = process_current(); + async_ctx_t* ctx = &process->async; + space_t* space = &process->space; + + if (async_ctx_acquire(ctx) == ERR) + { + errno = EBUSY; + return ERR; + } + + if (!(atomic_load(&ctx->flags) & ASYNC_CTX_MAPPED)) + { + async_ctx_release(ctx); + errno = EINVAL; + return ERR; + } + + if (async_ctx_unmap(ctx) == ERR) + { + async_ctx_release(ctx); + return ERR; + } + + async_ctx_release(ctx); + return 0; +} + +SYSCALL_DEFINE(SYS_ASYNC_NOTIFY, uint64_t, size_t amount, size_t wait) +{ + process_t* process = process_current(); + async_ctx_t* ctx = &process->async; + + return async_ctx_notify(ctx, amount, wait); +} \ No newline at end of file diff --git a/src/kernel/sync/task.c b/src/kernel/sync/task.c new file mode 100644 index 000000000..f0bae6bef --- /dev/null +++ b/src/kernel/sync/task.c @@ -0,0 +1,66 @@ +#include +#include + +#include + +PERCPU_DEFINE_CTOR(task_ctx_t, pcpu_tasks) +{ + task_ctx_t* tasks = SELF_PTR(pcpu_tasks); + + list_init(&tasks->timeouts); + lock_init(&tasks->lock); +} + +void task_timeout_add(task_t* task) +{ + task_ctx_t* tasks = SELF_PTR(pcpu_tasks); + LOCK_SCOPE(&tasks->lock); + + task->owner = tasks; + + task_t* entry; + LIST_FOR_EACH(entry, &tasks->timeouts, timeoutEntry) + { + if (task->deadline < entry->deadline) + { + list_prepend(&entry->entry, &task->timeoutEntry); + return; + } + } + + list_push_back(&tasks->timeouts, &task->timeoutEntry); +} + +void task_timeout_remove(task_t* task) +{ + assert(task->owner != NULL); + LOCK_SCOPE(&task->owner->lock); + + list_remove(&task->timeoutEntry); +} + +void task_timeouts_check(void) +{ + task_ctx_t* tasks = SELF_PTR(pcpu_tasks); + clock_t now = clock_uptime(); + + LOCK_SCOPE(&tasks->lock); + + task_t* task; + while (true) + { + task = CONTAINER_OF(list_first(&tasks->timeouts), task_t, timeoutEntry); + if (task == NULL) + { + break; + } + + if (task->deadline > now) + { + break; + } + + list_remove(&task->timeoutEntry); + /// @todo + } +} \ No newline at end of file diff --git a/src/kernel/sync/tasks.c b/src/kernel/sync/tasks.c new file mode 100644 index 000000000..966b49d07 --- /dev/null +++ b/src/kernel/sync/tasks.c @@ -0,0 +1,12 @@ +#include + +bool task_nop_cancel(task_nop_t* task) +{ + TASK_COMPLETE(task, 0); + return true; +} + +void task_nop_timeout(task_nop_t* task) +{ + TASK_COMPLETE(task, 0); +} \ No newline at end of file diff --git a/src/kernel/utils/fifo.c b/src/kernel/utils/fifo.c deleted file mode 100644 index 0dcf11a40..000000000 --- a/src/kernel/utils/fifo.c +++ /dev/null @@ -1,108 +0,0 @@ -#include - -#include -#include -#include - -void fifo_init(fifo_t* fifo, uint8_t* buffer, size_t size) -{ - fifo->buffer = buffer; - fifo->size = size; - fifo->head = 0; - fifo->tail = 0; -} - -void fifo_reset(fifo_t* fifo) -{ - fifo->head = 0; - fifo->tail = 0; -} - -size_t fifo_bytes_readable(const fifo_t* fifo) -{ - if (fifo->head >= fifo->tail) - { - return fifo->head - fifo->tail; - } - - return fifo->size - (fifo->tail - fifo->head); -} - -size_t fifo_bytes_writeable(const fifo_t* fifo) -{ - if (fifo->tail > fifo->head) - { - return fifo->tail - fifo->head - 1; - } - - return fifo->size - (fifo->head - fifo->tail) - 1; -} - -size_t fifo_read(fifo_t* fifo, void* buffer, size_t count) -{ - size_t readable = fifo_bytes_readable(fifo); - if (readable == 0) - { - return 0; - } - - if (count > readable) - { - count = readable; - } - - size_t firstSize = fifo->size - fifo->tail; - if (firstSize > count) - { - firstSize = count; - } - - memcpy(buffer, fifo->buffer + fifo->tail, firstSize); - fifo->tail = (fifo->tail + firstSize) % fifo->size; - - size_t remaining = count - firstSize; - if (remaining > 0) - { - memcpy((uint8_t*)buffer + firstSize, fifo->buffer + fifo->tail, remaining); - fifo->tail = (fifo->tail + remaining) % fifo->size; - } - - return count; -} - -size_t fifo_write(fifo_t* fifo, const void* buffer, size_t count) -{ - size_t writeable = fifo_bytes_writeable(fifo); - if (count > writeable) - { - count = writeable; - } - - size_t firstSize = fifo->size - fifo->head; - if (firstSize > count) - { - firstSize = count; - } - - memcpy(fifo->buffer + fifo->head, buffer, firstSize); - fifo->head = (fifo->head + firstSize) % fifo->size; - - size_t remaining = count - firstSize; - if (remaining > 0) - { - memcpy(fifo->buffer + fifo->head, (uint8_t*)buffer + firstSize, remaining); - fifo->head = (fifo->head + remaining) % fifo->size; - } - - return count; -} - -void fifo_advance_head(fifo_t* fifo, size_t count) -{ - fifo->head = (fifo->head + count) % fifo->size; -} - -void fifo_advance_tail(fifo_t* fifo, size_t count) -{ - fifo->tail = (fifo->tail + count) % fifo->size; -} diff --git a/src/libpatchwork/element.c b/src/libpatchwork/element.c index b79cf0bc5..db452e7b7 100644 --- a/src/libpatchwork/element.c +++ b/src/libpatchwork/element.c @@ -16,7 +16,7 @@ static uint64_t element_send_init(element_t* elem) } static element_t* element_new_raw(element_id_t id, const rect_t* rect, const char* text, element_flags_t flags, - procedure_t procedure, void* data) + procedure_t procedure, void* data) { element_t* elem = malloc(sizeof(element_t)); if (elem == NULL) @@ -46,7 +46,7 @@ static element_t* element_new_raw(element_id_t id, const rect_t* rect, const cha } element_t* element_new(element_t* parent, element_id_t id, const rect_t* rect, const char* text, element_flags_t flags, - procedure_t procedure, void* data) + procedure_t procedure, void* data) { if (parent == NULL || rect == NULL || text == NULL || procedure == NULL) { @@ -74,7 +74,7 @@ element_t* element_new(element_t* parent, element_id_t id, const rect_t* rect, c } element_t* element_new_root(window_t* win, element_id_t id, const rect_t* rect, const char* text, element_flags_t flags, - procedure_t procedure, void* data) + procedure_t procedure, void* data) { if (win == NULL || rect == NULL || text == NULL || procedure == NULL) { @@ -160,7 +160,7 @@ element_t* element_find(element_t* elem, element_id_t id) return NULL; } -void element_set_private(element_t* elem, void* data) +void element_set_private(element_t* elem, void* data) { if (elem == NULL) { diff --git a/src/libpatchwork/internal.h b/src/libpatchwork/internal.h index 0326f0cf3..f972c6f21 100644 --- a/src/libpatchwork/internal.h +++ b/src/libpatchwork/internal.h @@ -38,7 +38,7 @@ typedef struct element } element_t; element_t* element_new_root(window_t* win, element_id_t id, const rect_t* rect, const char* text, element_flags_t flags, - procedure_t procedure, void* data); + procedure_t procedure, void* data); typedef struct window { diff --git a/src/libpatchwork/window.c b/src/libpatchwork/window.c index 8f72b7e21..e5324284d 100644 --- a/src/libpatchwork/window.c +++ b/src/libpatchwork/window.c @@ -311,7 +311,7 @@ static uint64_t window_deco_procedure(window_t* win, element_t* elem, const even } window_t* window_new(display_t* disp, const char* name, const rect_t* rect, surface_type_t type, window_flags_t flags, - procedure_t procedure, void* data) + procedure_t procedure, void* data) { if (disp == NULL || name == NULL || rect == NULL || procedure == NULL || strnlen_s(name, MAX_NAME + 1) >= MAX_NAME) { diff --git a/src/libstd/common/print.h b/src/libstd/common/print.h index c0422a5ac..9300ea5ee 100644 --- a/src/libstd/common/print.h +++ b/src/libstd/common/print.h @@ -614,7 +614,7 @@ static inline int _print_format(_print_ctx_t* ctx) return ret; } -static inline int _print(const char* _RESTRICT format, size_t n, va_list arg, void* data) +static inline int _print(const char* _RESTRICT format, size_t n, va_list arg, void* data) { assert(format != NULL); diff --git a/src/libstd/common/scan.h b/src/libstd/common/scan.h index daf5636cc..d29428c93 100644 --- a/src/libstd/common/scan.h +++ b/src/libstd/common/scan.h @@ -663,7 +663,7 @@ static inline int _scan_format(_scan_ctx_t* ctx) return ret; } -static inline int _scan(const char* _RESTRICT format, va_list arg, void* data) +static inline int _scan(const char* _RESTRICT format, va_list arg, void* data) { assert(format != NULL); diff --git a/src/libstd/functions/elf/elf64_relocate.c b/src/libstd/functions/elf/elf64_relocate.c index 04bc2fe70..ab5d4525b 100644 --- a/src/libstd/functions/elf/elf64_relocate.c +++ b/src/libstd/functions/elf/elf64_relocate.c @@ -1,7 +1,7 @@ #include "common/elf.h" uint64_t elf64_relocate(const Elf64_File* elf, Elf64_Addr base, Elf64_Off offset, - void* (*resolve_symbol)(const char* name, void* data), void* data) + void* (*resolve_symbol)(const char* name, void* data), void* data) { for (uint64_t i = 0; i < elf->header->e_shnum; i++) { diff --git a/src/libstd/user/common/syscalls.h b/src/libstd/user/common/syscalls.h index ea6e0fe8b..f09a41247 100644 --- a/src/libstd/user/common/syscalls.h +++ b/src/libstd/user/common/syscalls.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -283,4 +284,20 @@ static inline uint64_t _syscall_umount(const char* mountpoint) static inline uint64_t _syscall_arch_prctl(arch_prctl_t code, uintptr_t addr) { return _SYSCALL2(uint64_t, SYS_ARCH_PRCTL, arch_prctl_t, code, uintptr_t, addr); +} + +static inline uint64_t _syscall_async_init(async_rings_t* rings, void* address, size_t sentries, size_t centries) +{ + return _SYSCALL4(uint64_t, SYS_ASYNC_INIT, async_rings_t*, rings, void*, address, size_t, sentries, size_t, + centries); +} + +static inline uint64_t _syscall_async_deinit(void) +{ + return _SYSCALL0(uint64_t, SYS_ASYNC_DEINIT); +} + +static inline uint64_t _syscall_async_notify(size_t amount, size_t wait) +{ + return _SYSCALL2(uint64_t, SYS_ASYNC_NOTIFY, size_t, amount, size_t, wait); } \ No newline at end of file diff --git a/src/libstd/user/common/threading.h b/src/libstd/user/common/threading.h index 228dbd4dc..8c1a97dd3 100644 --- a/src/libstd/user/common/threading.h +++ b/src/libstd/user/common/threading.h @@ -10,11 +10,11 @@ * @brief Threading. * @defgroup libstd_common_user_threading Threading * @ingroup libstd_common_user - * + * * @todo Write threading documentation. - * + * * @todo Implement Thread Local Storage (TLS). - * + * * @{ */ diff --git a/src/libstd/user/functions/async/async_deinit.c b/src/libstd/user/functions/async/async_deinit.c new file mode 100644 index 000000000..ad4cc6487 --- /dev/null +++ b/src/libstd/user/functions/async/async_deinit.c @@ -0,0 +1,13 @@ +#include + +#include "user/common/syscalls.h" + +uint64_t async_deinit(void) +{ + uint64_t result = _syscall_async_deinit(); + if (result == ERR) + { + errno = _syscall_errno(); + } + return result; +} \ No newline at end of file diff --git a/src/libstd/user/functions/async/async_init.c b/src/libstd/user/functions/async/async_init.c new file mode 100644 index 000000000..988434b25 --- /dev/null +++ b/src/libstd/user/functions/async/async_init.c @@ -0,0 +1,13 @@ +#include + +#include "user/common/syscalls.h" + +uint64_t async_init(async_rings_t* rings, void* address, size_t sentries, size_t centries) +{ + uint64_t result = _syscall_async_init(rings, address, sentries, centries); + if (result == ERR) + { + errno = _syscall_errno(); + } + return result; +} \ No newline at end of file diff --git a/src/libstd/user/functions/async/async_submit.c b/src/libstd/user/functions/async/async_submit.c new file mode 100644 index 000000000..25af9e004 --- /dev/null +++ b/src/libstd/user/functions/async/async_submit.c @@ -0,0 +1,13 @@ +#include + +#include "user/common/syscalls.h" + +uint64_t async_notify(size_t amount, size_t wait) +{ + uint64_t result = _syscall_async_notify(amount, wait); + if (result == ERR) + { + errno = _syscall_errno(); + } + return result; +} \ No newline at end of file diff --git a/src/modules/drivers/ps2/ps2.h b/src/modules/drivers/ps2/ps2.h index 8ba201ce4..f1f73b0a7 100644 --- a/src/modules/drivers/ps2/ps2.h +++ b/src/modules/drivers/ps2/ps2.h @@ -206,7 +206,7 @@ typedef struct irq_virt_t irq; ///< IRQ assigned to the device by ACPI. bool attached; ///< The device has been attached from ACPI. bool initialized; ///< The device has been initialized. - void* data; ///< Driver-specific private data. + void* data; ///< Driver-specific private data. } ps2_device_info_t; /** diff --git a/src/modules/fs/9p/9p.c b/src/modules/fs/9p/9p.c index bce207130..f9b26ff5c 100644 --- a/src/modules/fs/9p/9p.c +++ b/src/modules/fs/9p/9p.c @@ -49,7 +49,7 @@ static superblock_ops_t superOps = { .cleanup = ninep_super_cleanup, }; -static dentry_t* ninep_mount(filesystem_t* fs, const char* options, void* data) +static dentry_t* ninep_mount(filesystem_t* fs, const char* options, void* data) { UNUSED(data); diff --git a/src/modules/ipc/shmem/shmem.c b/src/modules/ipc/shmem/shmem.c index 6dc4262ed..a89bda5f3 100644 --- a/src/modules/ipc/shmem/shmem.c +++ b/src/modules/ipc/shmem/shmem.c @@ -86,7 +86,7 @@ static shmem_object_t* shmem_object_new(void) return shmem; } -static void shmem_vmm_callback( void* data) +static void shmem_vmm_callback(void* data) { shmem_object_t* shmem = data; if (shmem == NULL) From 376b8a94fe730438b7e7fe0b382f86fe2171939b Mon Sep 17 00:00:00 2001 From: KN Date: Sat, 17 Jan 2026 07:49:55 +0100 Subject: [PATCH 02/23] refactor(kernel:request): rename task to request --- include/kernel/sync/async.h | 4 +- include/kernel/sync/request.h | 246 +++++++++++++++++++++++++++++++++ include/kernel/sync/requests.h | 25 ++++ include/kernel/sync/task.h | 246 --------------------------------- include/kernel/sync/tasks.h | 25 ---- include/libstd/sys/async.h | 4 +- src/kernel/sync/async.c | 42 +++--- src/kernel/sync/request.c | 66 +++++++++ src/kernel/sync/requests.c | 12 ++ src/kernel/sync/task.c | 66 --------- src/kernel/sync/tasks.c | 12 -- 11 files changed, 374 insertions(+), 374 deletions(-) create mode 100644 include/kernel/sync/request.h create mode 100644 include/kernel/sync/requests.h delete mode 100644 include/kernel/sync/task.h delete mode 100644 include/kernel/sync/tasks.h create mode 100644 src/kernel/sync/request.c create mode 100644 src/kernel/sync/requests.c delete mode 100644 src/kernel/sync/task.c delete mode 100644 src/kernel/sync/tasks.c diff --git a/include/kernel/sync/async.h b/include/kernel/sync/async.h index 71dea5fe6..af31ce9f3 100644 --- a/include/kernel/sync/async.h +++ b/include/kernel/sync/async.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -36,7 +36,7 @@ typedef enum typedef struct async_ctx { async_rings_t rings; ///< Asynchronous rings information. - task_t* tasks; ///< A preallocated array of tasks, one for each CQE. + request_t* requests; ///< A preallocated array of requests, one for each CQE. list_t freeTasks; ///< Free list of tasks. void* userAddr; ///< Userspace address of the rings. void* kernelAddr; ///< Kernel address of the rings. diff --git a/include/kernel/sync/request.h b/include/kernel/sync/request.h new file mode 100644 index 000000000..9ea07002e --- /dev/null +++ b/include/kernel/sync/request.h @@ -0,0 +1,246 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +typedef struct process process_t; + +/** + * @brief Asynchronous Request Primitive + * @defgroup kernel_sync_request Request + * @ingroup kernel_sync + * + * @{ + */ + +/** + * @brief Per-CPU request queues. + * @struct request_ctx_t + */ +typedef struct request_ctx +{ + list_t timeouts; + list_t completed; + lock_t lock; +} request_ctx_t; + +/** + * @brief Task flags. + * @enum request_flags_t + */ +typedef enum +{ + REQUEST_DELAYED = 1 << 0, ///< The completion of the request has been delayed. + REQUEST_TIMEOUT = 1 << 1, ///< The request is in a timeout queue. +} request_flags_t; + +/** + * @brief Macro to define common request structure members. + * + * All requests contain the below common members: + * - `list_entry_t entry` - List entry for requests queues and completion queues. + * - `list_entry_t timeoutEntry` - List entry for timeout queues. + * - `request_ctx_t* ctx` - Pointer to the per-CPU request context storing this request for timeouts. + * - `process_t* process` - Pointer to the process that created the request. + * - `void* data` - Pointer to user data. + * - `void (*complete)(_type*)` - Completion callback. + * - `bool (*cancel)(_type*)` - Cancellation callback, should return `true` if the request was cancelled. + * - `void (*timeout)(_type*)` - Timeout callback. + * - `request_flags_t flags` - Task flags. + * - `errno_t err` - Error code for the request. + * - `clock_t deadline` - Deadline for the request. + * - `_resultType result` - Result of the request. + * + * @param _type The type of the request structure. + * @param _resultType The type of the request result. + */ +#define REQUEST_COMMON(_type, _resultType) \ + list_entry_t entry; \ + list_entry_t timeoutEntry; \ + request_ctx_t* ctx; \ + process_t* process; \ + void* data; \ + void (*complete)(_type*); \ + bool (*cancel)(_type*); \ + void (*timeout)(_type*); \ + request_flags_t flags; \ + errno_t err; \ + clock_t deadline; \ + _resultType result + +/** + * @brief Generic request structure. + * @struct request_t + * + * @warning Due to optimization done while allocating requests in the async system, no request structure should be larger than + * this structure. + */ +typedef struct request +{ + REQUEST_COMMON(struct request, uint64_t); + uint64_t _padding[4]; +} request_t; + +/** + * @brief Task queue structure. + * @struct request_queue_t + */ +typedef struct +{ + list_t requests; +} request_queue_t; + +/** + * @brief Initializes a request queue. + * + * @param queue Pointer to the request queue to initialize. + */ +static inline void request_queue_init(request_queue_t* queue) +{ + list_init(&queue->requests); +} + +/** + * @brief Adds a request to the per-CPU timeout queue. + * + * @param request Pointer to the request to add. + */ +void request_timeout_add(request_t* request); + +/** + * @brief Removes a request from the per-CPU timeout queue. + * + * @param request Pointer to the request to remove. + */ +void request_timeout_remove(request_t* request); + +/** + * @brief Checks for request timeouts on the current CPU and handles them. + * + * @warning Must be called with interrupts disabled. + */ +void request_timeouts_check(void); + +/** + * @brief Macro to initialize a requests common members. + * + * @param _request Pointer to the request to initialize. + */ +#define REQUEST_INIT(_request) \ + ({ \ + (_request)->entry = LIST_ENTRY_CREATE((_request)->entry); \ + (_request)->timeoutEntry = LIST_ENTRY_CREATE((_request)->timeoutEntry); \ + (_request)->ctx = NULL; \ + (_request)->process = NULL; \ + (_request)->data = NULL; \ + (_request)->complete = NULL; \ + (_request)->cancel = NULL; \ + (_request)->timeout = NULL; \ + (_request)->flags = 0; \ + (_request)->err = EOK; \ + (_request)->deadline = CLOCKS_NEVER; \ + (_request)->result = (typeof((_request)->result))0; \ + }) + +#define REQUEST_CALL(_request, _func) \ + ({ \ + typeof((_request)->result) result = _func(_request); \ + if ((_request)->err != EOK) \ + { \ + (_request)->flags &= ~REQUEST_DELAYED; \ + (_request)->complete(_request); \ + } \ + else if (!((_request)->flags & REQUEST_DELAYED)) \ + { \ + (_request)->result = result; \ + (_request)->complete(_request); \ + } \ + result; \ + }) + +#define REQUEST_DELAY_NO_QUEUE(_request) \ + ({ \ + uint64_t result = 0; \ + (_request)->flags |= REQUEST_DELAYED; \ + if ((_request)->deadline != CLOCKS_NEVER) \ + { \ + if ((_request)->timeout == NULL) \ + { \ + errno = EINVAL; \ + result = ERR; \ + } \ + else \ + { \ + (_request)->flags |= REQUEST_TIMEOUT; \ + request_timeout_add((request_t*)(_request)); \ + } \ + } \ + result; \ + }) + +#define REQUEST_DELAY(_request, _queue) \ + ({ \ + list_push_back(&(_queue)->requests, &(_request)->entry); \ + uint64_t result = REQUEST_DELAY_NO_QUEUE(_request); \ + if (result == ERR) \ + { \ + list_remove(&(_request)->entry); \ + } \ + result; \ + }) + +#define REQUEST_NEXT(_queue, _type) \ + (list_is_empty(&(_queue)->requests) ? NULL : CONTAINER_OF(list_first(&(_queue)->requests), _type, entry)) + +#define REQUEST_ERROR(_request, _errno) \ + ({ \ + if ((_request)->flags & REQUEST_TIMEOUT) \ + { \ + request_timeout_remove((request_t*)(_request)); \ + } \ + list_remove(&(_request)->entry); \ + (_request)->flags &= ~REQUEST_DELAYED; \ + (_request)->err = (_errno); \ + (_request)->complete((_request)); \ + }) + +#define REQUEST_COMPLETE(_request, _result) \ + ({ \ + if ((_request)->flags & REQUEST_TIMEOUT) \ + { \ + request_timeout_remove((request_t*)(_request)); \ + } \ + list_remove(&(_request)->entry); \ + (_request)->flags &= ~REQUEST_DELAYED; \ + (_request)->result = (_result); \ + (_request)->complete((_request)); \ + }) + +#define REQUEST_CANCEL(_request) \ + ({ \ + uint64_t result = 0; \ + if ((_request)->cancel == NULL) \ + { \ + errno = EINVAL; \ + result = ERR; \ + } \ + else \ + { \ + (_request)->err = ECANCELED; \ + if (!((_request)->cancel(_request))) \ + { \ + (_request)->err = EOK; \ + errno = EBUSY; \ + result = ERR; \ + } \ + } \ + result; \ + }) + +/** @} */ \ No newline at end of file diff --git a/include/kernel/sync/requests.h b/include/kernel/sync/requests.h new file mode 100644 index 000000000..f66845871 --- /dev/null +++ b/include/kernel/sync/requests.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +/** + * @brief Kernel Request Implementations + * @defgroup kernel_sync_requests Requests + * @ingroup kernel_sync + * + * @{ + */ + +/** + * @brief No-operation request structure. + * @struct request_nop_t + */ +typedef struct request_nop +{ + REQUEST_COMMON(struct request_nop, uint64_t); +} request_nop_t; + +bool request_nop_cancel(request_nop_t* request); +void request_nop_timeout(request_nop_t* request); + +/** @} */ \ No newline at end of file diff --git a/include/kernel/sync/task.h b/include/kernel/sync/task.h deleted file mode 100644 index 94ac22e86..000000000 --- a/include/kernel/sync/task.h +++ /dev/null @@ -1,246 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include - -typedef struct process process_t; - -/** - * @brief Asynchronous Task Primitive - * @defgroup kernel_sync_task Task - * @ingroup kernel_sync - * - * @{ - */ - -/** - * @brief Per-CPU task queues. - * @struct task_ctx_t - */ -typedef struct task_ctx -{ - list_t timeouts; - list_t completed; - lock_t lock; -} task_ctx_t; - -/** - * @brief Task flags. - * @enum task_flags_t - */ -typedef enum -{ - TASK_DELAYED = 1 << 0, ///< The completion of the task has been delayed. - TASK_TIMEOUT = 1 << 1, ///< The task is in a timeout queue. -} task_flags_t; - -/** - * @brief Macro to define common task structure members. - * - * All tasks contain the below common members: - * - `list_entry_t entry` - List entry for tasks queues and completion queues. - * - `list_entry_t timeoutEntry` - List entry for timeout queues. - * - `task_ctx_t* ctx` - Pointer to the per-CPU task context storing this task for timeouts. - * - `process_t* process` - Pointer to the process that created the task. - * - `void* data` - Pointer to user data. - * - `void (*complete)(_type*)` - Completion callback. - * - `bool (*cancel)(_type*)` - Cancellation callback, should return `true` if the task was cancelled. - * - `void (*timeout)(_type*)` - Timeout callback. - * - `task_flags_t flags` - Task flags. - * - `errno_t err` - Error code for the task. - * - `clock_t deadline` - Deadline for the task. - * - `_resultType result` - Result of the task. - * - * @param _type The type of the task structure. - * @param _resultType The type of the task result. - */ -#define TASK_COMMON(_type, _resultType) \ - list_entry_t entry; \ - list_entry_t timeoutEntry; \ - task_ctx_t* ctx; \ - process_t* process; \ - void* data; \ - void (*complete)(_type*); \ - bool (*cancel)(_type*); \ - void (*timeout)(_type*); \ - task_flags_t flags; \ - errno_t err; \ - clock_t deadline; \ - _resultType result - -/** - * @brief Generic task structure. - * @struct task_t - * - * @warning Due to optimization done while allocating tasks in the async system, no task structure should be larger than - * this structure. - */ -typedef struct task -{ - TASK_COMMON(struct task, uint64_t); - uint64_t _padding[4]; -} task_t; - -/** - * @brief Task queue structure. - * @struct task_queue_t - */ -typedef struct -{ - list_t tasks; -} task_queue_t; - -/** - * @brief Initializes a task queue. - * - * @param queue Pointer to the task queue to initialize. - */ -static inline void task_queue_init(task_queue_t* queue) -{ - list_init(&queue->tasks); -} - -/** - * @brief Adds a task to the per-CPU timeout queue. - * - * @param task Pointer to the task to add. - */ -void task_timeout_add(task_t* task); - -/** - * @brief Removes a task from the per-CPU timeout queue. - * - * @param task Pointer to the task to remove. - */ -void task_timeout_remove(task_t* task); - -/** - * @brief Checks for task timeouts on the current CPU and handles them. - * - * @warning Must be called with interrupts disabled. - */ -void task_timeouts_check(void); - -/** - * @brief Macro to initialize a tasks common members. - * - * @param _task Pointer to the task to initialize. - */ -#define TASK_INIT(_task) \ - ({ \ - (_task)->entry = LIST_ENTRY_CREATE((_task)->entry); \ - (_task)->timeoutEntry = LIST_ENTRY_CREATE((_task)->timeoutEntry); \ - (_task)->ctx = NULL; \ - (_task)->process = NULL; \ - (_task)->data = NULL; \ - (_task)->complete = NULL; \ - (_task)->cancel = NULL; \ - (_task)->timeout = NULL; \ - (_task)->flags = 0; \ - (_task)->err = EOK; \ - (_task)->deadline = CLOCKS_NEVER; \ - (_task)->result = (typeof((_task)->result))0; \ - }) - -#define TASK_CALL(_task, _func) \ - ({ \ - typeof((_task)->result) result = _func(_task); \ - if ((_task)->err != EOK) \ - { \ - (_task)->flags &= ~TASK_DELAYED; \ - (_task)->complete(_task); \ - } \ - else if (!((_task)->flags & TASK_DELAYED)) \ - { \ - (_task)->result = result; \ - (_task)->complete(_task); \ - } \ - result; \ - }) - -#define TASK_DELAY_NO_QUEUE(_task) \ - ({ \ - uint64_t result = 0; \ - (_task)->flags |= TASK_DELAYED; \ - if ((_task)->deadline != CLOCKS_NEVER) \ - { \ - if ((_task)->timeout == NULL) \ - { \ - errno = EINVAL; \ - result = ERR; \ - } \ - else \ - { \ - (_task)->flags |= TASK_TIMEOUT; \ - task_timeout_add((task_t*)(_task)); \ - } \ - } \ - result; \ - }) - -#define TASK_DELAY(_task, _queue) \ - ({ \ - list_push_back(&(_queue)->tasks, &(_task)->entry); \ - uint64_t result = TASK_DELAY_NO_QUEUE(_task); \ - if (result == ERR) \ - { \ - list_remove(&(_task)->entry); \ - } \ - result; \ - }) - -#define TASK_NEXT(_queue, _type) \ - (list_is_empty(&(_queue)->tasks) ? NULL : CONTAINER_OF(list_first(&(_queue)->tasks), _type, entry)) - -#define TASK_ERROR(_task, _errno) \ - ({ \ - if ((_task)->flags & TASK_TIMEOUT) \ - { \ - task_timeout_remove((task_t*)(_task)); \ - } \ - list_remove(&(_task)->entry); \ - (_task)->flags &= ~TASK_DELAYED; \ - (_task)->err = (_errno); \ - (_task)->complete((_task)); \ - }) - -#define TASK_COMPLETE(_task, _result) \ - ({ \ - if ((_task)->flags & TASK_TIMEOUT) \ - { \ - task_timeout_remove((task_t*)(_task)); \ - } \ - list_remove(&(_task)->entry); \ - (_task)->flags &= ~TASK_DELAYED; \ - (_task)->result = (_result); \ - (_task)->complete((_task)); \ - }) - -#define TASK_CANCEL(_task) \ - ({ \ - uint64_t result = 0; \ - if ((_task)->cancel == NULL) \ - { \ - errno = EINVAL; \ - result = ERR; \ - } \ - else \ - { \ - (_task)->err = ECANCELED; \ - if (!((_task)->cancel(_task))) \ - { \ - (_task)->err = EOK; \ - errno = EBUSY; \ - result = ERR; \ - } \ - } \ - result; \ - }) - -/** @} */ \ No newline at end of file diff --git a/include/kernel/sync/tasks.h b/include/kernel/sync/tasks.h deleted file mode 100644 index b96233699..000000000 --- a/include/kernel/sync/tasks.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include - -/** - * @brief Kernel Task Implementations - * @defgroup kernel_sync_tasks Tasks - * @ingroup kernel_sync - * - * @{ - */ - -/** - * @brief No-operation task structure. - * @struct task_nop_t - */ -typedef struct task_nop -{ - TASK_COMMON(struct task_nop, uint64_t); -} task_nop_t; - -bool task_nop_cancel(task_nop_t* task); -void task_nop_timeout(task_nop_t* task); - -/** @} */ \ No newline at end of file diff --git a/include/libstd/sys/async.h b/include/libstd/sys/async.h index a6f4e8f8c..4e7381fb9 100644 --- a/include/libstd/sys/async.h +++ b/include/libstd/sys/async.h @@ -1,10 +1,10 @@ -#include #ifndef _SYS_ASYNC_H #define _SYS_ASYNC_H 1 #include #include #include +#include #include #if defined(__cplusplus) @@ -50,7 +50,7 @@ typedef enum * * Used to modify the behavior of asynchronous operations. * - * @todo Implement `ASYNC_SEQ_LINK`. + * @todo Implement `ASYNC_SEQ_LINK` and `ASYNC_SEQ_IMMEDIATE`. */ typedef enum { diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index ffa4e9257..132f9a411 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include @@ -49,8 +49,8 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, async_rin return ERR; } - task_t* tasks = malloc(sizeof(task_t) * centries); - if (tasks == NULL) + request_t* requests = malloc(sizeof(request_t) * centries); + if (requests == NULL) { vmm_unmap(space, userAddr, pageAmount * PAGE_SIZE); vmm_unmap(NULL, kernelAddr, pageAmount * PAGE_SIZE); @@ -59,8 +59,8 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, async_rin for (size_t i = 0; i < centries; i++) { - TASK_INIT(&tasks[i]); - list_push_back(&ctx->freeTasks, &tasks[i].entry); + REQUEST_INIT(&requests[i]); + list_push_back(&ctx->freeTasks, &requests[i].entry); } async_shared_t* shared = (async_shared_t*)kernelAddr; @@ -85,7 +85,7 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, async_rin kernelRings->centries = centries; kernelRings->cmask = centries - 1; - ctx->tasks = tasks; + ctx->requests = requests; ctx->userAddr = userAddr; ctx->kernelAddr = kernelAddr; ctx->pageAmount = pageAmount; @@ -98,8 +98,8 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, async_rin static inline uint64_t async_ctx_unmap(async_ctx_t* ctx) { list_init(&ctx->freeTasks); - free(ctx->tasks); - ctx->tasks = NULL; + free(ctx->requests); + ctx->requests = NULL; vmm_unmap(ctx->space, ctx->userAddr, ctx->pageAmount * PAGE_SIZE); vmm_unmap(NULL, ctx->kernelAddr, ctx->pageAmount * PAGE_SIZE); @@ -108,19 +108,19 @@ static inline uint64_t async_ctx_unmap(async_ctx_t* ctx) return 0; } -static inline task_t* async_ctx_alloc_task(async_ctx_t* ctx) +static inline request_t* async_ctx_alloc_request(async_ctx_t* ctx) { if (list_is_empty(&ctx->freeTasks)) { return NULL; } - return CONTAINER_OF(list_pop_back(&ctx->freeTasks), task_t, entry); + return CONTAINER_OF(list_pop_back(&ctx->freeTasks), request_t, entry); } -static inline void async_ctx_free_task(async_ctx_t* ctx, task_t* task) +static inline void async_ctx_free_request(async_ctx_t* ctx, request_t* request) { - list_push_back(&ctx->freeTasks, &task->entry); + list_push_back(&ctx->freeTasks, &request->entry); } void async_ctx_init(async_ctx_t* ctx) @@ -160,7 +160,7 @@ void async_ctx_deinit(async_ctx_t* ctx) wait_queue_deinit(&ctx->waitQueue); } -static void async_nop_complete(task_nop_t* nop) +static void async_nop_complete(request_nop_t* nop) { async_cqe_t cqe; cqe.data = nop->data; @@ -169,14 +169,14 @@ static void async_nop_complete(task_nop_t* nop) process_t* process = nop->process; async_ctx_push_cqe(&process->async, &cqe); - async_ctx_free_task(&process->async, (task_t*)nop); + async_ctx_free_request(&process->async, (request_t*)nop); UNREF(nop->process); } static uint64_t async_handle_sqe(async_ctx_t* ctx, async_sqe_t* sqe) { - task_t* task = async_ctx_alloc_task(ctx); - if (task == NULL) + request_t* request = async_ctx_alloc_request(ctx); + if (request == NULL) { errno = ENOSPC; return ERR; @@ -186,17 +186,17 @@ static uint64_t async_handle_sqe(async_ctx_t* ctx, async_sqe_t* sqe) { case ASYNC_OP_NOP: { - task_nop_t* nop = (task_nop_t*)task; + request_nop_t* nop = (request_nop_t*)request; nop->data = sqe->data; nop->process = REF(process_current()); nop->complete = async_nop_complete; - nop->cancel = task_nop_cancel; - nop->timeout = task_nop_timeout; - TASK_DELAY_NO_QUEUE(nop); + nop->cancel = request_nop_cancel; + nop->timeout = request_nop_timeout; + REQUEST_DELAY_NO_QUEUE(nop); } break; default: - async_ctx_free_task(ctx, task); + async_ctx_free_request(ctx, request); errno = EINVAL; return ERR; } diff --git a/src/kernel/sync/request.c b/src/kernel/sync/request.c new file mode 100644 index 000000000..503773bed --- /dev/null +++ b/src/kernel/sync/request.c @@ -0,0 +1,66 @@ +#include +#include + +#include + +PERCPU_DEFINE_CTOR(request_ctx_t, pcpu_requests) +{ + request_ctx_t* requests = SELF_PTR(pcpu_requests); + + list_init(&requests->timeouts); + lock_init(&requests->lock); +} + +void request_timeout_add(request_t* request) +{ + request_ctx_t* requests = SELF_PTR(pcpu_requests); + LOCK_SCOPE(&requests->lock); + + request->ctx = requests; + + request_t* entry; + LIST_FOR_EACH(entry, &requests->timeouts, timeoutEntry) + { + if (request->deadline < entry->deadline) + { + list_prepend(&entry->entry, &request->timeoutEntry); + return; + } + } + + list_push_back(&requests->timeouts, &request->timeoutEntry); +} + +void request_timeout_remove(request_t* request) +{ + assert(request->owner != NULL); + LOCK_SCOPE(&request->ctx->lock); + + list_remove(&request->timeoutEntry); +} + +void request_timeouts_check(void) +{ + request_ctx_t* requests = SELF_PTR(pcpu_requests); + clock_t now = clock_uptime(); + + LOCK_SCOPE(&requests->lock); + + request_t* request; + while (true) + { + request = CONTAINER_OF(list_first(&requests->timeouts), request_t, timeoutEntry); + if (request == NULL) + { + break; + } + + if (request->deadline > now) + { + break; + } + + list_remove(&request->timeoutEntry); + /// @todo + } +} \ No newline at end of file diff --git a/src/kernel/sync/requests.c b/src/kernel/sync/requests.c new file mode 100644 index 000000000..74dbefda0 --- /dev/null +++ b/src/kernel/sync/requests.c @@ -0,0 +1,12 @@ +#include + +bool request_nop_cancel(request_nop_t* request) +{ + REQUEST_COMPLETE(request, 0); + return true; +} + +void request_nop_timeout(request_nop_t* request) +{ + REQUEST_COMPLETE(request, 0); +} \ No newline at end of file diff --git a/src/kernel/sync/task.c b/src/kernel/sync/task.c deleted file mode 100644 index f0bae6bef..000000000 --- a/src/kernel/sync/task.c +++ /dev/null @@ -1,66 +0,0 @@ -#include -#include - -#include - -PERCPU_DEFINE_CTOR(task_ctx_t, pcpu_tasks) -{ - task_ctx_t* tasks = SELF_PTR(pcpu_tasks); - - list_init(&tasks->timeouts); - lock_init(&tasks->lock); -} - -void task_timeout_add(task_t* task) -{ - task_ctx_t* tasks = SELF_PTR(pcpu_tasks); - LOCK_SCOPE(&tasks->lock); - - task->owner = tasks; - - task_t* entry; - LIST_FOR_EACH(entry, &tasks->timeouts, timeoutEntry) - { - if (task->deadline < entry->deadline) - { - list_prepend(&entry->entry, &task->timeoutEntry); - return; - } - } - - list_push_back(&tasks->timeouts, &task->timeoutEntry); -} - -void task_timeout_remove(task_t* task) -{ - assert(task->owner != NULL); - LOCK_SCOPE(&task->owner->lock); - - list_remove(&task->timeoutEntry); -} - -void task_timeouts_check(void) -{ - task_ctx_t* tasks = SELF_PTR(pcpu_tasks); - clock_t now = clock_uptime(); - - LOCK_SCOPE(&tasks->lock); - - task_t* task; - while (true) - { - task = CONTAINER_OF(list_first(&tasks->timeouts), task_t, timeoutEntry); - if (task == NULL) - { - break; - } - - if (task->deadline > now) - { - break; - } - - list_remove(&task->timeoutEntry); - /// @todo - } -} \ No newline at end of file diff --git a/src/kernel/sync/tasks.c b/src/kernel/sync/tasks.c deleted file mode 100644 index 966b49d07..000000000 --- a/src/kernel/sync/tasks.c +++ /dev/null @@ -1,12 +0,0 @@ -#include - -bool task_nop_cancel(task_nop_t* task) -{ - TASK_COMPLETE(task, 0); - return true; -} - -void task_nop_timeout(task_nop_t* task) -{ - TASK_COMPLETE(task, 0); -} \ No newline at end of file From 449330db9451bd95c5b726302daaa291c1d2ce62 Mon Sep 17 00:00:00 2001 From: KN Date: Sat, 17 Jan 2026 09:53:33 +0100 Subject: [PATCH 03/23] refactor(libstd:rings): use unix style verb names for async api --- include/kernel/cpu/syscall.h | 6 +- include/kernel/sched/timer.h | 4 +- include/kernel/sync/async.h | 10 +- include/kernel/sync/request.h | 19 +++- include/libstd/sys/{async.h => rings.h} | 97 +++++++++---------- src/kernel/cpu/interrupt.c | 4 +- src/kernel/sched/timer.c | 6 +- src/kernel/sync/async.c | 36 +++---- src/kernel/sync/request.c | 40 +++++--- src/libstd/user/common/syscalls.h | 14 +-- .../user/functions/async/async_deinit.c | 6 +- src/libstd/user/functions/async/async_init.c | 6 +- .../user/functions/async/async_submit.c | 6 +- src/programs/utils/ringstest/main.c | 37 +++++++ src/programs/utils/ringstest/ringstest.mk | 11 +++ 15 files changed, 186 insertions(+), 116 deletions(-) rename include/libstd/sys/{async.h => rings.h} (73%) create mode 100644 src/programs/utils/ringstest/main.c create mode 100644 src/programs/utils/ringstest/ringstest.mk diff --git a/include/kernel/cpu/syscall.h b/include/kernel/cpu/syscall.h index 6a09202a9..9b9833bc6 100644 --- a/include/kernel/cpu/syscall.h +++ b/include/kernel/cpu/syscall.h @@ -102,9 +102,9 @@ typedef enum SYS_MOUNT, SYS_UNMOUNT, SYS_ARCH_PRCTL, - SYS_ASYNC_INIT, - SYS_ASYNC_DEINIT, - SYS_ASYNC_NOTIFY, + SYS_SETUP, + SYS_TEARDOWN, + SYS_ENTER, SYS_TOTAL_AMOUNT } syscall_number_t; diff --git a/include/kernel/sched/timer.h b/include/kernel/sched/timer.h index a5bab73f1..7944a8dd9 100644 --- a/include/kernel/sched/timer.h +++ b/include/kernel/sched/timer.h @@ -113,10 +113,10 @@ uint64_t timer_source_amount(void); * @note Will never set the timeout to be less than `CONFIG_MIN_TIMER_TIMEOUT` to avoid spamming the CPU with timer * interrupts. * - * @param uptime The time since boot, we need to specify this as an argument to avoid inconsistency in the + * @param now The time since boot, we need to specify this as an argument to avoid inconsistency in the * timeout/deadline calculations. * @param deadline The desired deadline. */ -void timer_set(clock_t uptime, clock_t deadline); +void timer_set(clock_t now, clock_t deadline); /** @} */ diff --git a/include/kernel/sync/async.h b/include/kernel/sync/async.h index af31ce9f3..09ac85f3a 100644 --- a/include/kernel/sync/async.h +++ b/include/kernel/sync/async.h @@ -8,11 +8,11 @@ #include #include -#include +#include /** * @brief Asynchronous Rings - * @defgroup kernel_sync_rings Rings + * @defgroup kernel_sync_async Async * @ingroup kernel_sync * * @{ @@ -35,7 +35,7 @@ typedef enum */ typedef struct async_ctx { - async_rings_t rings; ///< Asynchronous rings information. + rings_t rings; ///< Asynchronous rings information. request_t* requests; ///< A preallocated array of requests, one for each CQE. list_t freeTasks; ///< Free list of tasks. void* userAddr; ///< Userspace address of the rings. @@ -104,9 +104,9 @@ static inline void async_ctx_release(async_ctx_t* ctx) * @param ctx Pointer to the async context. * @param cqe Pointer to the CQE to push. */ -static inline void async_ctx_push_cqe(async_ctx_t* ctx, async_cqe_t* cqe) +static inline void async_ctx_push_cqe(async_ctx_t* ctx, cqe_t* cqe) { - async_rings_t* rings = &ctx->rings; + rings_t* rings = &ctx->rings; uint32_t tail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); uint32_t head = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); diff --git a/include/kernel/sync/request.h b/include/kernel/sync/request.h index 9ea07002e..0d8b98ed5 100644 --- a/include/kernel/sync/request.h +++ b/include/kernel/sync/request.h @@ -16,6 +16,22 @@ typedef struct process process_t; * @defgroup kernel_sync_request Request * @ingroup kernel_sync * + * ## Callbacks + * + * Requests can define three callbacks, included is a list of their expected semantics. + * + * ### Completion Callback + * + * The `complete()` callback should be called when the request has been completed, the `complete()` implementation does not need to guarantee that the request structure will remain valid after a call to this function. + * + * ### Cancellation Callback + * + * The optional `cancel()` callback is called when attempting to cancel an in-progress request, if the request cannot be cancelled, the callback should return `false`, otherwise `true`. + * + * ### Timeout Callback + * + * The optional `timeout()` callback is called when a request has timed out, the request * will be removed from the timeout queue before this callback is called and it will never * be called more than once. + * * @{ */ @@ -26,7 +42,6 @@ typedef struct process process_t; typedef struct request_ctx { list_t timeouts; - list_t completed; lock_t lock; } request_ctx_t; @@ -50,7 +65,7 @@ typedef enum * - `process_t* process` - Pointer to the process that created the request. * - `void* data` - Pointer to user data. * - `void (*complete)(_type*)` - Completion callback. - * - `bool (*cancel)(_type*)` - Cancellation callback, should return `true` if the request was cancelled. + * - `bool (*cancel)(_type*)` - Cancellation callback. * - `void (*timeout)(_type*)` - Timeout callback. * - `request_flags_t flags` - Task flags. * - `errno_t err` - Error code for the request. diff --git a/include/libstd/sys/async.h b/include/libstd/sys/rings.h similarity index 73% rename from include/libstd/sys/async.h rename to include/libstd/sys/rings.h index 4e7381fb9..3e44fd380 100644 --- a/include/libstd/sys/async.h +++ b/include/libstd/sys/rings.h @@ -1,5 +1,5 @@ -#ifndef _SYS_ASYNC_H -#define _SYS_ASYNC_H 1 +#ifndef _SYS_RINGS_H +#define _SYS_RINGS_H 1 #include #include @@ -18,8 +18,8 @@ extern "C" #include "_internal/fd_t.h" /** - * @brief Asynchronous operations. - * @defgroup libstd_async Async + * @brief Ring-based submission/completion interface. + * @defgroup libstd_rings Rings * @ingroup libstd * * Asynchronous operations provide the core of all IO interfaces in PatchworkOS, all implemented in an interface @@ -37,41 +37,41 @@ extern "C" /** * @brief Asynchronous operation codes. - * @enum async_op_t + * @enum rings_op_t */ typedef enum { - ASYNC_OP_NOP = 0, ///< Never completes, can be used to implement a sleep equivalent. -} async_op_t; + RINGS_NOP = 0, ///< Never completes, can be used to implement a sleep equivalent. +} rings_op_t; /** * @brief Asynchronous sequence flags. - * @enum async_seq_flags_t + * @enum sqe_flags_t * * Used to modify the behavior of asynchronous operations. * - * @todo Implement `ASYNC_SEQ_LINK` and `ASYNC_SEQ_IMMEDIATE`. + * @todo Implement `SQE_LINK` and `SQE_IMMEDIATE`. */ typedef enum { - ASYNC_SEQ_NONE = 0, - ASYNC_SEQ_LINK = 1 << 0, ///< Must be completed before the next SQE in the submission queue is started. - ASYNC_SEQ_IMMEDIATE = 1 << 1, ///< Fail if the operation cannot be completed immediately. -} async_seq_flags_t; + SQE_DEFAULT = 0, + SQE_LINK = 1 << 0, ///< Must be completed before the next SQE in the submission queue is started. + SQE_IMMEDIATE = 1 << 1, ///< Fail if the operation cannot be completed immediately. +} sqe_flags_t; /** * @brief Asynchronous submission queue entry (SQE). - * @struct async_sqe_t + * @struct sqe_t * * @warning For operations such as `ASYNC_OP_OPEN`, it is the responsibility of userspace to ensure that any pointers * passed to the kernel remain valid until the operation is complete. */ -typedef struct async_sqe +typedef struct sqe { - void* data; ///< Private data for the operation, will be returned in the completion entry. - async_op_t opcode; ///< Operation code. - async_seq_flags_t flags; ///< Sequence flags. + rings_op_t opcode; ///< Operation code. + sqe_flags_t flags; ///< Sequence flags. clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. + void* data; ///< Private data for the operation, will be returned in the completion entry. union { struct { @@ -79,56 +79,54 @@ typedef struct async_sqe } nop; uint64_t _raw[5]; }; -} async_sqe_t; +} sqe_t; #ifdef static_assert -static_assert(sizeof(async_sqe_t) == 64, "async_sqe_t is not 64 bytes"); +static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); #endif /** * @brief Macro to create an asynchronous submission queue entry (SQE). * - * @param _id Unique identifier for the operation. * @param _opcode Operation code. * @param _flags Sequence flags. * @param _timeout Timeout for the operation, `CLOCKS_NEVER` for no timeout. * @param _data Private data for the operation. */ -#define ASYNC_SQE_CREATE(_id, _opcode, _flags, _timeout, _data) \ +#define SQE_CREATE(_opcode, _flags, _timeout, _data) \ { \ - .data = (_data), \ - .id = (_id), \ .opcode = (_opcode), \ .flags = (_flags), \ .timeout = (_timeout), \ + .data = (void*)(_data), \ } /** * @brief Asynchronous completion queue entry (CQE). - * @struct async_cqe_t + * @struct cqe_t */ -typedef struct ALIGNED(64) async_cqe +typedef struct ALIGNED(64) cqe { void* data; ///< Private data from the submission entry. - async_op_t opcode; ///< Operation code from the submission entry. + rings_op_t opcode; ///< Operation code from the submission entry. errno_t error; ///< Error code, if not equal to `EOK` an error occurred. union { size_t read; ///< The number of bytes read from `ASYNC_OP_READ`. uint64_t _raw; }; -} async_cqe_t; +} cqe_t; #ifdef static_assert -static_assert(sizeof(async_cqe_t) == 64, "async_cqe_t is not 64 bytes"); +static_assert(sizeof(cqe_t) == 64, "cqe_t is not 64 bytes"); #endif /** * @brief Shared asynchronous rings structure. - * @struct async_shared_t + * @struct rings_shared_t * * Used as the intermediate between userspace and the kernel. * */ -typedef struct ALIGNED(64) async_shared +typedef struct ALIGNED(64) rings_shared { atomic_uint32_t shead; ///< Submission head index, updated by the kernel. atomic_uint32_t ctail; ///< Completion tail index, updated by the kernel. @@ -136,39 +134,34 @@ typedef struct ALIGNED(64) async_shared sizeof(atomic_uint32_t) * 2]; ///< Padding to prevent false sharing between user space and the kernel. atomic_uint32_t stail; ///< Submission tail index, updated by userspace. atomic_uint32_t chead; ///< Completion head index, updated by userspace. -} async_shared_t; +} rings_shared_t; /** * @brief Asynchronous rings structure. - * @struct async_rings_t + * @struct rings_t * * The kernel and userspace will have their own instances of this structure. */ -typedef struct async_rings +typedef struct rings { - async_shared_t* shared; ///< Pointer to the shared structure. - async_sqe_t* squeue; ///< Pointer to the submission queue. + rings_shared_t* shared; ///< Pointer to the shared structure. + sqe_t* squeue; ///< Pointer to the submission queue. size_t sentries; ///< Number of entries in the submission queue. size_t smask; ///< Bitmask for submission queue (sentries - 1). - async_cqe_t* cqueue; ///< Pointer to the completion queue. + cqe_t* cqueue; ///< Pointer to the completion queue. size_t centries; ///< Number of entries in the completion queue. size_t cmask; ///< Bitmask for completion queue (centries - 1). -} async_rings_t; +} rings_t; /** * @brief Dont wait for any submissions to complete. */ -#define ASYNC_WAIT_NONE 0x0 +#define WAIT_NONE 0x0 /** * @brief Wait for at least one submission to complete. */ -#define ASYNC_WAIT_ONE 0x1 - -/** - * @brief Wait for all submissions to complete. - */ -#define ASYNC_WAIT_ALL SIZE_MAX +#define WAIT_ONE 0x1 /** * @brief System call to initialize the asynchronous rings. @@ -176,7 +169,7 @@ typedef struct async_rings * This system call will populate the given structure with the necessary pointers and metadata for the submission and * completion rings. * - * @note Since each process can only have one rings set, the `async_deinit()` system call must be used before calling + * @note Since each process can only have one rings set, the `teardown()` system call must be used before calling * this function again. * * @param rings Pointer to the structure to populate. @@ -185,14 +178,14 @@ typedef struct async_rings * @param centries Number of entries to allocate for the completion queue, must be a power of two. * @return On success, `0`. On failure, `ERR` and `errno` is set. */ -uint64_t async_init(async_rings_t* rings, void* address, size_t sentries, size_t centries); +uint64_t setup(rings_t* rings, void* address, size_t sentries, size_t centries); /** * @brief System call to deinitialize the asynchronous rings. * * @return On success, `0`. On failure, `ERR` and `errno` is set. */ -uint64_t async_deinit(void); +uint64_t teardown(void); /** * @brief System call to notify the kernel of new submission queue entries (SQEs). @@ -201,18 +194,18 @@ uint64_t async_deinit(void); * @param wait The minimum number of completion queue entries (CQEs) to wait for. * @return On success, the number of SQEs successfully processed. On failure, `ERR` and `errno` is set. */ -uint64_t async_notify(size_t amount, size_t wait); +uint64_t enter(size_t amount, size_t wait); /** * @brief Pushes a submission queue entry (SQE) to the submission queue. * - * After pushing SQEs, `async_notify()` must be called to notify the kernel of the new entries. + * After pushing SQEs, `enter()` must be called to notify the kernel of the new entries. * * @param rings Pointer to the asynchronous rings structure. * @param sqe Pointer to the SQE to push. * @return `true` if the SQE was pushed, `false` if the submission queue is full. */ -static inline bool async_push_sqe(async_rings_t* rings, async_sqe_t* sqe) +static inline bool sqe_push(rings_t* rings, sqe_t* sqe) { uint32_t tail = atomic_load_explicit(&rings->shared->stail, memory_order_relaxed); uint32_t head = atomic_load_explicit(&rings->shared->shead, memory_order_acquire); @@ -235,7 +228,7 @@ static inline bool async_push_sqe(async_rings_t* rings, async_sqe_t* sqe) * @param cqe Pointer to the CQE to pop. * @return `true` if a CQE was popped, `false` if the completion queue is empty. */ -static inline bool async_pop_cqe(async_rings_t* rings, async_cqe_t* cqe) +static inline bool cqe_pop(rings_t* rings, cqe_t* cqe) { uint32_t head = atomic_load_explicit(&rings->shared->chead, memory_order_relaxed); uint32_t tail = atomic_load_explicit(&rings->shared->ctail, memory_order_acquire); diff --git a/src/kernel/cpu/interrupt.c b/src/kernel/cpu/interrupt.c index 9134e8ded..c43b61fab 100644 --- a/src/kernel/cpu/interrupt.c +++ b/src/kernel/cpu/interrupt.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include @@ -221,7 +221,7 @@ void interrupt_handler(interrupt_frame_t* frame) } note_handle_pending(frame); - task_timeouts_check(); + request_timeouts_check(); wait_check_timeouts(frame); sched_do(frame); diff --git a/src/kernel/sched/timer.c b/src/kernel/sched/timer.c index 0253511df..f4dcd0379 100644 --- a/src/kernel/sched/timer.c +++ b/src/kernel/sched/timer.c @@ -136,7 +136,7 @@ uint64_t timer_source_amount(void) return amount; } -void timer_set(clock_t uptime, clock_t deadline) +void timer_set(clock_t now, clock_t deadline) { if (deadline == CLOCKS_NEVER) { @@ -145,7 +145,7 @@ void timer_set(clock_t uptime, clock_t deadline) RWLOCK_READ_SCOPE(&sourcesLock); - deadline = MAX(deadline, uptime + CONFIG_MIN_TIMER_TIMEOUT); + deadline = MAX(deadline, now + CONFIG_MIN_TIMER_TIMEOUT); if (pcpu_timer->deadline <= deadline) { @@ -155,6 +155,6 @@ void timer_set(clock_t uptime, clock_t deadline) if (bestSource != NULL) { - bestSource->set(VECTOR_TIMER, uptime, deadline - uptime); + bestSource->set(VECTOR_TIMER, now, deadline - now); } } \ No newline at end of file diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index 132f9a411..89985804e 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -8,14 +8,14 @@ #include #include -#include +#include #include -static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, async_rings_t* userRings, void* address, size_t sentries, size_t centries) +static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_t* userRings, void* address, size_t sentries, size_t centries) { - async_rings_t* kernelRings = &ctx->rings; + rings_t* kernelRings = &ctx->rings; - size_t pageAmount = BYTES_TO_PAGES(sizeof(async_shared_t) + (sentries * sizeof(async_sqe_t)) + (centries * sizeof(async_cqe_t))); + size_t pageAmount = BYTES_TO_PAGES(sizeof(rings_shared_t) + (sentries * sizeof(sqe_t)) + (centries * sizeof(cqe_t))); if (pageAmount >= CONFIG_MAX_ASYNC_PAGES) { errno = ENOMEM; @@ -63,25 +63,25 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, async_rin list_push_back(&ctx->freeTasks, &requests[i].entry); } - async_shared_t* shared = (async_shared_t*)kernelAddr; + rings_shared_t* shared = (rings_shared_t*)kernelAddr; atomic_init(&shared->shead, 0); atomic_init(&shared->stail, 0); atomic_init(&shared->ctail, 0); atomic_init(&shared->chead, 0); userRings->shared = userAddr; - userRings->squeue = (async_sqe_t*)((uintptr_t)userAddr + sizeof(async_shared_t)); + userRings->squeue = (sqe_t*)((uintptr_t)userAddr + sizeof(rings_shared_t)); userRings->sentries = sentries; userRings->smask = sentries - 1; - userRings->cqueue = (async_cqe_t*)((uintptr_t)userAddr + sizeof(async_shared_t) + (sentries * sizeof(async_sqe_t))); + userRings->cqueue = (cqe_t*)((uintptr_t)userAddr + sizeof(rings_shared_t) + (sentries * sizeof(sqe_t))); userRings->centries = centries; userRings->cmask = centries - 1; kernelRings->shared = kernelAddr; - kernelRings->squeue = (async_sqe_t*)((uintptr_t)kernelAddr + sizeof(async_shared_t)); + kernelRings->squeue = (sqe_t*)((uintptr_t)kernelAddr + sizeof(rings_shared_t)); kernelRings->sentries = sentries; kernelRings->smask = sentries - 1; - kernelRings->cqueue = (async_cqe_t*)((uintptr_t)kernelAddr + sizeof(async_shared_t) + (sentries * sizeof(async_sqe_t))); + kernelRings->cqueue = (cqe_t*)((uintptr_t)kernelAddr + sizeof(rings_shared_t) + (sentries * sizeof(sqe_t))); kernelRings->centries = centries; kernelRings->cmask = centries - 1; @@ -162,9 +162,9 @@ void async_ctx_deinit(async_ctx_t* ctx) static void async_nop_complete(request_nop_t* nop) { - async_cqe_t cqe; + cqe_t cqe; cqe.data = nop->data; - cqe.opcode = ASYNC_OP_NOP; + cqe.opcode = RINGS_NOP; cqe.error = EOK; process_t* process = nop->process; @@ -173,7 +173,7 @@ static void async_nop_complete(request_nop_t* nop) UNREF(nop->process); } -static uint64_t async_handle_sqe(async_ctx_t* ctx, async_sqe_t* sqe) +static uint64_t async_handle_sqe(async_ctx_t* ctx, sqe_t* sqe) { request_t* request = async_ctx_alloc_request(ctx); if (request == NULL) @@ -184,7 +184,7 @@ static uint64_t async_handle_sqe(async_ctx_t* ctx, async_sqe_t* sqe) switch (sqe->opcode) { - case ASYNC_OP_NOP: + case RINGS_NOP: { request_nop_t* nop = (request_nop_t*)request; nop->data = sqe->data; @@ -224,7 +224,7 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) return ERR; } - async_rings_t* rings = &ctx->rings; + rings_t* rings = &ctx->rings; size_t processed = 0; while (processed < amount) @@ -237,7 +237,7 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) break; } - async_sqe_t sqe = rings->squeue[shead & rings->smask]; + sqe_t sqe = rings->squeue[shead & rings->smask]; atomic_store_explicit(&rings->shared->shead, shead + 1, memory_order_release); if (async_handle_sqe(ctx, &sqe) == ERR) @@ -275,7 +275,7 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) return processed; } -SYSCALL_DEFINE(SYS_ASYNC_INIT, uint64_t, async_rings_t* userRings, void* address, size_t sentries, size_t centries) +SYSCALL_DEFINE(SYS_SETUP, uint64_t, rings_t* userRings, void* address, size_t sentries, size_t centries) { if (userRings == NULL || sentries == 0 || centries == 0 || !IS_POW2(sentries) || !IS_POW2(centries)) { @@ -310,7 +310,7 @@ SYSCALL_DEFINE(SYS_ASYNC_INIT, uint64_t, async_rings_t* userRings, void* address return 0; } -SYSCALL_DEFINE(SYS_ASYNC_DEINIT, uint64_t) +SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t) { process_t* process = process_current(); async_ctx_t* ctx = &process->async; @@ -339,7 +339,7 @@ SYSCALL_DEFINE(SYS_ASYNC_DEINIT, uint64_t) return 0; } -SYSCALL_DEFINE(SYS_ASYNC_NOTIFY, uint64_t, size_t amount, size_t wait) +SYSCALL_DEFINE(SYS_ENTER, uint64_t, size_t amount, size_t wait) { process_t* process = process_current(); async_ctx_t* ctx = &process->async; diff --git a/src/kernel/sync/request.c b/src/kernel/sync/request.c index 503773bed..f9a79d2db 100644 --- a/src/kernel/sync/request.c +++ b/src/kernel/sync/request.c @@ -1,25 +1,27 @@ #include +#include #include +#include #include PERCPU_DEFINE_CTOR(request_ctx_t, pcpu_requests) { - request_ctx_t* requests = SELF_PTR(pcpu_requests); + request_ctx_t* ctx = SELF_PTR(pcpu_requests); - list_init(&requests->timeouts); - lock_init(&requests->lock); + list_init(&ctx->timeouts); + lock_init(&ctx->lock); } void request_timeout_add(request_t* request) { - request_ctx_t* requests = SELF_PTR(pcpu_requests); - LOCK_SCOPE(&requests->lock); + request_ctx_t* ctx = SELF_PTR(pcpu_requests); + LOCK_SCOPE(&ctx->lock); - request->ctx = requests; + request->ctx = ctx; request_t* entry; - LIST_FOR_EACH(entry, &requests->timeouts, timeoutEntry) + LIST_FOR_EACH(entry, &ctx->timeouts, timeoutEntry) { if (request->deadline < entry->deadline) { @@ -28,12 +30,14 @@ void request_timeout_add(request_t* request) } } - list_push_back(&requests->timeouts, &request->timeoutEntry); + list_push_back(&ctx->timeouts, &request->timeoutEntry); + + timer_set(clock_uptime(), request->deadline); } void request_timeout_remove(request_t* request) { - assert(request->owner != NULL); + assert(request->ctx != NULL); LOCK_SCOPE(&request->ctx->lock); list_remove(&request->timeoutEntry); @@ -41,15 +45,17 @@ void request_timeout_remove(request_t* request) void request_timeouts_check(void) { - request_ctx_t* requests = SELF_PTR(pcpu_requests); + request_ctx_t* ctx = SELF_PTR(pcpu_requests); + assert(ctx != NULL); + clock_t now = clock_uptime(); - LOCK_SCOPE(&requests->lock); + lock_acquire(&ctx->lock); request_t* request; while (true) { - request = CONTAINER_OF(list_first(&requests->timeouts), request_t, timeoutEntry); + request = CONTAINER_OF_SAFE(list_first(&ctx->timeouts), request_t, timeoutEntry); if (request == NULL) { break; @@ -57,10 +63,18 @@ void request_timeouts_check(void) if (request->deadline > now) { + timer_set(now, request->deadline); break; } list_remove(&request->timeoutEntry); - /// @todo + lock_release(&ctx->lock); + + assert(request->timeout != NULL); + request->timeout(request); + + lock_acquire(&ctx->lock); } + + lock_release(&ctx->lock); } \ No newline at end of file diff --git a/src/libstd/user/common/syscalls.h b/src/libstd/user/common/syscalls.h index f09a41247..127a25481 100644 --- a/src/libstd/user/common/syscalls.h +++ b/src/libstd/user/common/syscalls.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include #include @@ -286,18 +286,18 @@ static inline uint64_t _syscall_arch_prctl(arch_prctl_t code, uintptr_t addr) return _SYSCALL2(uint64_t, SYS_ARCH_PRCTL, arch_prctl_t, code, uintptr_t, addr); } -static inline uint64_t _syscall_async_init(async_rings_t* rings, void* address, size_t sentries, size_t centries) +static inline uint64_t _syscall_setup(rings_t* rings, void* address, size_t sentries, size_t centries) { - return _SYSCALL4(uint64_t, SYS_ASYNC_INIT, async_rings_t*, rings, void*, address, size_t, sentries, size_t, + return _SYSCALL4(uint64_t, SYS_SETUP, rings_t*, rings, void*, address, size_t, sentries, size_t, centries); } -static inline uint64_t _syscall_async_deinit(void) +static inline uint64_t _syscall_teardown(void) { - return _SYSCALL0(uint64_t, SYS_ASYNC_DEINIT); + return _SYSCALL0(uint64_t, SYS_TEARDOWN); } -static inline uint64_t _syscall_async_notify(size_t amount, size_t wait) +static inline uint64_t _syscall_enter(size_t amount, size_t wait) { - return _SYSCALL2(uint64_t, SYS_ASYNC_NOTIFY, size_t, amount, size_t, wait); + return _SYSCALL2(uint64_t, SYS_ENTER, size_t, amount, size_t, wait); } \ No newline at end of file diff --git a/src/libstd/user/functions/async/async_deinit.c b/src/libstd/user/functions/async/async_deinit.c index ad4cc6487..9c5add982 100644 --- a/src/libstd/user/functions/async/async_deinit.c +++ b/src/libstd/user/functions/async/async_deinit.c @@ -1,10 +1,10 @@ -#include +#include #include "user/common/syscalls.h" -uint64_t async_deinit(void) +uint64_t teardown(void) { - uint64_t result = _syscall_async_deinit(); + uint64_t result = _syscall_teardown(); if (result == ERR) { errno = _syscall_errno(); diff --git a/src/libstd/user/functions/async/async_init.c b/src/libstd/user/functions/async/async_init.c index 988434b25..872d05a44 100644 --- a/src/libstd/user/functions/async/async_init.c +++ b/src/libstd/user/functions/async/async_init.c @@ -1,10 +1,10 @@ -#include +#include #include "user/common/syscalls.h" -uint64_t async_init(async_rings_t* rings, void* address, size_t sentries, size_t centries) +uint64_t setup(rings_t* rings, void* address, size_t sentries, size_t centries) { - uint64_t result = _syscall_async_init(rings, address, sentries, centries); + uint64_t result = _syscall_setup(rings, address, sentries, centries); if (result == ERR) { errno = _syscall_errno(); diff --git a/src/libstd/user/functions/async/async_submit.c b/src/libstd/user/functions/async/async_submit.c index 25af9e004..4aee5b915 100644 --- a/src/libstd/user/functions/async/async_submit.c +++ b/src/libstd/user/functions/async/async_submit.c @@ -1,10 +1,10 @@ -#include +#include #include "user/common/syscalls.h" -uint64_t async_notify(size_t amount, size_t wait) +uint64_t enter(size_t amount, size_t wait) { - uint64_t result = _syscall_async_notify(amount, wait); + uint64_t result = _syscall_enter(amount, wait); if (result == ERR) { errno = _syscall_errno(); diff --git a/src/programs/utils/ringstest/main.c b/src/programs/utils/ringstest/main.c new file mode 100644 index 000000000..b169ab013 --- /dev/null +++ b/src/programs/utils/ringstest/main.c @@ -0,0 +1,37 @@ +#include +#include +#include + +#define SENTRIES 64 +#define CENTRIES 64 + +int main() +{ + printf("setting up rings test...\n"); + rings_t rings; + setup(&rings, NULL, SENTRIES, CENTRIES); + + printf("pushing nop sqe...\n"); + sqe_t sqe = SQE_CREATE(RINGS_NOP, SQE_DEFAULT, CLOCKS_PER_SEC, 0x1234); + sqe_push(&rings, &sqe); + + printf("entering rings...\n"); + enter(1, 1); + + printf("popping cqe...\n"); + cqe_t cqe; + cqe_pop(&rings, &cqe); + if (cqe.error != EOK) + { + printf("cqe returned error: %d\n", cqe.error); + return 1; + } + + printf("cqe data: %p\n", cqe.data); + printf("cqe opcode: %d\n", cqe.opcode); + printf("cqe error: %d\n", cqe.error); + + printf("tearing down rings...\n"); + teardown(); + return 0; +} diff --git a/src/programs/utils/ringstest/ringstest.mk b/src/programs/utils/ringstest/ringstest.mk new file mode 100644 index 000000000..d03e4494c --- /dev/null +++ b/src/programs/utils/ringstest/ringstest.mk @@ -0,0 +1,11 @@ +include Make.defaults + +TARGET := $(BINDIR)/$(PROGRAM) + +LDFLAGS += + +all: $(TARGET) + +.PHONY: all + +include Make.rules From 3a7e31c17064c15f2dfa39f11fbc1834e36cb366 Mon Sep 17 00:00:00 2001 From: KN Date: Sat, 17 Jan 2026 10:19:13 +0100 Subject: [PATCH 04/23] feat(kernel:async): first async nop --- include/libstd/sys/rings.h | 5 +++-- src/kernel/sync/async.c | 46 ++++++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/rings.h index 3e44fd380..d7ed1e263 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/rings.h @@ -41,7 +41,9 @@ extern "C" */ typedef enum { + RINGS_MIN_OPCODE = 0, RINGS_NOP = 0, ///< Never completes, can be used to implement a sleep equivalent. + RINGS_MAX_OPCODE, } rings_op_t; /** @@ -50,13 +52,12 @@ typedef enum * * Used to modify the behavior of asynchronous operations. * - * @todo Implement `SQE_LINK` and `SQE_IMMEDIATE`. + * @todo Implement `SQE_LINK`. */ typedef enum { SQE_DEFAULT = 0, SQE_LINK = 1 << 0, ///< Must be completed before the next SQE in the submission queue is started. - SQE_IMMEDIATE = 1 << 1, ///< Fail if the operation cannot be completed immediately. } sqe_flags_t; /** diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index 89985804e..36108c844 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -6,8 +6,11 @@ #include #include #include +#include +#include #include +#include #include #include @@ -175,6 +178,12 @@ static void async_nop_complete(request_nop_t* nop) static uint64_t async_handle_sqe(async_ctx_t* ctx, sqe_t* sqe) { + if (sqe->opcode < RINGS_MIN_OPCODE || sqe->opcode >= RINGS_MAX_OPCODE) + { + errno = EINVAL; + return ERR; + } + request_t* request = async_ctx_alloc_request(ctx); if (request == NULL) { @@ -182,13 +191,16 @@ static uint64_t async_handle_sqe(async_ctx_t* ctx, sqe_t* sqe) return ERR; } + clock_t uptime = clock_uptime(); + request->data = sqe->data; + request->process = REF(process_current()); + request->deadline = CLOCKS_DEADLINE(sqe->timeout, uptime); + switch (sqe->opcode) { case RINGS_NOP: { request_nop_t* nop = (request_nop_t*)request; - nop->data = sqe->data; - nop->process = REF(process_current()); nop->complete = async_nop_complete; nop->cancel = request_nop_cancel; nop->timeout = request_nop_timeout; @@ -196,14 +208,21 @@ static uint64_t async_handle_sqe(async_ctx_t* ctx, sqe_t* sqe) } break; default: - async_ctx_free_request(ctx, request); - errno = EINVAL; - return ERR; + // Impossible due to above check. + panic(NULL, "Invalid opcode %d", sqe->opcode); } return 0; } +static inline uint64_t async_ctx_avail_cqes(async_ctx_t* ctx) +{ + rings_t* rings = &ctx->rings; + uint32_t ctail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); + uint32_t chead = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); + return ctail - chead; +} + uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) { if (amount == 0) @@ -254,21 +273,10 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) return processed; } - while (true) + if (WAIT_BLOCK(&ctx->waitQueue, async_ctx_avail_cqes(ctx) >= wait) == ERR) { - uint32_t ctail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); - uint32_t chead = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); - - if ((ctail - chead) >= wait) - { - break; - } - - if (WAIT_BLOCK(&ctx->waitQueue, false) == ERR) - { - async_ctx_release(ctx); - return processed > 0 ? processed : ERR; - } + async_ctx_release(ctx); + return processed > 0 ? processed : ERR; } async_ctx_release(ctx); From 5a1ade1db8342fef391c955098b58df0ec9e7ead Mon Sep 17 00:00:00 2001 From: KN Date: Sat, 17 Jan 2026 12:51:41 +0100 Subject: [PATCH 05/23] feat(libstd:rings): draft register system --- include/kernel/sync/async.h | 10 +-- include/kernel/sync/request.h | 73 ++++++++++++++++++---- include/kernel/sync/requests.h | 2 +- include/libstd/sys/rings.h | 94 +++++++++++++++++++++-------- src/kernel/sync/async.c | 26 ++++---- src/kernel/sync/request.c | 2 +- src/libstd/user/common/syscalls.h | 5 +- src/programs/utils/ringstest/main.c | 7 ++- 8 files changed, 160 insertions(+), 59 deletions(-) diff --git a/include/kernel/sync/async.h b/include/kernel/sync/async.h index 09ac85f3a..2ebc78236 100644 --- a/include/kernel/sync/async.h +++ b/include/kernel/sync/async.h @@ -1,11 +1,11 @@ #pragma once #include +#include #include #include #include #include -#include #include #include @@ -15,6 +15,8 @@ * @defgroup kernel_sync_async Async * @ingroup kernel_sync * + * @see libstd_rings for the userspace rings API. + * * @{ */ @@ -35,9 +37,9 @@ typedef enum */ typedef struct async_ctx { - rings_t rings; ///< Asynchronous rings information. - request_t* requests; ///< A preallocated array of requests, one for each CQE. - list_t freeTasks; ///< Free list of tasks. + rings_t rings; ///< Asynchronous rings information. + request_t* requests; ///< A preallocated array of requests, one for each CQE. + list_t freeTasks; ///< Free list of tasks. void* userAddr; ///< Userspace address of the rings. void* kernelAddr; ///< Kernel address of the rings. size_t pageAmount; ///< Amount of pages mapped for the rings. diff --git a/include/kernel/sync/request.h b/include/kernel/sync/request.h index 0d8b98ed5..7e138ed02 100644 --- a/include/kernel/sync/request.h +++ b/include/kernel/sync/request.h @@ -17,21 +17,24 @@ typedef struct process process_t; * @ingroup kernel_sync * * ## Callbacks - * + * * Requests can define three callbacks, included is a list of their expected semantics. - * + * * ### Completion Callback - * - * The `complete()` callback should be called when the request has been completed, the `complete()` implementation does not need to guarantee that the request structure will remain valid after a call to this function. - * + * + * The `complete()` callback should be called when the request has been completed, the `complete()` implementation does + * not need to guarantee that the request structure will remain valid after a call to this function. + * * ### Cancellation Callback - * - * The optional `cancel()` callback is called when attempting to cancel an in-progress request, if the request cannot be cancelled, the callback should return `false`, otherwise `true`. - * + * + * The optional `cancel()` callback is called when attempting to cancel an in-progress request, if the request cannot be + * cancelled, the callback should return `false`, otherwise `true`. + * * ### Timeout Callback * - * The optional `timeout()` callback is called when a request has timed out, the request * will be removed from the timeout queue before this callback is called and it will never * be called more than once. - * + * The optional `timeout()` callback is called when a request has timed out, the request * will be removed from the + * timeout queue before this callback is called and it will never * be called more than once. + * * @{ */ @@ -93,8 +96,8 @@ typedef enum * @brief Generic request structure. * @struct request_t * - * @warning Due to optimization done while allocating requests in the async system, no request structure should be larger than - * this structure. + * @warning Due to optimization done while allocating requests in the async system, no request structure should be + * larger than this structure. */ typedef struct request { @@ -163,6 +166,13 @@ void request_timeouts_check(void); (_request)->result = (typeof((_request)->result))0; \ }) +/** + * @brief Macro to call a function with a request and handle early completions. + * + * @param _request Pointer to the request. + * @param _func Function to call with the request. + * @return The result of the function call. + */ #define REQUEST_CALL(_request, _func) \ ({ \ typeof((_request)->result) result = _func(_request); \ @@ -179,6 +189,14 @@ void request_timeouts_check(void); result; \ }) +/** + * @brief Macro to delay the completion of a request without adding it to a queue. + * + * Primarily intended for use with timeout handling. + * + * @param _request Pointer to the request to delay. + * @return On success, `0`. On failure, `ERR` and `errno` is set. + */ #define REQUEST_DELAY_NO_QUEUE(_request) \ ({ \ uint64_t result = 0; \ @@ -199,6 +217,12 @@ void request_timeouts_check(void); result; \ }) +/** + * @brief Macro to delay the completion of a request. + * + * @param _request Pointer to the request to delay. + * @param _queue Pointer to the request queue to add the request to. + */ #define REQUEST_DELAY(_request, _queue) \ ({ \ list_push_back(&(_queue)->requests, &(_request)->entry); \ @@ -210,9 +234,22 @@ void request_timeouts_check(void); result; \ }) +/** + * @brief Macro to get the next request from a queue. + * + * @param _queue Pointer to the request queue. + * @param _type The type of the request structure. + * @return Pointer to the next request, or `NULL` if the queue is empty. + */ #define REQUEST_NEXT(_queue, _type) \ (list_is_empty(&(_queue)->requests) ? NULL : CONTAINER_OF(list_first(&(_queue)->requests), _type, entry)) +/** + * @brief Macro to complete a request with an error. + * + * @param _request Pointer to the request. + * @param _errno The errno code. + */ #define REQUEST_ERROR(_request, _errno) \ ({ \ if ((_request)->flags & REQUEST_TIMEOUT) \ @@ -225,6 +262,12 @@ void request_timeouts_check(void); (_request)->complete((_request)); \ }) +/** + * @brief Macro to complete a request. + * + * @param _request Pointer to the request. + * @param _result The result of the request. + */ #define REQUEST_COMPLETE(_request, _result) \ ({ \ if ((_request)->flags & REQUEST_TIMEOUT) \ @@ -237,6 +280,12 @@ void request_timeouts_check(void); (_request)->complete((_request)); \ }) +/** + * @brief Macro to cancel a request. + * + * @param _request Pointer to the request. + * @return On success, `0`. On failure, `ERR` and `errno` is set. + */ #define REQUEST_CANCEL(_request) \ ({ \ uint64_t result = 0; \ diff --git a/include/kernel/sync/requests.h b/include/kernel/sync/requests.h index f66845871..0ee017f7d 100644 --- a/include/kernel/sync/requests.h +++ b/include/kernel/sync/requests.h @@ -6,7 +6,7 @@ * @brief Kernel Request Implementations * @defgroup kernel_sync_requests Requests * @ingroup kernel_sync - * + * * @{ */ diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/rings.h index d7ed1e263..316431bf0 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/rings.h @@ -4,8 +4,8 @@ #include #include #include -#include #include +#include #if defined(__cplusplus) extern "C" @@ -18,16 +18,33 @@ extern "C" #include "_internal/fd_t.h" /** - * @brief Ring-based submission/completion interface. + * @brief Programmable submission/completion interface. * @defgroup libstd_rings Rings * @ingroup libstd * + * @todo The rings system is primarily a design document for now as it remains very work in progress and subject to + * change, currently being mostly unimplemented. + * * Asynchronous operations provide the core of all IO interfaces in PatchworkOS, all implemented in an interface * inspired by `io_uring()` from Linux. * * Synchronous operations are implemented on top of this API in userspace. * - * @todo The async system is not currently implemented, this is more just a draft for now. + * ## Registers + * + * Operations performed on a ring can load arguments from, and save their results to, seven 64-bit general purpose + * registers. All registers are initialized to zero. + * + * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for the result, and for + * each argument. Each specifier is stored as three bits, with a zero value indicating no-op and any other value + * representing the n-th register. The offset of the specifier specifies its meaning, for example, the first specifier + * (bits 0-2) specifies which register to save the result into, while the second specifier (bits 3-5) specifies which + * register to load into the first argument, and so on. + * + * This system, when combined with `SQE_LINK`, allows for multiple operations to be performed at once, for example, it + * would be possible to open a file, read from it, and close it, with a single `enter()` call. + * + * @see `sqe_flags_t` for more information about register specifiers and their formatting. * * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`. * @see [Manpages](https://man7.org/linux/man-pages/man7/io_uring.7.html) for more information about `io_uring`. @@ -36,49 +53,79 @@ extern "C" */ /** - * @brief Asynchronous operation codes. + * @brief Rings operation codes. * @enum rings_op_t */ typedef enum { RINGS_MIN_OPCODE = 0, - RINGS_NOP = 0, ///< Never completes, can be used to implement a sleep equivalent. - RINGS_MAX_OPCODE, + RINGS_NOP = 0, ///< Never completes, can be used to implement a sleep equivalent by specifying a timeout. + RINGS_MAX_OPCODE = 1, } rings_op_t; /** - * @brief Asynchronous sequence flags. - * @enum sqe_flags_t + * @brief Maximum number of arguments for a rings operation. + */ +#define SEQ_MAX_ARGS 5 + +/** + * @brief Rings register specifiers. + * @enum seq_regs_t * - * Used to modify the behavior of asynchronous operations. + * Used in the `sqe_flags_t` enum to specify which registers to load into arguments or save the result into. * - * @todo Implement `SQE_LINK`. */ typedef enum { - SQE_DEFAULT = 0, - SQE_LINK = 1 << 0, ///< Must be completed before the next SQE in the submission queue is started. + SQE_REG_NONE = 0, ///< Dont perform any save or load operation. + SQE_REG1 = 1, ///< The first register. + SQE_REG2 = 2, ///< The second register. + SQE_REG3 = 3, ///< The third register. + SQE_REG4 = 4, ///< The fourth register. + SQE_REG5 = 5, ///< The fifth register. + SQE_REG6 = 6, ///< The sixth register. + SQE_REG7 = 7, ///< The seventh register. + SQE_REG_SHIFT = 3, ///< The bitshift for each register specifier in a `sqe_flags_t`. + SQE_REG_MASK = 0b111, ///< The bitmask for a register specifier in a `sqe_flags_t`. +} seq_regs_t; + +/** + * @brief Submission queue entry (SQE) flags. + * @enum sqe_flags_t + */ +typedef enum +{ + SQE_SAVE = 0, ///< The offset to specify the register to save the result into. + SQE_LOAD0 = SQE_SAVE + SQE_REG_SHIFT, ///< The offset to specify which register to load into the first argument. + SQE_LOAD1 = SQE_LOAD0 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the second argument. + SQE_LOAD2 = SQE_LOAD1 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the third argument. + SQE_LOAD3 = SQE_LOAD2 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fourth argument. + SQE_LOAD4 = SQE_LOAD3 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fifth argument. + SQE_FLAGS_SHIFT = SQE_LOAD4 + SQE_REG_SHIFT, ///< The bitshift for where bit flags start in a `sqe_flags_t`. + SQE_LINK = 1 << SQE_FLAGS_SHIFT, ///< Link this operation to the next SQE, only process the next SQE if and when + ///< this one completes successfully. + SQE_RESET = 1 << (SQE_FLAGS_SHIFT + 1), ///< Reset registers before processing this SQE. } sqe_flags_t; /** * @brief Asynchronous submission queue entry (SQE). * @struct sqe_t * - * @warning For operations such as `ASYNC_OP_OPEN`, it is the responsibility of userspace to ensure that any pointers + * @warning It is the responsibility of userspace to ensure that any pointers * passed to the kernel remain valid until the operation is complete. */ typedef struct sqe { - rings_op_t opcode; ///< Operation code. - sqe_flags_t flags; ///< Sequence flags. - clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. - void* data; ///< Private data for the operation, will be returned in the completion entry. + rings_op_t opcode; ///< Operation code. + sqe_flags_t flags; ///< Submission flags. + clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. + void* data; ///< Private data for the operation, will be returned in the completion entry. union { struct { } nop; - uint64_t _raw[5]; + uint64_t args[SEQ_MAX_ARGS]; }; } sqe_t; @@ -90,7 +137,7 @@ static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); * @brief Macro to create an asynchronous submission queue entry (SQE). * * @param _opcode Operation code. - * @param _flags Sequence flags. + * @param _flags Submission flags. * @param _timeout Timeout for the operation, `CLOCKS_NEVER` for no timeout. * @param _data Private data for the operation. */ @@ -106,19 +153,18 @@ static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); * @brief Asynchronous completion queue entry (CQE). * @struct cqe_t */ -typedef struct ALIGNED(64) cqe +typedef struct ALIGNED(32) cqe { void* data; ///< Private data from the submission entry. rings_op_t opcode; ///< Operation code from the submission entry. errno_t error; ///< Error code, if not equal to `EOK` an error occurred. union { - size_t read; ///< The number of bytes read from `ASYNC_OP_READ`. uint64_t _raw; }; } cqe_t; #ifdef static_assert -static_assert(sizeof(cqe_t) == 64, "cqe_t is not 64 bytes"); +static_assert(sizeof(cqe_t) == 32, "cqe_t is not 32 bytes"); #endif /** * @brief Shared asynchronous rings structure. @@ -146,10 +192,10 @@ typedef struct ALIGNED(64) rings_shared typedef struct rings { rings_shared_t* shared; ///< Pointer to the shared structure. - sqe_t* squeue; ///< Pointer to the submission queue. + sqe_t* squeue; ///< Pointer to the submission queue. size_t sentries; ///< Number of entries in the submission queue. size_t smask; ///< Bitmask for submission queue (sentries - 1). - cqe_t* cqueue; ///< Pointer to the completion queue. + cqe_t* cqueue; ///< Pointer to the completion queue. size_t centries; ///< Number of entries in the completion queue. size_t cmask; ///< Bitmask for completion queue (centries - 1). } rings_t; diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index 36108c844..29e138dab 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -1,24 +1,26 @@ +#include +#include +#include #include -#include #include +#include #include +#include #include -#include -#include #include -#include -#include #include -#include -#include #include +#include +#include -static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_t* userRings, void* address, size_t sentries, size_t centries) +static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_t* userRings, void* address, + size_t sentries, size_t centries) { rings_t* kernelRings = &ctx->rings; - size_t pageAmount = BYTES_TO_PAGES(sizeof(rings_shared_t) + (sentries * sizeof(sqe_t)) + (centries * sizeof(cqe_t))); + size_t pageAmount = + BYTES_TO_PAGES(sizeof(rings_shared_t) + (sentries * sizeof(sqe_t)) + (centries * sizeof(cqe_t))); if (pageAmount >= CONFIG_MAX_ASYNC_PAGES) { errno = ENOMEM; @@ -85,7 +87,7 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_t* kernelRings->sentries = sentries; kernelRings->smask = sentries - 1; kernelRings->cqueue = (cqe_t*)((uintptr_t)kernelAddr + sizeof(rings_shared_t) + (sentries * sizeof(sqe_t))); - kernelRings->centries = centries; + kernelRings->centries = centries; kernelRings->cmask = centries - 1; ctx->requests = requests; @@ -169,7 +171,7 @@ static void async_nop_complete(request_nop_t* nop) cqe.data = nop->data; cqe.opcode = RINGS_NOP; cqe.error = EOK; - + process_t* process = nop->process; async_ctx_push_cqe(&process->async, &cqe); async_ctx_free_request(&process->async, (request_t*)nop); @@ -294,7 +296,7 @@ SYSCALL_DEFINE(SYS_SETUP, uint64_t, rings_t* userRings, void* address, size_t se process_t* process = process_current(); async_ctx_t* ctx = &process->async; space_t* space = &process->space; - + if (async_ctx_acquire(ctx) == ERR) { errno = EBUSY; diff --git a/src/kernel/sync/request.c b/src/kernel/sync/request.c index f9a79d2db..56532e246 100644 --- a/src/kernel/sync/request.c +++ b/src/kernel/sync/request.c @@ -1,7 +1,7 @@ #include +#include #include #include -#include #include diff --git a/src/libstd/user/common/syscalls.h b/src/libstd/user/common/syscalls.h index 127a25481..66ae7541b 100644 --- a/src/libstd/user/common/syscalls.h +++ b/src/libstd/user/common/syscalls.h @@ -3,9 +3,9 @@ #include #include -#include #include #include +#include #include #define _SYSCALL0(retType, num) \ @@ -288,8 +288,7 @@ static inline uint64_t _syscall_arch_prctl(arch_prctl_t code, uintptr_t addr) static inline uint64_t _syscall_setup(rings_t* rings, void* address, size_t sentries, size_t centries) { - return _SYSCALL4(uint64_t, SYS_SETUP, rings_t*, rings, void*, address, size_t, sentries, size_t, - centries); + return _SYSCALL4(uint64_t, SYS_SETUP, rings_t*, rings, void*, address, size_t, sentries, size_t, centries); } static inline uint64_t _syscall_teardown(void) diff --git a/src/programs/utils/ringstest/main.c b/src/programs/utils/ringstest/main.c index b169ab013..f7e496fcd 100644 --- a/src/programs/utils/ringstest/main.c +++ b/src/programs/utils/ringstest/main.c @@ -1,5 +1,5 @@ -#include #include +#include #include #define SENTRIES 64 @@ -12,7 +12,10 @@ int main() setup(&rings, NULL, SENTRIES, CENTRIES); printf("pushing nop sqe...\n"); - sqe_t sqe = SQE_CREATE(RINGS_NOP, SQE_DEFAULT, CLOCKS_PER_SEC, 0x1234); + sqe_t sqe = SQE_CREATE(RINGS_NOP, SQE_REG1 << SQE_SAVE, CLOCKS_PER_SEC, 0x1234); + sqe_push(&rings, &sqe); + + sqe = (sqe_t)SQE_CREATE(RINGS_NOP, SQE_REG1 << SQE_LOAD0, CLOCKS_PER_SEC, 0x5678); sqe_push(&rings, &sqe); printf("entering rings...\n"); From 91213b565c162bf6b15f0c1d42f109c2eef08260 Mon Sep 17 00:00:00 2001 From: KN Date: Sat, 17 Jan 2026 13:50:21 +0100 Subject: [PATCH 06/23] feat(kernel:async): support multiple rings per process --- include/kernel/proc/process.h | 2 +- include/kernel/sync/async.h | 5 +- include/kernel/sync/request.h | 29 +++--- include/libstd/sys/rings.h | 23 +++-- src/kernel/proc/process.c | 10 ++- src/kernel/sync/async.c | 89 +++++++++++++------ src/libstd/user/common/syscalls.h | 8 +- src/libstd/user/functions/async/async_init.c | 13 --- .../async/{async_submit.c => enter.c} | 4 +- src/libstd/user/functions/async/setup.c | 13 +++ .../async/{async_deinit.c => teardown.c} | 4 +- src/programs/utils/ringstest/main.c | 8 +- 12 files changed, 127 insertions(+), 81 deletions(-) delete mode 100644 src/libstd/user/functions/async/async_init.c rename src/libstd/user/functions/async/{async_submit.c => enter.c} (56%) create mode 100644 src/libstd/user/functions/async/setup.c rename src/libstd/user/functions/async/{async_deinit.c => teardown.c} (65%) diff --git a/include/kernel/proc/process.h b/include/kernel/proc/process.h index faf725663..3b47033fa 100644 --- a/include/kernel/proc/process.h +++ b/include/kernel/proc/process.h @@ -88,7 +88,7 @@ typedef struct process file_table_t fileTable; futex_ctx_t futexCtx; perf_process_ctx_t perf; - async_ctx_t async; + async_ctx_t async[CONFIG_MAX_ASYNC_RINGS]; note_handler_t noteHandler; wait_queue_t suspendQueue; wait_queue_t dyingQueue; diff --git a/include/kernel/sync/async.h b/include/kernel/sync/async.h index 2ebc78236..a87f56e0e 100644 --- a/include/kernel/sync/async.h +++ b/include/kernel/sync/async.h @@ -27,7 +27,7 @@ typedef enum { ASYNC_CTX_NONE = 0, ///< No flags set. - ASYNC_CTX_BUSY = 1 << 0, ///< Context is currently being used. + ASYNC_CTX_BUSY = 1 << 0, ///< Context is currently being used, used for fast locking. ASYNC_CTX_MAPPED = 1 << 1, ///< Context rings are mapped. } async_ctx_flags_t; @@ -39,7 +39,8 @@ typedef struct async_ctx { rings_t rings; ///< Asynchronous rings information. request_t* requests; ///< A preallocated array of requests, one for each CQE. - list_t freeTasks; ///< Free list of tasks. + list_t freeRequests; ///< Free list of requests. + uint64_t pending; ///< The amount of pending requests. void* userAddr; ///< Userspace address of the rings. void* kernelAddr; ///< Kernel address of the rings. size_t pageAmount; ///< Amount of pages mapped for the rings. diff --git a/include/kernel/sync/request.h b/include/kernel/sync/request.h index 7e138ed02..96310c293 100644 --- a/include/kernel/sync/request.h +++ b/include/kernel/sync/request.h @@ -9,6 +9,7 @@ #include #include +typedef struct async_ctx async_ctx_t; typedef struct process process_t; /** @@ -62,18 +63,18 @@ typedef enum * @brief Macro to define common request structure members. * * All requests contain the below common members: - * - `list_entry_t entry` - List entry for requests queues and completion queues. - * - `list_entry_t timeoutEntry` - List entry for timeout queues. - * - `request_ctx_t* ctx` - Pointer to the per-CPU request context storing this request for timeouts. - * - `process_t* process` - Pointer to the process that created the request. - * - `void* data` - Pointer to user data. - * - `void (*complete)(_type*)` - Completion callback. - * - `bool (*cancel)(_type*)` - Cancellation callback. - * - `void (*timeout)(_type*)` - Timeout callback. - * - `request_flags_t flags` - Task flags. - * - `errno_t err` - Error code for the request. - * - `clock_t deadline` - Deadline for the request. - * - `_resultType result` - Result of the request. + * - `list_entry_t entry`: List entry for requests queues and completion queues. + * - `list_entry_t timeoutEntry`: List entry for timeout queues. + * - `request_ctx_t* ctx`: Pointer to the per-CPU request context storing this request for timeouts. + * - `async_ctx_t* async`: Pointer to the async context that created this request, or `NULL`. + * - `void* data`: Pointer to user data. + * - `void (*complete)(_type*)`: Completion callback. + * - `bool (*cancel)(_type*)`: Cancellation callback. + * - `void (*timeout)(_type*)`: Timeout callback. + * - `request_flags_t flags`: Task flags. + * - `errno_t err`: Error code for the request. + * - `clock_t deadline`: Deadline for the request. + * - `_resultType result`: Result of the request. * * @param _type The type of the request structure. * @param _resultType The type of the request result. @@ -82,7 +83,7 @@ typedef enum list_entry_t entry; \ list_entry_t timeoutEntry; \ request_ctx_t* ctx; \ - process_t* process; \ + async_ctx_t* async; \ void* data; \ void (*complete)(_type*); \ bool (*cancel)(_type*); \ @@ -155,7 +156,7 @@ void request_timeouts_check(void); (_request)->entry = LIST_ENTRY_CREATE((_request)->entry); \ (_request)->timeoutEntry = LIST_ENTRY_CREATE((_request)->timeoutEntry); \ (_request)->ctx = NULL; \ - (_request)->process = NULL; \ + (_request)->async = NULL; \ (_request)->data = NULL; \ (_request)->complete = NULL; \ (_request)->cancel = NULL; \ diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/rings.h index 316431bf0..0d9b34b08 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/rings.h @@ -102,8 +102,7 @@ typedef enum SQE_LOAD3 = SQE_LOAD2 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fourth argument. SQE_LOAD4 = SQE_LOAD3 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fifth argument. SQE_FLAGS_SHIFT = SQE_LOAD4 + SQE_REG_SHIFT, ///< The bitshift for where bit flags start in a `sqe_flags_t`. - SQE_LINK = 1 << SQE_FLAGS_SHIFT, ///< Link this operation to the next SQE, only process the next SQE if and when - ///< this one completes successfully. + SQE_LINK = 1 << (SQE_FLAGS_SHIFT), ///< =nly process the next SQE if and when this one completes successfully. SQE_RESET = 1 << (SQE_FLAGS_SHIFT + 1), ///< Reset registers before processing this SQE. } sqe_flags_t; @@ -166,6 +165,12 @@ typedef struct ALIGNED(32) cqe #ifdef static_assert static_assert(sizeof(cqe_t) == 32, "cqe_t is not 32 bytes"); #endif + +/** + * @brief Rings ID type. + */ +typedef uint64_t rings_id_t; + /** * @brief Shared asynchronous rings structure. * @struct rings_shared_t @@ -192,6 +197,7 @@ typedef struct ALIGNED(64) rings_shared typedef struct rings { rings_shared_t* shared; ///< Pointer to the shared structure. + rings_id_t id; ///< The ID of the rings. sqe_t* squeue; ///< Pointer to the submission queue. size_t sentries; ///< Number of entries in the submission queue. size_t smask; ///< Bitmask for submission queue (sentries - 1). @@ -216,32 +222,31 @@ typedef struct rings * This system call will populate the given structure with the necessary pointers and metadata for the submission and * completion rings. * - * @note Since each process can only have one rings set, the `teardown()` system call must be used before calling - * this function again. - * * @param rings Pointer to the structure to populate. * @param address Desired address to allocate the rings, or `NULL` to let the kernel choose. * @param sentries Number of entires to allocate for the submission queue, must be a power of two. * @param centries Number of entries to allocate for the completion queue, must be a power of two. - * @return On success, `0`. On failure, `ERR` and `errno` is set. + * @return On success, the ring ID. On failure, `ERR` and `errno` is set. */ -uint64_t setup(rings_t* rings, void* address, size_t sentries, size_t centries); +rings_id_t setup(rings_t* rings, void* address, size_t sentries, size_t centries); /** * @brief System call to deinitialize the asynchronous rings. * + * @param id The ID of the rings to deinitialize. * @return On success, `0`. On failure, `ERR` and `errno` is set. */ -uint64_t teardown(void); +uint64_t teardown(rings_id_t id); /** * @brief System call to notify the kernel of new submission queue entries (SQEs). * + * @param id The ID of the rings to notify. * @param amount The number of SQEs that the kernel should process. * @param wait The minimum number of completion queue entries (CQEs) to wait for. * @return On success, the number of SQEs successfully processed. On failure, `ERR` and `errno` is set. */ -uint64_t enter(size_t amount, size_t wait); +uint64_t enter(rings_id_t id, size_t amount, size_t wait); /** * @brief Pushes a submission queue entry (SQE) to the submission queue. diff --git a/src/kernel/proc/process.c b/src/kernel/proc/process.c index fdad878a8..6e114a190 100644 --- a/src/kernel/proc/process.c +++ b/src/kernel/proc/process.c @@ -111,7 +111,10 @@ static void process_free(process_t* process) } space_deinit(&process->space); futex_ctx_deinit(&process->futexCtx); - async_ctx_deinit(&process->async); + for (uint64_t i = 0; i < CONFIG_MAX_ASYNC_RINGS; i++) + { + async_ctx_deinit(&process->async[i]); + } wait_queue_deinit(&process->dyingQueue); wait_queue_deinit(&process->suspendQueue); env_deinit(&process->env); @@ -151,7 +154,10 @@ process_t* process_new(priority_t priority, group_member_t* group, namespace_t* file_table_init(&process->fileTable); futex_ctx_init(&process->futexCtx); perf_process_ctx_init(&process->perf); - async_ctx_init(&process->async); + for (uint64_t i = 0; i < CONFIG_MAX_ASYNC_RINGS; i++) + { + async_ctx_init(&process->async[i]); + } note_handler_init(&process->noteHandler); wait_queue_init(&process->suspendQueue); wait_queue_init(&process->dyingQueue); diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index 29e138dab..f8caea10f 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -14,7 +14,7 @@ #include #include -static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_t* userRings, void* address, +static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_t id, rings_t* userRings, void* address, size_t sentries, size_t centries) { rings_t* kernelRings = &ctx->rings; @@ -65,7 +65,7 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_t* for (size_t i = 0; i < centries; i++) { REQUEST_INIT(&requests[i]); - list_push_back(&ctx->freeTasks, &requests[i].entry); + list_push_back(&ctx->freeRequests, &requests[i].entry); } rings_shared_t* shared = (rings_shared_t*)kernelAddr; @@ -75,6 +75,7 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_t* atomic_init(&shared->chead, 0); userRings->shared = userAddr; + userRings->id = id; userRings->squeue = (sqe_t*)((uintptr_t)userAddr + sizeof(rings_shared_t)); userRings->sentries = sentries; userRings->smask = sentries - 1; @@ -83,6 +84,7 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_t* userRings->cmask = centries - 1; kernelRings->shared = kernelAddr; + kernelRings->id = id; kernelRings->squeue = (sqe_t*)((uintptr_t)kernelAddr + sizeof(rings_shared_t)); kernelRings->sentries = sentries; kernelRings->smask = sentries - 1; @@ -102,7 +104,7 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_t* static inline uint64_t async_ctx_unmap(async_ctx_t* ctx) { - list_init(&ctx->freeTasks); + list_init(&ctx->freeRequests); free(ctx->requests); ctx->requests = NULL; @@ -115,17 +117,17 @@ static inline uint64_t async_ctx_unmap(async_ctx_t* ctx) static inline request_t* async_ctx_alloc_request(async_ctx_t* ctx) { - if (list_is_empty(&ctx->freeTasks)) + if (list_is_empty(&ctx->freeRequests)) { return NULL; } - return CONTAINER_OF(list_pop_back(&ctx->freeTasks), request_t, entry); + return CONTAINER_OF(list_pop_back(&ctx->freeRequests), request_t, entry); } static inline void async_ctx_free_request(async_ctx_t* ctx, request_t* request) { - list_push_back(&ctx->freeTasks, &request->entry); + list_push_back(&ctx->freeRequests, &request->entry); } void async_ctx_init(async_ctx_t* ctx) @@ -135,8 +137,14 @@ void async_ctx_init(async_ctx_t* ctx) return; } - memset(ctx, 0, sizeof(async_ctx_t)); - list_init(&ctx->freeTasks); + ctx->rings = (rings_t){0}; + ctx->requests = NULL; + list_init(&ctx->freeRequests); + ctx->pending = 0; + ctx->userAddr = NULL; + ctx->kernelAddr = NULL; + ctx->pageAmount = 0; + ctx->space = NULL; wait_queue_init(&ctx->waitQueue); atomic_init(&ctx->flags, ASYNC_CTX_NONE); } @@ -172,10 +180,16 @@ static void async_nop_complete(request_nop_t* nop) cqe.opcode = RINGS_NOP; cqe.error = EOK; - process_t* process = nop->process; - async_ctx_push_cqe(&process->async, &cqe); - async_ctx_free_request(&process->async, (request_t*)nop); - UNREF(nop->process); + async_ctx_t* async = nop->async; + assert(async != NULL); + assert(async->rings.id < CONFIG_MAX_ASYNC_RINGS); + + //process_t* process = CONTAINER_OF(async, process_t, async[async->rings.id]); + + async_ctx_push_cqe(async, &cqe); + async_ctx_free_request(async, (request_t*)nop); + + /// @todo What to do if the process gets killed? } static uint64_t async_handle_sqe(async_ctx_t* ctx, sqe_t* sqe) @@ -195,7 +209,7 @@ static uint64_t async_handle_sqe(async_ctx_t* ctx, sqe_t* sqe) clock_t uptime = clock_uptime(); request->data = sqe->data; - request->process = REF(process_current()); + request->async = ctx; request->deadline = CLOCKS_DEADLINE(sqe->timeout, uptime); switch (sqe->opcode) @@ -232,6 +246,10 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) return 0; } + /// @todo Implement the register state logic. + + /// @todo Implement SQE_LINK. + if (async_ctx_acquire(ctx) == ERR) { errno = EBUSY; @@ -285,7 +303,7 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) return processed; } -SYSCALL_DEFINE(SYS_SETUP, uint64_t, rings_t* userRings, void* address, size_t sentries, size_t centries) +SYSCALL_DEFINE(SYS_SETUP, rings_id_t, rings_t* userRings, void* address, size_t sentries, size_t centries) { if (userRings == NULL || sentries == 0 || centries == 0 || !IS_POW2(sentries) || !IS_POW2(centries)) { @@ -294,37 +312,46 @@ SYSCALL_DEFINE(SYS_SETUP, uint64_t, rings_t* userRings, void* address, size_t se } process_t* process = process_current(); - async_ctx_t* ctx = &process->async; space_t* space = &process->space; - if (async_ctx_acquire(ctx) == ERR) + async_ctx_t* ctx = NULL; + rings_id_t id = 0; + for (id = 0; id < CONFIG_MAX_ASYNC_RINGS; id++) { - errno = EBUSY; - return ERR; + async_ctx_flags_t expected = ASYNC_CTX_NONE; + if (atomic_compare_exchange_strong(&process->async[id].flags, &expected, ASYNC_CTX_BUSY)) + { + ctx = &process->async[id]; + break; + } } - if (atomic_load(&ctx->flags) & ASYNC_CTX_MAPPED) + if (ctx == NULL) { - async_ctx_release(ctx); - errno = EBUSY; + errno = EMFILE; return ERR; } - if (async_ctx_map(ctx, space, userRings, address, sentries, centries) == ERR) + if (async_ctx_map(ctx, space, id, userRings, address, sentries, centries) == ERR) { async_ctx_release(ctx); return ERR; } async_ctx_release(ctx); - return 0; + return id; } -SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t) +SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, rings_id_t id) { + if (id >= CONFIG_MAX_ASYNC_RINGS) + { + errno = EINVAL; + return ERR; + } + process_t* process = process_current(); - async_ctx_t* ctx = &process->async; - space_t* space = &process->space; + async_ctx_t* ctx = &process->async[id]; if (async_ctx_acquire(ctx) == ERR) { @@ -349,10 +376,16 @@ SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t) return 0; } -SYSCALL_DEFINE(SYS_ENTER, uint64_t, size_t amount, size_t wait) +SYSCALL_DEFINE(SYS_ENTER, uint64_t, rings_id_t id, size_t amount, size_t wait) { + if (id >= CONFIG_MAX_ASYNC_RINGS) + { + errno = EINVAL; + return ERR; + } + process_t* process = process_current(); - async_ctx_t* ctx = &process->async; + async_ctx_t* ctx = &process->async[id]; return async_ctx_notify(ctx, amount, wait); } \ No newline at end of file diff --git a/src/libstd/user/common/syscalls.h b/src/libstd/user/common/syscalls.h index 66ae7541b..000baca15 100644 --- a/src/libstd/user/common/syscalls.h +++ b/src/libstd/user/common/syscalls.h @@ -291,12 +291,12 @@ static inline uint64_t _syscall_setup(rings_t* rings, void* address, size_t sent return _SYSCALL4(uint64_t, SYS_SETUP, rings_t*, rings, void*, address, size_t, sentries, size_t, centries); } -static inline uint64_t _syscall_teardown(void) +static inline uint64_t _syscall_teardown(rings_id_t id) { - return _SYSCALL0(uint64_t, SYS_TEARDOWN); + return _SYSCALL1(uint64_t, SYS_TEARDOWN, rings_id_t, id); } -static inline uint64_t _syscall_enter(size_t amount, size_t wait) +static inline uint64_t _syscall_enter(rings_id_t id, size_t amount, size_t wait) { - return _SYSCALL2(uint64_t, SYS_ENTER, size_t, amount, size_t, wait); + return _SYSCALL3(uint64_t, SYS_ENTER, rings_id_t, id, size_t, amount, size_t, wait); } \ No newline at end of file diff --git a/src/libstd/user/functions/async/async_init.c b/src/libstd/user/functions/async/async_init.c deleted file mode 100644 index 872d05a44..000000000 --- a/src/libstd/user/functions/async/async_init.c +++ /dev/null @@ -1,13 +0,0 @@ -#include - -#include "user/common/syscalls.h" - -uint64_t setup(rings_t* rings, void* address, size_t sentries, size_t centries) -{ - uint64_t result = _syscall_setup(rings, address, sentries, centries); - if (result == ERR) - { - errno = _syscall_errno(); - } - return result; -} \ No newline at end of file diff --git a/src/libstd/user/functions/async/async_submit.c b/src/libstd/user/functions/async/enter.c similarity index 56% rename from src/libstd/user/functions/async/async_submit.c rename to src/libstd/user/functions/async/enter.c index 4aee5b915..865335dba 100644 --- a/src/libstd/user/functions/async/async_submit.c +++ b/src/libstd/user/functions/async/enter.c @@ -2,9 +2,9 @@ #include "user/common/syscalls.h" -uint64_t enter(size_t amount, size_t wait) +uint64_t enter(rings_id_t id, size_t amount, size_t wait) { - uint64_t result = _syscall_enter(amount, wait); + uint64_t result = _syscall_enter(id, amount, wait); if (result == ERR) { errno = _syscall_errno(); diff --git a/src/libstd/user/functions/async/setup.c b/src/libstd/user/functions/async/setup.c new file mode 100644 index 000000000..6dcaea1d8 --- /dev/null +++ b/src/libstd/user/functions/async/setup.c @@ -0,0 +1,13 @@ +#include + +#include "user/common/syscalls.h" + +rings_id_t setup(rings_t* rings, void* address, size_t sentries, size_t centries) +{ + rings_id_t result = _syscall_setup(rings, address, sentries, centries); + if (result == ERR) + { + errno = _syscall_errno(); + } + return result; +} \ No newline at end of file diff --git a/src/libstd/user/functions/async/async_deinit.c b/src/libstd/user/functions/async/teardown.c similarity index 65% rename from src/libstd/user/functions/async/async_deinit.c rename to src/libstd/user/functions/async/teardown.c index 9c5add982..0213e3ea4 100644 --- a/src/libstd/user/functions/async/async_deinit.c +++ b/src/libstd/user/functions/async/teardown.c @@ -2,9 +2,9 @@ #include "user/common/syscalls.h" -uint64_t teardown(void) +uint64_t teardown(rings_id_t id) { - uint64_t result = _syscall_teardown(); + uint64_t result = _syscall_teardown(id); if (result == ERR) { errno = _syscall_errno(); diff --git a/src/programs/utils/ringstest/main.c b/src/programs/utils/ringstest/main.c index f7e496fcd..f8eac90ed 100644 --- a/src/programs/utils/ringstest/main.c +++ b/src/programs/utils/ringstest/main.c @@ -9,9 +9,9 @@ int main() { printf("setting up rings test...\n"); rings_t rings; - setup(&rings, NULL, SENTRIES, CENTRIES); + rings_id_t id = setup(&rings, NULL, SENTRIES, CENTRIES); - printf("pushing nop sqe...\n"); + printf("pushing nop sqe to rings %llu...\n", id); sqe_t sqe = SQE_CREATE(RINGS_NOP, SQE_REG1 << SQE_SAVE, CLOCKS_PER_SEC, 0x1234); sqe_push(&rings, &sqe); @@ -19,7 +19,7 @@ int main() sqe_push(&rings, &sqe); printf("entering rings...\n"); - enter(1, 1); + enter(id, 1, 1); printf("popping cqe...\n"); cqe_t cqe; @@ -35,6 +35,6 @@ int main() printf("cqe error: %d\n", cqe.error); printf("tearing down rings...\n"); - teardown(); + teardown(id); return 0; } From 75b2809fdff758ef4533ebe85a0caf033224d0a4 Mon Sep 17 00:00:00 2001 From: KN Date: Sat, 17 Jan 2026 16:15:51 +0100 Subject: [PATCH 07/23] feat(kernel:async): implement SQE_LINK --- include/kernel/sync/async.h | 1 - include/kernel/sync/request.h | 98 +++++++++++--------------- include/kernel/sync/requests.h | 1 - include/libstd/sys/rings.h | 22 +++--- src/kernel/sync/async.c | 105 +++++++++++++++++++++------- src/kernel/sync/request.c | 21 +++--- src/kernel/sync/requests.c | 5 -- src/programs/utils/ringstest/main.c | 38 ++++++---- 8 files changed, 170 insertions(+), 121 deletions(-) diff --git a/include/kernel/sync/async.h b/include/kernel/sync/async.h index a87f56e0e..253383c54 100644 --- a/include/kernel/sync/async.h +++ b/include/kernel/sync/async.h @@ -40,7 +40,6 @@ typedef struct async_ctx rings_t rings; ///< Asynchronous rings information. request_t* requests; ///< A preallocated array of requests, one for each CQE. list_t freeRequests; ///< Free list of requests. - uint64_t pending; ///< The amount of pending requests. void* userAddr; ///< Userspace address of the rings. void* kernelAddr; ///< Kernel address of the rings. size_t pageAmount; ///< Amount of pages mapped for the rings. diff --git a/include/kernel/sync/request.h b/include/kernel/sync/request.h index 96310c293..89a95c19f 100644 --- a/include/kernel/sync/request.h +++ b/include/kernel/sync/request.h @@ -7,6 +7,7 @@ #include #include #include +#include #include typedef struct async_ctx async_ctx_t; @@ -17,28 +18,26 @@ typedef struct process process_t; * @defgroup kernel_sync_request Request * @ingroup kernel_sync * - * ## Callbacks - * - * Requests can define three callbacks, included is a list of their expected semantics. - * - * ### Completion Callback + * ## Completion Callback * * The `complete()` callback should be called when the request has been completed, the `complete()` implementation does * not need to guarantee that the request structure will remain valid after a call to this function. * - * ### Cancellation Callback + * Generally, the completion callback should be implemented by the creator of the request while the `cancel()` callback is implemented by the subsystem processing the request. + * + * ## Cancellation Callback * - * The optional `cancel()` callback is called when attempting to cancel an in-progress request, if the request cannot be + * The optional `cancel()` callback is called when attempting to cancel an in-progress request or when its deadline expires, if the request cannot be * cancelled, the callback should return `false`, otherwise `true`. * - * ### Timeout Callback - * - * The optional `timeout()` callback is called when a request has timed out, the request * will be removed from the - * timeout queue before this callback is called and it will never * be called more than once. - * * @{ */ +/** + * @brief Represents that there is no next request in a chain. + */ +#define REQUEST_NO_INDEX UINT16_MAX + /** * @brief Per-CPU request queues. * @struct request_ctx_t @@ -63,34 +62,40 @@ typedef enum * @brief Macro to define common request structure members. * * All requests contain the below common members: - * - `list_entry_t entry`: List entry for requests queues and completion queues. + * - `list_entry_t entry`: List entry made available for use by any subsystem. * - `list_entry_t timeoutEntry`: List entry for timeout queues. - * - `request_ctx_t* ctx`: Pointer to the per-CPU request context storing this request for timeouts. + * - `uint16_t flags`: Request flags, see `request_flags_t`. + * - `uint16_t err`: Errno value for the request, or `EOK`. + * - `cpu_id_t cpu`: The ID of the cpu with the request context storing this request for timeouts. + * - `uint16_t next`: Index of the next request in a chain, or `REQUEST_NO_INDEX`. Used since a pointer would be to large. * - `async_ctx_t* async`: Pointer to the async context that created this request, or `NULL`. * - `void* data`: Pointer to user data. * - `void (*complete)(_type*)`: Completion callback. * - `bool (*cancel)(_type*)`: Cancellation callback. - * - `void (*timeout)(_type*)`: Timeout callback. - * - `request_flags_t flags`: Task flags. - * - `errno_t err`: Error code for the request. - * - `clock_t deadline`: Deadline for the request. + * - `union { clock_t deadline; clock_t timeout; }`: Before a request is queued for processing, this member represents the timeout duration for the request. After being queued, it represents the absolute deadline for the request. * - `_resultType result`: Result of the request. * + * @note The ordering here is important to avoid padding and keeping the full `request_t` structure at 128 bytes. + * * @param _type The type of the request structure. * @param _resultType The type of the request result. */ #define REQUEST_COMMON(_type, _resultType) \ list_entry_t entry; \ list_entry_t timeoutEntry; \ - request_ctx_t* ctx; \ + uint16_t flags; \ + uint16_t err; \ + cpu_id_t cpu; \ + uint16_t next; \ async_ctx_t* async; \ void* data; \ void (*complete)(_type*); \ bool (*cancel)(_type*); \ - void (*timeout)(_type*); \ - request_flags_t flags; \ - errno_t err; \ - clock_t deadline; \ + union \ + { \ + clock_t deadline; \ + clock_t timeout; \ + }; \ _resultType result /** @@ -103,27 +108,12 @@ typedef enum typedef struct request { REQUEST_COMMON(struct request, uint64_t); - uint64_t _padding[4]; + uint64_t _padding[SEQ_MAX_ARGS]; } request_t; -/** - * @brief Task queue structure. - * @struct request_queue_t - */ -typedef struct -{ - list_t requests; -} request_queue_t; - -/** - * @brief Initializes a request queue. - * - * @param queue Pointer to the request queue to initialize. - */ -static inline void request_queue_init(request_queue_t* queue) -{ - list_init(&queue->requests); -} +#ifdef static_assert +static_assert(sizeof(request_t) == 128, "request_t is not 128 bytes"); +#endif /** * @brief Adds a request to the per-CPU timeout queue. @@ -155,14 +145,14 @@ void request_timeouts_check(void); ({ \ (_request)->entry = LIST_ENTRY_CREATE((_request)->entry); \ (_request)->timeoutEntry = LIST_ENTRY_CREATE((_request)->timeoutEntry); \ - (_request)->ctx = NULL; \ + (_request)->flags = 0; \ + (_request)->err = EOK; \ + (_request)->cpu = CPU_ID_INVALID; \ + (_request)->next = REQUEST_NO_INDEX; \ (_request)->async = NULL; \ (_request)->data = NULL; \ (_request)->complete = NULL; \ (_request)->cancel = NULL; \ - (_request)->timeout = NULL; \ - (_request)->flags = 0; \ - (_request)->err = EOK; \ (_request)->deadline = CLOCKS_NEVER; \ (_request)->result = (typeof((_request)->result))0; \ }) @@ -204,16 +194,8 @@ void request_timeouts_check(void); (_request)->flags |= REQUEST_DELAYED; \ if ((_request)->deadline != CLOCKS_NEVER) \ { \ - if ((_request)->timeout == NULL) \ - { \ - errno = EINVAL; \ - result = ERR; \ - } \ - else \ - { \ - (_request)->flags |= REQUEST_TIMEOUT; \ - request_timeout_add((request_t*)(_request)); \ - } \ + (_request)->flags |= REQUEST_TIMEOUT; \ + request_timeout_add((request_t*)(_request)); \ } \ result; \ }) @@ -222,11 +204,11 @@ void request_timeouts_check(void); * @brief Macro to delay the completion of a request. * * @param _request Pointer to the request to delay. - * @param _queue Pointer to the request queue to add the request to. + * @param _queue Pointer to a list to add the request to. */ #define REQUEST_DELAY(_request, _queue) \ ({ \ - list_push_back(&(_queue)->requests, &(_request)->entry); \ + list_push_back(_queue, &(_request)->entry); \ uint64_t result = REQUEST_DELAY_NO_QUEUE(_request); \ if (result == ERR) \ { \ diff --git a/include/kernel/sync/requests.h b/include/kernel/sync/requests.h index 0ee017f7d..cc9255aa1 100644 --- a/include/kernel/sync/requests.h +++ b/include/kernel/sync/requests.h @@ -20,6 +20,5 @@ typedef struct request_nop } request_nop_t; bool request_nop_cancel(request_nop_t* request); -void request_nop_timeout(request_nop_t* request); /** @} */ \ No newline at end of file diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/rings.h index 0d9b34b08..4387af1b0 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/rings.h @@ -30,10 +30,13 @@ extern "C" * * Synchronous operations are implemented on top of this API in userspace. * + * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`. + * @see [Manpages](https://man7.org/linux/man-pages/man7/io_uring.7.html) for more information about `io_uring`. + * * ## Registers * * Operations performed on a ring can load arguments from, and save their results to, seven 64-bit general purpose - * registers. All registers are initialized to zero. + * registers. All registers are stored in the shared area of the rings structure, as such they can be inspected and modified by user space. * * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for the result, and for * each argument. Each specifier is stored as three bits, with a zero value indicating no-op and any other value @@ -45,9 +48,7 @@ extern "C" * would be possible to open a file, read from it, and close it, with a single `enter()` call. * * @see `sqe_flags_t` for more information about register specifiers and their formatting. - * - * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`. - * @see [Manpages](https://man7.org/linux/man-pages/man7/io_uring.7.html) for more information about `io_uring`. + * * @{ */ @@ -87,6 +88,7 @@ typedef enum SQE_REG7 = 7, ///< The seventh register. SQE_REG_SHIFT = 3, ///< The bitshift for each register specifier in a `sqe_flags_t`. SQE_REG_MASK = 0b111, ///< The bitmask for a register specifier in a `sqe_flags_t`. + SEQ_REGS_MAX = 7, ///< The maximum number of registers. } seq_regs_t; /** @@ -102,7 +104,7 @@ typedef enum SQE_LOAD3 = SQE_LOAD2 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fourth argument. SQE_LOAD4 = SQE_LOAD3 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fifth argument. SQE_FLAGS_SHIFT = SQE_LOAD4 + SQE_REG_SHIFT, ///< The bitshift for where bit flags start in a `sqe_flags_t`. - SQE_LINK = 1 << (SQE_FLAGS_SHIFT), ///< =nly process the next SQE if and when this one completes successfully. + SQE_LINK = 1 << (SQE_FLAGS_SHIFT), ///< Only process the next SQE if and when this one completes successfully, only applies within one `enter()` call. SQE_RESET = 1 << (SQE_FLAGS_SHIFT + 1), ///< Reset registers before processing this SQE. } sqe_flags_t; @@ -158,6 +160,7 @@ typedef struct ALIGNED(32) cqe rings_op_t opcode; ///< Operation code from the submission entry. errno_t error; ///< Error code, if not equal to `EOK` an error occurred. union { + uint64_t nop; uint64_t _raw; }; } cqe_t; @@ -177,15 +180,16 @@ typedef uint64_t rings_id_t; * * Used as the intermediate between userspace and the kernel. * + * @note The structure is aligned in such a way to reduce false sharing. + * */ typedef struct ALIGNED(64) rings_shared { atomic_uint32_t shead; ///< Submission head index, updated by the kernel. atomic_uint32_t ctail; ///< Completion tail index, updated by the kernel. - uint8_t _padding[64 - - sizeof(atomic_uint32_t) * 2]; ///< Padding to prevent false sharing between user space and the kernel. - atomic_uint32_t stail; ///< Submission tail index, updated by userspace. - atomic_uint32_t chead; ///< Completion head index, updated by userspace. + atomic_uint32_t stail ALIGNED(64); ///< Submission tail index, updated by userspace. + atomic_uint32_t chead; ///< Completion head index, updated by userspace. + atomic_uint64_t regs[SEQ_REGS_MAX] ALIGNED(64); ///< General purpose registers. } rings_shared_t; /** diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index f8caea10f..a9e463658 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -73,6 +73,10 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_ atomic_init(&shared->stail, 0); atomic_init(&shared->ctail, 0); atomic_init(&shared->chead, 0); + for (size_t i = 0; i < SEQ_REGS_MAX; i++) + { + atomic_init(&shared->regs[i], 0); + } userRings->shared = userAddr; userRings->id = id; @@ -130,6 +134,15 @@ static inline void async_ctx_free_request(async_ctx_t* ctx, request_t* request) list_push_back(&ctx->freeRequests, &request->entry); } + +static inline uint64_t async_ctx_avail_cqes(async_ctx_t* ctx) +{ + rings_t* rings = &ctx->rings; + uint32_t ctail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); + uint32_t chead = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); + return ctail - chead; +} + void async_ctx_init(async_ctx_t* ctx) { if (ctx == NULL) @@ -140,7 +153,6 @@ void async_ctx_init(async_ctx_t* ctx) ctx->rings = (rings_t){0}; ctx->requests = NULL; list_init(&ctx->freeRequests); - ctx->pending = 0; ctx->userAddr = NULL; ctx->kernelAddr = NULL; ctx->pageAmount = 0; @@ -173,8 +185,22 @@ void async_ctx_deinit(async_ctx_t* ctx) wait_queue_deinit(&ctx->waitQueue); } +typedef struct +{ + async_ctx_t* ctx; + list_t requests; + request_t* link; +} async_notify_ctx_t; + +static void async_dispatch(request_t* request) +{ + REQUEST_DELAY_NO_QUEUE(request); +} + static void async_nop_complete(request_nop_t* nop) { + LOG_INFO("completing NOP request data=%p\n", nop->data); + cqe_t cqe; cqe.data = nop->data; cqe.opcode = RINGS_NOP; @@ -183,35 +209,46 @@ static void async_nop_complete(request_nop_t* nop) async_ctx_t* async = nop->async; assert(async != NULL); assert(async->rings.id < CONFIG_MAX_ASYNC_RINGS); + + async_ctx_push_cqe(async, &cqe); - //process_t* process = CONTAINER_OF(async, process_t, async[async->rings.id]); + if (nop->next != REQUEST_NO_INDEX) + { + request_t* next = &async->requests[nop->next]; + async_dispatch(next); + nop->next = REQUEST_NO_INDEX; + } - async_ctx_push_cqe(async, &cqe); async_ctx_free_request(async, (request_t*)nop); + // process_t* process = CONTAINER_OF(async, process_t, async[async->rings.id]); /// @todo What to do if the process gets killed? } -static uint64_t async_handle_sqe(async_ctx_t* ctx, sqe_t* sqe) +static uint64_t async_handle_sqe(async_notify_ctx_t* notify, sqe_t* sqe) { if (sqe->opcode < RINGS_MIN_OPCODE || sqe->opcode >= RINGS_MAX_OPCODE) { - errno = EINVAL; - return ERR; + cqe_t cqe; + cqe.data = sqe->data; + cqe.opcode = sqe->opcode; + cqe.error = EINVAL; + cqe._raw = ERR; + async_ctx_push_cqe(notify->ctx, &cqe); + return 0; } - request_t* request = async_ctx_alloc_request(ctx); + request_t* request = async_ctx_alloc_request(notify->ctx); if (request == NULL) { errno = ENOSPC; return ERR; } - clock_t uptime = clock_uptime(); request->data = sqe->data; - request->async = ctx; - request->deadline = CLOCKS_DEADLINE(sqe->timeout, uptime); - + request->async = notify->ctx; + request->timeout = sqe->timeout; + switch (sqe->opcode) { case RINGS_NOP: @@ -219,8 +256,6 @@ static uint64_t async_handle_sqe(async_ctx_t* ctx, sqe_t* sqe) request_nop_t* nop = (request_nop_t*)request; nop->complete = async_nop_complete; nop->cancel = request_nop_cancel; - nop->timeout = request_nop_timeout; - REQUEST_DELAY_NO_QUEUE(nop); } break; default: @@ -228,15 +263,24 @@ static uint64_t async_handle_sqe(async_ctx_t* ctx, sqe_t* sqe) panic(NULL, "Invalid opcode %d", sqe->opcode); } - return 0; -} + if (notify->link != NULL) + { + size_t index = request - notify->ctx->requests; + assert(index < REQUEST_NO_INDEX); + notify->link->next = (uint16_t)index; + notify->link = NULL; + } + else + { + list_push_back(¬ify->requests, &request->entry); + } -static inline uint64_t async_ctx_avail_cqes(async_ctx_t* ctx) -{ - rings_t* rings = &ctx->rings; - uint32_t ctail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); - uint32_t chead = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); - return ctail - chead; + if (sqe->flags & SQE_LINK) + { + notify->link = request; + } + + return 0; } uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) @@ -248,8 +292,6 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) /// @todo Implement the register state logic. - /// @todo Implement SQE_LINK. - if (async_ctx_acquire(ctx) == ERR) { errno = EBUSY; @@ -266,6 +308,12 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) rings_t* rings = &ctx->rings; size_t processed = 0; + async_notify_ctx_t notify = { + .ctx = ctx, + .requests = LIST_CREATE(notify.requests), + .link = NULL, + }; + while (processed < amount) { uint32_t stail = atomic_load_explicit(&rings->shared->stail, memory_order_acquire); @@ -279,14 +327,19 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) sqe_t sqe = rings->squeue[shead & rings->smask]; atomic_store_explicit(&rings->shared->shead, shead + 1, memory_order_release); - if (async_handle_sqe(ctx, &sqe) == ERR) + if (async_handle_sqe(¬ify, &sqe) == ERR) { - async_ctx_release(ctx); - return ERR; + break; } processed++; } + while (!list_is_empty(¬ify.requests)) + { + request_t* request = CONTAINER_OF(list_pop_front(¬ify.requests), request_t, entry); + async_dispatch(request); + } + if (wait == 0) { async_ctx_release(ctx); diff --git a/src/kernel/sync/request.c b/src/kernel/sync/request.c index 56532e246..4cd3a51f1 100644 --- a/src/kernel/sync/request.c +++ b/src/kernel/sync/request.c @@ -18,7 +18,10 @@ void request_timeout_add(request_t* request) request_ctx_t* ctx = SELF_PTR(pcpu_requests); LOCK_SCOPE(&ctx->lock); - request->ctx = ctx; + request->cpu = SELF->id; + + clock_t now = clock_uptime(); + request->deadline = CLOCKS_DEADLINE(request->timeout, now); request_t* entry; LIST_FOR_EACH(entry, &ctx->timeouts, timeoutEntry) @@ -26,20 +29,21 @@ void request_timeout_add(request_t* request) if (request->deadline < entry->deadline) { list_prepend(&entry->entry, &request->timeoutEntry); + timer_set(now, request->deadline); return; } } list_push_back(&ctx->timeouts, &request->timeoutEntry); - - timer_set(clock_uptime(), request->deadline); + timer_set(now, request->deadline); } void request_timeout_remove(request_t* request) { - assert(request->ctx != NULL); - LOCK_SCOPE(&request->ctx->lock); - + request_ctx_t* ctx = CPU_PTR(request->cpu, pcpu_requests); + assert(ctx != NULL); + + LOCK_SCOPE(&ctx->lock); list_remove(&request->timeoutEntry); } @@ -70,8 +74,9 @@ void request_timeouts_check(void) list_remove(&request->timeoutEntry); lock_release(&ctx->lock); - assert(request->timeout != NULL); - request->timeout(request); + assert(request->cancel != NULL); + request->err = ETIMEDOUT; + request->cancel(request); lock_acquire(&ctx->lock); } diff --git a/src/kernel/sync/requests.c b/src/kernel/sync/requests.c index 74dbefda0..81f9e6962 100644 --- a/src/kernel/sync/requests.c +++ b/src/kernel/sync/requests.c @@ -4,9 +4,4 @@ bool request_nop_cancel(request_nop_t* request) { REQUEST_COMPLETE(request, 0); return true; -} - -void request_nop_timeout(request_nop_t* request) -{ - REQUEST_COMPLETE(request, 0); } \ No newline at end of file diff --git a/src/programs/utils/ringstest/main.c b/src/programs/utils/ringstest/main.c index f8eac90ed..77a2e7624 100644 --- a/src/programs/utils/ringstest/main.c +++ b/src/programs/utils/ringstest/main.c @@ -3,37 +3,49 @@ #include #define SENTRIES 64 -#define CENTRIES 64 +#define CENTRIES 128 int main() { printf("setting up rings test...\n"); rings_t rings; rings_id_t id = setup(&rings, NULL, SENTRIES, CENTRIES); + if (id == ERR) + { + printf("failed to set up rings\n"); + return errno; + } printf("pushing nop sqe to rings %llu...\n", id); - sqe_t sqe = SQE_CREATE(RINGS_NOP, SQE_REG1 << SQE_SAVE, CLOCKS_PER_SEC, 0x1234); + sqe_t sqe = SQE_CREATE(RINGS_NOP, SQE_LINK, CLOCKS_PER_SEC, 0x1234); sqe_push(&rings, &sqe); - sqe = (sqe_t)SQE_CREATE(RINGS_NOP, SQE_REG1 << SQE_LOAD0, CLOCKS_PER_SEC, 0x5678); + printf("pushing nop sqe to rings %llu...\n", id); + sqe = (sqe_t)SQE_CREATE(RINGS_NOP, SQE_LINK, CLOCKS_PER_SEC, 0x5678); sqe_push(&rings, &sqe); printf("entering rings...\n"); - enter(id, 1, 1); + if (enter(id, 2, 2) == ERR) + { + printf("failed to enter rings\n"); + return errno; + } - printf("popping cqe...\n"); cqe_t cqe; - cqe_pop(&rings, &cqe); - if (cqe.error != EOK) + while (cqe_pop(&rings, &cqe)) { - printf("cqe returned error: %d\n", cqe.error); - return 1; + printf("popped cqe...\n"); + if (cqe.error != EOK) + { + printf("cqe returned error\n"); + return cqe.error; + } + + printf("cqe data: %p\n", cqe.data); + printf("cqe opcode: %d\n", cqe.opcode); + printf("cqe error: %d\n", cqe.error); } - printf("cqe data: %p\n", cqe.data); - printf("cqe opcode: %d\n", cqe.opcode); - printf("cqe error: %d\n", cqe.error); - printf("tearing down rings...\n"); teardown(id); return 0; From 45a119f2f9c883053005133ae589aa0864ff326a Mon Sep 17 00:00:00 2001 From: KN Date: Sat, 17 Jan 2026 21:13:13 +0100 Subject: [PATCH 08/23] feat(kernel:request): implement request pool structure --- include/kernel/sync/async.h | 69 ++-------- include/kernel/sync/request.h | 193 +++++++++++++++++++--------- include/libstd/sys/rings.h | 44 +++++-- src/kernel/sync/async.c | 173 ++++++++++++++----------- src/kernel/sync/request.c | 59 ++++++++- src/programs/utils/ringstest/main.c | 9 +- 6 files changed, 334 insertions(+), 213 deletions(-) diff --git a/include/kernel/sync/async.h b/include/kernel/sync/async.h index 253383c54..3b1900ff3 100644 --- a/include/kernel/sync/async.h +++ b/include/kernel/sync/async.h @@ -37,14 +37,14 @@ typedef enum */ typedef struct async_ctx { - rings_t rings; ///< Asynchronous rings information. - request_t* requests; ///< A preallocated array of requests, one for each CQE. - list_t freeRequests; ///< Free list of requests. - void* userAddr; ///< Userspace address of the rings. - void* kernelAddr; ///< Kernel address of the rings. - size_t pageAmount; ///< Amount of pages mapped for the rings. - space_t* space; ///< Pointer to the owning address space. - wait_queue_t waitQueue; ///< Wait queue for completions. + rings_t rings; ///< Asynchronous rings information. + request_pool_t* requests; ///< Pool of preallocated requests. + void* userAddr; ///< Userspace address of the rings. + void* kernelAddr; ///< Kernel address of the rings. + size_t pageAmount; ///< Amount of pages mapped for the rings. + space_t* space; ///< Pointer to the owning address space. + wait_queue_t waitQueue; ///< Wait queue for completions. + process_t* process; ///< Holds a reference to the owner process while there are pending requests. _Atomic(async_ctx_flags_t) flags; } async_ctx_t; @@ -72,57 +72,4 @@ void async_ctx_deinit(async_ctx_t* ctx); */ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait); -/** - * @brief Acquire a async context. - * - * @param ctx Pointer to the context to acquire. - * @return On success, `0`. On failure, `ERR`. - */ -static inline uint64_t async_ctx_acquire(async_ctx_t* ctx) -{ - async_ctx_flags_t expected = atomic_load(&ctx->flags); - if (!(expected & ASYNC_CTX_BUSY) && - atomic_compare_exchange_strong(&ctx->flags, &expected, expected | ASYNC_CTX_BUSY)) - { - return 0; - } - - return ERR; -} - -/** - * @brief Release a async context. - * - * @param ctx Pointer to the context to release. - */ -static inline void async_ctx_release(async_ctx_t* ctx) -{ - atomic_fetch_and(&ctx->flags, ~ASYNC_CTX_BUSY); -} - -/** - * @brief Push a completion queue entry (CQE) to the completion queue. - * - * @param ctx Pointer to the async context. - * @param cqe Pointer to the CQE to push. - */ -static inline void async_ctx_push_cqe(async_ctx_t* ctx, cqe_t* cqe) -{ - rings_t* rings = &ctx->rings; - - uint32_t tail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); - uint32_t head = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); - - if ((tail - head) >= rings->centries) - { - /// @todo Handle overflow properly. - panic(NULL, "Async completion queue overflow"); - } - - rings->cqueue[tail & rings->cmask] = *cqe; - atomic_store_explicit(&rings->shared->ctail, tail + 1, memory_order_release); - - wait_unblock(&ctx->waitQueue, WAIT_ALL, EOK); -} - /** @} */ \ No newline at end of file diff --git a/include/kernel/sync/request.h b/include/kernel/sync/request.h index 89a95c19f..603957f0d 100644 --- a/include/kernel/sync/request.h +++ b/include/kernel/sync/request.h @@ -18,35 +18,39 @@ typedef struct process process_t; * @defgroup kernel_sync_request Request * @ingroup kernel_sync * + * The request primitive is designed to be generic enough to be used by any system in the kernel, however it is + * primarily used by the asynchronous rings system. + * + * @warning The request system is not thread-safe, it is the responsibility of the caller to ensure proper + * synchronization. + * + * @see kernel_sync_async for the asynchronous rings system. + * * ## Completion Callback * * The `complete()` callback should be called when the request has been completed, the `complete()` implementation does * not need to guarantee that the request structure will remain valid after a call to this function. * - * Generally, the completion callback should be implemented by the creator of the request while the `cancel()` callback is implemented by the subsystem processing the request. - * + * Generally, the completion callback should be implemented by the creator of the request while the `cancel()` callback + * is implemented by the subsystem processing the request. + * * ## Cancellation Callback * - * The optional `cancel()` callback is called when attempting to cancel an in-progress request or when its deadline expires, if the request cannot be - * cancelled, the callback should return `false`, otherwise `true`. + * The optional `cancel()` callback is called when attempting to cancel an in-progress request or when its deadline + * expires, if the request cannot be cancelled, the callback should return `false`, otherwise `true`. * * @{ */ /** - * @brief Represents that there is no next request in a chain. + * @brief Request ID type. */ -#define REQUEST_NO_INDEX UINT16_MAX +typedef uint16_t request_id_t; /** - * @brief Per-CPU request queues. - * @struct request_ctx_t + * @brief The maximum id value for requests. */ -typedef struct request_ctx -{ - list_t timeouts; - lock_t lock; -} request_ctx_t; +#define REQUEST_ID_MAX UINT16_MAX /** * @brief Task flags. @@ -61,59 +65,145 @@ typedef enum /** * @brief Macro to define common request structure members. * - * All requests contain the below common members: - * - `list_entry_t entry`: List entry made available for use by any subsystem. - * - `list_entry_t timeoutEntry`: List entry for timeout queues. - * - `uint16_t flags`: Request flags, see `request_flags_t`. - * - `uint16_t err`: Errno value for the request, or `EOK`. - * - `cpu_id_t cpu`: The ID of the cpu with the request context storing this request for timeouts. - * - `uint16_t next`: Index of the next request in a chain, or `REQUEST_NO_INDEX`. Used since a pointer would be to large. - * - `async_ctx_t* async`: Pointer to the async context that created this request, or `NULL`. - * - `void* data`: Pointer to user data. - * - `void (*complete)(_type*)`: Completion callback. - * - `bool (*cancel)(_type*)`: Cancellation callback. - * - `union { clock_t deadline; clock_t timeout; }`: Before a request is queued for processing, this member represents the timeout duration for the request. After being queued, it represents the absolute deadline for the request. - * - `_resultType result`: Result of the request. - * * @note The ordering here is important to avoid padding and keeping the full `request_t` structure at 128 bytes. - * + * * @param _type The type of the request structure. * @param _resultType The type of the request result. */ #define REQUEST_COMMON(_type, _resultType) \ + static_assert(sizeof(_resultType) == sizeof(uint64_t), "result type must be 64 bits"); \ list_entry_t entry; \ list_entry_t timeoutEntry; \ - uint16_t flags; \ - uint16_t err; \ - cpu_id_t cpu; \ - uint16_t next; \ - async_ctx_t* async; \ - void* data; \ void (*complete)(_type*); \ bool (*cancel)(_type*); \ - union \ - { \ + union { \ clock_t deadline; \ clock_t timeout; \ }; \ - _resultType result + request_id_t index; \ + request_id_t next; \ + cpu_id_t cpu; \ + uint8_t flags; \ + uint8_t type; \ + uint8_t err; \ + void* data; \ + _resultType result; /** * @brief Generic request structure. * @struct request_t * - * @warning Due to optimization done while allocating requests in the async system, no request structure should be + * @warning Due to optimization for the request pools, no request structure should be * larger than this structure. */ typedef struct request { REQUEST_COMMON(struct request, uint64_t); - uint64_t _padding[SEQ_MAX_ARGS]; + uint64_t _raw[SEQ_MAX_ARGS]; ///< Should be used by requests to store data. } request_t; -#ifdef static_assert static_assert(sizeof(request_t) == 128, "request_t is not 128 bytes"); -#endif + +/** + * @brief Request pool structure. + * @struct request_pool_t + */ +typedef struct request_pool +{ + void* ctx; + size_t used; + list_t free; + request_t requests[]; +} request_pool_t; + +/** + * @brief Allocate a new request pool. + * + * @param size The amount of requests to allocate. + * @param ctx The context of the request pool. + * @return On success, a pointer to the new request pool. On failure, `NULL` and `errno` is set. + */ +request_pool_t* request_pool_new(size_t size, void* ctx); + +/** + * @brief Free a request pool. + * + * @param pool Pointer to the request pool to free. + */ +void request_pool_free(request_pool_t* pool); + +/** + * @brief Retrieve the request pool that a request was allocated from. + * + * @param request Pointer to the request. + * @return Pointer to the request pool. + */ +static inline request_pool_t* request_get_pool(request_t* request) +{ + return CONTAINER_OF(request, request_pool_t, requests[request->index]); +} + +/** + * @brief Retrieve the context of the request pool that a request was allocated from. + * + * @param request Pointer to the request. + * @return Pointer to the context. + */ +static inline void* request_get_ctx(request_t* request) +{ + return request_get_pool(request)->ctx; +} + +/** + * @brief Retrieve the next request and clear the next field. + * + * @param request Pointer to the current request. + * @return Pointer to the next request, or `NULL` if there is no next request. + */ +static inline request_t* request_next(request_t* request) +{ + request_pool_t* pool = request_get_pool(request); + if (request->next == REQUEST_ID_MAX) + { + return NULL; + } + + request_t* next = &pool->requests[request->next]; + request->next = REQUEST_ID_MAX; + return next; +} + +/** + * @brief Allocate a new request from a pool. + * + * The pool that the request was allocated from, and its context, can be retrieved using the `request_get_pool()` + * function. + * + * @param pool Pointer to the request pool. + * @return On success, a pointer to the allocated request. On failure, `NULL`. + */ +static inline request_t* request_new(request_pool_t* pool) +{ + if (list_is_empty(&pool->free)) + { + return NULL; + } + + pool->used++; + return CONTAINER_OF(list_pop_back(&pool->free), request_t, entry); +} + +/** + * @brief Free a request back to its pool. + * + * @param request Pointer to the request to free. + */ +static inline void request_free(request_t* request) +{ + request_pool_t* pool = request_get_pool(request); + pool->used--; + list_push_back(&pool->free, &request->entry); +} /** * @brief Adds a request to the per-CPU timeout queue. @@ -136,27 +226,6 @@ void request_timeout_remove(request_t* request); */ void request_timeouts_check(void); -/** - * @brief Macro to initialize a requests common members. - * - * @param _request Pointer to the request to initialize. - */ -#define REQUEST_INIT(_request) \ - ({ \ - (_request)->entry = LIST_ENTRY_CREATE((_request)->entry); \ - (_request)->timeoutEntry = LIST_ENTRY_CREATE((_request)->timeoutEntry); \ - (_request)->flags = 0; \ - (_request)->err = EOK; \ - (_request)->cpu = CPU_ID_INVALID; \ - (_request)->next = REQUEST_NO_INDEX; \ - (_request)->async = NULL; \ - (_request)->data = NULL; \ - (_request)->complete = NULL; \ - (_request)->cancel = NULL; \ - (_request)->deadline = CLOCKS_NEVER; \ - (_request)->result = (typeof((_request)->result))0; \ - }) - /** * @brief Macro to call a function with a request and handle early completions. * diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/rings.h index 4387af1b0..6df6cb69e 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/rings.h @@ -36,7 +36,8 @@ extern "C" * ## Registers * * Operations performed on a ring can load arguments from, and save their results to, seven 64-bit general purpose - * registers. All registers are stored in the shared area of the rings structure, as such they can be inspected and modified by user space. + * registers. All registers are stored in the shared area of the rings structure, as such they can be inspected and + * modified by user space. * * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for the result, and for * each argument. Each specifier is stored as three bits, with a zero value indicating no-op and any other value @@ -47,8 +48,32 @@ extern "C" * This system, when combined with `SQE_LINK`, allows for multiple operations to be performed at once, for example, it * would be possible to open a file, read from it, and close it, with a single `enter()` call. * + * ## Errors + * + * The majority of errors are returned in the completion queue entries, certain errors (such as `ENOMEM`) may be + * reported directly from the `enter()` call. + * + * Certain error values that may be returned in a completion queue entry include: + * - `EOK`: Success. + * - `ECANCELED`: The operation was cancelled. + * - `ETIMEDOUT`: The operation timed out. + * - Other values may be returned depending on the operation. + * * @see `sqe_flags_t` for more information about register specifiers and their formatting. - + * + * ## Syncronization + * + * The rings structure is designed to be safe under the assumption that there is a single producer (one user-space + * thread) and a single consumer (the kernel). + * + * If a rings structure needs multiple producers (needs to be accessed by multiple threads) it is the responsibility of + * the caller to ensure proper synchronization. + * + * @note The reason for this limitation is optimization for the common case, as the syncronization logic for multiple + * producers would add significant overhead. + * + * Regarding the rings structure itself, the structure can only be torndown as long as nothing is using it and there are + * no pending operations. * * @{ */ @@ -104,7 +129,8 @@ typedef enum SQE_LOAD3 = SQE_LOAD2 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fourth argument. SQE_LOAD4 = SQE_LOAD3 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fifth argument. SQE_FLAGS_SHIFT = SQE_LOAD4 + SQE_REG_SHIFT, ///< The bitshift for where bit flags start in a `sqe_flags_t`. - SQE_LINK = 1 << (SQE_FLAGS_SHIFT), ///< Only process the next SQE if and when this one completes successfully, only applies within one `enter()` call. + SQE_LINK = 1 << (SQE_FLAGS_SHIFT), ///< Only process the next SQE if and when this one completes successfully, only + ///< applies within one `enter()` call. SQE_RESET = 1 << (SQE_FLAGS_SHIFT + 1), ///< Reset registers before processing this SQE. } sqe_flags_t; @@ -156,9 +182,9 @@ static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); */ typedef struct ALIGNED(32) cqe { - void* data; ///< Private data from the submission entry. rings_op_t opcode; ///< Operation code from the submission entry. errno_t error; ///< Error code, if not equal to `EOK` an error occurred. + void* data; ///< Private data from the submission entry. union { uint64_t nop; uint64_t _raw; @@ -181,14 +207,14 @@ typedef uint64_t rings_id_t; * Used as the intermediate between userspace and the kernel. * * @note The structure is aligned in such a way to reduce false sharing. - * + * */ typedef struct ALIGNED(64) rings_shared { - atomic_uint32_t shead; ///< Submission head index, updated by the kernel. - atomic_uint32_t ctail; ///< Completion tail index, updated by the kernel. - atomic_uint32_t stail ALIGNED(64); ///< Submission tail index, updated by userspace. - atomic_uint32_t chead; ///< Completion head index, updated by userspace. + atomic_uint32_t shead; ///< Submission head index, updated by the kernel. + atomic_uint32_t ctail; ///< Completion tail index, updated by the kernel. + atomic_uint32_t stail ALIGNED(64); ///< Submission tail index, updated by userspace. + atomic_uint32_t chead; ///< Completion head index, updated by userspace. atomic_uint64_t regs[SEQ_REGS_MAX] ALIGNED(64); ///< General purpose registers. } rings_shared_t; diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index a9e463658..891d8ea91 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -14,6 +15,23 @@ #include #include +static inline uint64_t async_ctx_acquire(async_ctx_t* ctx) +{ + async_ctx_flags_t expected = atomic_load(&ctx->flags); + if (!(expected & ASYNC_CTX_BUSY) && + atomic_compare_exchange_strong(&ctx->flags, &expected, expected | ASYNC_CTX_BUSY)) + { + return 0; + } + + return ERR; +} + +static inline void async_ctx_release(async_ctx_t* ctx) +{ + atomic_fetch_and(&ctx->flags, ~ASYNC_CTX_BUSY); +} + static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_t id, rings_t* userRings, void* address, size_t sentries, size_t centries) { @@ -27,6 +45,12 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_ return ERR; } + if (centries >= REQUEST_ID_MAX) + { + errno = EINVAL; + return ERR; + } + void* pages[CONFIG_MAX_ASYNC_PAGES]; if (pmm_alloc_pages(pages, pageAmount) == ERR) { @@ -54,7 +78,7 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_ return ERR; } - request_t* requests = malloc(sizeof(request_t) * centries); + request_pool_t* requests = request_pool_new(centries, ctx); if (requests == NULL) { vmm_unmap(space, userAddr, pageAmount * PAGE_SIZE); @@ -62,12 +86,6 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_ return ERR; } - for (size_t i = 0; i < centries; i++) - { - REQUEST_INIT(&requests[i]); - list_push_back(&ctx->freeRequests, &requests[i].entry); - } - rings_shared_t* shared = (rings_shared_t*)kernelAddr; atomic_init(&shared->shead, 0); atomic_init(&shared->stail, 0); @@ -108,8 +126,7 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_ static inline uint64_t async_ctx_unmap(async_ctx_t* ctx) { - list_init(&ctx->freeRequests); - free(ctx->requests); + request_pool_free(ctx->requests); ctx->requests = NULL; vmm_unmap(ctx->space, ctx->userAddr, ctx->pageAmount * PAGE_SIZE); @@ -119,22 +136,6 @@ static inline uint64_t async_ctx_unmap(async_ctx_t* ctx) return 0; } -static inline request_t* async_ctx_alloc_request(async_ctx_t* ctx) -{ - if (list_is_empty(&ctx->freeRequests)) - { - return NULL; - } - - return CONTAINER_OF(list_pop_back(&ctx->freeRequests), request_t, entry); -} - -static inline void async_ctx_free_request(async_ctx_t* ctx, request_t* request) -{ - list_push_back(&ctx->freeRequests, &request->entry); -} - - static inline uint64_t async_ctx_avail_cqes(async_ctx_t* ctx) { rings_t* rings = &ctx->rings; @@ -152,7 +153,6 @@ void async_ctx_init(async_ctx_t* ctx) ctx->rings = (rings_t){0}; ctx->requests = NULL; - list_init(&ctx->freeRequests); ctx->userAddr = NULL; ctx->kernelAddr = NULL; ctx->pageAmount = 0; @@ -187,87 +187,106 @@ void async_ctx_deinit(async_ctx_t* ctx) typedef struct { - async_ctx_t* ctx; list_t requests; request_t* link; } async_notify_ctx_t; static void async_dispatch(request_t* request) { - REQUEST_DELAY_NO_QUEUE(request); + switch (request->type) + { + case RINGS_NOP: + { + request_nop_t* nop = (request_nop_t*)request; + nop->cancel = request_nop_cancel; + REQUEST_DELAY_NO_QUEUE(nop); + } + break; + default: + // Impossible due to the check in async_handle_sqe. + panic(NULL, "Invalid opcode %d", request->type); + break; + } } -static void async_nop_complete(request_nop_t* nop) +static void async_ctx_push_cqe(async_ctx_t* ctx, rings_op_t opcode, errno_t error, void* data, uint64_t result) { - LOG_INFO("completing NOP request data=%p\n", nop->data); + rings_t* rings = &ctx->rings; - cqe_t cqe; - cqe.data = nop->data; - cqe.opcode = RINGS_NOP; - cqe.error = EOK; + uint32_t tail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); + uint32_t head = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); - async_ctx_t* async = nop->async; - assert(async != NULL); - assert(async->rings.id < CONFIG_MAX_ASYNC_RINGS); - - async_ctx_push_cqe(async, &cqe); + if ((tail - head) >= rings->centries) + { + /// @todo Handle overflow properly. + panic(NULL, "Async completion queue overflow"); + } + + cqe_t* cqe = &rings->cqueue[tail & rings->cmask]; + cqe->opcode = opcode; + cqe->error = error; + cqe->data = data; + cqe->_raw = result; - if (nop->next != REQUEST_NO_INDEX) + atomic_store_explicit(&rings->shared->ctail, tail + 1, memory_order_release); + + wait_unblock(&ctx->waitQueue, WAIT_ALL, EOK); +} + +static void async_request_complete(request_t* request) +{ + async_ctx_t* ctx = request_get_ctx(request); + + async_ctx_push_cqe(ctx, request->type, request->err, request->data, request->result); + + request_t* next = request_next(request); + if (next != NULL) { - request_t* next = &async->requests[nop->next]; async_dispatch(next); - nop->next = REQUEST_NO_INDEX; } - async_ctx_free_request(async, (request_t*)nop); + request_free(request); - // process_t* process = CONTAINER_OF(async, process_t, async[async->rings.id]); - /// @todo What to do if the process gets killed? + if (ctx->requests->used == 0) + { + UNREF(ctx->process); + ctx->process = NULL; + } } -static uint64_t async_handle_sqe(async_notify_ctx_t* notify, sqe_t* sqe) +static uint64_t async_handle_sqe(async_ctx_t* ctx, async_notify_ctx_t* notify, sqe_t* sqe) { if (sqe->opcode < RINGS_MIN_OPCODE || sqe->opcode >= RINGS_MAX_OPCODE) { - cqe_t cqe; - cqe.data = sqe->data; - cqe.opcode = sqe->opcode; - cqe.error = EINVAL; - cqe._raw = ERR; - async_ctx_push_cqe(notify->ctx, &cqe); + async_ctx_push_cqe(ctx, sqe->opcode, EINVAL, sqe->data, ERR); return 0; } - request_t* request = async_ctx_alloc_request(notify->ctx); + request_t* request = request_new(ctx->requests); if (request == NULL) { errno = ENOSPC; return ERR; } - request->data = sqe->data; - request->async = notify->ctx; - request->timeout = sqe->timeout; - - switch (sqe->opcode) + if (ctx->requests->used == 1) { - case RINGS_NOP: - { - request_nop_t* nop = (request_nop_t*)request; - nop->complete = async_nop_complete; - nop->cancel = request_nop_cancel; - } - break; - default: - // Impossible due to above check. - panic(NULL, "Invalid opcode %d", sqe->opcode); + process_t* process = process_current(); + assert(process->async[0] <= ctx && ctx <= process->async[CONFIG_MAX_ASYNC_RINGS - 1]); + ctx->process = REF(process); } + request->complete = async_request_complete; + request->cancel = NULL; + request->timeout = sqe->timeout; + request->type = sqe->opcode; + request->err = EOK; + request->data = sqe->data; + request->result = 0; + if (notify->link != NULL) { - size_t index = request - notify->ctx->requests; - assert(index < REQUEST_NO_INDEX); - notify->link->next = (uint16_t)index; + notify->link->next = request->index; notify->link = NULL; } else @@ -309,11 +328,10 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) size_t processed = 0; async_notify_ctx_t notify = { - .ctx = ctx, .requests = LIST_CREATE(notify.requests), .link = NULL, }; - + while (processed < amount) { uint32_t stail = atomic_load_explicit(&rings->shared->stail, memory_order_acquire); @@ -327,7 +345,7 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) sqe_t sqe = rings->squeue[shead & rings->smask]; atomic_store_explicit(&rings->shared->shead, shead + 1, memory_order_release); - if (async_handle_sqe(¬ify, &sqe) == ERR) + if (async_handle_sqe(ctx, ¬ify, &sqe) == ERR) { break; } @@ -419,6 +437,13 @@ SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, rings_id_t id) return ERR; } + if (ctx->requests != NULL && ctx->requests->used > 0) + { + async_ctx_release(ctx); + errno = EBUSY; + return ERR; + } + if (async_ctx_unmap(ctx) == ERR) { async_ctx_release(ctx); diff --git a/src/kernel/sync/request.c b/src/kernel/sync/request.c index 4cd3a51f1..2ddb51146 100644 --- a/src/kernel/sync/request.c +++ b/src/kernel/sync/request.c @@ -5,6 +5,12 @@ #include +typedef struct request_ctx +{ + list_t timeouts; + lock_t lock; +} request_ctx_t; + PERCPU_DEFINE_CTOR(request_ctx_t, pcpu_requests) { request_ctx_t* ctx = SELF_PTR(pcpu_requests); @@ -13,6 +19,57 @@ PERCPU_DEFINE_CTOR(request_ctx_t, pcpu_requests) lock_init(&ctx->lock); } +request_pool_t* request_pool_new(size_t size, void* ctx) +{ + if (size == 0 || size >= REQUEST_ID_MAX) + { + errno = EINVAL; + return NULL; + } + + request_pool_t* pool = malloc(sizeof(request_pool_t) + (sizeof(request_t) * size)); + if (pool == NULL) + { + errno = ENOMEM; + return NULL; + } + + pool->ctx = ctx; + pool->used = 0; + list_init(&pool->free); + for (request_id_t i = 0; i < (request_id_t)size; i++) + { + request_t* request = &pool->requests[i]; + + list_entry_init(&request->entry); + list_entry_init(&request->timeoutEntry); + request->complete = NULL; + request->cancel = NULL; + request->deadline = CLOCKS_NEVER; + request->index = i; + request->next = REQUEST_ID_MAX; + request->cpu = CPU_ID_INVALID; + request->flags = 0; + request->type = 0; + request->err = 0; + request->data = NULL; + request->result = 0; + for (size_t j = 0; j < ARRAY_SIZE(request->_raw); j++) + { + request->_raw[j] = 0; + } + + list_push_back(&pool->free, &pool->requests[i].entry); + } + + return pool; +} + +void request_pool_free(request_pool_t* pool) +{ + free(pool); +} + void request_timeout_add(request_t* request) { request_ctx_t* ctx = SELF_PTR(pcpu_requests); @@ -42,7 +99,7 @@ void request_timeout_remove(request_t* request) { request_ctx_t* ctx = CPU_PTR(request->cpu, pcpu_requests); assert(ctx != NULL); - + LOCK_SCOPE(&ctx->lock); list_remove(&request->timeoutEntry); } diff --git a/src/programs/utils/ringstest/main.c b/src/programs/utils/ringstest/main.c index 77a2e7624..c73f3dc57 100644 --- a/src/programs/utils/ringstest/main.c +++ b/src/programs/utils/ringstest/main.c @@ -1,5 +1,6 @@ #include #include +#include #include #define SENTRIES 64 @@ -35,15 +36,11 @@ int main() while (cqe_pop(&rings, &cqe)) { printf("popped cqe...\n"); - if (cqe.error != EOK) - { - printf("cqe returned error\n"); - return cqe.error; - } printf("cqe data: %p\n", cqe.data); printf("cqe opcode: %d\n", cqe.opcode); - printf("cqe error: %d\n", cqe.error); + printf("cqe error: %s\n", strerror(cqe.error)); + printf("cqe result: %llu\n", cqe._raw); } printf("tearing down rings...\n"); From 8e4fcf0622636cdc1a9864ce26ffbc6bbb3d868e Mon Sep 17 00:00:00 2001 From: KN Date: Sat, 17 Jan 2026 21:42:59 +0100 Subject: [PATCH 09/23] feat(kernel:async): implement register system --- include/kernel/sync/request.h | 34 ++++++-------------------- include/libstd/sys/rings.h | 37 +++++++++++++++-------------- src/kernel/sync/async.c | 22 ++++++++++++++++- src/kernel/sync/request.c | 6 ++--- src/programs/utils/ringstest/main.c | 12 ++++++++-- 5 files changed, 60 insertions(+), 51 deletions(-) diff --git a/include/kernel/sync/request.h b/include/kernel/sync/request.h index 603957f0d..162f9c880 100644 --- a/include/kernel/sync/request.h +++ b/include/kernel/sync/request.h @@ -52,16 +52,6 @@ typedef uint16_t request_id_t; */ #define REQUEST_ID_MAX UINT16_MAX -/** - * @brief Task flags. - * @enum request_flags_t - */ -typedef enum -{ - REQUEST_DELAYED = 1 << 0, ///< The completion of the request has been delayed. - REQUEST_TIMEOUT = 1 << 1, ///< The request is in a timeout queue. -} request_flags_t; - /** * @brief Macro to define common request structure members. * @@ -80,10 +70,10 @@ typedef enum clock_t deadline; \ clock_t timeout; \ }; \ + cpu_id_t cpu; \ request_id_t index; \ request_id_t next; \ - cpu_id_t cpu; \ - uint8_t flags; \ + uint32_t flags; \ uint8_t type; \ uint8_t err; \ void* data; \ @@ -94,12 +84,13 @@ typedef enum * @struct request_t * * @warning Due to optimization for the request pools, no request structure should be - * larger than this structure. + * larger than this structure. Additionally, due to how arguments are handled, all arguments should be aligned to 64 + * bits. */ typedef struct request { REQUEST_COMMON(struct request, uint64_t); - uint64_t _raw[SEQ_MAX_ARGS]; ///< Should be used by requests to store data. + uint64_t args[SEQ_MAX_ARGS]; ///< Should be used by requests to store data. } request_t; static_assert(sizeof(request_t) == 128, "request_t is not 128 bytes"); @@ -238,7 +229,6 @@ void request_timeouts_check(void); typeof((_request)->result) result = _func(_request); \ if ((_request)->err != EOK) \ { \ - (_request)->flags &= ~REQUEST_DELAYED; \ (_request)->complete(_request); \ } \ else if (!((_request)->flags & REQUEST_DELAYED)) \ @@ -260,10 +250,8 @@ void request_timeouts_check(void); #define REQUEST_DELAY_NO_QUEUE(_request) \ ({ \ uint64_t result = 0; \ - (_request)->flags |= REQUEST_DELAYED; \ if ((_request)->deadline != CLOCKS_NEVER) \ { \ - (_request)->flags |= REQUEST_TIMEOUT; \ request_timeout_add((request_t*)(_request)); \ } \ result; \ @@ -304,12 +292,8 @@ void request_timeouts_check(void); */ #define REQUEST_ERROR(_request, _errno) \ ({ \ - if ((_request)->flags & REQUEST_TIMEOUT) \ - { \ - request_timeout_remove((request_t*)(_request)); \ - } \ + request_timeout_remove((request_t*)(_request)); \ list_remove(&(_request)->entry); \ - (_request)->flags &= ~REQUEST_DELAYED; \ (_request)->err = (_errno); \ (_request)->complete((_request)); \ }) @@ -322,12 +306,8 @@ void request_timeouts_check(void); */ #define REQUEST_COMPLETE(_request, _result) \ ({ \ - if ((_request)->flags & REQUEST_TIMEOUT) \ - { \ - request_timeout_remove((request_t*)(_request)); \ - } \ + request_timeout_remove((request_t*)(_request)); \ list_remove(&(_request)->entry); \ - (_request)->flags &= ~REQUEST_DELAYED; \ (_request)->result = (_result); \ (_request)->complete((_request)); \ }) diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/rings.h index 6df6cb69e..05008452d 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/rings.h @@ -39,14 +39,15 @@ extern "C" * registers. All registers are stored in the shared area of the rings structure, as such they can be inspected and * modified by user space. * - * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for the result, and for - * each argument. Each specifier is stored as three bits, with a zero value indicating no-op and any other value - * representing the n-th register. The offset of the specifier specifies its meaning, for example, the first specifier - * (bits 0-2) specifies which register to save the result into, while the second specifier (bits 3-5) specifies which - * register to load into the first argument, and so on. + * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for each argument and + * one for the result. Each specifier is stored as three bits, with a `SQE_REG_NONE` value indicating no-op and any + * other value representing the n-th register. The offset of the specifier specifies its meaning, for example, bits + * `0-2` specify the register to load into the first argument, bits `3-5` specify the register to load into the second + * argument, and so on until bits `15-18` which specify the register to save the result into. * * This system, when combined with `SQE_LINK`, allows for multiple operations to be performed at once, for example, it - * would be possible to open a file, read from it, and close it, with a single `enter()` call. + * would be possible to open a file, read from it, seek to a new position, write to it, and finally close the file, with + * a single `enter()` call. * * ## Errors * @@ -103,17 +104,17 @@ typedef enum */ typedef enum { - SQE_REG_NONE = 0, ///< Dont perform any save or load operation. - SQE_REG1 = 1, ///< The first register. - SQE_REG2 = 2, ///< The second register. - SQE_REG3 = 3, ///< The third register. - SQE_REG4 = 4, ///< The fourth register. - SQE_REG5 = 5, ///< The fifth register. - SQE_REG6 = 6, ///< The sixth register. - SQE_REG7 = 7, ///< The seventh register. + SQE_REG0 = 0, ///< The first register. + SQE_REG1 = 1, ///< The second register. + SQE_REG2 = 2, ///< The third register. + SQE_REG3 = 3, ///< The fourth register. + SQE_REG4 = 4, ///< The fifth register. + SQE_REG5 = 5, ///< The sixth register. + SQE_REG6 = 6, ///< The seventh register. + SQE_REG_NONE = 7, ///< No register. + SEQ_REGS_MAX = 7, ///< The maximum number of registers. SQE_REG_SHIFT = 3, ///< The bitshift for each register specifier in a `sqe_flags_t`. SQE_REG_MASK = 0b111, ///< The bitmask for a register specifier in a `sqe_flags_t`. - SEQ_REGS_MAX = 7, ///< The maximum number of registers. } seq_regs_t; /** @@ -122,13 +123,13 @@ typedef enum */ typedef enum { - SQE_SAVE = 0, ///< The offset to specify the register to save the result into. - SQE_LOAD0 = SQE_SAVE + SQE_REG_SHIFT, ///< The offset to specify which register to load into the first argument. + SQE_LOAD0 = 0, ///< The offset to specify which register to load into the first argument. SQE_LOAD1 = SQE_LOAD0 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the second argument. SQE_LOAD2 = SQE_LOAD1 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the third argument. SQE_LOAD3 = SQE_LOAD2 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fourth argument. SQE_LOAD4 = SQE_LOAD3 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fifth argument. - SQE_FLAGS_SHIFT = SQE_LOAD4 + SQE_REG_SHIFT, ///< The bitshift for where bit flags start in a `sqe_flags_t`. + SQE_SAVE = SQE_LOAD4 + SQE_REG_SHIFT, ///< The offset to specify the register to save the result into. + SQE_FLAGS_SHIFT = SQE_SAVE + SQE_REG_SHIFT, ///< The bitshift for where bit flags start in a `sqe_flags_t`. SQE_LINK = 1 << (SQE_FLAGS_SHIFT), ///< Only process the next SQE if and when this one completes successfully, only ///< applies within one `enter()` call. SQE_RESET = 1 << (SQE_FLAGS_SHIFT + 1), ///< Reset registers before processing this SQE. diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index 891d8ea91..431228edb 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -193,6 +193,19 @@ typedef struct static void async_dispatch(request_t* request) { + async_ctx_t* ctx = request_get_ctx(request); + + for (uint64_t i = 0; i < SEQ_MAX_ARGS; i++) + { + seq_regs_t reg = (request->flags >> (i * SQE_REG_SHIFT)) & SQE_REG_MASK; + if (reg == SQE_REG_NONE) + { + continue; + } + + request->args[i] = atomic_load_explicit(&ctx->rings.shared->regs[reg], memory_order_acquire); + } + switch (request->type) { case RINGS_NOP: @@ -237,6 +250,12 @@ static void async_request_complete(request_t* request) { async_ctx_t* ctx = request_get_ctx(request); + seq_regs_t reg = (request->flags >> SQE_SAVE) & SQE_REG_MASK; + if (reg != SQE_REG_NONE) + { + atomic_store_explicit(&ctx->rings.shared->regs[reg], request->result, memory_order_release); + } + async_ctx_push_cqe(ctx, request->type, request->err, request->data, request->result); request_t* next = request_next(request); @@ -272,13 +291,14 @@ static uint64_t async_handle_sqe(async_ctx_t* ctx, async_notify_ctx_t* notify, s if (ctx->requests->used == 1) { process_t* process = process_current(); - assert(process->async[0] <= ctx && ctx <= process->async[CONFIG_MAX_ASYNC_RINGS - 1]); + assert(&process->async[0] <= ctx && ctx <= &process->async[CONFIG_MAX_ASYNC_RINGS - 1]); ctx->process = REF(process); } request->complete = async_request_complete; request->cancel = NULL; request->timeout = sqe->timeout; + request->flags = sqe->flags; request->type = sqe->opcode; request->err = EOK; request->data = sqe->data; diff --git a/src/kernel/sync/request.c b/src/kernel/sync/request.c index 2ddb51146..e47ed2719 100644 --- a/src/kernel/sync/request.c +++ b/src/kernel/sync/request.c @@ -46,17 +46,17 @@ request_pool_t* request_pool_new(size_t size, void* ctx) request->complete = NULL; request->cancel = NULL; request->deadline = CLOCKS_NEVER; + request->cpu = CPU_ID_INVALID; request->index = i; request->next = REQUEST_ID_MAX; - request->cpu = CPU_ID_INVALID; request->flags = 0; request->type = 0; request->err = 0; request->data = NULL; request->result = 0; - for (size_t j = 0; j < ARRAY_SIZE(request->_raw); j++) + for (size_t j = 0; j < ARRAY_SIZE(request->args); j++) { - request->_raw[j] = 0; + request->args[j] = 0; } list_push_back(&pool->free, &pool->requests[i].entry); diff --git a/src/programs/utils/ringstest/main.c b/src/programs/utils/ringstest/main.c index c73f3dc57..cbc874f35 100644 --- a/src/programs/utils/ringstest/main.c +++ b/src/programs/utils/ringstest/main.c @@ -17,8 +17,10 @@ int main() return errno; } + memset(&rings.shared->regs, -1, sizeof(rings.shared->regs)); + printf("pushing nop sqe to rings %llu...\n", id); - sqe_t sqe = SQE_CREATE(RINGS_NOP, SQE_LINK, CLOCKS_PER_SEC, 0x1234); + sqe_t sqe = SQE_CREATE(RINGS_NOP, SQE_LINK | (SQE_REG0 << SQE_SAVE), CLOCKS_PER_SEC, 0x1234); sqe_push(&rings, &sqe); printf("pushing nop sqe to rings %llu...\n", id); @@ -35,7 +37,7 @@ int main() cqe_t cqe; while (cqe_pop(&rings, &cqe)) { - printf("popped cqe...\n"); + printf("cqe:\n"); printf("cqe data: %p\n", cqe.data); printf("cqe opcode: %d\n", cqe.opcode); @@ -43,6 +45,12 @@ int main() printf("cqe result: %llu\n", cqe._raw); } + printf("registers:\n"); + for (uint64_t i = 0; i < SEQ_REGS_MAX; i++) + { + printf("reg[%llu]: %llu\n", i, rings.shared->regs[i]); + } + printf("tearing down rings...\n"); teardown(id); return 0; From da45c80013fb7cd4c17885a62d50244ac97fb3a4 Mon Sep 17 00:00:00 2001 From: KN Date: Sat, 17 Jan 2026 22:32:54 +0100 Subject: [PATCH 10/23] feat(kernel:async): add SQE_HARDLINK --- include/libstd/sys/rings.h | 6 +++--- src/kernel/sync/async.c | 29 +++++++++++++++++++++++++---- src/programs/utils/ringstest/main.c | 2 +- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/rings.h index 05008452d..7a3191398 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/rings.h @@ -130,9 +130,9 @@ typedef enum SQE_LOAD4 = SQE_LOAD3 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fifth argument. SQE_SAVE = SQE_LOAD4 + SQE_REG_SHIFT, ///< The offset to specify the register to save the result into. SQE_FLAGS_SHIFT = SQE_SAVE + SQE_REG_SHIFT, ///< The bitshift for where bit flags start in a `sqe_flags_t`. - SQE_LINK = 1 << (SQE_FLAGS_SHIFT), ///< Only process the next SQE if and when this one completes successfully, only - ///< applies within one `enter()` call. - SQE_RESET = 1 << (SQE_FLAGS_SHIFT + 1), ///< Reset registers before processing this SQE. + SQE_LINK = 1 << (SQE_FLAGS_SHIFT), ///< Only process the next SQE when this one completes successfully, only + /// applies within one `enter()` call. + SQE_HARDLINK = 1 << (SQE_FLAGS_SHIFT + 1), ///< Like `SQE_LINK`, but will process the next SQE even if this one fails. } sqe_flags_t; /** diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index 431228edb..901398ec6 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -258,10 +258,26 @@ static void async_request_complete(request_t* request) async_ctx_push_cqe(ctx, request->type, request->err, request->data, request->result); - request_t* next = request_next(request); - if (next != NULL) + if (request->err != EOK && !(request->flags & SQE_HARDLINK)) { - async_dispatch(next); + while (true) + { + request_t* next = request_next(request); + if (next == NULL) + { + break; + } + + request_free(next); + } + } + else + { + request_t* next = request_next(request); + if (next != NULL) + { + async_dispatch(next); + } } request_free(request); @@ -304,6 +320,11 @@ static uint64_t async_handle_sqe(async_ctx_t* ctx, async_notify_ctx_t* notify, s request->data = sqe->data; request->result = 0; + for (uint64_t i = 0; i < SEQ_MAX_ARGS; i++) + { + request->args[i] = sqe->args[i]; + } + if (notify->link != NULL) { notify->link->next = request->index; @@ -314,7 +335,7 @@ static uint64_t async_handle_sqe(async_ctx_t* ctx, async_notify_ctx_t* notify, s list_push_back(¬ify->requests, &request->entry); } - if (sqe->flags & SQE_LINK) + if (sqe->flags & SQE_LINK || sqe->flags & SQE_HARDLINK) { notify->link = request; } diff --git a/src/programs/utils/ringstest/main.c b/src/programs/utils/ringstest/main.c index cbc874f35..42aaf0ba4 100644 --- a/src/programs/utils/ringstest/main.c +++ b/src/programs/utils/ringstest/main.c @@ -20,7 +20,7 @@ int main() memset(&rings.shared->regs, -1, sizeof(rings.shared->regs)); printf("pushing nop sqe to rings %llu...\n", id); - sqe_t sqe = SQE_CREATE(RINGS_NOP, SQE_LINK | (SQE_REG0 << SQE_SAVE), CLOCKS_PER_SEC, 0x1234); + sqe_t sqe = SQE_CREATE(RINGS_NOP, SQE_HARDLINK | (SQE_REG0 << SQE_SAVE), CLOCKS_PER_SEC, 0x1234); sqe_push(&rings, &sqe); printf("pushing nop sqe to rings %llu...\n", id); From c14b8b43c6090a03544a7cdb66697bbfaf3b02e0 Mon Sep 17 00:00:00 2001 From: KN Date: Mon, 19 Jan 2026 17:45:29 +0100 Subject: [PATCH 11/23] feat(kernel:irp): implement I/O Request Packets --- include/kernel/fs/dentry.h | 6 +- include/kernel/fs/path.h | 1 + include/kernel/proc/process.h | 2 +- include/kernel/sync/async.h | 113 +++++- include/kernel/sync/irp.h | 574 ++++++++++++++++++++++++++++ include/kernel/sync/request.h | 342 ----------------- include/kernel/sync/requests.h | 24 -- include/libstd/sys/defs.h | 4 - include/libstd/sys/io.h | 2 +- include/libstd/sys/rings.h | 196 ++++------ meta/doxy/Doxyfile | 2 +- src/kernel/cpu/interrupt.c | 4 +- src/kernel/fs/devfs.c | 2 +- src/kernel/fs/filesystem.c | 4 +- src/kernel/fs/netfs.c | 8 +- src/kernel/fs/procfs.c | 28 +- src/kernel/fs/sysfs.c | 2 +- src/kernel/fs/tmpfs.c | 6 +- src/kernel/fs/vfs.c | 4 +- src/kernel/init/init.c | 5 +- src/kernel/linker.lds | 8 + src/kernel/module/module.c | 2 +- src/kernel/proc/process.c | 4 +- src/kernel/sync/async.c | 249 ++++++------ src/kernel/sync/irp.c | 304 +++++++++++++++ src/kernel/sync/request.c | 142 ------- src/kernel/sync/requests.c | 7 - src/programs/utils/ringstest/main.c | 8 +- src/programs/utils/stat/main.c | 2 +- 29 files changed, 1209 insertions(+), 846 deletions(-) create mode 100644 include/kernel/sync/irp.h delete mode 100644 include/kernel/sync/request.h delete mode 100644 include/kernel/sync/requests.h create mode 100644 src/kernel/sync/irp.c delete mode 100644 src/kernel/sync/request.c delete mode 100644 src/kernel/sync/requests.c diff --git a/include/kernel/fs/dentry.h b/include/kernel/fs/dentry.h index d0f2c8a9a..737451408 100644 --- a/include/kernel/fs/dentry.h +++ b/include/kernel/fs/dentry.h @@ -67,12 +67,12 @@ typedef uint64_t dentry_id_t; #define DENTRY_IS_POSITIVE(dentry) ((dentry)->inode != NULL) /** - * @brief Check if the inode associated with a dentry is a file. + * @brief Check if the inode associated with a dentry is a regular file. * * @param dentry The dentry to check. - * @return true if the dentry is a file, false otherwise or if the dentry is negative. + * @return true if the dentry is a regular file, false otherwise or if the dentry is negative. */ -#define DENTRY_IS_FILE(dentry) (DENTRY_IS_POSITIVE(dentry) && (dentry)->inode->type == INODE_FILE) +#define DENTRY_IS_REGULAR(dentry) (DENTRY_IS_POSITIVE(dentry) && (dentry)->inode->type == INODE_REGULAR) /** * @brief Check if the inode associated with a dentry is a directory. diff --git a/include/kernel/fs/path.h b/include/kernel/fs/path.h index f2f9a89b1..ee04fbb7d 100644 --- a/include/kernel/fs/path.h +++ b/include/kernel/fs/path.h @@ -12,6 +12,7 @@ typedef struct path path_t; typedef struct mount mount_t; typedef struct dentry dentry_t; typedef struct namespace namespace_t; +typedef struct file file_t; // clang-format off /** diff --git a/include/kernel/proc/process.h b/include/kernel/proc/process.h index 3b47033fa..fe92b1d49 100644 --- a/include/kernel/proc/process.h +++ b/include/kernel/proc/process.h @@ -88,7 +88,7 @@ typedef struct process file_table_t fileTable; futex_ctx_t futexCtx; perf_process_ctx_t perf; - async_ctx_t async[CONFIG_MAX_ASYNC_RINGS]; + async_t async[CONFIG_MAX_ASYNC_RINGS]; note_handler_t noteHandler; wait_queue_t suspendQueue; wait_queue_t dyingQueue; diff --git a/include/kernel/sync/async.h b/include/kernel/sync/async.h index 3b1900ff3..093e0a3bb 100644 --- a/include/kernel/sync/async.h +++ b/include/kernel/sync/async.h @@ -4,63 +4,136 @@ #include #include #include +#include #include -#include #include #include /** - * @brief Asynchronous Rings - * @defgroup kernel_sync_async Async + * @brief Programmable submission/completion interface. + * @defgroup kernel_sync_async Asynchronous Rings * @ingroup kernel_sync * - * @see libstd_rings for the userspace rings API. + * @todo The rings system is primarily a design document for now as it remains very work in progress and subject to + * change, currently being mostly unimplemented. + * + * The Asynchronous rings provide the core of all interfaces in PatchworkOS, all implemented in an interface + * inspired by `io_uring()` from Linux. + * + * Synchronous operations are implemented on top of this API in userspace. + * + * @see libstd_sys_rings for the userspace interface to the asynchronous rings. + * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`. + * @see [Manpages](https://man7.org/linux/man-pages/man7/io_uring.7.html) for more information about `io_uring`. + * + * ## Syncronization + * + * The rings structure is designed to be safe under the assumption that there is a single producer (one user-space + * thread) and a single consumer (the kernel). + * + * If a rings structure needs multiple producers (needs to be accessed by multiple threads) it is the responsibility of + * the caller to ensure proper synchronization. + * + * @note The reason for this limitation is optimization for the common case, as the syncronization logic for multiple + * producers would add significant overhead. + * + * Regarding the rings structure itself, the structure can only be torndown as long as nothing is using it and there are + * no pending operations. + * + * ## Registers + * + * Operations performed on a ring can load arguments from, and save their results to, seven 64-bit general purpose + * registers. All registers are stored in the shared area of the rings structure, as such they can be inspected and + * modified by user space. + * + * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for each argument and + * one for the result. Each specifier is stored as three bits, with a `SQE_REG_NONE` value indicating no-op and any + * other value representing the n-th register. The offset of the specifier specifies its meaning, for example, bits + * `0-2` specify the register to load into the first argument, bits `3-5` specify the register to load into the second + * argument, and so on until bits `15-18` which specify the register to save the result into. + * + * This system, when combined with `SQE_LINK`, allows for multiple operations to be performed at once, for example, it + * would be possible to open a file, read from it, seek to a new position, write to it, and finally close the file, with + * a single `enter()` call. + * + * @see `sqe_flags_t` for more information about register specifiers and their formatting. + * + * ## Errors + * + * The majority of errors are returned in the completion queue entries, certain errors (such as `ENOMEM`) may be + * reported directly from the `enter()` call. + * + * Certain error values that may be returned in a completion queue entry include: + * - `EOK`: Success. + * - `ECANCELED`: The operation was cancelled. + * - `ETIMEDOUT`: The operation timed out. + * - Other values may be returned depending on the operation. + * + * ## Verbs + * + * A verb specifies the operation to perform. Included is a list of currently defines verbs. + * + * ### `VERB_NOP` + * + * Never completes, can be used to implement a sleep equivalent by specifying a timeout. + * + * @param None + * @return Always `0`. + * + * ### `VERB_OPEN` + * + * Opens a file, including regular files, directories, symlinks, etc. + * + * @param from The file descriptor to open the file relative to, or `FD_NONE` to open from the current working + * directory. + * @param path Pointer to a null-terminated string containing the path to the file to open + * @return The file descriptor of the opened file. * * @{ */ /** * @brief Async context flags. - * @enum async_ctx_flags_t + * @enum async_flags_t */ typedef enum { ASYNC_CTX_NONE = 0, ///< No flags set. ASYNC_CTX_BUSY = 1 << 0, ///< Context is currently being used, used for fast locking. ASYNC_CTX_MAPPED = 1 << 1, ///< Context rings are mapped. -} async_ctx_flags_t; +} async_flags_t; /** * @brief The kernel-side asynchronous context structure. - * @struct async_ctx_t + * @struct async_t */ typedef struct async_ctx { - rings_t rings; ///< Asynchronous rings information. - request_pool_t* requests; ///< Pool of preallocated requests. - void* userAddr; ///< Userspace address of the rings. - void* kernelAddr; ///< Kernel address of the rings. - size_t pageAmount; ///< Amount of pages mapped for the rings. - space_t* space; ///< Pointer to the owning address space. - wait_queue_t waitQueue; ///< Wait queue for completions. - process_t* process; ///< Holds a reference to the owner process while there are pending requests. - _Atomic(async_ctx_flags_t) flags; -} async_ctx_t; + rings_t rings; ///< Asynchronous rings information. + irp_pool_t* irps; ///< Pool of preallocated IRPs. + void* userAddr; ///< Userspace address of the rings. + void* kernelAddr; ///< Kernel address of the rings. + size_t pageAmount; ///< Amount of pages mapped for the rings. + space_t* space; ///< Pointer to the owning address space. + wait_queue_t waitQueue; ///< Wait queue for completions. + process_t* process; ///< Holds a reference to the owner process while there are pending requests. + _Atomic(async_flags_t) flags; +} async_t; /** * @brief Initialize a async context. * * @param ctx Pointer to the context to initialize. */ -void async_ctx_init(async_ctx_t* ctx); +void async_init(async_t* ctx); /** * @brief Deinitialize a async context. * * @param ctx Pointer to the context to deinitialize. */ -void async_ctx_deinit(async_ctx_t* ctx); +void async_deinit(async_t* ctx); /** * @brief Notify the context of new SQEs. @@ -70,6 +143,6 @@ void async_ctx_deinit(async_ctx_t* ctx); * @param wait The minimum number of CQEs to wait for. * @return On success, the number of SQEs processed. On failure, `ERR` and `errno` is set. */ -uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait); +uint64_t async_notify(async_t* ctx, size_t amount, size_t wait); /** @} */ \ No newline at end of file diff --git a/include/kernel/sync/irp.h b/include/kernel/sync/irp.h new file mode 100644 index 000000000..27d3ed5df --- /dev/null +++ b/include/kernel/sync/irp.h @@ -0,0 +1,574 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct async_ctx async_t; +typedef struct process process_t; +typedef struct file file_t; + +typedef struct irp irp_t; + +/** + * @brief I/O Request Packet. + * @defgroup kernel_sync_irp I/O Request Packet + * @ingroup kernel_sync + * + * The I/O Request Packet is a lock-less, self-contained, layered, completion-based request that act as the primary + * structure used internally by the kernel for asynchronous operations. + * + * The IRP system is designed to be generic enough to be used by any system in the kernel, however it is primarily used + * by the asynchronous rings system. + * + * @warning The IRP system is not thread-safe, it is the responsibility of the caller to ensure proper synchronization. + * + * ## Completion + * + * The IRP system is designed around the concept of layered completions as it may take more than one subsystem within + * the kernel to complete a IRP. + * + * Consider a traditional synchronous set of functions: + * + * ``` + * int fun_c(void) + * { + * wait_until_data_ready(); + * return data; + * } + * + * int fun_b(int val) + * { + * return fun_c(val) + 1; + * } + * + * int fun_a(int val) + * { + * return fun_b(val) * 2; + * } + * + * int result = fun_a(); + * ``` + * + * When the code is executed, `fun_a()` would be called, which calls `fun_b()`, which in turn calls `fun_c()`. At this + * point `fun_c()` will block, causing the scheduler to switch to another thread until the data is ready. Once the data + * is ready, `fun_c()` will "complete" and return, followed by `fun_b()` and finally `fun_a()`, with the final result + * being stored in `result`. + * + * The above may seem obvious, but in a asynchronous kernel we are not allowed to block but must still be able to + * achieve the same result. As such, we need a way of representing the layered calls and their completions. + * + * @note In practice its possible that more than just one layer needs to block, as such the IRP system needs to handle + * such cases as well. + * + * Using the IRP system, the above code would be written as: + * + * ``` + * void fun_c_complete(irp_t* irp, void* ctx) + * { + * irp->result = get_data(); + * irp_complete(irp); + * } + * + * void fun_b_complete(irp_t* irp, void* ctx) + * { + * irp->result += 1; + * irp_complete(irp); + * } + * + * void fun_a_complete(irp_t* irp, void* ctx) + * { + * irp->result *= 2; + * irp_complete(irp); + * } + * + * void fun_c(irp_t* irp) + * { + * if (can_complete_now()) + * { + * irp->result = get_data(); + * irp_complete(irp); + * } + * else + * { + * irp_push(irp, fun_c_complete, NULL); + * } + * } + * + * void fun_b(irp_t* irp) + * { + * irp_push(irp, fun_b_complete, NULL); + * fun_c(irp); + * } + * + * void fun_a(irp_t* irp) + * { + * irp_push(irp, fun_a_complete, NULL); + * fun_b(irp); + * } + * + * irp_t* irp = irp_new(pool); + * // We could add our own complete here to handle the final result. + * fun_a_do(irp); + * // Continue executing even if fun_c() cannot complete immediately. + * ``` + * + * When `fun_a()` is called, it pushes its completion onto the IRP stack, followed by `fun_b()` pushing its completion, + * and finally `fun_c()` which may either complete immediately or push its completion if it cannot complete right away. + * + * Each time a completion is called via `irp_complete()`, the next completion on the stack is called until the stack is + * empty, at which point the IRP is considered fully completed. + * + * A real world example of this would be the Async Rings system allocating a IRP, pushing a completion which will add a + * `cqe_t` to its Rings, before passing the IRP to the VFS which may pass it to a filesystem. Each layer pushing its own + * completion to handle its part of the operation. + * + * ## Cancellation + * + * The current owner of a IRP is responsible for handling cancellation. The current owner being the last subsystem to + * push a completion onto the IRP stack. + * + * @note Intuitively, we can think of "cancelling" a IRP to be equivalent to causing the last completion to fail, thus + * resulting in all the other completions to fail as well. In the examples from the Completion section, it would be as + * though the synchronous `fun_c()` returned an error code instead of the data. + * + * The owner implements cancellation by calling `irp_set_cancel()` to set a cancellation callback when it pushes its + * completion. When a IRP is to be cancelled or timedout the cancellation callback will be invoked and atomically + * exchanged with a `IRP_CANCELLED` sentinel value. At which point the owner should cleanup the IRP and call + * `irp_complete()`. + * + * It is not possible for the IRP system to perform this atomic exchange for completions. As such, to avoid race + * conditions while completing a IRP, it is vital that the owner of the IRP atomically exchanges the cancellation + * callback with the `IRP_CANCELLED` sentinel value. For the sake of convenience, the `irp_claim()` function is provided + * to perform this operation. + * + * Below is an example of how to safely implement a completion with an associated cancellation callback: + * + * ``` + * void my_completion(irp_t* irp, void* ctx) + * { + * if (!irp_claim(irp)) + * { + * // The IRP has already been cancelled, nothing to do here. + * return; + * } + * + * // Complete the IRP... + * + * irp_complete(irp); + * } + * + * uint64_t my_cancel(irp_t* irp) + * { + * // Cancellation callback is automatically cleared. + * + * if (irp->err == ETIMEDOUT) + * { + * // We timed out. + * } + * if (irp->err == ECANCELED) + * { + * // We were explicitly cancelled. + * } + * + * // Perform cancellation... + * + * uint64_t result = ...; + * if (result == ERR) // If an error occurs we can reassign the cancellation callback. + * { + * irp_set_cancel(irp, my_cancel); + * return ERR; + * } + * + * irp_complete(irp); + * return 0; + * } + * ``` + * + * ## Error Values + * + * The IRP system uses the `err` field to indicate both the current state of the IRP as well as any error that may have + * occurred during its processing. + * + * Included below are a list of "special" values which the IRP system will recognize: + * + * - `EOK`: Operation completed successfully. + * - `ECANCELED`: Operation was cancelled. + * - `ETIMEDOUT`: Operation timed out. + * - `EINPROGRESS`: Operation is in a timeout queue. + * + * @see kernel_sync_async for the asynchronous rings system. + * @see [Wikipedia](https://en.wikipedia.org/wiki/I/O_request_packet) for more information about IRPs. + * @{ + */ + +/** + * @brief Represents the index of a IRP in a IRP pool. + * + * Used to save space in a IRP, by storing indexes instead of pointers. + */ +typedef uint16_t irp_idx_t; + +#define IRP_IDX_MAX UINT16_MAX ///< The maximum index value for IRP. + +#define IRP_TAG_INC ((uint64_t)(IRP_IDX_MAX) + 1) ///< The amount to increment the tag by in the tagged free list. + +#define IRP_LOC_MAX 8 ///< The maximum number of locations in a IRP. + +#define IRP_ARGS_MAX 5 ///< The maximum number of arguments in a IRP. + +/** + * @brief IRP completion callback type. + * + * @param irp Pointer to the IRP. + * @param ctx Context pointer. + */ +typedef void (*irp_complete_t)(irp_t* irp, void* ctx); + +/** + * @brief IRP cancellation callback type. + * + * @param irp Pointer to the IRP. + * @return On success, `0`. On failure, `ERR`. + */ +typedef uint64_t (*irp_cancel_t)(irp_t* irp); + +/** + * @brief Sentinel value indicating that the IRP has been cancelled. + */ +#define IRP_CANCELLED ((irp_cancel_t)1) + +/** + * @brief IRP location structure. + * @struct irp_loc + */ +typedef struct irp_loc +{ + void* ctx; + irp_complete_t complete; +} irp_loc_t; + +/** + * @brief I/O Request Packet structure. + * @struct irp_t + * + * @note We need the ability to store both the original arguments from a SQE and the parsed arguments. For example, + * opening a `fd_t` into a `file_t*`. As such, to avoid using another cache line, the SQE is stored in a union with the + * parsed arguments. + */ +typedef struct ALIGNED(64) irp +{ + list_entry_t entry; ///< Used to store the IRP in various lists. + list_entry_t timeoutEntry; ///< Used to store the IRP in the timeout queue. + _Atomic(irp_cancel_t) cancel; ///< Cancellation callback, must be atomic to ensure a IRP is only cancelled once. + union { + struct + { + verb_t verb; ///< Verb specifying the action to perform. + uint8_t _reserved1[3]; + sqe_flags_t flags; ///< Submission flags. + union { + clock_t timeout; ///< The timeout starting from when the IRP is added to a timeout queue. + clock_t deadline; ///< The time at which the IRP will be removed from a timeout queue. + }; + void* data; ///< Private data for the operation, will be returned in the completion entry. + union { + struct + { + file_t* from; + char* path; + } open; + uint64_t _args[IRP_ARGS_MAX]; + }; + }; + sqe_t sqe; ///< The original SQE for this IRP. + }; + uint64_t result; ///< Result of the IRP. + errno_t err; ///< The error code of the operation, also used to specify its current state. + irp_idx_t index; ///< Index of the IRP in its pool. + irp_idx_t next; ///< Index of the next IRP in a chain or in the free list. + cpu_id_t cpu; ///< The CPU whose timeout queue the IRP is in. + uint8_t location; ///< The index of the current location in the stack. + uint8_t _reserved2[5]; + irp_loc_t stack[IRP_LOC_MAX]; ///< The location stack, grows downwards. +} irp_t; + +static_assert(offsetof(irp_t, verb) == offsetof(irp_t, sqe.verb), "verb offset mismatch"); +static_assert(offsetof(irp_t, flags) == offsetof(irp_t, sqe.flags), "flags offset mismatch"); +static_assert(offsetof(irp_t, timeout) == offsetof(irp_t, sqe.timeout), "timeout offset mismatch"); +static_assert(offsetof(irp_t, data) == offsetof(irp_t, sqe.data), "data offset mismatch"); +static_assert(offsetof(irp_t, _args) == offsetof(irp_t, sqe._args), "args offset mismatch"); + +/** + * @brief Request pool structure. + * @struct irp_pool + */ +typedef struct irp_pool +{ + void* ctx; ///< Context pointer. + atomic_size_t used; ///< Number of used IRPs. + atomic_uint64_t free; ///< The tagged head of the free list. + irp_t irps[]; ///< Array of IRPs. +} irp_pool_t; + +/** + * @brief Allocate a new IRP pool. + * + * @param size The amount of requests to allocate. + * @param ctx The context of the IRP pool. + * @return On success, a pointer to the new IRP pool. On failure, `NULL` and `errno` is set. + */ +irp_pool_t* irp_pool_new(size_t size, void* ctx); + +/** + * @brief Free a IRP pool. + * + * @param pool Pointer to the IRP pool to free. + */ +void irp_pool_free(irp_pool_t* pool); + +/** + * @brief Retrieve the IRP pool that an IRP was allocated from. + * + * @param irp Pointer to the IRP. + * @return Pointer to the IRP pool. + */ +static inline irp_pool_t* irp_pool_get(irp_t* irp) +{ + return (irp_pool_t*)((uintptr_t)irp - (irp->index * sizeof(irp_t)) - offsetof(irp_pool_t, irps)); +} + +/** + * @brief Add an IRP to a per-CPU timeout queue with the timeout specified in the IRP. + * + * @param irp Pointer to the IRP to add. + */ +void irp_timeout_add(irp_t* irp); + +/** + * @brief Remove an IRP from its per-CPU timeout queue. + * + * @param irp Pointer to the IRP to remove. + */ +void irp_timeout_remove(irp_t* irp); + +/** + * @brief Check and handle expired IRP timeouts on the current CPU. + */ +void irp_timeouts_check(void); + +/** + * @brief Allocate a new IRP from a pool. + * + * The pool that the IRP was allocated from, and its context, can be retrieved using the `irp_pool_get()` + * function. + * + * @param pool Pointer to the IRP pool. + * @return On success, a pointer to the allocated IRP. On failure, `NULL`. + */ +irp_t* irp_new(irp_pool_t* pool); + +/** + * @brief Free a IRP back to its pool. + * + * @param irp Pointer to the IRP to free. + */ +void irp_free(irp_t* irp); + +/** + * @brief Set the cancellation callback for an IRP. + * + * @param irp Pointer to the IRP. + * @param cancel The cancellation callback. + * @return The previous cancellation callback. + */ +static inline irp_cancel_t irp_set_cancel(irp_t* irp, irp_cancel_t cancel) +{ + irp_cancel_t expected = atomic_load(&irp->cancel); + while (expected != IRP_CANCELLED) + { + if (atomic_compare_exchange_weak(&irp->cancel, &expected, cancel)) + { + return expected; + } + } + return IRP_CANCELLED; +} + +/** + * @brief Attempt to claim an IRP for completion. + * + * @param irp Pointer to the IRP. + * @return `true` if the IRP was successfully claimed, `false` if it was already cancelled or claimed. + */ +static inline bool irp_claim(irp_t* irp) +{ + return irp_set_cancel(irp, NULL) != IRP_CANCELLED; +} + +/** + * @brief Retrieve the context of the IRP pool that a IRP was allocated from. + * + * @param irp Pointer to the IRP. + * @return Pointer to the context. + */ +static inline void* irp_get_ctx(irp_t* irp) +{ + return irp_pool_get(irp)->ctx; +} + +/** + * @brief Retrieve the next IRP and clear the next field. + * + * @param irp Pointer to the current IRP. + * @return Pointer to the next IRP, or `NULL` if there is no next IRP. + */ +static inline irp_t* irp_next(irp_t* irp) +{ + irp_pool_t* pool = irp_pool_get(irp); + if (irp->next == IRP_IDX_MAX) + { + return NULL; + } + + irp_t* next = &pool->irps[irp->next]; + irp->next = IRP_IDX_MAX; + return next; +} + +/** + * @brief Retrieve the current location in the IRP stack. + * + * @param irp Pointer to the IRP to retrieve the location from. + * @return Pointer to the current location. + */ +static inline irp_loc_t* irp_current(irp_t* irp) +{ + assert(irp->location <= IRP_LOC_MAX); + return &irp->stack[irp->location]; +} + +/** + * @brief Retrieve the next location in the IRP stack. + * + * @param irp Pointer to the IRP to retrieve the location from. + * @return Pointer to the next location, or `NULL` if we are at the bottom of the stack. + */ +static inline irp_loc_t* irp_next_loc(irp_t* irp) +{ + if (irp->location == 0) + { + return NULL; + } + return &irp->stack[irp->location - 1]; +} + +/** + * @brief Push a new location onto the IRP stack. + * + * @param irp Pointer to the IRP to push to. + * @param complete The completion callback. + * @param ctx The context pointer. + */ +static inline void irp_push(irp_t* irp, irp_complete_t complete, void* ctx) +{ + assert(irp->location > 0); + assert(complete != NULL); + irp_loc_t* loc = &irp->stack[irp->location - 1]; + loc->complete = complete; + loc->ctx = ctx; + irp->location--; +} + +/** + * @brief Complete the current location in the IRP stack. + * + * @param irp Pointer to the IRP to complete. + */ +static inline void irp_complete(irp_t* irp) +{ + if (irp->location == IRP_LOC_MAX) + { + return; + } + + irp_loc_t* loc = irp_current(irp); + irp->location++; + + if (irp->location == IRP_LOC_MAX) + { + irp_timeout_remove(irp); + } + + loc->complete(irp, loc->ctx); +} + +/** + * @brief Attempt to cancel an IRP. + * + * @param irp Pointer to the IRP to cancel. + * @return On success, `0`. On failure, `ERR` and `errno` is set. + */ +uint64_t irp_cancel(irp_t* irp); + +/** + * @brief Dispatch an IRP to the appropriate handler. + * + * @param irp Pointer to the IRP to dispatch. + */ +void irp_dispatch(irp_t* irp); + +/** + * @brief Sort and validate the IRP handlers table. + */ +void irp_table_init(void); + +/** + * @brief IRP handler structure. + * @struct irp_handler_t + */ +typedef struct +{ + verb_t verb; + void (*handler)(irp_t* irp); +} irp_handler_t; + +/** + * @brief Linker defined start of the IRP handlers table. + */ +extern irp_handler_t _irp_table_start[]; + +/** + * @brief Linker defined end of the IRP handlers table. + */ +extern irp_handler_t _irp_table_end[]; + +/** + * @brief Macro to register a IRP handler to a verb using the `._irp_table` section. + * + * @param _verb The verb to register the handler for. + * @param _handler The handler function. + */ +#define IRP_REGISTER(_verb, _handler) \ + static irp_handler_t __irp_##_verb __attribute__((section("._irp_table"), used)) = { \ + .verb = (_verb), \ + .handler = (_handler), \ + }; + +/** + * @brief Function to asynchronously do nothing. + * + * Usefull as a sleep or delay operation. + * + * @param irp Pointer to a IRP to do nothing with. + */ +void nop_do(irp_t* irp); + +/** @} */ \ No newline at end of file diff --git a/include/kernel/sync/request.h b/include/kernel/sync/request.h deleted file mode 100644 index 162f9c880..000000000 --- a/include/kernel/sync/request.h +++ /dev/null @@ -1,342 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include - -typedef struct async_ctx async_ctx_t; -typedef struct process process_t; - -/** - * @brief Asynchronous Request Primitive - * @defgroup kernel_sync_request Request - * @ingroup kernel_sync - * - * The request primitive is designed to be generic enough to be used by any system in the kernel, however it is - * primarily used by the asynchronous rings system. - * - * @warning The request system is not thread-safe, it is the responsibility of the caller to ensure proper - * synchronization. - * - * @see kernel_sync_async for the asynchronous rings system. - * - * ## Completion Callback - * - * The `complete()` callback should be called when the request has been completed, the `complete()` implementation does - * not need to guarantee that the request structure will remain valid after a call to this function. - * - * Generally, the completion callback should be implemented by the creator of the request while the `cancel()` callback - * is implemented by the subsystem processing the request. - * - * ## Cancellation Callback - * - * The optional `cancel()` callback is called when attempting to cancel an in-progress request or when its deadline - * expires, if the request cannot be cancelled, the callback should return `false`, otherwise `true`. - * - * @{ - */ - -/** - * @brief Request ID type. - */ -typedef uint16_t request_id_t; - -/** - * @brief The maximum id value for requests. - */ -#define REQUEST_ID_MAX UINT16_MAX - -/** - * @brief Macro to define common request structure members. - * - * @note The ordering here is important to avoid padding and keeping the full `request_t` structure at 128 bytes. - * - * @param _type The type of the request structure. - * @param _resultType The type of the request result. - */ -#define REQUEST_COMMON(_type, _resultType) \ - static_assert(sizeof(_resultType) == sizeof(uint64_t), "result type must be 64 bits"); \ - list_entry_t entry; \ - list_entry_t timeoutEntry; \ - void (*complete)(_type*); \ - bool (*cancel)(_type*); \ - union { \ - clock_t deadline; \ - clock_t timeout; \ - }; \ - cpu_id_t cpu; \ - request_id_t index; \ - request_id_t next; \ - uint32_t flags; \ - uint8_t type; \ - uint8_t err; \ - void* data; \ - _resultType result; - -/** - * @brief Generic request structure. - * @struct request_t - * - * @warning Due to optimization for the request pools, no request structure should be - * larger than this structure. Additionally, due to how arguments are handled, all arguments should be aligned to 64 - * bits. - */ -typedef struct request -{ - REQUEST_COMMON(struct request, uint64_t); - uint64_t args[SEQ_MAX_ARGS]; ///< Should be used by requests to store data. -} request_t; - -static_assert(sizeof(request_t) == 128, "request_t is not 128 bytes"); - -/** - * @brief Request pool structure. - * @struct request_pool_t - */ -typedef struct request_pool -{ - void* ctx; - size_t used; - list_t free; - request_t requests[]; -} request_pool_t; - -/** - * @brief Allocate a new request pool. - * - * @param size The amount of requests to allocate. - * @param ctx The context of the request pool. - * @return On success, a pointer to the new request pool. On failure, `NULL` and `errno` is set. - */ -request_pool_t* request_pool_new(size_t size, void* ctx); - -/** - * @brief Free a request pool. - * - * @param pool Pointer to the request pool to free. - */ -void request_pool_free(request_pool_t* pool); - -/** - * @brief Retrieve the request pool that a request was allocated from. - * - * @param request Pointer to the request. - * @return Pointer to the request pool. - */ -static inline request_pool_t* request_get_pool(request_t* request) -{ - return CONTAINER_OF(request, request_pool_t, requests[request->index]); -} - -/** - * @brief Retrieve the context of the request pool that a request was allocated from. - * - * @param request Pointer to the request. - * @return Pointer to the context. - */ -static inline void* request_get_ctx(request_t* request) -{ - return request_get_pool(request)->ctx; -} - -/** - * @brief Retrieve the next request and clear the next field. - * - * @param request Pointer to the current request. - * @return Pointer to the next request, or `NULL` if there is no next request. - */ -static inline request_t* request_next(request_t* request) -{ - request_pool_t* pool = request_get_pool(request); - if (request->next == REQUEST_ID_MAX) - { - return NULL; - } - - request_t* next = &pool->requests[request->next]; - request->next = REQUEST_ID_MAX; - return next; -} - -/** - * @brief Allocate a new request from a pool. - * - * The pool that the request was allocated from, and its context, can be retrieved using the `request_get_pool()` - * function. - * - * @param pool Pointer to the request pool. - * @return On success, a pointer to the allocated request. On failure, `NULL`. - */ -static inline request_t* request_new(request_pool_t* pool) -{ - if (list_is_empty(&pool->free)) - { - return NULL; - } - - pool->used++; - return CONTAINER_OF(list_pop_back(&pool->free), request_t, entry); -} - -/** - * @brief Free a request back to its pool. - * - * @param request Pointer to the request to free. - */ -static inline void request_free(request_t* request) -{ - request_pool_t* pool = request_get_pool(request); - pool->used--; - list_push_back(&pool->free, &request->entry); -} - -/** - * @brief Adds a request to the per-CPU timeout queue. - * - * @param request Pointer to the request to add. - */ -void request_timeout_add(request_t* request); - -/** - * @brief Removes a request from the per-CPU timeout queue. - * - * @param request Pointer to the request to remove. - */ -void request_timeout_remove(request_t* request); - -/** - * @brief Checks for request timeouts on the current CPU and handles them. - * - * @warning Must be called with interrupts disabled. - */ -void request_timeouts_check(void); - -/** - * @brief Macro to call a function with a request and handle early completions. - * - * @param _request Pointer to the request. - * @param _func Function to call with the request. - * @return The result of the function call. - */ -#define REQUEST_CALL(_request, _func) \ - ({ \ - typeof((_request)->result) result = _func(_request); \ - if ((_request)->err != EOK) \ - { \ - (_request)->complete(_request); \ - } \ - else if (!((_request)->flags & REQUEST_DELAYED)) \ - { \ - (_request)->result = result; \ - (_request)->complete(_request); \ - } \ - result; \ - }) - -/** - * @brief Macro to delay the completion of a request without adding it to a queue. - * - * Primarily intended for use with timeout handling. - * - * @param _request Pointer to the request to delay. - * @return On success, `0`. On failure, `ERR` and `errno` is set. - */ -#define REQUEST_DELAY_NO_QUEUE(_request) \ - ({ \ - uint64_t result = 0; \ - if ((_request)->deadline != CLOCKS_NEVER) \ - { \ - request_timeout_add((request_t*)(_request)); \ - } \ - result; \ - }) - -/** - * @brief Macro to delay the completion of a request. - * - * @param _request Pointer to the request to delay. - * @param _queue Pointer to a list to add the request to. - */ -#define REQUEST_DELAY(_request, _queue) \ - ({ \ - list_push_back(_queue, &(_request)->entry); \ - uint64_t result = REQUEST_DELAY_NO_QUEUE(_request); \ - if (result == ERR) \ - { \ - list_remove(&(_request)->entry); \ - } \ - result; \ - }) - -/** - * @brief Macro to get the next request from a queue. - * - * @param _queue Pointer to the request queue. - * @param _type The type of the request structure. - * @return Pointer to the next request, or `NULL` if the queue is empty. - */ -#define REQUEST_NEXT(_queue, _type) \ - (list_is_empty(&(_queue)->requests) ? NULL : CONTAINER_OF(list_first(&(_queue)->requests), _type, entry)) - -/** - * @brief Macro to complete a request with an error. - * - * @param _request Pointer to the request. - * @param _errno The errno code. - */ -#define REQUEST_ERROR(_request, _errno) \ - ({ \ - request_timeout_remove((request_t*)(_request)); \ - list_remove(&(_request)->entry); \ - (_request)->err = (_errno); \ - (_request)->complete((_request)); \ - }) - -/** - * @brief Macro to complete a request. - * - * @param _request Pointer to the request. - * @param _result The result of the request. - */ -#define REQUEST_COMPLETE(_request, _result) \ - ({ \ - request_timeout_remove((request_t*)(_request)); \ - list_remove(&(_request)->entry); \ - (_request)->result = (_result); \ - (_request)->complete((_request)); \ - }) - -/** - * @brief Macro to cancel a request. - * - * @param _request Pointer to the request. - * @return On success, `0`. On failure, `ERR` and `errno` is set. - */ -#define REQUEST_CANCEL(_request) \ - ({ \ - uint64_t result = 0; \ - if ((_request)->cancel == NULL) \ - { \ - errno = EINVAL; \ - result = ERR; \ - } \ - else \ - { \ - (_request)->err = ECANCELED; \ - if (!((_request)->cancel(_request))) \ - { \ - (_request)->err = EOK; \ - errno = EBUSY; \ - result = ERR; \ - } \ - } \ - result; \ - }) - -/** @} */ \ No newline at end of file diff --git a/include/kernel/sync/requests.h b/include/kernel/sync/requests.h deleted file mode 100644 index cc9255aa1..000000000 --- a/include/kernel/sync/requests.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include - -/** - * @brief Kernel Request Implementations - * @defgroup kernel_sync_requests Requests - * @ingroup kernel_sync - * - * @{ - */ - -/** - * @brief No-operation request structure. - * @struct request_nop_t - */ -typedef struct request_nop -{ - REQUEST_COMMON(struct request_nop, uint64_t); -} request_nop_t; - -bool request_nop_cancel(request_nop_t* request); - -/** @} */ \ No newline at end of file diff --git a/include/libstd/sys/defs.h b/include/libstd/sys/defs.h index 7f41a45ab..faa9af476 100644 --- a/include/libstd/sys/defs.h +++ b/include/libstd/sys/defs.h @@ -75,10 +75,6 @@ * @return The concatenated token. */ #define CONCAT(a, b) CONCAT_INNER(a, b) - -/** - * @brief Inner helper macro for token concatenation. - */ #define CONCAT_INNER(a, b) a##b /** diff --git a/include/libstd/sys/io.h b/include/libstd/sys/io.h index c7a811dce..5c2fa60d4 100644 --- a/include/libstd/sys/io.h +++ b/include/libstd/sys/io.h @@ -340,7 +340,7 @@ poll_events_t poll1(fd_t fd, poll_events_t events, clock_t timeout); */ typedef enum { - INODE_FILE, ///< Is a file. + INODE_REGULAR, ///< Is a regular file. INODE_DIR, ///< Is a directory. INODE_SYMLINK, ///< Is a symbolic link. } itype_t; diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/rings.h index 7a3191398..2cd825fb5 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/rings.h @@ -12,128 +12,53 @@ extern "C" { #endif +#include "_internal/MAX_NAME.h" #include "_internal/MAX_PATH.h" #include "_internal/clock_t.h" #include "_internal/errno_t.h" #include "_internal/fd_t.h" /** - * @brief Programmable submission/completion interface. - * @defgroup libstd_rings Rings - * @ingroup libstd - * - * @todo The rings system is primarily a design document for now as it remains very work in progress and subject to - * change, currently being mostly unimplemented. - * - * Asynchronous operations provide the core of all IO interfaces in PatchworkOS, all implemented in an interface - * inspired by `io_uring()` from Linux. - * - * Synchronous operations are implemented on top of this API in userspace. - * - * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`. - * @see [Manpages](https://man7.org/linux/man-pages/man7/io_uring.7.html) for more information about `io_uring`. - * - * ## Registers - * - * Operations performed on a ring can load arguments from, and save their results to, seven 64-bit general purpose - * registers. All registers are stored in the shared area of the rings structure, as such they can be inspected and - * modified by user space. - * - * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for each argument and - * one for the result. Each specifier is stored as three bits, with a `SQE_REG_NONE` value indicating no-op and any - * other value representing the n-th register. The offset of the specifier specifies its meaning, for example, bits - * `0-2` specify the register to load into the first argument, bits `3-5` specify the register to load into the second - * argument, and so on until bits `15-18` which specify the register to save the result into. - * - * This system, when combined with `SQE_LINK`, allows for multiple operations to be performed at once, for example, it - * would be possible to open a file, read from it, seek to a new position, write to it, and finally close the file, with - * a single `enter()` call. - * - * ## Errors - * - * The majority of errors are returned in the completion queue entries, certain errors (such as `ENOMEM`) may be - * reported directly from the `enter()` call. - * - * Certain error values that may be returned in a completion queue entry include: - * - `EOK`: Success. - * - `ECANCELED`: The operation was cancelled. - * - `ETIMEDOUT`: The operation timed out. - * - Other values may be returned depending on the operation. - * - * @see `sqe_flags_t` for more information about register specifiers and their formatting. - * - * ## Syncronization - * - * The rings structure is designed to be safe under the assumption that there is a single producer (one user-space - * thread) and a single consumer (the kernel). - * - * If a rings structure needs multiple producers (needs to be accessed by multiple threads) it is the responsibility of - * the caller to ensure proper synchronization. - * - * @note The reason for this limitation is optimization for the common case, as the syncronization logic for multiple - * producers would add significant overhead. - * - * Regarding the rings structure itself, the structure can only be torndown as long as nothing is using it and there are - * no pending operations. - * + * @addtogroup kernel_sync_async * @{ */ -/** - * @brief Rings operation codes. - * @enum rings_op_t - */ -typedef enum -{ - RINGS_MIN_OPCODE = 0, - RINGS_NOP = 0, ///< Never completes, can be used to implement a sleep equivalent by specifying a timeout. - RINGS_MAX_OPCODE = 1, -} rings_op_t; - -/** - * @brief Maximum number of arguments for a rings operation. - */ -#define SEQ_MAX_ARGS 5 - -/** - * @brief Rings register specifiers. - * @enum seq_regs_t - * - * Used in the `sqe_flags_t` enum to specify which registers to load into arguments or save the result into. - * - */ -typedef enum -{ - SQE_REG0 = 0, ///< The first register. - SQE_REG1 = 1, ///< The second register. - SQE_REG2 = 2, ///< The third register. - SQE_REG3 = 3, ///< The fourth register. - SQE_REG4 = 4, ///< The fifth register. - SQE_REG5 = 5, ///< The sixth register. - SQE_REG6 = 6, ///< The seventh register. - SQE_REG_NONE = 7, ///< No register. - SEQ_REGS_MAX = 7, ///< The maximum number of registers. - SQE_REG_SHIFT = 3, ///< The bitshift for each register specifier in a `sqe_flags_t`. - SQE_REG_MASK = 0b111, ///< The bitmask for a register specifier in a `sqe_flags_t`. -} seq_regs_t; - -/** - * @brief Submission queue entry (SQE) flags. - * @enum sqe_flags_t - */ -typedef enum -{ - SQE_LOAD0 = 0, ///< The offset to specify which register to load into the first argument. - SQE_LOAD1 = SQE_LOAD0 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the second argument. - SQE_LOAD2 = SQE_LOAD1 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the third argument. - SQE_LOAD3 = SQE_LOAD2 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fourth argument. - SQE_LOAD4 = SQE_LOAD3 + SQE_REG_SHIFT, ///< The offset to specify which register to load into the fifth argument. - SQE_SAVE = SQE_LOAD4 + SQE_REG_SHIFT, ///< The offset to specify the register to save the result into. - SQE_FLAGS_SHIFT = SQE_SAVE + SQE_REG_SHIFT, ///< The bitshift for where bit flags start in a `sqe_flags_t`. - SQE_LINK = 1 << (SQE_FLAGS_SHIFT), ///< Only process the next SQE when this one completes successfully, only - /// applies within one `enter()` call. - SQE_HARDLINK = 1 << (SQE_FLAGS_SHIFT + 1), ///< Like `SQE_LINK`, but will process the next SQE even if this one fails. -} sqe_flags_t; +typedef uint8_t verb_t; ///< Verb type. + +#define VERB_NOP 0 ///< No-op verb. +#define VERB_OPEN 1 ///< Open file verb. +#define VERB_MAX 1 ///< Maximum verb. + +#define SEQ_MAX_ARGS 5 ///< Maximum number of arguments for a rings operation. + +typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. + +#define SQE_REG0 (0) ///< The first register. +#define SQE_REG1 (1) ///< The second register. +#define SQE_REG2 (2) ///< The third register. +#define SQE_REG3 (3) ///< The fourth register. +#define SQE_REG4 (4) ///< The fifth register. +#define SQE_REG5 (5) ///< The sixth register. +#define SQE_REG6 (6) ///< The seventh register. +#define SQE_REG_NONE (7) ///< No register. +#define SEQ_REGS_MAX (7) ///< The maximum number of registers. +#define SQE_REG_SHIFT (3) ///< The bitshift for each register specifier in a `sqe_flags_t`. +#define SQE_REG_MASK (0b111) ///< The bitmask for a register specifier in a `sqe_flags_t`. + +#define SQE_LOAD0 (0) ///< The offset to specify which register to load into the first argument. +#define SQE_LOAD1 \ + (SQE_LOAD0 + SQE_REG_SHIFT) ///< The offset to specify which register to load into the second argument. +#define SQE_LOAD2 (SQE_LOAD1 + SQE_REG_SHIFT) ///< The offset to specify which register to load into the third argument. +#define SQE_LOAD3 \ + (SQE_LOAD2 + SQE_REG_SHIFT) ///< The offset to specify which register to load into the fourth argument. +#define SQE_LOAD4 (SQE_LOAD3 + SQE_REG_SHIFT) ///< The offset to specify which register to load into the fifth argument. +#define SQE_SAVE (SQE_LOAD4 + SQE_REG_SHIFT) ///< The offset to specify the register to save the result into. +#define SQE_FLAGS_SHIFT (SQE_SAVE + SQE_REG_SHIFT) ///< The bitshift for where bit flags start in a `sqe_flags_t`. +#define SQE_LINK \ + (1 << (SQE_FLAGS_SHIFT)) ///< Only process the next SQE when this one completes successfully) only + /// applies within one `enter()` call. +#define SQE_HARDLINK \ + (1 << (SQE_FLAGS_SHIFT + 1)) ///< Like `SQE_LINK`) but will process the next SQE even if this one fails. /** * @brief Asynchronous submission queue entry (SQE). @@ -141,19 +66,27 @@ typedef enum * * @warning It is the responsibility of userspace to ensure that any pointers * passed to the kernel remain valid until the operation is complete. + * + * @see kernel_sync_async for more information on the possible operations. */ typedef struct sqe { - rings_op_t opcode; ///< Operation code. + verb_t verb; ///< Verb specifying the action to perform. + uint8_t _reserved[3]; sqe_flags_t flags; ///< Submission flags. clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. void* data; ///< Private data for the operation, will be returned in the completion entry. union { struct { - + uint64_t none; } nop; - uint64_t args[SEQ_MAX_ARGS]; + struct + { + fd_t from; + char* path; + } open; + uint64_t _args[SEQ_MAX_ARGS]; }; } sqe_t; @@ -164,14 +97,14 @@ static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); /** * @brief Macro to create an asynchronous submission queue entry (SQE). * - * @param _opcode Operation code. + * @param _verb Operation verb. * @param _flags Submission flags. * @param _timeout Timeout for the operation, `CLOCKS_NEVER` for no timeout. * @param _data Private data for the operation. */ -#define SQE_CREATE(_opcode, _flags, _timeout, _data) \ +#define SQE_CREATE(_verb, _flags, _timeout, _data) \ { \ - .opcode = (_opcode), \ + .verb = (_verb), \ .flags = (_flags), \ .timeout = (_timeout), \ .data = (void*)(_data), \ @@ -180,15 +113,19 @@ static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); /** * @brief Asynchronous completion queue entry (CQE). * @struct cqe_t + * + * @see kernel_sync_async for more information on the possible operations. */ typedef struct ALIGNED(32) cqe { - rings_op_t opcode; ///< Operation code from the submission entry. - errno_t error; ///< Error code, if not equal to `EOK` an error occurred. - void* data; ///< Private data from the submission entry. + verb_t verb; ///< Verb specifying the action that was performed. + uint8_t _reserved[3]; + errno_t error; ///< Error code, if not equal to `EOK` an error occurred. + void* data; ///< Private data from the submission entry. union { uint64_t nop; - uint64_t _raw; + fd_t open; + uint64_t _result; }; } cqe_t; @@ -237,6 +174,19 @@ typedef struct rings size_t cmask; ///< Bitmask for completion queue (centries - 1). } rings_t; +/** + * @} + * @brief User-side asynchronous rings interface. + * @defgroup libstd_sys_rings User Asynchronous Rings + * @ingroup libstd + * + * The rings interface acts as the interface for all asynchronous operations in the kernel. + * + * @see kernel_sync_async for more information about the asynchronous rings system. + * + * @{ + */ + /** * @brief Dont wait for any submissions to complete. */ diff --git a/meta/doxy/Doxyfile b/meta/doxy/Doxyfile index 2441f4e20..b9a782793 100644 --- a/meta/doxy/Doxyfile +++ b/meta/doxy/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = "PatchworkOS" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "9ec3d2a3-dirty" +PROJECT_NUMBER = "de3685e6-dirty" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewers a diff --git a/src/kernel/cpu/interrupt.c b/src/kernel/cpu/interrupt.c index c43b61fab..cf57120b5 100644 --- a/src/kernel/cpu/interrupt.c +++ b/src/kernel/cpu/interrupt.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include @@ -221,7 +221,7 @@ void interrupt_handler(interrupt_frame_t* frame) } note_handle_pending(frame); - request_timeouts_check(); + irp_timeouts_check(); wait_check_timeouts(frame); sched_do(frame); diff --git a/src/kernel/fs/devfs.c b/src/kernel/fs/devfs.c index ea1e0d289..4538b2a09 100644 --- a/src/kernel/fs/devfs.c +++ b/src/kernel/fs/devfs.c @@ -149,7 +149,7 @@ dentry_t* devfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* } UNREF_DEFER(dentry); - inode_t* inode = inode_new(parent->superblock, vfs_id_get(), INODE_FILE, inodeOps, fileOps); + inode_t* inode = inode_new(parent->superblock, vfs_id_get(), INODE_REGULAR, inodeOps, fileOps); if (inode == NULL) { return NULL; diff --git a/src/kernel/fs/filesystem.c b/src/kernel/fs/filesystem.c index 9fb4b28aa..08b996a9f 100644 --- a/src/kernel/fs/filesystem.c +++ b/src/kernel/fs/filesystem.c @@ -100,7 +100,7 @@ static uint64_t filesystem_lookup(inode_t* dir, dentry_t* dentry) } inode_t* inode = - inode_new(dentry->superblock, ino_gen(dir->number, dentry->name), INODE_FILE, NULL, &sbFileOps); + inode_new(dentry->superblock, ino_gen(dir->number, dentry->name), INODE_REGULAR, NULL, &sbFileOps); if (inode == NULL) { return ERR; @@ -136,7 +136,7 @@ static uint64_t filesystem_iterate(dentry_t* dentry, dir_ctx_t* ctx) char name[MAX_NAME]; snprintf(name, MAX_NAME, "%llu", sb->id); - if (!ctx->emit(ctx, name, ino_gen(dentry->inode->number, name), INODE_FILE)) + if (!ctx->emit(ctx, name, ino_gen(dentry->inode->number, name), INODE_REGULAR)) { return 0; } diff --git a/src/kernel/fs/netfs.c b/src/kernel/fs/netfs.c index 0c3cf31ae..e70025595 100644 --- a/src/kernel/fs/netfs.c +++ b/src/kernel/fs/netfs.c @@ -334,7 +334,7 @@ static uint64_t netfs_socket_lookup(inode_t* dir, dentry_t* dentry) continue; } - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, socketFiles[i].name), INODE_FILE, NULL, + inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, socketFiles[i].name), INODE_REGULAR, NULL, socketFiles[i].fileOps); if (inode == NULL) { @@ -375,7 +375,7 @@ static uint64_t netfs_socket_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, socketFiles[i].name, ino_gen(dentry->inode->number, socketFiles[i].name), INODE_FILE)) + if (!ctx->emit(ctx, socketFiles[i].name, ino_gen(dentry->inode->number, socketFiles[i].name), INODE_REGULAR)) { return 0; } @@ -553,7 +553,7 @@ static uint64_t netfs_family_lookup(inode_t* dir, dentry_t* dentry) continue; } - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, familyFiles[i].name), INODE_FILE, + inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, familyFiles[i].name), INODE_REGULAR, &familyFileInodeOps, familyFiles[i].fileOps); if (inode == NULL) { @@ -642,7 +642,7 @@ static uint64_t netfs_family_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, familyFiles[i].name, ino_gen(dentry->inode->number, familyFiles[i].name), INODE_FILE)) + if (!ctx->emit(ctx, familyFiles[i].name, ino_gen(dentry->inode->number, familyFiles[i].name), INODE_REGULAR)) { return 0; } diff --git a/src/kernel/fs/procfs.c b/src/kernel/fs/procfs.c index c97b6ce9f..e1f04bc64 100644 --- a/src/kernel/fs/procfs.c +++ b/src/kernel/fs/procfs.c @@ -843,7 +843,7 @@ static uint64_t procfs_env_lookup(inode_t* dir, dentry_t* target) return 0; } - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, target->name), INODE_FILE, NULL, &envVarOps); + inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, target->name), INODE_REGULAR, NULL, &envVarOps); if (inode == NULL) { return ERR; @@ -872,7 +872,7 @@ static uint64_t procfs_env_create(inode_t* dir, dentry_t* target, mode_t mode) return ERR; } - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, target->name), INODE_FILE, NULL, &envVarOps); + inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, target->name), INODE_REGULAR, NULL, &envVarOps); if (inode == NULL) { return ERR; @@ -923,7 +923,7 @@ static uint64_t procfs_env_iterate(dentry_t* dentry, dir_ctx_t* ctx) } if (!ctx->emit(ctx, process->env.vars[i].key, ino_gen(dentry->inode->number, process->env.vars[i].key), - INODE_FILE)) + INODE_REGULAR)) { return 0; } @@ -967,63 +967,63 @@ typedef struct static const procfs_entry_t pidEntries[] = { { .name = "prio", - .type = INODE_FILE, + .type = INODE_REGULAR, .fileOps = &prioOps, .dentryOps = &hideDentryOps, }, { .name = "cwd", - .type = INODE_FILE, + .type = INODE_REGULAR, .fileOps = &cwdOps, .dentryOps = &hideDentryOps, }, { .name = "cmdline", - .type = INODE_FILE, + .type = INODE_REGULAR, .fileOps = &cmdlineOps, }, { .name = "note", - .type = INODE_FILE, + .type = INODE_REGULAR, .fileOps = ¬eOps, .dentryOps = &hideDentryOps, }, { .name = "notegroup", - .type = INODE_FILE, + .type = INODE_REGULAR, .fileOps = ¬egroupOps, .dentryOps = &hideDentryOps, }, { .name = "group", - .type = INODE_FILE, + .type = INODE_REGULAR, .fileOps = &groupOps, .dentryOps = &hideDentryOps, }, { .name = "pid", - .type = INODE_FILE, + .type = INODE_REGULAR, .fileOps = &pidOps, }, { .name = "wait", - .type = INODE_FILE, + .type = INODE_REGULAR, .fileOps = &waitOps, }, { .name = "perf", - .type = INODE_FILE, + .type = INODE_REGULAR, .fileOps = &perfOps, }, { .name = "ns", - .type = INODE_FILE, + .type = INODE_REGULAR, .fileOps = &nsOps, .dentryOps = &hideDentryOps, }, { .name = "ctl", - .type = INODE_FILE, + .type = INODE_REGULAR, .fileOps = &ctlOps, .dentryOps = &hideDentryOps, }, diff --git a/src/kernel/fs/sysfs.c b/src/kernel/fs/sysfs.c index 7684fdc34..0ae524570 100644 --- a/src/kernel/fs/sysfs.c +++ b/src/kernel/fs/sysfs.c @@ -181,7 +181,7 @@ dentry_t* sysfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* } UNREF_DEFER(dentry); - inode_t* inode = inode_new(parent->superblock, vfs_id_get(), INODE_FILE, inodeOps, fileOps); + inode_t* inode = inode_new(parent->superblock, vfs_id_get(), INODE_REGULAR, inodeOps, fileOps); if (inode == NULL) { return NULL; diff --git a/src/kernel/fs/tmpfs.c b/src/kernel/fs/tmpfs.c index 129cc891f..79b289651 100644 --- a/src/kernel/fs/tmpfs.c +++ b/src/kernel/fs/tmpfs.c @@ -93,7 +93,7 @@ static uint64_t tmpfs_create(inode_t* dir, dentry_t* target, mode_t mode) { MUTEX_SCOPE(&dir->mutex); - inode_t* inode = tmpfs_inode_new(dir->superblock, mode & MODE_DIRECTORY ? INODE_DIR : INODE_FILE, NULL, 0); + inode_t* inode = tmpfs_inode_new(dir->superblock, mode & MODE_DIRECTORY ? INODE_DIR : INODE_REGULAR, NULL, 0); if (inode == NULL) { return ERR; @@ -164,7 +164,7 @@ static uint64_t tmpfs_remove(inode_t* dir, dentry_t* target) { MUTEX_SCOPE(&dir->mutex); - if (target->inode->type == INODE_FILE || target->inode->type == INODE_SYMLINK) + if (target->inode->type == INODE_REGULAR || target->inode->type == INODE_SYMLINK) { tmpfs_dentry_remove(target); } @@ -228,7 +228,7 @@ static dentry_t* tmpfs_load_file(superblock_t* superblock, dentry_t* parent, con tmpfs_dentry_add(dentry); - inode_t* inode = tmpfs_inode_new(superblock, INODE_FILE, in->data, in->size); + inode_t* inode = tmpfs_inode_new(superblock, INODE_REGULAR, in->data, in->size); if (inode == NULL) { panic(NULL, "Failed to create tmpfs file inode"); diff --git a/src/kernel/fs/vfs.c b/src/kernel/fs/vfs.c index 986c8e4f0..d9b334877 100644 --- a/src/kernel/fs/vfs.c +++ b/src/kernel/fs/vfs.c @@ -200,7 +200,7 @@ uint64_t vfs_open2(const pathname_t* pathname, file_t* files[2], process_t* proc return ERR; } - if (pathname->mode & MODE_TRUNCATE && files[0]->inode->type == INODE_FILE) + if (pathname->mode & MODE_TRUNCATE && files[0]->inode->type == INODE_REGULAR) { inode_truncate(files[0]->inode); } @@ -258,7 +258,7 @@ file_t* vfs_openat(const path_t* from, const pathname_t* pathname, process_t* pr return NULL; } - if (pathname->mode & MODE_TRUNCATE && file->inode->type == INODE_FILE) + if (pathname->mode & MODE_TRUNCATE && file->inode->type == INODE_REGULAR) { inode_truncate(file->inode); } diff --git a/src/kernel/init/init.c b/src/kernel/init/init.c index d9420dc8e..391e8d6f1 100644 --- a/src/kernel/init/init.c +++ b/src/kernel/init/init.c @@ -63,6 +63,9 @@ void init_early(void) vmm_kernel_space_load(); + irp_table_init(); + syscall_table_init(); + _std_init(); PERCPU_INIT(); @@ -104,8 +107,6 @@ static void init_finalize(void) perf_init(); - syscall_table_init(); - boot_info_t* bootInfo = boot_info_get(); if (bootInfo->gop.virtAddr != NULL) diff --git a/src/kernel/linker.lds b/src/kernel/linker.lds index 9fb35ad2a..641c8491f 100644 --- a/src/kernel/linker.lds +++ b/src/kernel/linker.lds @@ -100,6 +100,14 @@ SECTIONS _syscall_table_end = .; } :data + . = ALIGN(4K); + ._irp_table : AT(ADDR(._irp_table) - KERNEL_START) + { + _irp_table_start = .; + KEEP(*(._irp_table)) + _irp_table_end = .; + } :data + . = ALIGN(4K); _kernel_end = .; diff --git a/src/kernel/module/module.c b/src/kernel/module/module.c index 6b43ce3bb..11cfc5752 100644 --- a/src/kernel/module/module.c +++ b/src/kernel/module/module.c @@ -624,7 +624,7 @@ static uint64_t module_cache_build(void) for (uint64_t i = 0; i < readCount / sizeof(dirent_t); i++) { - if (buffer[i].path[0] == '.' || buffer[i].type != INODE_FILE) + if (buffer[i].path[0] == '.' || buffer[i].type != INODE_REGULAR) { continue; } diff --git a/src/kernel/proc/process.c b/src/kernel/proc/process.c index 6e114a190..700dd20a9 100644 --- a/src/kernel/proc/process.c +++ b/src/kernel/proc/process.c @@ -113,7 +113,7 @@ static void process_free(process_t* process) futex_ctx_deinit(&process->futexCtx); for (uint64_t i = 0; i < CONFIG_MAX_ASYNC_RINGS; i++) { - async_ctx_deinit(&process->async[i]); + async_deinit(&process->async[i]); } wait_queue_deinit(&process->dyingQueue); wait_queue_deinit(&process->suspendQueue); @@ -156,7 +156,7 @@ process_t* process_new(priority_t priority, group_member_t* group, namespace_t* perf_process_ctx_init(&process->perf); for (uint64_t i = 0; i < CONFIG_MAX_ASYNC_RINGS; i++) { - async_ctx_init(&process->async[i]); + async_init(&process->async[i]); } note_handler_init(&process->noteHandler); wait_queue_init(&process->suspendQueue); diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index 901398ec6..56e8bed3d 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -7,17 +7,16 @@ #include #include #include -#include -#include +#include #include #include #include #include -static inline uint64_t async_ctx_acquire(async_ctx_t* ctx) +static inline uint64_t async_acquire(async_t* ctx) { - async_ctx_flags_t expected = atomic_load(&ctx->flags); + async_flags_t expected = atomic_load(&ctx->flags); if (!(expected & ASYNC_CTX_BUSY) && atomic_compare_exchange_strong(&ctx->flags, &expected, expected | ASYNC_CTX_BUSY)) { @@ -27,12 +26,12 @@ static inline uint64_t async_ctx_acquire(async_ctx_t* ctx) return ERR; } -static inline void async_ctx_release(async_ctx_t* ctx) +static inline void async_release(async_t* ctx) { atomic_fetch_and(&ctx->flags, ~ASYNC_CTX_BUSY); } -static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_t id, rings_t* userRings, void* address, +static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, rings_t* userRings, void* address, size_t sentries, size_t centries) { rings_t* kernelRings = &ctx->rings; @@ -45,7 +44,7 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_ return ERR; } - if (centries >= REQUEST_ID_MAX) + if (centries >= IRP_IDX_MAX) { errno = EINVAL; return ERR; @@ -78,8 +77,8 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_ return ERR; } - request_pool_t* requests = request_pool_new(centries, ctx); - if (requests == NULL) + irp_pool_t* irps = irp_pool_new(centries, ctx); + if (irps == NULL) { vmm_unmap(space, userAddr, pageAmount * PAGE_SIZE); vmm_unmap(NULL, kernelAddr, pageAmount * PAGE_SIZE); @@ -114,7 +113,7 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_ kernelRings->centries = centries; kernelRings->cmask = centries - 1; - ctx->requests = requests; + ctx->irps = irps; ctx->userAddr = userAddr; ctx->kernelAddr = kernelAddr; ctx->pageAmount = pageAmount; @@ -124,10 +123,10 @@ static inline uint64_t async_ctx_map(async_ctx_t* ctx, space_t* space, rings_id_ return 0; } -static inline uint64_t async_ctx_unmap(async_ctx_t* ctx) +static inline uint64_t async_unmap(async_t* ctx) { - request_pool_free(ctx->requests); - ctx->requests = NULL; + irp_pool_free(ctx->irps); + ctx->irps = NULL; vmm_unmap(ctx->space, ctx->userAddr, ctx->pageAmount * PAGE_SIZE); vmm_unmap(NULL, ctx->kernelAddr, ctx->pageAmount * PAGE_SIZE); @@ -136,7 +135,7 @@ static inline uint64_t async_ctx_unmap(async_ctx_t* ctx) return 0; } -static inline uint64_t async_ctx_avail_cqes(async_ctx_t* ctx) +static inline uint64_t async_avail_cqes(async_t* ctx) { rings_t* rings = &ctx->rings; uint32_t ctail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); @@ -144,7 +143,7 @@ static inline uint64_t async_ctx_avail_cqes(async_ctx_t* ctx) return ctail - chead; } -void async_ctx_init(async_ctx_t* ctx) +void async_init(async_t* ctx) { if (ctx == NULL) { @@ -152,7 +151,7 @@ void async_ctx_init(async_ctx_t* ctx) } ctx->rings = (rings_t){0}; - ctx->requests = NULL; + ctx->irps = NULL; ctx->userAddr = NULL; ctx->kernelAddr = NULL; ctx->pageAmount = 0; @@ -161,71 +160,45 @@ void async_ctx_init(async_ctx_t* ctx) atomic_init(&ctx->flags, ASYNC_CTX_NONE); } -void async_ctx_deinit(async_ctx_t* ctx) +void async_deinit(async_t* ctx) { if (ctx == NULL) { return; } - if (async_ctx_acquire(ctx) == ERR) + if (async_acquire(ctx) == ERR) { panic(NULL, "failed to acquire async context for deinitialization"); } if (atomic_load(&ctx->flags) & ASYNC_CTX_MAPPED) { - if (async_ctx_unmap(ctx) == ERR) + if (async_unmap(ctx) == ERR) { panic(NULL, "failed to deinitialize async context"); } } - async_ctx_release(ctx); + async_release(ctx); wait_queue_deinit(&ctx->waitQueue); } -typedef struct -{ - list_t requests; - request_t* link; -} async_notify_ctx_t; +static void async_dispatch(irp_t* irp); -static void async_dispatch(request_t* request) +static void async_complete(irp_t* irp, void* _ptr) { - async_ctx_t* ctx = request_get_ctx(request); - - for (uint64_t i = 0; i < SEQ_MAX_ARGS; i++) - { - seq_regs_t reg = (request->flags >> (i * SQE_REG_SHIFT)) & SQE_REG_MASK; - if (reg == SQE_REG_NONE) - { - continue; - } + UNUSED(_ptr); - request->args[i] = atomic_load_explicit(&ctx->rings.shared->regs[reg], memory_order_acquire); - } + async_t* ctx = irp_get_ctx(irp); - switch (request->type) - { - case RINGS_NOP: + sqe_flags_t reg = (irp->flags >> SQE_SAVE) & SQE_REG_MASK; + if (reg != SQE_REG_NONE) { - request_nop_t* nop = (request_nop_t*)request; - nop->cancel = request_nop_cancel; - REQUEST_DELAY_NO_QUEUE(nop); - } - break; - default: - // Impossible due to the check in async_handle_sqe. - panic(NULL, "Invalid opcode %d", request->type); - break; + atomic_store_explicit(&ctx->rings.shared->regs[reg], irp->result, memory_order_release); } -} -static void async_ctx_push_cqe(async_ctx_t* ctx, rings_op_t opcode, errno_t error, void* data, uint64_t result) -{ rings_t* rings = &ctx->rings; - uint32_t tail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); uint32_t head = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); @@ -236,123 +209,133 @@ static void async_ctx_push_cqe(async_ctx_t* ctx, rings_op_t opcode, errno_t erro } cqe_t* cqe = &rings->cqueue[tail & rings->cmask]; - cqe->opcode = opcode; - cqe->error = error; - cqe->data = data; - cqe->_raw = result; + cqe->verb = irp->verb; + cqe->error = irp->err; + cqe->data = irp->data; + cqe->_result = irp->result; atomic_store_explicit(&rings->shared->ctail, tail + 1, memory_order_release); - wait_unblock(&ctx->waitQueue, WAIT_ALL, EOK); -} - -static void async_request_complete(request_t* request) -{ - async_ctx_t* ctx = request_get_ctx(request); - seq_regs_t reg = (request->flags >> SQE_SAVE) & SQE_REG_MASK; - if (reg != SQE_REG_NONE) - { - atomic_store_explicit(&ctx->rings.shared->regs[reg], request->result, memory_order_release); - } - - async_ctx_push_cqe(ctx, request->type, request->err, request->data, request->result); - - if (request->err != EOK && !(request->flags & SQE_HARDLINK)) + if (irp->err != EOK && !(irp->flags & SQE_HARDLINK)) { while (true) { - request_t* next = request_next(request); + irp_t* next = irp_next(irp); if (next == NULL) { break; } - request_free(next); + irp_free(next); } } else { - request_t* next = request_next(request); + irp_t* next = irp_next(irp); if (next != NULL) { async_dispatch(next); } } - request_free(request); + irp_free(irp); - if (ctx->requests->used == 0) + if (atomic_load(&ctx->irps->used) == 0) { UNREF(ctx->process); ctx->process = NULL; } } -static uint64_t async_handle_sqe(async_ctx_t* ctx, async_notify_ctx_t* notify, sqe_t* sqe) +static void async_dispatch(irp_t* irp) { - if (sqe->opcode < RINGS_MIN_OPCODE || sqe->opcode >= RINGS_MAX_OPCODE) + async_t* ctx = irp_get_ctx(irp); + + for (uint64_t i = 0; i < SEQ_MAX_ARGS; i++) { - async_ctx_push_cqe(ctx, sqe->opcode, EINVAL, sqe->data, ERR); - return 0; + sqe_flags_t reg = (irp->flags >> (i * SQE_REG_SHIFT)) & SQE_REG_MASK; + if (reg == SQE_REG_NONE) + { + continue; + } + + irp->sqe._args[i] = atomic_load_explicit(&ctx->rings.shared->regs[reg], memory_order_acquire); } - request_t* request = request_new(ctx->requests); - if (request == NULL) + switch (irp->verb) + { + case VERB_NOP: + break; + default: + break; + } + + irp_push(irp, async_complete, NULL); + irp_dispatch(irp); +} + +typedef struct +{ + list_t irps; + irp_t* link; +} async_notify_ctx_t; + +static uint64_t async_sqe_pop(async_t* ctx, async_notify_ctx_t* notify) +{ + irp_t* irp = irp_new(ctx->irps); + if (irp == NULL) { errno = ENOSPC; return ERR; } - if (ctx->requests->used == 1) + if (atomic_load(&ctx->irps->used) == 1) { process_t* process = process_current(); assert(&process->async[0] <= ctx && ctx <= &process->async[CONFIG_MAX_ASYNC_RINGS - 1]); ctx->process = REF(process); } - request->complete = async_request_complete; - request->cancel = NULL; - request->timeout = sqe->timeout; - request->flags = sqe->flags; - request->type = sqe->opcode; - request->err = EOK; - request->data = sqe->data; - request->result = 0; + rings_t* rings = &ctx->rings; + uint32_t stail = atomic_load_explicit(&rings->shared->stail, memory_order_acquire); + uint32_t shead = atomic_load_explicit(&rings->shared->shead, memory_order_relaxed); - for (uint64_t i = 0; i < SEQ_MAX_ARGS; i++) + if (shead == stail) { - request->args[i] = sqe->args[i]; + irp_free(irp); + return ERR; } + irp->sqe = rings->squeue[shead & rings->smask]; + atomic_store_explicit(&rings->shared->shead, shead + 1, memory_order_release); + if (notify->link != NULL) { - notify->link->next = request->index; + notify->link->next = irp->index; notify->link = NULL; } else { - list_push_back(¬ify->requests, &request->entry); + list_push_back(¬ify->irps, &irp->entry); } - if (sqe->flags & SQE_LINK || sqe->flags & SQE_HARDLINK) + if (irp->sqe.flags & SQE_LINK || irp->sqe.flags & SQE_HARDLINK) { - notify->link = request; + notify->link = irp; } return 0; } -uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) +uint64_t async_notify(async_t* ctx, size_t amount, size_t wait) { if (amount == 0) { return 0; } - /// @todo Implement the register state logic. - - if (async_ctx_acquire(ctx) == ERR) + if (async_acquire(ctx) == ERR) { errno = EBUSY; return ERR; @@ -360,58 +343,46 @@ uint64_t async_ctx_notify(async_ctx_t* ctx, size_t amount, size_t wait) if (!(atomic_load(&ctx->flags) & ASYNC_CTX_MAPPED)) { - async_ctx_release(ctx); + async_release(ctx); errno = EINVAL; return ERR; } - rings_t* rings = &ctx->rings; size_t processed = 0; async_notify_ctx_t notify = { - .requests = LIST_CREATE(notify.requests), + .irps = LIST_CREATE(notify.irps), .link = NULL, }; while (processed < amount) { - uint32_t stail = atomic_load_explicit(&rings->shared->stail, memory_order_acquire); - uint32_t shead = atomic_load_explicit(&rings->shared->shead, memory_order_relaxed); - - if (shead == stail) - { - break; - } - - sqe_t sqe = rings->squeue[shead & rings->smask]; - atomic_store_explicit(&rings->shared->shead, shead + 1, memory_order_release); - - if (async_handle_sqe(ctx, ¬ify, &sqe) == ERR) + if (async_sqe_pop(ctx, ¬ify) == ERR) { break; } processed++; } - while (!list_is_empty(¬ify.requests)) + while (!list_is_empty(¬ify.irps)) { - request_t* request = CONTAINER_OF(list_pop_front(¬ify.requests), request_t, entry); - async_dispatch(request); + irp_t* irp = CONTAINER_OF(list_pop_front(¬ify.irps), irp_t, entry); + async_dispatch(irp); } if (wait == 0) { - async_ctx_release(ctx); + async_release(ctx); return processed; } - if (WAIT_BLOCK(&ctx->waitQueue, async_ctx_avail_cqes(ctx) >= wait) == ERR) + if (WAIT_BLOCK(&ctx->waitQueue, async_avail_cqes(ctx) >= wait) == ERR) { - async_ctx_release(ctx); + async_release(ctx); return processed > 0 ? processed : ERR; } - async_ctx_release(ctx); + async_release(ctx); return processed; } @@ -426,11 +397,11 @@ SYSCALL_DEFINE(SYS_SETUP, rings_id_t, rings_t* userRings, void* address, size_t process_t* process = process_current(); space_t* space = &process->space; - async_ctx_t* ctx = NULL; + async_t* ctx = NULL; rings_id_t id = 0; for (id = 0; id < CONFIG_MAX_ASYNC_RINGS; id++) { - async_ctx_flags_t expected = ASYNC_CTX_NONE; + async_flags_t expected = ASYNC_CTX_NONE; if (atomic_compare_exchange_strong(&process->async[id].flags, &expected, ASYNC_CTX_BUSY)) { ctx = &process->async[id]; @@ -444,13 +415,13 @@ SYSCALL_DEFINE(SYS_SETUP, rings_id_t, rings_t* userRings, void* address, size_t return ERR; } - if (async_ctx_map(ctx, space, id, userRings, address, sentries, centries) == ERR) + if (async_map(ctx, space, id, userRings, address, sentries, centries) == ERR) { - async_ctx_release(ctx); + async_release(ctx); return ERR; } - async_ctx_release(ctx); + async_release(ctx); return id; } @@ -463,9 +434,9 @@ SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, rings_id_t id) } process_t* process = process_current(); - async_ctx_t* ctx = &process->async[id]; + async_t* ctx = &process->async[id]; - if (async_ctx_acquire(ctx) == ERR) + if (async_acquire(ctx) == ERR) { errno = EBUSY; return ERR; @@ -473,25 +444,25 @@ SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, rings_id_t id) if (!(atomic_load(&ctx->flags) & ASYNC_CTX_MAPPED)) { - async_ctx_release(ctx); + async_release(ctx); errno = EINVAL; return ERR; } - if (ctx->requests != NULL && ctx->requests->used > 0) + if (ctx->irps != NULL && atomic_load(&ctx->irps->used) != 0) { - async_ctx_release(ctx); + async_release(ctx); errno = EBUSY; return ERR; } - if (async_ctx_unmap(ctx) == ERR) + if (async_unmap(ctx) == ERR) { - async_ctx_release(ctx); + async_release(ctx); return ERR; } - async_ctx_release(ctx); + async_release(ctx); return 0; } @@ -504,7 +475,7 @@ SYSCALL_DEFINE(SYS_ENTER, uint64_t, rings_id_t id, size_t amount, size_t wait) } process_t* process = process_current(); - async_ctx_t* ctx = &process->async[id]; + async_t* ctx = &process->async[id]; - return async_ctx_notify(ctx, amount, wait); + return async_notify(ctx, amount, wait); } \ No newline at end of file diff --git a/src/kernel/sync/irp.c b/src/kernel/sync/irp.c new file mode 100644 index 000000000..ef8eeded3 --- /dev/null +++ b/src/kernel/sync/irp.c @@ -0,0 +1,304 @@ +#include <_internal/clock_t.h> +#include +#include +#include +#include +#include +#include + +#include + +typedef struct irp_ctx +{ + list_t timeouts; + lock_t lock; +} irp_ctx_t; + +PERCPU_DEFINE_CTOR(irp_ctx_t, pcpu_irps) +{ + irp_ctx_t* ctx = SELF_PTR(pcpu_irps); + + list_init(&ctx->timeouts); + lock_init(&ctx->lock); +} + +irp_pool_t* irp_pool_new(size_t size, void* ctx) +{ + if (size == 0 || size >= IRP_IDX_MAX) + { + errno = EINVAL; + return NULL; + } + + irp_pool_t* pool = malloc(sizeof(irp_pool_t) + (sizeof(irp_t) * size)); + if (pool == NULL) + { + errno = ENOMEM; + return NULL; + } + + pool->ctx = ctx; + atomic_init(&pool->used, 0); + atomic_init(&pool->free, 0); + for (irp_idx_t i = 0; i < (irp_idx_t)size; i++) + { + irp_t* irp = &pool->irps[i]; + list_entry_init(&irp->entry); + list_entry_init(&irp->timeoutEntry); + atomic_init(&irp->cancel, NULL); + irp->verb = VERB_MAX; + irp->flags = 0; + irp->timeout = CLOCKS_NEVER; + irp->data = NULL; + for (size_t j = 0; j < IRP_ARGS_MAX; j++) + { + irp->sqe._args[j] = 0; + } + irp->result = 0; + irp->err = EOK; + irp->index = i; + irp->next = i < size - 1 ? i + 1 : IRP_IDX_MAX; + irp->location = IRP_LOC_MAX; + irp->cpu = CPU_ID_INVALID; + for (size_t j = 0; j < IRP_LOC_MAX; j++) + { + irp->stack[j].ctx = NULL; + irp->stack[j].complete = NULL; + } + } + + return pool; +} + +void irp_pool_free(irp_pool_t* pool) +{ + free(pool); +} + +irp_t* irp_new(irp_pool_t* pool) +{ + irp_idx_t idx; + for (;;) + { + uint64_t head = atomic_load_explicit(&pool->free, memory_order_acquire); + idx = (irp_idx_t)(head & IRP_IDX_MAX); + if (idx == IRP_IDX_MAX) + { + return NULL; + } + + uint64_t newHead = ((head & ~IRP_IDX_MAX) + IRP_TAG_INC) | pool->irps[idx].next; + if (atomic_compare_exchange_weak_explicit(&pool->free, &head, newHead, memory_order_acquire, + memory_order_relaxed)) + { + break; + } + ASM("pause"); + } + + atomic_fetch_add_explicit(&pool->used, 1, memory_order_relaxed); + irp_t* irp = &pool->irps[idx]; + irp->location = IRP_LOC_MAX; + irp->next = IRP_IDX_MAX; + irp->err = EINPROGRESS; + irp->result = 0; + irp->sqe = (sqe_t){0}; + atomic_store_explicit(&irp->cancel, NULL, memory_order_relaxed); + irp->next = IRP_IDX_MAX; + irp->cpu = CPU_ID_INVALID; + for (size_t j = 0; j < IRP_LOC_MAX; j++) + { + irp->stack[j].ctx = NULL; + irp->stack[j].complete = NULL; + } + return irp; +} + +void irp_free(irp_t* irp) +{ + irp_pool_t* pool = irp_pool_get(irp); + + irp_idx_t idx = irp->index; + for (;;) + { + uint64_t head = atomic_load_explicit(&pool->free, memory_order_relaxed); + irp->next = (irp_idx_t)(head & IRP_IDX_MAX); + uint64_t newHead = ((head & ~IRP_IDX_MAX) + IRP_TAG_INC) | idx; + + if (atomic_compare_exchange_weak_explicit(&pool->free, &head, newHead, memory_order_release, + memory_order_relaxed)) + { + break; + } + ASM("pause"); + } + atomic_fetch_sub_explicit(&pool->used, 1, memory_order_relaxed); +} + +uint64_t irp_cancel(irp_t* irp) +{ + irp_cancel_t handler = atomic_exchange(&irp->cancel, IRP_CANCELLED); + if (handler == IRP_CANCELLED) + { + errno = EBUSY; + return ERR; + } + + if (handler == NULL) + { + errno = EBUSY; + return ERR; + } + + irp->err = ECANCELED; + return handler(irp); +} + +void irp_timeout_add(irp_t* irp) +{ + if (irp->timeout == CLOCKS_NEVER) + { + return; + } + + irp_ctx_t* ctx = SELF_PTR(pcpu_irps); + LOCK_SCOPE(&ctx->lock); + + irp->cpu = SELF->id; + + clock_t now = clock_uptime(); + irp->deadline = CLOCKS_DEADLINE(irp->timeout, now); + + irp_t* entry; + LIST_FOR_EACH(entry, &ctx->timeouts, timeoutEntry) + { + if (irp->deadline < entry->deadline) + { + list_prepend(&entry->timeoutEntry, &irp->timeoutEntry); + timer_set(now, irp->deadline); + return; + } + } + + list_push_back(&ctx->timeouts, &irp->timeoutEntry); + timer_set(now, irp->deadline); +} + +void irp_timeout_remove(irp_t* irp) +{ + cpu_id_t cpu = irp->cpu; + if (cpu == CPU_ID_INVALID) + { + return; + } + + irp_ctx_t* ctx = CPU_PTR(cpu, pcpu_irps); + assert(ctx != NULL); + + LOCK_SCOPE(&ctx->lock); + if (irp->cpu != cpu) + { + return; + } + + list_remove(&irp->timeoutEntry); + irp->cpu = CPU_ID_INVALID; +} + +void irp_timeouts_check(void) +{ + irp_ctx_t* ctx = SELF_PTR(pcpu_irps); + assert(ctx != NULL); + + clock_t now = clock_uptime(); + + lock_acquire(&ctx->lock); + + irp_t* irp; + while (true) + { + irp = CONTAINER_OF_SAFE(list_first(&ctx->timeouts), irp_t, timeoutEntry); + if (irp == NULL) + { + break; + } + + if (irp->deadline > now) + { + timer_set(now, irp->deadline); + break; + } + + list_remove(&irp->timeoutEntry); + irp->deadline = CLOCKS_NEVER; + irp->cpu = CPU_ID_INVALID; + lock_release(&ctx->lock); + + irp_cancel_t handler = atomic_exchange(&irp->cancel, IRP_CANCELLED); + if (handler == IRP_CANCELLED) + { + // Already cancelled + } + else if (handler != NULL) + { + irp->err = ETIMEDOUT; + handler(irp); + } + + lock_acquire(&ctx->lock); + } + + lock_release(&ctx->lock); +} + +void irp_dispatch(irp_t* irp) +{ + if (irp->verb >= VERB_MAX || _irp_table_start[irp->verb].handler == NULL) + { + irp->err = ENOSYS; + irp_complete(irp); + return; + } + + _irp_table_start[irp->verb].handler(irp); +} + +static int irp_handler_cmp(const void* a, const void* b) +{ + const irp_handler_t* irpA = (const irp_handler_t*)a; + const irp_handler_t* irpB = (const irp_handler_t*)b; + return irpA->verb - irpB->verb; +} + +void irp_table_init(void) +{ + const uint64_t irpsInTable = (((uint64_t)_irp_table_end - (uint64_t)_irp_table_start) / sizeof(irp_handler_t)); + assert(irpsInTable == VERB_MAX); + + LOG_INFO("sorting IRP table, total IRPs %d\n", VERB_MAX); + qsort(_irp_table_start, irpsInTable, sizeof(irp_handler_t), irp_handler_cmp); + + for (uint64_t i = 0; i < irpsInTable; i++) + { + assert(_irp_table_start[i].verb == i); + } +} + +static uint64_t _nop_cancel(irp_t* irp) +{ + irp_complete(irp); + return 0; +} + +void nop_do(irp_t* irp) +{ + irp_set_cancel(irp, _nop_cancel); + + if (irp->timeout != CLOCKS_NEVER) + { + irp_timeout_add(irp); + return; + } +} + +IRP_REGISTER(VERB_NOP, nop_do); \ No newline at end of file diff --git a/src/kernel/sync/request.c b/src/kernel/sync/request.c deleted file mode 100644 index e47ed2719..000000000 --- a/src/kernel/sync/request.c +++ /dev/null @@ -1,142 +0,0 @@ -#include -#include -#include -#include - -#include - -typedef struct request_ctx -{ - list_t timeouts; - lock_t lock; -} request_ctx_t; - -PERCPU_DEFINE_CTOR(request_ctx_t, pcpu_requests) -{ - request_ctx_t* ctx = SELF_PTR(pcpu_requests); - - list_init(&ctx->timeouts); - lock_init(&ctx->lock); -} - -request_pool_t* request_pool_new(size_t size, void* ctx) -{ - if (size == 0 || size >= REQUEST_ID_MAX) - { - errno = EINVAL; - return NULL; - } - - request_pool_t* pool = malloc(sizeof(request_pool_t) + (sizeof(request_t) * size)); - if (pool == NULL) - { - errno = ENOMEM; - return NULL; - } - - pool->ctx = ctx; - pool->used = 0; - list_init(&pool->free); - for (request_id_t i = 0; i < (request_id_t)size; i++) - { - request_t* request = &pool->requests[i]; - - list_entry_init(&request->entry); - list_entry_init(&request->timeoutEntry); - request->complete = NULL; - request->cancel = NULL; - request->deadline = CLOCKS_NEVER; - request->cpu = CPU_ID_INVALID; - request->index = i; - request->next = REQUEST_ID_MAX; - request->flags = 0; - request->type = 0; - request->err = 0; - request->data = NULL; - request->result = 0; - for (size_t j = 0; j < ARRAY_SIZE(request->args); j++) - { - request->args[j] = 0; - } - - list_push_back(&pool->free, &pool->requests[i].entry); - } - - return pool; -} - -void request_pool_free(request_pool_t* pool) -{ - free(pool); -} - -void request_timeout_add(request_t* request) -{ - request_ctx_t* ctx = SELF_PTR(pcpu_requests); - LOCK_SCOPE(&ctx->lock); - - request->cpu = SELF->id; - - clock_t now = clock_uptime(); - request->deadline = CLOCKS_DEADLINE(request->timeout, now); - - request_t* entry; - LIST_FOR_EACH(entry, &ctx->timeouts, timeoutEntry) - { - if (request->deadline < entry->deadline) - { - list_prepend(&entry->entry, &request->timeoutEntry); - timer_set(now, request->deadline); - return; - } - } - - list_push_back(&ctx->timeouts, &request->timeoutEntry); - timer_set(now, request->deadline); -} - -void request_timeout_remove(request_t* request) -{ - request_ctx_t* ctx = CPU_PTR(request->cpu, pcpu_requests); - assert(ctx != NULL); - - LOCK_SCOPE(&ctx->lock); - list_remove(&request->timeoutEntry); -} - -void request_timeouts_check(void) -{ - request_ctx_t* ctx = SELF_PTR(pcpu_requests); - assert(ctx != NULL); - - clock_t now = clock_uptime(); - - lock_acquire(&ctx->lock); - - request_t* request; - while (true) - { - request = CONTAINER_OF_SAFE(list_first(&ctx->timeouts), request_t, timeoutEntry); - if (request == NULL) - { - break; - } - - if (request->deadline > now) - { - timer_set(now, request->deadline); - break; - } - - list_remove(&request->timeoutEntry); - lock_release(&ctx->lock); - - assert(request->cancel != NULL); - request->err = ETIMEDOUT; - request->cancel(request); - - lock_acquire(&ctx->lock); - } - - lock_release(&ctx->lock); -} \ No newline at end of file diff --git a/src/kernel/sync/requests.c b/src/kernel/sync/requests.c deleted file mode 100644 index 81f9e6962..000000000 --- a/src/kernel/sync/requests.c +++ /dev/null @@ -1,7 +0,0 @@ -#include - -bool request_nop_cancel(request_nop_t* request) -{ - REQUEST_COMPLETE(request, 0); - return true; -} \ No newline at end of file diff --git a/src/programs/utils/ringstest/main.c b/src/programs/utils/ringstest/main.c index 42aaf0ba4..cc33ea78b 100644 --- a/src/programs/utils/ringstest/main.c +++ b/src/programs/utils/ringstest/main.c @@ -20,11 +20,11 @@ int main() memset(&rings.shared->regs, -1, sizeof(rings.shared->regs)); printf("pushing nop sqe to rings %llu...\n", id); - sqe_t sqe = SQE_CREATE(RINGS_NOP, SQE_HARDLINK | (SQE_REG0 << SQE_SAVE), CLOCKS_PER_SEC, 0x1234); + sqe_t sqe = SQE_CREATE(VERB_NOP, SQE_HARDLINK | (SQE_REG0 << SQE_SAVE), CLOCKS_PER_SEC, 0x1234); sqe_push(&rings, &sqe); printf("pushing nop sqe to rings %llu...\n", id); - sqe = (sqe_t)SQE_CREATE(RINGS_NOP, SQE_LINK, CLOCKS_PER_SEC, 0x5678); + sqe = (sqe_t)SQE_CREATE(VERB_NOP, SQE_LINK, CLOCKS_PER_SEC, 0x5678); sqe_push(&rings, &sqe); printf("entering rings...\n"); @@ -40,9 +40,9 @@ int main() printf("cqe:\n"); printf("cqe data: %p\n", cqe.data); - printf("cqe opcode: %d\n", cqe.opcode); + printf("cqe verb: %d\n", cqe.verb); printf("cqe error: %s\n", strerror(cqe.error)); - printf("cqe result: %llu\n", cqe._raw); + printf("cqe result: %llu\n", cqe._result); } printf("registers:\n"); diff --git a/src/programs/utils/stat/main.c b/src/programs/utils/stat/main.c index b85d36a1d..0985abc1b 100644 --- a/src/programs/utils/stat/main.c +++ b/src/programs/utils/stat/main.c @@ -10,7 +10,7 @@ static const char* type_to_string(itype_t type) { switch (type) { - case INODE_FILE: + case INODE_REGULAR: return "file"; case INODE_DIR: return "directory"; From 9837728fafaf4d4faee8d42b85e27f9e2f087617 Mon Sep 17 00:00:00 2001 From: KN Date: Tue, 20 Jan 2026 12:46:34 +0100 Subject: [PATCH 12/23] feat(kernel:pmm): new pmm with reference counting --- include/kernel/mem/mem_desc.h | 30 +++ include/kernel/mem/pmm.h | 148 +++++++++----- include/kernel/mem/pmm_bitmap.h | 71 ------- include/kernel/mem/pmm_stack.h | 80 -------- include/kernel/mem/pool.h | 74 +++++++ include/kernel/sync/async.h | 8 +- include/kernel/sync/irp.h | 38 ++-- include/libstd/sys/bitmap.h | 19 +- include/libstd/sys/rings.h | 8 +- src/kernel/cpu/syscall.c | 2 +- src/kernel/drivers/perf.c | 4 +- src/kernel/init/boot_info.c | 2 +- src/kernel/init/init.c | 2 +- src/kernel/log/panic.c | 4 +- src/kernel/mem/mdl.c | 0 src/kernel/mem/pmm.c | 329 ++++++++++++++++++++++--------- src/kernel/mem/pmm_bitmap.c | 41 ---- src/kernel/mem/pmm_stack.c | 59 ------ src/kernel/mem/pool.c | 67 +++++++ src/kernel/mem/vmm.c | 26 +-- src/kernel/sync/async.c | 55 ++++-- src/kernel/sync/irp.c | 60 ++---- src/modules/acpi/acpi.c | 2 +- src/modules/acpi/tables.c | 4 +- src/modules/drivers/apic/lapic.c | 3 +- src/modules/drivers/hpet/hpet.c | 4 +- src/modules/drivers/pci/config.c | 4 +- 27 files changed, 632 insertions(+), 512 deletions(-) create mode 100644 include/kernel/mem/mem_desc.h delete mode 100644 include/kernel/mem/pmm_bitmap.h delete mode 100644 include/kernel/mem/pmm_stack.h create mode 100644 include/kernel/mem/pool.h create mode 100644 src/kernel/mem/mdl.c delete mode 100644 src/kernel/mem/pmm_bitmap.c delete mode 100644 src/kernel/mem/pmm_stack.c create mode 100644 src/kernel/mem/pool.c diff --git a/include/kernel/mem/mem_desc.h b/include/kernel/mem/mem_desc.h new file mode 100644 index 000000000..8f6463945 --- /dev/null +++ b/include/kernel/mem/mem_desc.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +#include + +typedef struct process process_t; + +/** + * @brief Memory Descriptor. + * @defgroup kernel_mem_mem_desc Memory Descriptor + * @ingroup kernel_mem + * + * @{ + */ + +typedef struct mem_seg +{ + pfn_t page; + uint32_t length; + uint32_t offset; +} mem_seg_t; + +typedef struct ALIGNED(64) mem_desc +{ + +} mem_desc_t; + +/** @} */ \ No newline at end of file diff --git a/include/kernel/mem/pmm.h b/include/kernel/mem/pmm.h index 14e36902f..39d77d15f 100644 --- a/include/kernel/mem/pmm.h +++ b/include/kernel/mem/pmm.h @@ -2,120 +2,162 @@ #include +#include + /** * @brief Physical Memory Manager (PMM). * @defgroup kernel_mem_pmm PMM * @ingroup kernel_mem * - * The Physical Memory Manager (PMM) is responsible for managing physical memory pages. It uses a free stack for - * allocating single pages in constant-time, and for more specialized allocations (requiring a specific address range - * or alignment) a bitmap allocator is used. The bitmap allocator should only be used when no other option is available. + * The Physical Memory Manager (PMM) is responsible for allocating and freeing physical memory pages. + * + * @note All physical pages are identity mapped to the higher half of the address space, as such, the PMM will always + * return valid and usable higher half addresses. + * + * ## The Free Page Stack + * + * For most allocations, the PMM uses a fast `O(1)` stack-based allocator to manage free pages, with the limitation that + * only single pages can be allocated or freed at a time. + * + * ## The Bitmap Allocator + * + * For larger, contiguous or aligned allocations, the PMM uses a bitmap allocator. This allocator is slower (`O(n)`) but + * is usefull for more specialized allocations. + * + * ## Reference Counting * - * All physical memory is identity mapped to the beginning of the higher half of the address space. This means that for - * example `NULL` is always an invalid address and that the PMM returns addresses in the higher half. + * All allocations from the PMM are referenced counted, meaning that a page is only freed when its reference count + * reaches zero. This allows pages to be passed around between subsystems without fear of double frees or + * use-after-frees. * - * The free stack provides some advantages over for instance a free list, mainly due to cache improvements. + * @see kernel_mem_mem_desc * * @{ */ /** - * @brief Initializes the Physical Memory Manager. + * @brief The type used for page reference counts. + */ +typedef uint16_t pmm_ref_t; + +/** + * @brief Invalid page reference count. + */ +#define PAGE_REF_MAX UINT16_MAX + +/** + * @brief Maximum number of free pages that can be stored in a free page. + */ +#define FREE_PAGE_MAX (PAGE_SIZE / sizeof(void*) - 1) + +/** + * @brief Stored in free pages to form a stack of free pages. + * @struct page_stack_t + */ +typedef struct page_stack +{ + struct page_stack* next; + void* pages[FREE_PAGE_MAX]; +} page_stack_t; + +static_assert(sizeof(page_stack_t) == PAGE_SIZE, "page_stack_t must be exactly one page in size"); + +/** + * @brief Read the boot info memory map and initialize the PMM. */ void pmm_init(void); /** - * @brief Allocates a single physical page. + * @brief Allocate a single page of physical memory. * - * The returned page will not be zeroed. + * Will by default use the free stack allocator, but if no pages are available there, it will fall back to the bitmap + * allocator. * - * @return On success, returns the higher half physical address of the allocated page. On failure, returns `NULL`. + * @return Pointer to the allocated page, or `NULL` if no memory is available. */ void* pmm_alloc(void); /** - * @brief Allocates multiple physical pages. - * - * The `pmm_alloc_pages` function allocates `count` non-contiguous physical pages from the free stack, by using this - * function its possible to avoid holding the lock for each page allocation, improving performance when allocating many - * pages at once. + * @brief Allocate multiple pages of physical memory. * - * The returned pages will not be zeroed. + * Usefull for reducing overhead from locking when allocating many pages. * - * @param addresses An array where the higher half physical addresses of the allocated pages will be stored. - * @param count The number of pages to allocate. - * @return On success, `0`. On failure, `ERR` and `errno` is set. + * @param addresses Array to store the allocated page addresses. + * @param count Number of pages to allocate. + * @return On success, `0`. On failure, `ERR` and no pages are allocated. */ uint64_t pmm_alloc_pages(void** addresses, size_t count); /** - * @brief Allocates a contiguous region of physical pages managed by the bitmap. + * @brief Allocate a contiguous region of physical memory using the bitmap. * - * The `pmm_alloc_bitmap` function allocates a contiguous block of `count` physical pages from the memory region - * managed by the bitmap. It also enforces a maximum address and alignment for the allocation. - * - * The returned pages will not be zeroed. - * - * @param count The number of contiguous pages to allocate. - * @param maxAddr The maximum physical address (exclusive) for the allocation. - * @param alignment The required alignment for the allocated region, in bytes. - * @return On success, returns the higher half physical address of the allocated region. On failure, returns `NULL`. + * @param count Number of pages to allocate. + * @param maxAddr Maximum address to allocate up to (exclusive). + * @param alignment Alignment of the region. + * @return Pointer to the first allocated page, or `NULL` if no memory is available. */ void* pmm_alloc_bitmap(size_t count, uintptr_t maxAddr, uint64_t alignment); /** - * @brief Frees a single physical page. + * @brief Free a single page of physical memory. * - * The `pmm_free` function frees a page returning ownership of it to the PMM. The PMM will determine based on the - * address if it's owned by the bitmap or the free stack. + * The page will only be reclaimed if its reference count reaches zero. * - * @param address The higher half physical address of the page to free. + * @param address Pointer to the page to free. */ void pmm_free(void* address); /** - * @brief Frees multiple physical pages. + * @brief Free multiple pages of physical memory. * - * The `pmm_free_pages` function frees `count` physical pages returning ownership of them to the PMM. The PMM will - * determine based on the addresses if they're owned by the bitmap or the free stack. + * Useful for reducing overhead from locking when freeing many pages. * - * @param addresses An array containing the higher half physical addresses of the pages to free. - * @param count The number of pages to free. + * The pages will only be reclaimed if its reference count reaches zero. + * + * @param addresses Array of pointers to the pages to free. + * @param count Number of pages to free. */ void pmm_free_pages(void** addresses, size_t count); /** - * @brief Frees a contiguous region of physical pages. + * @brief Free a contiguous region of physical memory. * - * The `pmm_free_region` function frees a contiguous block of `count` physical pages, returning ownership of them to the - * PMM. The PMM will determine based on the address if it's owned by the bitmap or the free stack. + * The pages will only be reclaimed if its reference count reaches zero. * - * @param address The higher half physical address of the first page in the region to free. - * @param count The number of pages to free. + * @param address Pointer to the first page of the region to free. + * @param count Number of pages to free. */ void pmm_free_region(void* address, size_t count); /** - * @brief Retrieves the total amount of physical memory managed by the PMM. + * @brief Increment the reference count of a physical page. + * + * Will fail if the page is not allocated. * - * @return The total amount of physical memory in pages. + * @param address Address of the physical page. + * @return On success, the new reference count. On failure, `ERR`. */ -size_t pmm_total_amount(void); +uint64_t pmm_ref_inc(void* address); /** - * @brief Retrieves the amount of free physical memory. + * @brief Get the total number of physical pages. * - * @return The amount of currently free physical memory in pages. + * @return Total number of physical pages. */ -size_t pmm_free_amount(void); +size_t pmm_total_pages(void); /** - * @brief Retrieves the amount of reserved physical memory. + * @brief Get the number of available physical pages. * - * Reserved memory includes memory that is not available for allocation (e.g., kernel code, hardware regions). + * @return Number of available physical pages. + */ +size_t pmm_avail_pages(void); + +/** + * @brief Get the number of used physical pages. * - * @return The amount of reserved physical memory in pages. + * @return Number of used physical pages. */ -size_t pmm_used_amount(void); +size_t pmm_used_pages(void); /** @} */ diff --git a/include/kernel/mem/pmm_bitmap.h b/include/kernel/mem/pmm_bitmap.h deleted file mode 100644 index 4f1fd8308..000000000 --- a/include/kernel/mem/pmm_bitmap.h +++ /dev/null @@ -1,71 +0,0 @@ -#pragma once - -#include -#include - -/** - * @brief A generic bitmap page allocator. - * @defgroup kernel_mem_pmm_bitmap PMM Bitmap - * @ingroup kernel_mem - * - * The PMM bitmap provides a flexible allocator for more specific allocations, for example it can handle contiguous - * pages, specific alignments and allocating below some specified address. This flexibility comes at the cost of - * performance, so the bitmap should only be used when necessary. - * - * @{ - */ - -/** - * @brief Represents a bitmap allocator's state. - */ -typedef struct -{ - /** - * @brief The underlying bitmap used for tracking page status. - */ - bitmap_t bitmap; - /** - * @brief The number of free pages in the bitmap. - */ - uint64_t free; - /** - * @brief The total number of pages managed by the bitmap. - */ - uint64_t total; - /** - * @brief The maximum address managed by the bitmap. - */ - uintptr_t maxAddr; -} pmm_bitmap_t; - -/** - * @brief Initializes a PMM bitmap. - * - * @param bitmap The bitmap to initialize. - * @param buffer The buffer to use for the bitmap data. - * @param size The number of pages to manage. - * @param maxAddr The maximum address to manage. - */ -void pmm_bitmap_init(pmm_bitmap_t* bitmap, void* buffer, uint64_t size, uintptr_t maxAddr); - -/** - * @brief Allocates a contiguous region of pages from the bitmap. - * - * @param bitmap The bitmap to allocate from. - * @param count The number of pages to allocate. - * @param maxAddr The maximum address for the allocation. - * @param alignment The required alignment for the allocation. - * @return On success, a pointer to the allocated region. On failure `NULL` and errno is set. - */ -void* pmm_bitmap_alloc(pmm_bitmap_t* bitmap, uint64_t count, uintptr_t maxAddr, uint64_t alignment); - -/** - * @brief Frees a region of pages, returning them to the bitmap. - * - * @param bitmap The bitmap to free to. - * @param address The address of the region to free. - * @param count The number of pages to free. - */ -void pmm_bitmap_free(pmm_bitmap_t* bitmap, void* address, uint64_t count); - -/** @} */ diff --git a/include/kernel/mem/pmm_stack.h b/include/kernel/mem/pmm_stack.h deleted file mode 100644 index f6c371226..000000000 --- a/include/kernel/mem/pmm_stack.h +++ /dev/null @@ -1,80 +0,0 @@ -#pragma once - -#include - -/** - * @brief A generic free stack page allocator. - * @defgroup kernel_mem_pmm_stack PMM Stack - * @ingroup kernel_mem - * - * The PMM stack provides a fast, O(1) allocator for single pages. It uses freed pages to store metadata about other - * free pages, forming a stack of page buffers. - * - * @{ - */ - -/** - * @brief Structure for a page buffer in the PMM stack. - * - * The `page_buffer_t` structure is stored in free pages and keeps track of pages that are currently freed. - */ -typedef struct page_buffer -{ - /** - * @brief Pointer to the previous page buffer in the stack. - */ - struct page_buffer* prev; - /** - * @brief Flexible array member to store free physical pages. - */ - void* pages[]; -} page_buffer_t; - -/** - * @brief The maximum number of pages that can be stored in a `page_buffer_t`. - */ -#define PMM_BUFFER_MAX ((PAGE_SIZE - sizeof(page_buffer_t)) / sizeof(void*)) - -/** - * @brief PMM stack structure for managing higher physical memory. - */ -typedef struct -{ - /** - * @brief Pointer to the last page buffer in the stack. - */ - page_buffer_t* last; - /** - * @brief Current index within the `pages` array of the `last` page buffer. - */ - uint64_t index; - /** - * @brief The number of free pages in the stack. - */ - uint64_t free; -} pmm_stack_t; - -/** - * @brief Initializes a PMM stack. - * - * @param stack The stack to initialize. - */ -void pmm_stack_init(pmm_stack_t* stack); - -/** - * @brief Allocates a single page from the stack. - * - * @param stack The stack to allocate from. - * @return On success, a pointer to the allocated page. On failure `NULL` and errno is set. - */ -void* pmm_stack_alloc(pmm_stack_t* stack); - -/** - * @brief Frees a single page, returning it to the stack. - * - * @param stack The stack to free to. - * @param address The address of the page to free. - */ -void pmm_stack_free(pmm_stack_t* stack, void* address); - -/** @} */ diff --git a/include/kernel/mem/pool.h b/include/kernel/mem/pool.h new file mode 100644 index 000000000..966e55cdc --- /dev/null +++ b/include/kernel/mem/pool.h @@ -0,0 +1,74 @@ +#pragma once + +#include +#include +#include + +#include + +/** + * @brief Lock-free memory pool. + * @defgroup kernel_mem_pool Pool + * @ingroup kernel_mem + * + * The memory pool system provides a lock-free allocator using a pre-allocated array. Its intended to be used for + * performance-critical structures and as a even more specialized alternative to the Object Cache. + * + * In addition to its performance advantages, since the pool uses an array to store its objects, it is possible for + * certain structures to avoid storing full pointers to objects allocated from a pool instead using a `pool_idx_t` and + * thus saving memory or allowing better caching. + * + * @{ + */ + +/** + * @brief Pool index type. + */ +typedef uint16_t pool_idx_t; + +#define POOL_IDX_MAX UINT16_MAX ///< The maximum index value for pool. + +#define POOL_TAG_INC ((uint64_t)(POOL_IDX_MAX) + 1) ///< The amount to increment the tag by in the tagged free list. + +/** + * @brief Pool structure. + * @struct pool_t + */ +typedef struct +{ + atomic_size_t used; ///< Number of used elements. + atomic_uint64_t free; ///< The tagged head of the free list. + void* elements; ///< Pointer to the elements array. + size_t elementSize; ///< Size of each element. + size_t nextOffset; ///< Offset of a `pool_idx_t` variable within each element used for the free list. + size_t capacity; ///< Maximum number of elements. +} pool_t; + +/** + * @brief Initialize a pool. + * + * @param pool Pointer to the pool structure to initialize. + * @param elements Pointer to the elements array. + * @param capacity Maximum number of elements. + * @param elementSize Size of each element. + * @param nextOffset Offset of a `pool_idx_t` variable within each element used for the free list. + */ +void pool_init(pool_t* pool, void* elements, size_t capacity, size_t elementSize, size_t nextOffset); + +/** + * @brief Allocate an element from the pool. + * + * @param pool Pointer to the pool to allocate from. + * @return The index of the allocated element, or `POOL_IDX_MAX` if the pool is full. + */ +pool_idx_t pool_alloc(pool_t* pool); + +/** + * @brief Free an element back to the pool. + * + * @param pool Pointer to the pool to free to. + * @param idx The index of the element to free. + */ +void pool_free(pool_t* pool, pool_idx_t idx); + +/** @} */ \ No newline at end of file diff --git a/include/kernel/sync/async.h b/include/kernel/sync/async.h index 093e0a3bb..04413271f 100644 --- a/include/kernel/sync/async.h +++ b/include/kernel/sync/async.h @@ -99,16 +99,16 @@ */ typedef enum { - ASYNC_CTX_NONE = 0, ///< No flags set. - ASYNC_CTX_BUSY = 1 << 0, ///< Context is currently being used, used for fast locking. - ASYNC_CTX_MAPPED = 1 << 1, ///< Context rings are mapped. + ASYNC_NONE = 0, ///< No flags set. + ASYNC_BUSY = 1 << 0, ///< Context is currently being used, used for fast locking. + ASYNC_MAPPED = 1 << 1, ///< Context rings are mapped. } async_flags_t; /** * @brief The kernel-side asynchronous context structure. * @struct async_t */ -typedef struct async_ctx +typedef struct async { rings_t rings; ///< Asynchronous rings information. irp_pool_t* irps; ///< Pool of preallocated IRPs. diff --git a/include/kernel/sync/irp.h b/include/kernel/sync/irp.h index 27d3ed5df..8ad36dd8d 100644 --- a/include/kernel/sync/irp.h +++ b/include/kernel/sync/irp.h @@ -1,5 +1,7 @@ #pragma once +#include +#include #include #include @@ -11,9 +13,10 @@ #include #include -typedef struct async_ctx async_t; +typedef struct async async_t; typedef struct process process_t; typedef struct file file_t; +typedef struct pathname pathname_t; typedef struct irp irp_t; @@ -209,17 +212,6 @@ typedef struct irp irp_t; * @{ */ -/** - * @brief Represents the index of a IRP in a IRP pool. - * - * Used to save space in a IRP, by storing indexes instead of pointers. - */ -typedef uint16_t irp_idx_t; - -#define IRP_IDX_MAX UINT16_MAX ///< The maximum index value for IRP. - -#define IRP_TAG_INC ((uint64_t)(IRP_IDX_MAX) + 1) ///< The amount to increment the tag by in the tagged free list. - #define IRP_LOC_MAX 8 ///< The maximum number of locations in a IRP. #define IRP_ARGS_MAX 5 ///< The maximum number of arguments in a IRP. @@ -271,8 +263,7 @@ typedef struct ALIGNED(64) irp union { struct { - verb_t verb; ///< Verb specifying the action to perform. - uint8_t _reserved1[3]; + verb_t verb; ///< Verb specifying the action to perform. sqe_flags_t flags; ///< Submission flags. union { clock_t timeout; ///< The timeout starting from when the IRP is added to a timeout queue. @@ -283,7 +274,7 @@ typedef struct ALIGNED(64) irp struct { file_t* from; - char* path; + pathname_t* path; } open; uint64_t _args[IRP_ARGS_MAX]; }; @@ -292,8 +283,8 @@ typedef struct ALIGNED(64) irp }; uint64_t result; ///< Result of the IRP. errno_t err; ///< The error code of the operation, also used to specify its current state. - irp_idx_t index; ///< Index of the IRP in its pool. - irp_idx_t next; ///< Index of the next IRP in a chain or in the free list. + pool_idx_t index; ///< Index of the IRP in its pool. + pool_idx_t next; ///< Index of the next IRP in a chain or in the free list. cpu_id_t cpu; ///< The CPU whose timeout queue the IRP is in. uint8_t location; ///< The index of the current location in the stack. uint8_t _reserved2[5]; @@ -312,10 +303,9 @@ static_assert(offsetof(irp_t, _args) == offsetof(irp_t, sqe._args), "args offset */ typedef struct irp_pool { - void* ctx; ///< Context pointer. - atomic_size_t used; ///< Number of used IRPs. - atomic_uint64_t free; ///< The tagged head of the free list. - irp_t irps[]; ///< Array of IRPs. + void* ctx; + pool_t pool; + irp_t irps[]; } irp_pool_t; /** @@ -433,13 +423,13 @@ static inline void* irp_get_ctx(irp_t* irp) static inline irp_t* irp_next(irp_t* irp) { irp_pool_t* pool = irp_pool_get(irp); - if (irp->next == IRP_IDX_MAX) + if (irp->next == POOL_IDX_MAX) { return NULL; } irp_t* next = &pool->irps[irp->next]; - irp->next = IRP_IDX_MAX; + irp->next = POOL_IDX_MAX; return next; } @@ -521,6 +511,8 @@ uint64_t irp_cancel(irp_t* irp); /** * @brief Dispatch an IRP to the appropriate handler. * + * If `irp->err != EINPROGRESS` the IRP is immediately completed. + * * @param irp Pointer to the IRP to dispatch. */ void irp_dispatch(irp_t* irp); diff --git a/include/libstd/sys/bitmap.h b/include/libstd/sys/bitmap.h index c51702fdd..9ce03b36c 100644 --- a/include/libstd/sys/bitmap.h +++ b/include/libstd/sys/bitmap.h @@ -84,6 +84,16 @@ typedef struct uint64_t name##Buffer[BITMAP_BITS_TO_QWORDS(bits)] = {0}; \ bitmap_t name = {.firstZeroIdx = 0, .length = (bits), .buffer = name##Buffer} +/** + * @brief Define and create a one-initialized bitmap and its buffer. + * + * @param name Name of the bitmap. + * @param bits Length of the bitmap in bits. + */ +#define BITMAP_CREATE_ONE(name, bits) \ + uint64_t name##Buffer[BITMAP_BITS_TO_QWORDS(bits)] = {-1ULL}; \ + bitmap_t name = {.firstZeroIdx = 0, .length = (bits), .buffer = name##Buffer} + /** * @brief Define a bitmap and its buffer. * @@ -311,7 +321,8 @@ static inline uint64_t bitmap_find_first_clear(bitmap_t* map, uint64_t startIdx, uint64_t maskedQword = qword | ((1ULL << bitIdx) - 1); if (maskedQword != ~0ULL) { - return qwordIdx * 64 + __builtin_ctzll(~maskedQword); + uint64_t res = qwordIdx * 64 + __builtin_ctzll(~maskedQword); + return res < endIdx ? res : map->length; } qwordIdx++; } @@ -320,7 +331,8 @@ static inline uint64_t bitmap_find_first_clear(bitmap_t* map, uint64_t startIdx, { if (map->buffer[i] != ~0ULL) { - return i * 64 + __builtin_ctzll(~map->buffer[i]); + uint64_t res = i * 64 + __builtin_ctzll(~map->buffer[i]); + return res < endIdx ? res : map->length; } } @@ -356,7 +368,8 @@ static inline uint64_t bitmap_find_first_set(bitmap_t* map, uint64_t startIdx, u if (qword != 0) { - return startQwordIdx * 64 + __builtin_ctzll(qword); + uint64_t res = startQwordIdx * 64 + __builtin_ctzll(qword); + return res < endIdx ? res : map->length; } startQwordIdx++; diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/rings.h index 2cd825fb5..3e4fa5deb 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/rings.h @@ -23,7 +23,7 @@ extern "C" * @{ */ -typedef uint8_t verb_t; ///< Verb type. +typedef uint32_t verb_t; ///< Verb type. #define VERB_NOP 0 ///< No-op verb. #define VERB_OPEN 1 ///< Open file verb. @@ -71,8 +71,7 @@ typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. */ typedef struct sqe { - verb_t verb; ///< Verb specifying the action to perform. - uint8_t _reserved[3]; + verb_t verb; ///< Verb specifying the action to perform. sqe_flags_t flags; ///< Submission flags. clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. void* data; ///< Private data for the operation, will be returned in the completion entry. @@ -118,8 +117,7 @@ static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); */ typedef struct ALIGNED(32) cqe { - verb_t verb; ///< Verb specifying the action that was performed. - uint8_t _reserved[3]; + verb_t verb; ///< Verb specifying the action that was performed. errno_t error; ///< Error code, if not equal to `EOK` an error occurred. void* data; ///< Private data from the submission entry. union { diff --git a/src/kernel/cpu/syscall.c b/src/kernel/cpu/syscall.c index a66a858a4..11831b781 100644 --- a/src/kernel/cpu/syscall.c +++ b/src/kernel/cpu/syscall.c @@ -43,7 +43,7 @@ static int syscall_descriptor_cmp(const void* a, const void* b) { const syscall_descriptor_t* sysA = (const syscall_descriptor_t*)a; const syscall_descriptor_t* sysB = (const syscall_descriptor_t*)b; - return sysA->number - sysB->number; + return (int32_t)sysA->number - (int32_t)sysB->number; } void syscall_table_init(void) diff --git a/src/kernel/drivers/perf.c b/src/kernel/drivers/perf.c index b38b255a0..6ddf7a4ed 100644 --- a/src/kernel/drivers/perf.c +++ b/src/kernel/drivers/perf.c @@ -116,8 +116,8 @@ static size_t perf_mem_read(file_t* file, void* buffer, size_t count, size_t* of return ERR; } - int length = sprintf(string, "total_pages %lu\nfree_pages %lu\nused_pages %lu", pmm_total_amount(), - pmm_free_amount(), pmm_used_amount()); + int length = sprintf(string, "total_pages %lu\nfree_pages %lu\nused_pages %lu", pmm_total_pages(), + pmm_avail_pages(), pmm_used_pages()); if (length < 0) { free(string); diff --git a/src/kernel/init/boot_info.c b/src/kernel/init/boot_info.c index 7cac165a5..5c9daa7e8 100644 --- a/src/kernel/init/boot_info.c +++ b/src/kernel/init/boot_info.c @@ -103,7 +103,7 @@ void boot_info_free(void) if (desc->Type == EfiLoaderData) { - LOG_INFO("free boot memory [0x%016lx-0x%016lx]\n", desc->VirtualStart, + LOG_INFO("free boot memory [%p-%p]\n", desc->VirtualStart, (uintptr_t)desc->VirtualStart + (desc->NumberOfPages * PAGE_SIZE)); #ifndef NDEBUG // Clear the memory to deliberately cause corruption if the memory is actually being used. diff --git a/src/kernel/init/init.c b/src/kernel/init/init.c index 391e8d6f1..10545388a 100644 --- a/src/kernel/init/init.c +++ b/src/kernel/init/init.c @@ -153,7 +153,7 @@ static void init_finalize(void) panic(NULL, "No IPI chip registered, most likely no IPI chips with a provided driver was found"); } - LOG_INFO("kernel initalized using %llu kb of memory\n", pmm_used_amount() * PAGE_SIZE / 1024); + LOG_INFO("kernel initalized using %llu kb of memory\n", pmm_used_pages() * PAGE_SIZE / 1024); } static inline void init_process_spawn(void) diff --git a/src/kernel/log/panic.c b/src/kernel/log/panic.c index 89a64cc58..2b9b7190f 100644 --- a/src/kernel/log/panic.c +++ b/src/kernel/log/panic.c @@ -343,8 +343,8 @@ void panic(const interrupt_frame_t* frame, const char* format, ...) LOG_PANIC("last errno: %d (%s)\n", err, strerror(err)); - uint64_t freePages = pmm_free_amount(); - uint64_t reservedPages = pmm_used_amount(); + uint64_t freePages = pmm_avail_pages(); + uint64_t reservedPages = pmm_used_pages(); uint64_t totalPages = freePages + reservedPages; LOG_PANIC("memory: %lluK/%lluK available (%lluK kernel code/data, %lluK reserved)\n", diff --git a/src/kernel/mem/mdl.c b/src/kernel/mem/mdl.c new file mode 100644 index 000000000..e69de29bb diff --git a/src/kernel/mem/pmm.c b/src/kernel/mem/pmm.c index 091e993ca..fe1684063 100644 --- a/src/kernel/mem/pmm.c +++ b/src/kernel/mem/pmm.c @@ -1,23 +1,22 @@ +#include #include #include #include #include #include -#include -#include #include #include +#include +#include +#include #include #include #include #include -extern char _kernel_start; -extern char _kernel_end; - static const char* efiMemTypeToString[] = { "reserved", "loader code", @@ -36,19 +35,20 @@ static const char* efiMemTypeToString[] = { "persistent", }; -#define PMM_BITMAP_SIZE (CONFIG_PMM_BITMAP_MAX_ADDR / PAGE_SIZE) +static pmm_ref_t* refs = NULL; -static pmm_stack_t stack; -static pmm_bitmap_t bitmap; +static page_stack_t* stack = NULL; +static size_t location = FREE_PAGE_MAX; -// Stores the bitmap data -static uint64_t bitmapBuffer[BITMAP_BITS_TO_QWORDS(PMM_BITMAP_SIZE)]; +BITMAP_CREATE_ONE(bitmap, CONFIG_PMM_BITMAP_MAX_ADDR / PAGE_SIZE); -static uint64_t pageAmount = 0; +static uintptr_t highest = 0; +static size_t total = 0; +static size_t avail = 0; static lock_t lock = LOCK_CREATE(); -static bool pmm_is_efi_mem_available(EFI_MEMORY_TYPE type) +static bool pmm_is_mem_avail(EFI_MEMORY_TYPE type) { if (!boot_is_mem_ram(type)) { @@ -68,50 +68,111 @@ static bool pmm_is_efi_mem_available(EFI_MEMORY_TYPE type) } } -static void pmm_free_unlocked(void* address) +static inline pmm_ref_t* pmm_ref_get(void* address) { - if (address >= (void*)PML_LOWER_TO_HIGHER(CONFIG_PMM_BITMAP_MAX_ADDR)) + return &refs[PML_HIGHER_TO_LOWER(address) / PAGE_SIZE]; +} + +static inline void pmm_ref_set(void* address, size_t count, pmm_ref_t value) +{ + pmm_ref_t* ref = pmm_ref_get(address); + for (size_t i = 0; i < count; i++) { - pmm_stack_free(&stack, address); + ref[i] = value; } - else if (address >= (void*)PML_LOWER_TO_HIGHER(0)) +} + +static inline size_t pmm_refs_size(void) +{ + return (PML_HIGHER_TO_LOWER(highest) / PAGE_SIZE) * sizeof(pmm_ref_t); +} + +static inline void pmm_stack_push(void* addr) +{ + if (stack == NULL || location == 0) { - pmm_bitmap_free(&bitmap, address, 1); + page_stack_t* page = addr; + page->next = stack; + stack = page; + location = FREE_PAGE_MAX; + + return; } - else + + stack->pages[--location] = addr; +} + +static inline void* pmm_stack_pop(void) +{ + if (location == FREE_PAGE_MAX) { - panic(NULL, "pmm: attempt to free lower half address %p", address); + if (stack == NULL) + { + return NULL; + } + + void* addr = stack; + stack = stack->next; + location = (stack == NULL) ? FREE_PAGE_MAX : 0; + return addr; } + + return stack->pages[location++]; } -static void pmm_free_pages_unlocked(void* address, size_t count) +static inline void* pmm_bitmap_set(size_t count, uintptr_t maxAddr, size_t alignment) { - address = (void*)ROUND_DOWN(address, PAGE_SIZE); + alignment = MAX(ROUND_UP(alignment, PAGE_SIZE), PAGE_SIZE); + maxAddr = MIN(maxAddr, CONFIG_PMM_BITMAP_MAX_ADDR); - uintptr_t physStart = PML_HIGHER_TO_LOWER(address); - uintptr_t physEnd = physStart + count * PAGE_SIZE; - - if (physEnd <= CONFIG_PMM_BITMAP_MAX_ADDR) + size_t index = bitmap_find_clear_region_and_set(&bitmap, 0, maxAddr / PAGE_SIZE, count, alignment / PAGE_SIZE); + if (index == bitmap.length) { - pmm_bitmap_free(&bitmap, address, count); + return NULL; } - else if (physStart < CONFIG_PMM_BITMAP_MAX_ADDR) + + return (void*)PML_LOWER_TO_HIGHER(index * PAGE_SIZE); +} + +static inline void pmm_bitmap_clear(void* addr, size_t pageAmount) +{ + addr = (void*)ROUND_DOWN(addr, PAGE_SIZE); + + size_t index = PML_HIGHER_TO_LOWER(addr) / PAGE_SIZE; + bitmap_clear_range(&bitmap, index, pageAmount); +} + +static void pmm_free_unlocked(void* address) +{ + pmm_ref_t* ref = pmm_ref_get(address); + assert(*ref > 0); + (*ref)--; + if (*ref > 0) { - size_t bitmapPageCount = (CONFIG_PMM_BITMAP_MAX_ADDR - physStart) / PAGE_SIZE; - pmm_bitmap_free(&bitmap, address, bitmapPageCount); + return; + } - uintptr_t stackAddr = PML_LOWER_TO_HIGHER(CONFIG_PMM_BITMAP_MAX_ADDR); - for (size_t i = 0; i < count - bitmapPageCount; i++) - { - pmm_stack_free(&stack, (void*)(stackAddr + i * PAGE_SIZE)); - } + if (address >= (void*)PML_LOWER_TO_HIGHER(CONFIG_PMM_BITMAP_MAX_ADDR)) + { + pmm_stack_push(address); + } + else if (address >= (void*)PML_LOWER_TO_HIGHER(0)) + { + pmm_bitmap_clear(address, 1); } else { - for (size_t i = 0; i < count; i++) - { - pmm_stack_free(&stack, (void*)((uintptr_t)address + i * PAGE_SIZE)); - } + panic(NULL, "Attempt to free lower half address %p", address); + } + + avail++; +} + +static void pmm_free_region_unlocked(void* address, size_t count) +{ + for (size_t i = 0; i < count; i++) + { + pmm_free_unlocked((void*)((uintptr_t)address + (i * PAGE_SIZE))); } } @@ -125,11 +186,32 @@ static void pmm_detect_memory(const boot_memory_map_t* map) if (boot_is_mem_ram(desc->Type)) { - pageAmount += desc->NumberOfPages; + total += desc->NumberOfPages; + } + highest = MAX(highest, (uintptr_t)desc->VirtualStart + (desc->NumberOfPages * PAGE_SIZE)); + } + + LOG_INFO("page amount %llu\n", total); + LOG_INFO("highest address %p\n", highest); +} + +static void pmm_init_refs(const boot_memory_map_t* map) +{ + size_t size = pmm_refs_size(); + size_t pages = BYTES_TO_PAGES(size); + for (size_t i = 0; i < map->length; i++) + { + const EFI_MEMORY_DESCRIPTOR* desc = BOOT_MEMORY_MAP_GET_DESCRIPTOR(map, i); + if (desc->Type == EfiConventionalMemory && desc->NumberOfPages >= pages) + { + refs = (pmm_ref_t*)desc->VirtualStart; + memset(refs, -1, pages * PAGE_SIZE); + LOG_INFO("pmm ref [%p-%p]\n", refs, (uintptr_t)refs + pages * PAGE_SIZE); + return; } } - LOG_INFO("page amount %llu\n", pageAmount); + panic(NULL, "Failed to allocate pmm refs array"); } static void pmm_load_memory(const boot_memory_map_t* map) @@ -138,24 +220,35 @@ static void pmm_load_memory(const boot_memory_map_t* map) { const EFI_MEMORY_DESCRIPTOR* desc = BOOT_MEMORY_MAP_GET_DESCRIPTOR(map, i); - if (pmm_is_efi_mem_available(desc->Type)) + uintptr_t address = desc->VirtualStart; + size_t pages = desc->NumberOfPages; + if (address == (uintptr_t)refs) + { + // Skip the refs array. + size_t refPages = BYTES_TO_PAGES(pmm_refs_size()); + assert(pages >= refPages); + address += refPages * PAGE_SIZE; + pages -= refPages; + } + + if (pmm_is_mem_avail(desc->Type)) { #ifndef NDEBUG - // Clear the memory to deliberatly cause corruption if the memory is actually being used. - memset((void*)desc->VirtualStart, 0xCC, desc->NumberOfPages * PAGE_SIZE); + // Clear the memory to deliberately cause corruption if the memory is actually being used. + memset((void*)address, 0xCC, pages * PAGE_SIZE); #endif - pmm_free_pages_unlocked((void*)desc->VirtualStart, desc->NumberOfPages); + pmm_ref_set((void*)address, pages, 1); + pmm_free_region_unlocked((void*)address, pages); } else { - LOG_INFO("reserve [0x%016lx-0x%016lx] pages=%d type=%s\n", desc->VirtualStart, - (uint64_t)desc->VirtualStart + desc->NumberOfPages * PAGE_SIZE, desc->NumberOfPages, + LOG_INFO("reserve [%p-%p] pages=%d type=%s\n", address, (uintptr_t)address + (pages * PAGE_SIZE), pages, efiMemTypeToString[desc->Type]); } } - LOG_INFO("memory %llu MB (usable %llu MB reserved %llu MB)\n", (pageAmount * PAGE_SIZE) / 1000000, - (pmm_free_amount() * PAGE_SIZE) / 1000000, ((pageAmount - pmm_free_amount()) * PAGE_SIZE) / 1000000); + LOG_INFO("memory %llu MB (usable %llu MB reserved %llu MB)\n", (total * PAGE_SIZE) / 1000000, + (pmm_avail_pages() * PAGE_SIZE) / 1000000, ((total - pmm_avail_pages()) * PAGE_SIZE) / 1000000); } void pmm_init(void) @@ -164,97 +257,157 @@ void pmm_init(void) const boot_memory_map_t* map = &bootInfo->memory.map; pmm_detect_memory(map); - - pmm_stack_init(&stack); - pmm_bitmap_init(&bitmap, bitmapBuffer, PMM_BITMAP_SIZE, CONFIG_PMM_BITMAP_MAX_ADDR); - + pmm_init_refs(map); pmm_load_memory(map); } void* pmm_alloc(void) { - LOCK_SCOPE(&lock); - void* address = pmm_stack_alloc(&stack); - if (address == NULL) + lock_acquire(&lock); + void* addr = pmm_stack_pop(); + if (addr == NULL) { - LOG_WARN("failed to allocate single page, there are %llu pages left\n", stack.free); - errno = ENOMEM; - return NULL; + addr = pmm_bitmap_set(1, CONFIG_PMM_BITMAP_MAX_ADDR, PAGE_SIZE); } - return address; + + if (addr != NULL) + { + pmm_ref_t* ref = pmm_ref_get(addr); + assert(*ref == 0); + *ref = 1; + avail--; + } + lock_release(&lock); + + if (addr == NULL) + { + LOG_WARN("out of memory in pmm_alloc()\n"); + } + + return addr; } uint64_t pmm_alloc_pages(void** addresses, size_t count) { - LOCK_SCOPE(&lock); + lock_acquire(&lock); for (size_t i = 0; i < count; i++) { - void* address = pmm_stack_alloc(&stack); - if (address == NULL) + addresses[i] = pmm_stack_pop(); + if (addresses[i] == NULL) + { + addresses[i] = pmm_bitmap_set(1, CONFIG_PMM_BITMAP_MAX_ADDR, PAGE_SIZE); + } + + if (addresses[i] == NULL) { - LOG_WARN("failed to allocate page %llu of %llu, there are %llu pages left\n", i, count, stack.free); - // Free previously allocated pages. + LOG_WARN("out of memory in pmm_alloc_pages()\n"); for (size_t j = 0; j < i; j++) { - pmm_stack_free(&stack, addresses[j]); - addresses[j] = NULL; + if (addresses[j] >= (void*)PML_LOWER_TO_HIGHER(CONFIG_PMM_BITMAP_MAX_ADDR)) + { + pmm_stack_push(addresses[j]); + } + else + { + pmm_bitmap_clear(addresses[j], 1); + } } - errno = ENOMEM; + lock_release(&lock); return ERR; } - addresses[i] = address; } + + for (size_t i = 0; i < count; i++) + { + pmm_ref_t* ref = pmm_ref_get(addresses[i]); + assert(*ref == 0); + *ref = 1; + } + + avail -= count; + lock_release(&lock); return 0; } void* pmm_alloc_bitmap(size_t count, uintptr_t maxAddr, uint64_t alignment) { - LOCK_SCOPE(&lock); - void* address = pmm_bitmap_alloc(&bitmap, count, maxAddr, alignment); - if (address == NULL) + lock_acquire(&lock); + void* addr = pmm_bitmap_set(count, maxAddr, alignment); + if (addr != NULL) { - LOG_WARN("failed to allocate %llu pages from bitmap, there are %llu bitmap pages left\n", count, bitmap.free); - errno = ENOMEM; - return NULL; + for (size_t i = 0; i < count; i++) + { + pmm_ref_t* ref = pmm_ref_get((void*)((uintptr_t)addr + (i * PAGE_SIZE))); + assert(*ref == 0); + *ref = 1; + } + avail -= count; } - return address; + lock_release(&lock); + + if (addr == NULL) + { + LOG_WARN("out of memory in pmm_alloc_bitmap()\n"); + } + + return addr; } void pmm_free(void* address) { - LOCK_SCOPE(&lock); + lock_acquire(&lock); pmm_free_unlocked(address); + lock_release(&lock); } void pmm_free_pages(void** addresses, size_t count) { - LOCK_SCOPE(&lock); + lock_acquire(&lock); for (size_t i = 0; i < count; i++) { pmm_free_unlocked(addresses[i]); } + lock_release(&lock); } void pmm_free_region(void* address, size_t count) { - LOCK_SCOPE(&lock); - pmm_free_pages_unlocked(address, count); + lock_acquire(&lock); + pmm_free_region_unlocked(address, count); + lock_release(&lock); } -size_t pmm_total_amount(void) +uint64_t pmm_ref_inc(void* address) { - LOCK_SCOPE(&lock); - return pageAmount; + lock_acquire(&lock); + pmm_ref_t* ref = pmm_ref_get(address); + assert(*ref != PAGE_REF_MAX); + (*ref)++; + uint64_t ret = *ref; + lock_release(&lock); + return ret; } -size_t pmm_free_amount(void) +size_t pmm_total_pages(void) { - LOCK_SCOPE(&lock); - return stack.free + bitmap.free; + lock_acquire(&lock); + size_t ret = total; + lock_release(&lock); + return ret; } -size_t pmm_used_amount(void) +size_t pmm_avail_pages(void) { - LOCK_SCOPE(&lock); - return pageAmount - (stack.free + bitmap.free); + lock_acquire(&lock); + size_t ret = avail; + lock_release(&lock); + return ret; } + +size_t pmm_used_pages(void) +{ + lock_acquire(&lock); + size_t ret = total - avail; + lock_release(&lock); + return ret; +} \ No newline at end of file diff --git a/src/kernel/mem/pmm_bitmap.c b/src/kernel/mem/pmm_bitmap.c deleted file mode 100644 index 0b5506b4c..000000000 --- a/src/kernel/mem/pmm_bitmap.c +++ /dev/null @@ -1,41 +0,0 @@ -#include -#include - -#include -#include - -void pmm_bitmap_init(pmm_bitmap_t* bitmap, void* buffer, uint64_t size, uintptr_t maxAddr) -{ - assert(size >= maxAddr / PAGE_SIZE); - bitmap_init(&bitmap->bitmap, buffer, size); - bitmap->free = 0; - bitmap->total = maxAddr / PAGE_SIZE; - bitmap->maxAddr = maxAddr; -} - -void* pmm_bitmap_alloc(pmm_bitmap_t* bitmap, uint64_t count, uintptr_t maxAddr, uint64_t alignment) -{ - alignment = MAX(ROUND_UP(alignment, PAGE_SIZE), PAGE_SIZE); - maxAddr = MIN(maxAddr, bitmap->maxAddr); - - uint64_t index = - bitmap_find_clear_region_and_set(&bitmap->bitmap, 0, maxAddr / PAGE_SIZE, count, alignment / PAGE_SIZE); - if (index == bitmap->bitmap.length) - { - return NULL; - } - - bitmap->free -= count; - return (void*)(index * PAGE_SIZE + PML_HIGHER_HALF_START); -} - -void pmm_bitmap_free(pmm_bitmap_t* bitmap, void* address, uint64_t count) -{ - address = (void*)ROUND_DOWN(address, PAGE_SIZE); - - uint64_t index = (uint64_t)PML_HIGHER_TO_LOWER(address) / PAGE_SIZE; - assert(index < bitmap->maxAddr / PAGE_SIZE); - - bitmap_clear_range(&bitmap->bitmap, index, index + count); - bitmap->free += count; -} diff --git a/src/kernel/mem/pmm_stack.c b/src/kernel/mem/pmm_stack.c deleted file mode 100644 index 8c3e21549..000000000 --- a/src/kernel/mem/pmm_stack.c +++ /dev/null @@ -1,59 +0,0 @@ -#include - -#include - -void pmm_stack_init(pmm_stack_t* stack) -{ - stack->last = NULL; - stack->index = 0; - stack->free = 0; -} - -void* pmm_stack_alloc(pmm_stack_t* stack) -{ - if (stack->last == NULL) - { - return NULL; - } - - void* address; - if (stack->index == 0) - { - address = stack->last; - stack->last = stack->last->prev; - stack->index = PMM_BUFFER_MAX; - } - else - { - address = stack->last->pages[--stack->index]; - } - - stack->free--; - - return address; -} - -void pmm_stack_free(pmm_stack_t* stack, void* address) -{ - address = (void*)ROUND_DOWN(address, PAGE_SIZE); - - if (stack->last == NULL) - { - stack->last = address; - stack->last->prev = NULL; - stack->index = 0; - } - else if (stack->index == PMM_BUFFER_MAX) - { - page_buffer_t* next = address; - next->prev = stack->last; - stack->last = next; - stack->index = 0; - } - else - { - stack->last->pages[stack->index++] = address; - } - - stack->free++; -} diff --git a/src/kernel/mem/pool.c b/src/kernel/mem/pool.c new file mode 100644 index 000000000..348d3cf7a --- /dev/null +++ b/src/kernel/mem/pool.c @@ -0,0 +1,67 @@ +#include + +void pool_init(pool_t* pool, void* elements, size_t capacity, size_t elementSize, size_t nextOffset) +{ + atomic_init(&pool->used, 0); + atomic_init(&pool->free, 0); + pool->elements = elements; + pool->elementSize = elementSize; + pool->nextOffset = nextOffset; + pool->capacity = capacity; + + for (size_t i = 0; i < capacity; i++) + { + void* element = (void*)((uintptr_t)elements + (i * elementSize)); + pool_idx_t* next = (pool_idx_t*)((uintptr_t)element + nextOffset); + *next = (i == capacity - 1) ? POOL_IDX_MAX : (pool_idx_t)(i + 1); + } +} + +pool_idx_t pool_alloc(pool_t* pool) +{ + pool_idx_t idx; + while (true) + { + uint64_t head = atomic_load_explicit(&pool->free, memory_order_acquire); + idx = (pool_idx_t)(head & POOL_IDX_MAX); + if (idx == POOL_IDX_MAX) + { + return POOL_IDX_MAX; + } + + void* element = (void*)((uintptr_t)pool->elements + (idx * pool->elementSize)); + pool_idx_t next = *(pool_idx_t*)((uintptr_t)element + pool->nextOffset); + + uint64_t newHead = ((head & ~POOL_IDX_MAX) + POOL_TAG_INC) | next; + if (atomic_compare_exchange_weak_explicit(&pool->free, &head, newHead, memory_order_acquire, + memory_order_relaxed)) + { + break; + } + ASM("pause"); + } + + atomic_fetch_add_explicit(&pool->used, 1, memory_order_relaxed); + return idx; +} + +void pool_free(pool_t* pool, pool_idx_t idx) +{ + void* element = (void*)((uintptr_t)pool->elements + (idx * pool->elementSize)); + pool_idx_t* next = (pool_idx_t*)((uintptr_t)element + pool->nextOffset); + + while (true) + { + uint64_t head = atomic_load_explicit(&pool->free, memory_order_relaxed); + *next = (pool_idx_t)(head & POOL_IDX_MAX); + uint64_t newHead = ((head & ~POOL_IDX_MAX) + POOL_TAG_INC) | idx; + + if (atomic_compare_exchange_weak_explicit(&pool->free, &head, newHead, memory_order_release, + memory_order_relaxed)) + { + break; + } + ASM("pause"); + } + atomic_fetch_sub_explicit(&pool->used, 1, memory_order_relaxed); +} \ No newline at end of file diff --git a/src/kernel/mem/vmm.c b/src/kernel/mem/vmm.c index 3ab1ef22b..a4b4b62ad 100644 --- a/src/kernel/mem/vmm.c +++ b/src/kernel/mem/vmm.c @@ -57,11 +57,11 @@ void vmm_init(void) } LOG_DEBUG("address space layout:\n"); - LOG_DEBUG(" kernel binary: 0x%016lx-0x%016lx\n", VMM_KERNEL_BINARY_MIN, VMM_KERNEL_BINARY_MAX); - LOG_DEBUG(" kernel stacks: 0x%016lx-0x%016lx\n", VMM_KERNEL_STACKS_MIN, VMM_KERNEL_STACKS_MAX); - LOG_DEBUG(" kernel heap: 0x%016lx-0x%016lx\n", VMM_KERNEL_HEAP_MIN, VMM_KERNEL_HEAP_MAX); - LOG_DEBUG(" identity map: 0x%016lx-0x%016lx\n", VMM_IDENTITY_MAPPED_MIN, VMM_IDENTITY_MAPPED_MAX); - LOG_DEBUG(" user space: 0x%016lx-0x%016lx\n", VMM_USER_SPACE_MIN, VMM_USER_SPACE_MAX); + LOG_DEBUG(" kernel binary: %p-%p\n", VMM_KERNEL_BINARY_MIN, VMM_KERNEL_BINARY_MAX); + LOG_DEBUG(" kernel stacks: %p-%p\n", VMM_KERNEL_STACKS_MIN, VMM_KERNEL_STACKS_MAX); + LOG_DEBUG(" kernel heap: %p-%p\n", VMM_KERNEL_HEAP_MIN, VMM_KERNEL_HEAP_MAX); + LOG_DEBUG(" identity map: %p-%p\n", VMM_IDENTITY_MAPPED_MIN, VMM_IDENTITY_MAPPED_MAX); + LOG_DEBUG(" user space: %p-%p\n", VMM_USER_SPACE_MIN, VMM_USER_SPACE_MAX); LOG_INFO("kernel pml4 allocated at 0x%lx\n", kernelSpace.pageTable.pml4); @@ -76,18 +76,18 @@ void vmm_init(void) const EFI_MEMORY_DESCRIPTOR* desc = BOOT_MEMORY_MAP_GET_DESCRIPTOR(&memory->map, i); if (desc->VirtualStart < PML_HIGHER_HALF_START) { - panic(NULL, "Memory descriptor %d has invalid virtual address 0x%016lx", i, desc->VirtualStart); + panic(NULL, "Memory descriptor %d has invalid virtual address %p", i, desc->VirtualStart); } if (desc->PhysicalStart > PML_LOWER_HALF_END) { - panic(NULL, "Memory descriptor %d has invalid physical address 0x%016lx", i, desc->PhysicalStart); + panic(NULL, "Memory descriptor %d has invalid physical address %p", i, desc->PhysicalStart); } if (page_table_map(&kernelSpace.pageTable, (void*)desc->VirtualStart, (void*)desc->PhysicalStart, desc->NumberOfPages, PML_WRITE | PML_GLOBAL | PML_PRESENT, PML_CALLBACK_NONE) == ERR) { - panic(NULL, "Failed to map memory descriptor %d (phys=0x%016lx-0x%016lx virt=0x%016lx)", i, - desc->PhysicalStart, desc->PhysicalStart + desc->NumberOfPages * PAGE_SIZE, desc->VirtualStart); + panic(NULL, "Failed to map memory descriptor %d (phys=%p-%p virt=%p)", i, desc->PhysicalStart, + desc->PhysicalStart + desc->NumberOfPages * PAGE_SIZE, desc->VirtualStart); } } @@ -96,16 +96,16 @@ void vmm_init(void) elf64_get_loadable_bounds(&kernel->elf, &minVaddr, &maxVaddr); uint64_t kernelPageAmount = BYTES_TO_PAGES(maxVaddr - minVaddr); - LOG_INFO("kernel virt=[0x%016lx-0x%016lx] phys=[0x%016lx-0x%016lx]\n", minVaddr, maxVaddr, - (uintptr_t)kernel->physAddr, (uintptr_t)kernel->physAddr + kernelPageAmount * PAGE_SIZE); + LOG_INFO("kernel virt=[%p-%p] phys=[%p-%p]\n", minVaddr, maxVaddr, (uintptr_t)kernel->physAddr, + (uintptr_t)kernel->physAddr + kernelPageAmount * PAGE_SIZE); if (page_table_map(&kernelSpace.pageTable, (void*)minVaddr, kernel->physAddr, kernelPageAmount, PML_WRITE | PML_PRESENT, PML_CALLBACK_NONE) == ERR) { panic(NULL, "Failed to map kernel memory"); } - LOG_INFO("GOP virt=[0x%016lx-0x%016lx] phys=[0x%016lx-0x%016lx]\n", gop->virtAddr, gop->virtAddr + gop->size, - gop->physAddr, gop->physAddr + gop->size); + LOG_INFO("GOP virt=[%p-%p] phys=[%p-%p]\n", gop->virtAddr, gop->virtAddr + gop->size, gop->physAddr, + gop->physAddr + gop->size); if (page_table_map(&kernelSpace.pageTable, (void*)gop->virtAddr, (void*)gop->physAddr, BYTES_TO_PAGES(gop->size), PML_WRITE | PML_GLOBAL | PML_PRESENT, PML_CALLBACK_NONE) == ERR) { diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index 56e8bed3d..490cc49fa 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -1,4 +1,7 @@ +#include <_internal/fd_t.h> #include +#include +#include #include #include #include @@ -17,8 +20,7 @@ static inline uint64_t async_acquire(async_t* ctx) { async_flags_t expected = atomic_load(&ctx->flags); - if (!(expected & ASYNC_CTX_BUSY) && - atomic_compare_exchange_strong(&ctx->flags, &expected, expected | ASYNC_CTX_BUSY)) + if (!(expected & ASYNC_BUSY) && atomic_compare_exchange_strong(&ctx->flags, &expected, expected | ASYNC_BUSY)) { return 0; } @@ -28,7 +30,7 @@ static inline uint64_t async_acquire(async_t* ctx) static inline void async_release(async_t* ctx) { - atomic_fetch_and(&ctx->flags, ~ASYNC_CTX_BUSY); + atomic_fetch_and(&ctx->flags, ~ASYNC_BUSY); } static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, rings_t* userRings, void* address, @@ -44,7 +46,7 @@ static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, ri return ERR; } - if (centries >= IRP_IDX_MAX) + if (centries >= POOL_IDX_MAX) { errno = EINVAL; return ERR; @@ -119,7 +121,7 @@ static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, ri ctx->pageAmount = pageAmount; ctx->space = space; - atomic_fetch_or(&ctx->flags, ASYNC_CTX_MAPPED); + atomic_fetch_or(&ctx->flags, ASYNC_MAPPED); return 0; } @@ -131,7 +133,7 @@ static inline uint64_t async_unmap(async_t* ctx) vmm_unmap(ctx->space, ctx->userAddr, ctx->pageAmount * PAGE_SIZE); vmm_unmap(NULL, ctx->kernelAddr, ctx->pageAmount * PAGE_SIZE); - atomic_fetch_and(&ctx->flags, ~ASYNC_CTX_MAPPED); + atomic_fetch_and(&ctx->flags, ~ASYNC_MAPPED); return 0; } @@ -157,7 +159,7 @@ void async_init(async_t* ctx) ctx->pageAmount = 0; ctx->space = NULL; wait_queue_init(&ctx->waitQueue); - atomic_init(&ctx->flags, ASYNC_CTX_NONE); + atomic_init(&ctx->flags, ASYNC_NONE); } void async_deinit(async_t* ctx) @@ -172,7 +174,7 @@ void async_deinit(async_t* ctx) panic(NULL, "failed to acquire async context for deinitialization"); } - if (atomic_load(&ctx->flags) & ASYNC_CTX_MAPPED) + if (atomic_load(&ctx->flags) & ASYNC_MAPPED) { if (async_unmap(ctx) == ERR) { @@ -241,7 +243,7 @@ static void async_complete(irp_t* irp, void* _ptr) irp_free(irp); - if (atomic_load(&ctx->irps->used) == 0) + if (atomic_load(&ctx->irps->pool.used) == 0) { UNREF(ctx->process); ctx->process = NULL; @@ -267,6 +269,29 @@ static void async_dispatch(irp_t* irp) { case VERB_NOP: break; + case VERB_OPEN: + { + file_t* from = NULL; + if (irp->sqe.open.from != FD_NONE) + { + from = file_table_get(&ctx->process->fileTable, irp->sqe.open.from); + if (from == NULL) + { + irp->err = EBADF; + break; + } + } + + irp->open.from = from; + irp->open.path = malloc(sizeof(pathname_t)); + if (irp->open.path == NULL) + { + UNREF(from); + irp->err = ENOMEM; + break; + } + } + break; default: break; } @@ -290,7 +315,7 @@ static uint64_t async_sqe_pop(async_t* ctx, async_notify_ctx_t* notify) return ERR; } - if (atomic_load(&ctx->irps->used) == 1) + if (atomic_load(&ctx->irps->pool.used) == 1) { process_t* process = process_current(); assert(&process->async[0] <= ctx && ctx <= &process->async[CONFIG_MAX_ASYNC_RINGS - 1]); @@ -341,7 +366,7 @@ uint64_t async_notify(async_t* ctx, size_t amount, size_t wait) return ERR; } - if (!(atomic_load(&ctx->flags) & ASYNC_CTX_MAPPED)) + if (!(atomic_load(&ctx->flags) & ASYNC_MAPPED)) { async_release(ctx); errno = EINVAL; @@ -401,8 +426,8 @@ SYSCALL_DEFINE(SYS_SETUP, rings_id_t, rings_t* userRings, void* address, size_t rings_id_t id = 0; for (id = 0; id < CONFIG_MAX_ASYNC_RINGS; id++) { - async_flags_t expected = ASYNC_CTX_NONE; - if (atomic_compare_exchange_strong(&process->async[id].flags, &expected, ASYNC_CTX_BUSY)) + async_flags_t expected = ASYNC_NONE; + if (atomic_compare_exchange_strong(&process->async[id].flags, &expected, ASYNC_BUSY)) { ctx = &process->async[id]; break; @@ -442,14 +467,14 @@ SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, rings_id_t id) return ERR; } - if (!(atomic_load(&ctx->flags) & ASYNC_CTX_MAPPED)) + if (!(atomic_load(&ctx->flags) & ASYNC_MAPPED)) { async_release(ctx); errno = EINVAL; return ERR; } - if (ctx->irps != NULL && atomic_load(&ctx->irps->used) != 0) + if (ctx->irps != NULL && atomic_load(&ctx->irps->pool.used) != 0) { async_release(ctx); errno = EBUSY; diff --git a/src/kernel/sync/irp.c b/src/kernel/sync/irp.c index ef8eeded3..2aaac9602 100644 --- a/src/kernel/sync/irp.c +++ b/src/kernel/sync/irp.c @@ -24,7 +24,7 @@ PERCPU_DEFINE_CTOR(irp_ctx_t, pcpu_irps) irp_pool_t* irp_pool_new(size_t size, void* ctx) { - if (size == 0 || size >= IRP_IDX_MAX) + if (size == 0 || size >= POOL_IDX_MAX) { errno = EINVAL; return NULL; @@ -38,9 +38,7 @@ irp_pool_t* irp_pool_new(size_t size, void* ctx) } pool->ctx = ctx; - atomic_init(&pool->used, 0); - atomic_init(&pool->free, 0); - for (irp_idx_t i = 0; i < (irp_idx_t)size; i++) + for (pool_idx_t i = 0; i < (pool_idx_t)size; i++) { irp_t* irp = &pool->irps[i]; list_entry_init(&irp->entry); @@ -57,7 +55,7 @@ irp_pool_t* irp_pool_new(size_t size, void* ctx) irp->result = 0; irp->err = EOK; irp->index = i; - irp->next = i < size - 1 ? i + 1 : IRP_IDX_MAX; + irp->next = i < size - 1 ? i + 1 : POOL_IDX_MAX; irp->location = IRP_LOC_MAX; irp->cpu = CPU_ID_INVALID; for (size_t j = 0; j < IRP_LOC_MAX; j++) @@ -67,6 +65,8 @@ irp_pool_t* irp_pool_new(size_t size, void* ctx) } } + pool_init(&pool->pool, pool->irps, size, sizeof(irp_t), offsetof(irp_t, next)); + return pool; } @@ -77,34 +77,21 @@ void irp_pool_free(irp_pool_t* pool) irp_t* irp_new(irp_pool_t* pool) { - irp_idx_t idx; - for (;;) + pool_idx_t idx = pool_alloc(&pool->pool); + if (idx == POOL_IDX_MAX) { - uint64_t head = atomic_load_explicit(&pool->free, memory_order_acquire); - idx = (irp_idx_t)(head & IRP_IDX_MAX); - if (idx == IRP_IDX_MAX) - { - return NULL; - } - - uint64_t newHead = ((head & ~IRP_IDX_MAX) + IRP_TAG_INC) | pool->irps[idx].next; - if (atomic_compare_exchange_weak_explicit(&pool->free, &head, newHead, memory_order_acquire, - memory_order_relaxed)) - { - break; - } - ASM("pause"); + errno = ENOSPC; + return NULL; } - atomic_fetch_add_explicit(&pool->used, 1, memory_order_relaxed); irp_t* irp = &pool->irps[idx]; irp->location = IRP_LOC_MAX; - irp->next = IRP_IDX_MAX; + irp->next = POOL_IDX_MAX; irp->err = EINPROGRESS; irp->result = 0; irp->sqe = (sqe_t){0}; atomic_store_explicit(&irp->cancel, NULL, memory_order_relaxed); - irp->next = IRP_IDX_MAX; + irp->next = POOL_IDX_MAX; irp->cpu = CPU_ID_INVALID; for (size_t j = 0; j < IRP_LOC_MAX; j++) { @@ -117,22 +104,7 @@ irp_t* irp_new(irp_pool_t* pool) void irp_free(irp_t* irp) { irp_pool_t* pool = irp_pool_get(irp); - - irp_idx_t idx = irp->index; - for (;;) - { - uint64_t head = atomic_load_explicit(&pool->free, memory_order_relaxed); - irp->next = (irp_idx_t)(head & IRP_IDX_MAX); - uint64_t newHead = ((head & ~IRP_IDX_MAX) + IRP_TAG_INC) | idx; - - if (atomic_compare_exchange_weak_explicit(&pool->free, &head, newHead, memory_order_release, - memory_order_relaxed)) - { - break; - } - ASM("pause"); - } - atomic_fetch_sub_explicit(&pool->used, 1, memory_order_relaxed); + pool_free(&pool->pool, irp->index); } uint64_t irp_cancel(irp_t* irp) @@ -253,6 +225,12 @@ void irp_timeouts_check(void) void irp_dispatch(irp_t* irp) { + if (irp->err != EINPROGRESS) + { + irp_complete(irp); + return; + } + if (irp->verb >= VERB_MAX || _irp_table_start[irp->verb].handler == NULL) { irp->err = ENOSYS; @@ -267,7 +245,7 @@ static int irp_handler_cmp(const void* a, const void* b) { const irp_handler_t* irpA = (const irp_handler_t*)a; const irp_handler_t* irpB = (const irp_handler_t*)b; - return irpA->verb - irpB->verb; + return (int32_t)irpA->verb - (int32_t)irpB->verb; } void irp_table_init(void) diff --git a/src/modules/acpi/acpi.c b/src/modules/acpi/acpi.c index d74b9dc73..29e135a4f 100644 --- a/src/modules/acpi/acpi.c +++ b/src/modules/acpi/acpi.c @@ -70,7 +70,7 @@ void acpi_reclaim_memory(const boot_memory_map_t* map) if (desc->Type == EfiACPIReclaimMemory) { pmm_free_region((void*)PML_LOWER_TO_HIGHER(desc->PhysicalStart), desc->NumberOfPages); - LOG_INFO("reclaim memory [0x%016lx-0x%016lx]\n", desc->PhysicalStart, + LOG_INFO("reclaim memory [%p-%p]\n", desc->PhysicalStart, ((uintptr_t)desc->PhysicalStart) + desc->NumberOfPages * PAGE_SIZE); } } diff --git a/src/modules/acpi/tables.c b/src/modules/acpi/tables.c index 2f8d55422..a0c1327d4 100644 --- a/src/modules/acpi/tables.c +++ b/src/modules/acpi/tables.c @@ -132,7 +132,7 @@ static uint64_t acpi_tables_push(sdt_header_t* table) } cachedTables[tableAmount++].table = cachedTable; - LOG_INFO("%.*s 0x%016lx 0x%06x v%02X %.*s\n", SDT_SIGNATURE_LENGTH, cachedTable->signature, cachedTable, + LOG_INFO("%.*s %p 0x%06x v%02X %.*s\n", SDT_SIGNATURE_LENGTH, cachedTable->signature, cachedTable, cachedTable->length, cachedTable->revision, SDT_OEM_ID_LENGTH, cachedTable->oemId); return 0; } @@ -200,7 +200,7 @@ uint64_t acpi_tables_init(rsdp_t* rsdp) } xsdt_t* xsdt = (xsdt_t*)PML_ENSURE_HIGHER_HALF(rsdp->xsdtAddress); - LOG_INFO("located XSDT at 0x%016lx\n", rsdp->xsdtAddress); + LOG_INFO("located XSDT at %p\n", rsdp->xsdtAddress); if (acpi_tables_load_from_xsdt(xsdt) == ERR) { diff --git a/src/modules/drivers/apic/lapic.c b/src/modules/drivers/apic/lapic.c index bc1b5550d..d0648678b 100644 --- a/src/modules/drivers/apic/lapic.c +++ b/src/modules/drivers/apic/lapic.c @@ -106,8 +106,7 @@ uint64_t lapic_global_init(void) return ERR; } - LOG_INFO("local apic mapped base=0x%016lx phys=0x%016lx\n", lapicBase, - (uintptr_t)madt->localInterruptControllerAddress); + LOG_INFO("local apic mapped base=%p phys=%p\n", lapicBase, (uintptr_t)madt->localInterruptControllerAddress); if (ipi_chip_register(&lapicIpiChip) == ERR) { diff --git a/src/modules/drivers/hpet/hpet.c b/src/modules/drivers/hpet/hpet.c index 55eb15475..2d66a8c57 100644 --- a/src/modules/drivers/hpet/hpet.c +++ b/src/modules/drivers/hpet/hpet.c @@ -221,7 +221,7 @@ static uint64_t hpet_init(void) if (vmm_map(NULL, (void*)address, (void*)hpet->address, PAGE_SIZE, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == NULL) { - LOG_ERR("failed to map HPET memory at 0x%016lx\n", hpet->address); + LOG_ERR("failed to map HPET memory at %p\n", hpet->address); return ERR; } @@ -234,7 +234,7 @@ static uint64_t hpet_init(void) return ERR; } - LOG_INFO("started HPET timer phys=0x%016lx virt=0x%016lx period=%lluns timers=%u %s-bit\n", hpet->address, address, + LOG_INFO("started HPET timer phys=%p virt=%p period=%lluns timers=%u %s-bit\n", hpet->address, address, period / (HPET_FEMTOSECONDS_PER_SECOND / CLOCKS_PER_SEC), hpet->comparatorCount + 1, hpet->counterIs64Bit ? "64" : "32"); diff --git a/src/modules/drivers/pci/config.c b/src/modules/drivers/pci/config.c index 8c2990bb2..64df4c2a7 100644 --- a/src/modules/drivers/pci/config.c +++ b/src/modules/drivers/pci/config.c @@ -46,11 +46,11 @@ static uint64_t pci_config_init(void) if (vmm_map(NULL, virtAddr, (void*)entry->base, length, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == NULL) { - LOG_ERR("failed to map PCI-e configuration space at 0x%016lx\n", entry->base); + LOG_ERR("failed to map PCI-e configuration space at %p\n", entry->base); return ERR; } - LOG_INFO("mapped PCI-e config space 0x%016lx (segment=%u bus=%u-%u)\n", entry->base, entry->segmentGroup, + LOG_INFO("mapped PCI-e config space %p (segment=%u bus=%u-%u)\n", entry->base, entry->segmentGroup, entry->startBus, entry->endBus); } From 5972b200e69227f514edef964b74da0d5dabe516 Mon Sep 17 00:00:00 2001 From: KN Date: Tue, 20 Jan 2026 15:57:48 +0100 Subject: [PATCH 13/23] refactor: use Page Frame Numbers instead of void* for pages --- include/boot/boot_info.h | 4 +- include/kernel/mem/mem_desc.h | 111 +++++++- include/kernel/mem/paging.h | 301 ++++++++++------------ include/kernel/mem/paging_types.h | 63 ++++- include/kernel/mem/pmm.h | 41 ++- include/kernel/mem/pool.h | 4 +- include/kernel/mem/space.h | 6 +- include/kernel/mem/vmm.h | 12 +- include/kernel/sync/async.h | 2 + include/kernel/sync/irp.h | 12 +- include/libstd/sys/rings.h | 1 + src/boot/main.c | 19 +- src/kernel/cpu/simd.c | 9 +- src/kernel/init/boot_info.c | 6 +- src/kernel/mem/mdl.c | 0 src/kernel/mem/mem_desc.c | 58 +++++ src/kernel/mem/pmm.c | 153 ++++++----- src/kernel/mem/space.c | 44 ++-- src/kernel/mem/vmm.c | 34 +-- src/kernel/sync/async.c | 31 ++- src/modules/acpi/acpi.c | 2 +- src/modules/acpi/aml/runtime/field_unit.c | 3 +- src/modules/drivers/apic/ioapic.c | 5 +- src/modules/drivers/apic/lapic.c | 2 +- src/modules/drivers/gop/gop.c | 6 +- src/modules/drivers/hpet/hpet.c | 2 +- src/modules/drivers/pci/config.c | 2 +- src/modules/ipc/pipe/pipe.c | 6 +- src/modules/ipc/shmem/shmem.c | 38 ++- src/modules/smp/trampoline.c | 16 +- 30 files changed, 599 insertions(+), 394 deletions(-) delete mode 100644 src/kernel/mem/mdl.c create mode 100644 src/kernel/mem/mem_desc.c diff --git a/include/boot/boot_info.h b/include/boot/boot_info.h index c441495cf..5948ad91e 100644 --- a/include/boot/boot_info.h +++ b/include/boot/boot_info.h @@ -41,7 +41,7 @@ static UNUSED_FUNC bool boot_is_mem_ram(EFI_MEMORY_TYPE type) } typedef struct { - uint32_t* physAddr; + phys_addr_t physAddr; uint32_t* virtAddr; size_t size; size_t width; @@ -87,7 +87,7 @@ typedef struct boot_info boot_info_t; typedef struct { Elf64_File elf; - void* physAddr; + phys_addr_t physAddr; } boot_kernel_t; typedef struct diff --git a/include/kernel/mem/mem_desc.h b/include/kernel/mem/mem_desc.h index 8f6463945..290660cf6 100644 --- a/include/kernel/mem/mem_desc.h +++ b/include/kernel/mem/mem_desc.h @@ -3,6 +3,8 @@ #include #include +#include +#include #include typedef struct process process_t; @@ -15,16 +17,117 @@ typedef struct process process_t; * @{ */ +/** + * @brief Amount of memory segments statically allocated for small descriptors. + */ +#define MEM_SEGS_SMALL_MAX 2 + +/** + * @brief Memory Segment structure. + * @struct mem_seg_t + */ typedef struct mem_seg { - pfn_t page; - uint32_t length; - uint32_t offset; + void* page; ///< Pointer to the first page of the segment in the higher half. + uint32_t length; ///< Length of the segment in bytes. + uint32_t offset; ///< Offset in bytes within the first page. } mem_seg_t; +/** + * @brief Memory Descriptor structure. + * @struct mem_desc_t + */ typedef struct ALIGNED(64) mem_desc { - + pool_idx_t next; ///< Index of the next descriptor or used for the pools free list. + pool_idx_t index; ///< Index of this descriptor in its pool. + uint32_t amount; ///< Number of memory segments. + uint32_t capacity; ///< Capacity of the `large` array. + size_t size; ///< Total size of the memory region in bytes. + mem_seg_t small[MEM_SEGS_SMALL_MAX]; ///< Statically allocated segments for small regions. + mem_seg_t* large; ///< Pointer to additional segments for large regions. } mem_desc_t; +/** + * @brief Memory Descriptor Pool structure. + * @struct mem_desc_pool_t + */ +typedef struct +{ + pool_t pool; + mem_desc_t descs[]; +} mem_desc_pool_t; + +/** + * @brief Allocate a new Memory Descriptor pool. + * + * @param size The amount of descriptors to allocate. + * @return On success, a pointer to the new Memory Descriptor pool. On failure, `NULL` and `errno` is set. + */ +mem_desc_pool_t* mem_desc_pool_new(size_t size); + +/** + * @brief Free a Memory Descriptor pool. + * + * @param pool Pointer to the Memory Descriptor pool to free. + */ +void mem_desc_pool_free(mem_desc_pool_t* pool); + +/** + * @brief Retrieve the Memory Descriptor pool that a Memory Descriptor was allocated from. + * + * @param desc Pointer to the Memory Descriptor. + * @return Pointer to the Memory Descriptor pool. + */ +static inline mem_desc_pool_t* mem_desc_pool_get(mem_desc_t* desc) +{ + return CONTAINER_OF(desc, mem_desc_pool_t, descs[desc->index]); +} + +/** + * @brief Allocate a new Memory Descriptor from a pool. + * + * @param pool Pointer to the Memory Descriptor pool. + * @return On success, a pointer to the allocated Memory Descriptor. On failure, `NULL` and `errno` is set. + */ +static inline mem_desc_t* mem_desc_new(mem_desc_pool_t* pool) +{ + pool_idx_t idx = pool_alloc(&pool->pool); + if (idx == POOL_IDX_MAX) + { + errno = ENOSPC; + return NULL; + } + + mem_desc_t* desc = &pool->descs[idx]; + desc->next = POOL_IDX_MAX; + desc->amount = 0; + desc->capacity = 0; + desc->size = 0; + desc->large = NULL; + return desc; +} + +/** + * @brief Free a Memory Descriptor back to its pool. + * + * @param desc Pointer to the Memory Descriptor to free. + */ +static inline void mem_desc_free(mem_desc_t* desc) +{ + mem_desc_pool_t* pool = mem_desc_pool_get(desc); + pool_free(&pool->pool, desc->next); +} + +static inline size_t mem_desc_size(mem_desc_t* desc) +{ + return desc->size; +} + +uint64_t mem_desc_add(mem_desc_t* desc, void* addr, size_t size); + +static inline uint64_t mem_desc_populate_user(mem_desc_t* desc, process_t* process, void* addr, size_t size) +{ +} + /** @} */ \ No newline at end of file diff --git a/include/kernel/mem/paging.h b/include/kernel/mem/paging.h index c1922c3ae..0d559f253 100644 --- a/include/kernel/mem/paging.h +++ b/include/kernel/mem/paging.h @@ -21,47 +21,29 @@ * our changes are detected we must invalidate this cache using `invlpg` or if many pages are changed, a full TLB flush * by reloading CR3. * - * @param virtAddr The virtual address of the page to invalidate. + * @param addr The starting virtual address of the region. + * @param amount The number of pages to invalidate. */ -static inline void tlb_invalidate(void* virtAddr, uint64_t pageCount) +static inline void tlb_invalidate(void* addr, size_t amount) { - if (pageCount == 0) + if (amount == 0) { return; } - if (pageCount > 16) + if (amount > 16) { cr3_write(cr3_read()); } else { - for (uint64_t i = 0; i < pageCount; i++) + for (uint64_t i = 0; i < amount; i++) { - ASM("invlpg (%0)" ::"r"(virtAddr + i * PAGE_SIZE) : "memory"); + ASM("invlpg (%0)" ::"r"(addr + (i * PAGE_SIZE)) : "memory"); } } } -/** - * @brief Retrieves the address from a page table entry and converts it to an accessible address. - * - * The accessible address depends on if we are in the kernel or the bootloader as the bootloader has physical memory - * identity mapped to the higher half of the address space, while the kernel does not and instead has the higher half - * mapped to the lower half of the address space. - * - * @param entry The page table entry. - * @return The accessible address contained in the entry. - */ -static inline uintptr_t pml_accessible_addr(pml_entry_t entry) -{ -#ifdef _BOOT_ - return entry.addr << PML_ADDR_OFFSET_BITS; -#else - return PML_LOWER_TO_HIGHER(entry.addr << PML_ADDR_OFFSET_BITS); -#endif -} - /** * @brief Checks if a page table level is empty (all entries are 0). * @@ -91,11 +73,12 @@ static inline bool pml_is_empty(pml_t* pml) */ static inline uint64_t pml_new(page_table_t* table, pml_t** outPml) { - pml_t* pml; - if (table->allocPages((void**)&pml, 1) == ERR) + pfn_t pfn; + if (table->allocPages(&pfn, 1) == ERR) { return ERR; } + pml_t* pml = PFN_TO_VIRT(pfn); memset(pml, 0, PAGE_SIZE); *outPml = pml; return 0; @@ -125,16 +108,17 @@ static inline void pml_free(page_table_t* table, pml_t* pml, pml_level_t level) if (level > PML1) { - pml_free(table, (pml_t*)pml_accessible_addr(*entry), level - 1); + pml_free(table, PFN_TO_VIRT(entry->pfn), level - 1); } else if (entry->owned) { - void* addr = (void*)pml_accessible_addr(*entry); - table->freePages(&addr, 1); + pfn_t pfn = entry->pfn; + table->freePages(&pfn, 1); } } - table->freePages((void**)&pml, 1); + pfn_t pfn = VIRT_TO_PFN(pml); + table->freePages(&pfn, 1); } /** @@ -189,31 +173,32 @@ static inline void page_table_load(page_table_t* table) * set, it returns `ERR`. * * @param table The page table. - * @param currentPml The current page table level. + * @param current The current page table level. * @param index The index within the current page table level. * @param flags The flags to assign to a newly allocated page table level, if applicable. - * @param outPml Will be filled with the retrieved or newly allocated page table level. + * @param out Will be filled with the retrieved or newly allocated page table level. * @return On success, `0`. On failure, `ERR`. */ -static inline uint64_t page_table_get_pml(page_table_t* table, pml_t* currentPml, pml_index_t index, pml_flags_t flags, - pml_t** outPml) +static inline uint64_t page_table_get_pml(page_table_t* table, pml_t* current, pml_index_t index, pml_flags_t flags, + pml_t** out) { - pml_entry_t* entry = ¤tPml->entries[index]; + pml_entry_t* entry = ¤t->entries[index]; if (entry->present) { - *outPml = (pml_t*)pml_accessible_addr(*entry); + *out = PFN_TO_VIRT(entry->pfn); return 0; } if (flags & PML_PRESENT) { - pml_t* nextPml; - if (pml_new(table, &nextPml) == ERR) + pml_t* next; + if (pml_new(table, &next) == ERR) { return ERR; } - currentPml->entries[index].raw = (flags & PML_FLAGS_MASK) | (PML_ENSURE_LOWER_HALF(nextPml) & PML_ADDR_MASK); - *outPml = nextPml; + current->entries[index].raw = flags & PML_FLAGS_MASK; + current->entries[index].pfn = VIRT_TO_PFN(next); + *out = next; return 0; } @@ -262,15 +247,15 @@ typedef struct * * @param table The page table. * @param traverse The helper structure used to cache each layer. - * @param virtAddr The target virtual address. + * @param addr The target virtual address. * @param flags The flags to assigned to newly allocated levels, if the present flag is not set then dont allocate new * levels. * @return On success, `0`. On failure, `ERR`. */ -static inline uint64_t page_table_traverse(page_table_t* table, page_table_traverse_t* traverse, uintptr_t virtAddr, +static inline uint64_t page_table_traverse(page_table_t* table, page_table_traverse_t* traverse, const void* addr, pml_flags_t flags) { - pml_index_t newIdx3 = PML_ADDR_TO_INDEX(virtAddr, PML4); + pml_index_t newIdx3 = PML_ADDR_TO_INDEX(addr, PML4); if (!traverse->pml3Valid || traverse->oldIdx3 != newIdx3) { if (page_table_get_pml(table, table->pml4, newIdx3, (flags | PML_WRITE | PML_USER) & ~PML_GLOBAL, @@ -282,7 +267,7 @@ static inline uint64_t page_table_traverse(page_table_t* table, page_table_trave traverse->pml2Valid = false; // Invalidate cache for lower levels } - pml_index_t newIdx2 = PML_ADDR_TO_INDEX(virtAddr, PML3); + pml_index_t newIdx2 = PML_ADDR_TO_INDEX(addr, PML3); if (!traverse->pml2Valid || traverse->oldIdx2 != newIdx2) { if (page_table_get_pml(table, traverse->pml3, newIdx2, flags | PML_WRITE | PML_USER, &traverse->pml2) == ERR) @@ -293,7 +278,7 @@ static inline uint64_t page_table_traverse(page_table_t* table, page_table_trave traverse->pml1Valid = false; // Invalidate cache for lower levels } - pml_index_t newIdx1 = PML_ADDR_TO_INDEX(virtAddr, PML2); + pml_index_t newIdx1 = PML_ADDR_TO_INDEX(addr, PML2); if (!traverse->pml1Valid || traverse->oldIdx1 != newIdx1) { if (page_table_get_pml(table, traverse->pml2, newIdx1, flags | PML_WRITE | PML_USER, &traverse->pml1) == ERR) @@ -303,28 +288,26 @@ static inline uint64_t page_table_traverse(page_table_t* table, page_table_trave traverse->oldIdx1 = newIdx1; } - traverse->entry = &traverse->pml1->entries[PML_ADDR_TO_INDEX(virtAddr, PML1)]; + traverse->entry = &traverse->pml1->entries[PML_ADDR_TO_INDEX(addr, PML1)]; return 0; } /** * @brief Retrieves the physical address mapped to a given virtual address. * - * If the virtual address is not mapped, the function returns `ERR`. - * * @param table The page table. - * @param virtAddr The virtual address to look up. - * @param outPhysAddr Will be filled with the corresponding physical address on success. + * @param addr The virtual address to look up. + * @param out Will be filled with the corresponding physical address on success. * @return On success, `0`. On failure, `ERR`. */ -static inline uint64_t page_table_get_phys_addr(page_table_t* table, const void* virtAddr, void** outPhysAddr) +static inline uint64_t page_table_get_phys_addr(page_table_t* table, void* addr, phys_addr_t* out) { - uint64_t offset = ((uint64_t)virtAddr) % PAGE_SIZE; - virtAddr = (void*)ROUND_DOWN(virtAddr, PAGE_SIZE); + size_t offset = ((uintptr_t)addr) % PAGE_SIZE; + addr = (void*)ROUND_DOWN(addr, PAGE_SIZE); page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - if (page_table_traverse(table, &traverse, (uintptr_t)virtAddr, PML_NONE) == ERR) + if (page_table_traverse(table, &traverse, addr, PML_NONE) == ERR) { return ERR; } @@ -334,7 +317,7 @@ static inline uint64_t page_table_get_phys_addr(page_table_t* table, const void* return ERR; } - *outPhysAddr = (void*)((traverse.entry->addr << PML_ADDR_OFFSET_BITS) + offset); + *out = PFN_TO_PHYS(traverse.entry->pfn) + offset; return 0; } @@ -344,16 +327,16 @@ static inline uint64_t page_table_get_phys_addr(page_table_t* table, const void* * If any page in the range is not mapped, the function returns `false`. * * @param table The page table. - * @param virtAddr The starting virtual address. - * @param pageAmount The number of pages to check. + * @param addr The starting virtual address. + * @param amount The number of pages to check. * @return `true` if the entire range is mapped, `false` otherwise. */ -static inline bool page_table_is_mapped(page_table_t* table, const void* virtAddr, size_t pageAmount) +static inline bool page_table_is_mapped(page_table_t* table, const void* addr, size_t amount) { page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - for (uint64_t i = 0; i < pageAmount; i++) + for (uint64_t i = 0; i < amount; i++) { - if (page_table_traverse(table, &traverse, (uintptr_t)virtAddr + i * PAGE_SIZE, PML_NONE) == ERR) + if (page_table_traverse(table, &traverse, addr + i * PAGE_SIZE, PML_NONE) == ERR) { return false; } @@ -373,17 +356,17 @@ static inline bool page_table_is_mapped(page_table_t* table, const void* virtAdd * If any page in the range is mapped, the function returns `false`. * * @param table The page table. - * @param virtAddr The starting virtual address. - * @param pageAmount The number of pages to check. + * @param addr The starting virtual address. + * @param amount The number of pages to check. * @return `true` if the entire range is unmapped, `false` otherwise. */ -static inline bool page_table_is_unmapped(page_table_t* table, const void* virtAddr, size_t pageAmount) +static inline bool page_table_is_unmapped(page_table_t* table, void* addr, size_t amount) { page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - for (uint64_t i = 0; i < pageAmount; i++) + for (uint64_t i = 0; i < amount; i++) { - if (page_table_traverse(table, &traverse, (uintptr_t)virtAddr + i * PAGE_SIZE, PML_NONE) == ERR) + if (page_table_traverse(table, &traverse, addr + i * PAGE_SIZE, PML_NONE) == ERR) { continue; } @@ -403,14 +386,14 @@ static inline bool page_table_is_unmapped(page_table_t* table, const void* virtA * If any page in the range is already mapped, the function will fail and return `ERR`. * * @param table The page table. - * @param virtAddr The starting virtual address. - * @param physAddr The starting physical address. - * @param pageAmount The number of pages to map. + * @param addr The starting virtual address. + * @param phys The starting physical address. + * @param amount The number of pages to map. * @param flags The flags to set for the mapped pages. Must include `PML_PRESENT`. * @param callbackId The callback ID to associate with the mapped pages or `PML_CALLBACK_NONE`. * @return On success, `0`. On failure, `ERR`. */ -static inline uint64_t page_table_map(page_table_t* table, void* virtAddr, void* physAddr, size_t pageAmount, +static inline uint64_t page_table_map(page_table_t* table, void* addr, phys_addr_t phys, size_t amount, pml_flags_t flags, pml_callback_id_t callbackId) { if (!(flags & PML_PRESENT)) @@ -420,9 +403,9 @@ static inline uint64_t page_table_map(page_table_t* table, void* virtAddr, void* page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - for (uint64_t i = 0; i < pageAmount; i++) + for (uint64_t i = 0; i < amount; i++) { - if (page_table_traverse(table, &traverse, (uintptr_t)virtAddr, flags) == ERR) + if (page_table_traverse(table, &traverse, addr, flags) == ERR) { return ERR; } @@ -433,12 +416,12 @@ static inline uint64_t page_table_map(page_table_t* table, void* virtAddr, void* } traverse.entry->raw = flags; - traverse.entry->addr = ((uintptr_t)PML_ENSURE_LOWER_HALF(physAddr)) >> PML_ADDR_OFFSET_BITS; + traverse.entry->pfn = PHYS_TO_PFN(phys); traverse.entry->lowCallbackId = callbackId & 1; traverse.entry->highCallbackId = callbackId >> 1; - physAddr = (void*)((uintptr_t)physAddr + PAGE_SIZE); - virtAddr = (void*)((uintptr_t)virtAddr + PAGE_SIZE); + phys += PAGE_SIZE; + addr += PAGE_SIZE; } return 0; @@ -450,14 +433,14 @@ static inline uint64_t page_table_map(page_table_t* table, void* virtAddr, void* * If any page in the range is already mapped, the function will fail and return `ERR`. * * @param table The page table. - * @param virtAddr The starting virtual address. - * @param pages Array of physical page addresses to map. - * @param pageAmount The number of pages in the array to map. + * @param addr The starting virtual address. + * @param pfns Array of page frame numbers to map. + * @param amount The number of pages in the array to map. * @param flags The flags to set for the mapped pages. Must include `PML_PRESENT`. * @param callbackId The callback ID to associate with the mapped pages or `PML_CALLBACK_NONE`. * @return On success, `0`. On failure, `ERR`. */ -static inline uint64_t page_table_map_pages(page_table_t* table, void* virtAddr, void** pages, size_t pageAmount, +static inline uint64_t page_table_map_pages(page_table_t* table, void* addr, const pfn_t* pfns, size_t amount, pml_flags_t flags, pml_callback_id_t callbackId) { if (!(flags & PML_PRESENT)) @@ -467,9 +450,9 @@ static inline uint64_t page_table_map_pages(page_table_t* table, void* virtAddr, page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - for (uint64_t i = 0; i < pageAmount; i++) + for (uint64_t i = 0; i < amount; i++) { - if (page_table_traverse(table, &traverse, (uintptr_t)virtAddr, flags) == ERR) + if (page_table_traverse(table, &traverse, addr, flags) == ERR) { return ERR; } @@ -480,11 +463,11 @@ static inline uint64_t page_table_map_pages(page_table_t* table, void* virtAddr, } traverse.entry->raw = flags; - traverse.entry->addr = ((uintptr_t)PML_ENSURE_LOWER_HALF(pages[i])) >> PML_ADDR_OFFSET_BITS; + traverse.entry->pfn = pfns[i]; traverse.entry->lowCallbackId = callbackId & 1; traverse.entry->highCallbackId = callbackId >> 1; - virtAddr = (void*)((uintptr_t)virtAddr + PAGE_SIZE); + addr = (void*)((uintptr_t)addr + PAGE_SIZE); } return 0; @@ -500,16 +483,16 @@ static inline uint64_t page_table_map_pages(page_table_t* table, void* virtAddr, * free owned pages separately. * * @param table The page table. - * @param virtAddr The starting virtual address. - * @param pageAmount The number of pages to unmap. + * @param addr The starting virtual address. + * @param amount The number of pages to unmap. */ -static inline void page_table_unmap(page_table_t* table, void* virtAddr, size_t pageAmount) +static inline void page_table_unmap(page_table_t* table, void* addr, size_t amount) { page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - for (uint64_t i = 0; i < pageAmount; i++) + for (uint64_t i = 0; i < amount; i++) { - if (page_table_traverse(table, &traverse, (uintptr_t)virtAddr + i * PAGE_SIZE, PML_NONE) == ERR) + if (page_table_traverse(table, &traverse, addr + i * PAGE_SIZE, PML_NONE) == ERR) { continue; } @@ -522,7 +505,7 @@ static inline void page_table_unmap(page_table_t* table, void* virtAddr, size_t traverse.entry->present = 0; } - tlb_invalidate(virtAddr, pageAmount); + tlb_invalidate(addr, amount); } /** @@ -531,8 +514,8 @@ static inline void page_table_unmap(page_table_t* table, void* virtAddr, size_t */ typedef struct { - void* pages[PML_PAGE_BUFFER_SIZE]; - uint64_t pageCount; + pfn_t pfns[PML_PAGE_BUFFER_SIZE]; + uint64_t amount; } page_table_page_buffer_t; /** @@ -546,13 +529,13 @@ typedef struct */ static inline void page_table_page_buffer_push(page_table_t* table, page_table_page_buffer_t* buffer, void* address) { - buffer->pages[buffer->pageCount] = address; - buffer->pageCount++; + buffer->pfns[buffer->amount] = VIRT_TO_PFN(address); + buffer->amount++; - if (buffer->pageCount >= PML_PAGE_BUFFER_SIZE) + if (buffer->amount >= PML_PAGE_BUFFER_SIZE) { - table->freePages(buffer->pages, buffer->pageCount); - buffer->pageCount = 0; + table->freePages(buffer->pfns, buffer->amount); + buffer->amount = 0; } } @@ -566,10 +549,10 @@ static inline void page_table_page_buffer_push(page_table_t* table, page_table_p */ static inline void page_table_page_buffer_flush(page_table_t* table, page_table_page_buffer_t* buffer) { - if (buffer->pageCount > 0) + if (buffer->amount > 0) { - table->freePages(buffer->pages, buffer->pageCount); - buffer->pageCount = 0; + table->freePages(buffer->pfns, buffer->amount); + buffer->amount = 0; } } @@ -614,22 +597,20 @@ static inline void page_table_clear_pml1_pml2_pml3(page_table_t* table, page_tab * All unskipped entries will be fully cleared (set to 0). * * @param table The page table. - * @param virtAddr The starting virtual address. - * @param pageAmount The number of pages to clear. + * @param addr The starting virtual address. + * @param amount The number of pages to clear. */ -static inline void page_table_clear(page_table_t* table, void* virtAddr, size_t pageAmount) +static inline void page_table_clear(page_table_t* table, void* addr, size_t amount) { - page_table_page_buffer_t pageBuffer = {.pageCount = 0}; + page_table_page_buffer_t pageBuffer = {0}; page_table_traverse_t prevTraverse = PAGE_TABLE_TRAVERSE_CREATE; page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - for (uint64_t i = 0; i < pageAmount; i++) + for (uint64_t i = 0; i < amount; i++) { - uintptr_t currentVirtAddr = (uintptr_t)virtAddr + i * PAGE_SIZE; - page_table_clear_pml1_pml2_pml3(table, &prevTraverse, &traverse, &pageBuffer); - if (page_table_traverse(table, &traverse, currentVirtAddr, PML_NONE) == ERR) + if (page_table_traverse(table, &traverse, addr + i * PAGE_SIZE, PML_NONE) == ERR) { prevTraverse.pml1Valid = false; prevTraverse.pml2Valid = false; @@ -645,7 +626,7 @@ static inline void page_table_clear(page_table_t* table, void* virtAddr, size_t if (traverse.entry->owned) { - page_table_page_buffer_push(table, &pageBuffer, (void*)pml_accessible_addr(*traverse.entry)); + page_table_page_buffer_push(table, &pageBuffer, PFN_TO_VIRT(traverse.entry->pfn)); } traverse.entry->raw = 0; @@ -659,18 +640,18 @@ static inline void page_table_clear(page_table_t* table, void* virtAddr, size_t * @brief Collects the number of pages associated with each callback ID in the specified range. * * @param table The page table. - * @param virtAddr The starting virtual address. - * @param pageAmount The number of pages to check. + * @param addr The starting virtual address. + * @param amount The number of pages to check. * @param callbacks An array of size `PML_MAX_CALLBACK` that will be filled with the occurrences of each callback ID. */ -static inline void page_table_collect_callbacks(page_table_t* table, void* virtAddr, size_t pageAmount, +static inline void page_table_collect_callbacks(page_table_t* table, void* addr, size_t amount, uint64_t* callbacks) { page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - for (uint64_t i = 0; i < pageAmount; i++) + for (uint64_t i = 0; i < amount; i++) { - if (page_table_traverse(table, &traverse, (uintptr_t)virtAddr + i * PAGE_SIZE, PML_NONE) == ERR) + if (page_table_traverse(table, &traverse, addr + i * PAGE_SIZE, PML_NONE) == ERR) { continue; } @@ -694,18 +675,18 @@ static inline void page_table_collect_callbacks(page_table_t* table, void* virtA * If a page is not currently mapped, it is skipped. * * @param table The page table. - * @param virtAddr The starting virtual address. - * @param pageAmount The number of pages to update. + * @param addr The starting virtual address. + * @param amount The number of pages to update. * @param flags The new flags to set. The `PML_OWNED` flag is preserved. * @return On success, `0`. On failure, `ERR`. */ -static inline uint64_t page_table_set_flags(page_table_t* table, void* virtAddr, size_t pageAmount, pml_flags_t flags) +static inline uint64_t page_table_set_flags(page_table_t* table, void* addr, size_t amount, pml_flags_t flags) { page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - for (uint64_t i = 0; i < pageAmount; i++) + for (uint64_t i = 0; i < amount; i++) { - if (page_table_traverse(table, &traverse, (uintptr_t)virtAddr + i * PAGE_SIZE, PML_NONE) == ERR) + if (page_table_traverse(table, &traverse, addr + i * PAGE_SIZE, PML_NONE) == ERR) { continue; } @@ -724,7 +705,7 @@ static inline uint64_t page_table_set_flags(page_table_t* table, void* virtAddr, traverse.entry->raw = (traverse.entry->raw & ~PML_FLAGS_MASK) | (flags & PML_FLAGS_MASK); } - tlb_invalidate(virtAddr, pageAmount); + tlb_invalidate(addr, amount); return 0; } @@ -740,13 +721,13 @@ static inline uint64_t page_table_set_flags(page_table_t* table, void* virtAddr, * @param table The page table. * @param startAddr The start address to begin searching (inclusive). * @param endAddr The end address of the search range (exclusive). - * @param pageAmount The number of consecutive unmapped pages needed. + * @param amount The number of consecutive unmapped pages needed. * @param alignment The required alignment for the region in bytes. * @param outAddr Will be filled with the start address of the unmapped region if found. * @return On success, `0`. If no suitable region is found, `ERR`. */ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void* startAddr, void* endAddr, - size_t pageAmount, size_t alignment, void** outAddr) + size_t amount, size_t alignment, void** outAddr) { uintptr_t currentAddr = ROUND_DOWN((uintptr_t)startAddr, PAGE_SIZE); uintptr_t end = (uintptr_t)endAddr; @@ -756,7 +737,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void alignment = PAGE_SIZE; } - if (pageAmount >= (PML3_SIZE / PAGE_SIZE)) + if (amount >= (PML3_SIZE / PAGE_SIZE)) { while (currentAddr < end) { @@ -777,7 +758,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void continue; } - pml_t* pml3 = (pml_t*)pml_accessible_addr(*entry4); + pml_t* pml3 = PFN_TO_VIRT(entry4->pfn); pml_entry_t* entry3 = &pml3->entries[idx3]; if (!entry3->present) @@ -796,7 +777,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void return ERR; } - if (pageAmount >= (PML2_SIZE / PAGE_SIZE)) + if (amount >= (PML2_SIZE / PAGE_SIZE)) { while (currentAddr < end) { @@ -816,7 +797,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void continue; } - pml_t* pml3 = (pml_t*)pml_accessible_addr(*entry4); + pml_t* pml3 = PFN_TO_VIRT(entry4->pfn); pml_index_t idx3 = PML_ADDR_TO_INDEX(currentAddr, PML3); pml_entry_t* entry3 = &pml3->entries[idx3]; @@ -833,7 +814,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void continue; } - pml_t* pml2 = (pml_t*)pml_accessible_addr(*entry3); + pml_t* pml2 = PFN_TO_VIRT(entry3->pfn); pml_index_t idx2 = PML_ADDR_TO_INDEX(currentAddr, PML2); pml_entry_t* entry2 = &pml2->entries[idx2]; @@ -872,7 +853,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void uint64_t skippedPages = (MIN(skipTo, end) - currentAddr) / PAGE_SIZE; consecutiveUnmapped += skippedPages; - if (consecutiveUnmapped >= pageAmount) + if (consecutiveUnmapped >= amount) { *outAddr = (void*)regionStart; return 0; @@ -882,7 +863,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void continue; } - pml_t* pml3 = (pml_t*)pml_accessible_addr(*entry4); + pml_t* pml3 = PFN_TO_VIRT(entry4->pfn); pml_index_t idx3 = PML_ADDR_TO_INDEX(currentAddr, PML3); pml_entry_t* entry3 = &pml3->entries[idx3]; @@ -905,7 +886,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void consecutiveUnmapped += skippedPages; } - if (consecutiveUnmapped >= pageAmount) + if (consecutiveUnmapped >= amount) { *outAddr = (void*)regionStart; return 0; @@ -915,7 +896,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void continue; } - pml_t* pml2 = (pml_t*)pml_accessible_addr(*entry3); + pml_t* pml2 = PFN_TO_VIRT(entry3->pfn); pml_index_t idx2 = PML_ADDR_TO_INDEX(currentAddr, PML2); pml_entry_t* entry2 = &pml2->entries[idx2]; @@ -938,7 +919,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void consecutiveUnmapped += skippedPages; } - if (consecutiveUnmapped >= pageAmount) + if (consecutiveUnmapped >= amount) { *outAddr = (void*)regionStart; return 0; @@ -948,7 +929,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void continue; } - pml_t* pml1 = (pml_t*)pml_accessible_addr(*entry2); + pml_t* pml1 = PFN_TO_VIRT(entry2->pfn); pml_index_t idx1 = PML_ADDR_TO_INDEX(currentAddr, PML1); for (; idx1 < PML_INDEX_AMOUNT && currentAddr < end; idx1++, currentAddr += PAGE_SIZE) @@ -969,7 +950,7 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void consecutiveUnmapped++; } - if (consecutiveUnmapped >= pageAmount) + if (consecutiveUnmapped >= amount) { *outAddr = (void*)regionStart; return 0; @@ -989,16 +970,16 @@ static inline uint64_t page_table_find_unmapped_region(page_table_t* table, void * @brief Checks if any page in a range is pinned. * * @param table The page table. - * @param virtAddr The starting virtual address. - * @param pageAmount The number of pages to check. + * @param addr The starting virtual address. + * @param amount The number of pages to check. * @return `true` if any page in the range us pinned, `false` otherwise. */ -static inline bool page_table_is_pinned(page_table_t* table, const void* virtAddr, size_t pageAmount) +static inline bool page_table_is_pinned(page_table_t* table, void* addr, size_t amount) { page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - for (uint64_t i = 0; i < pageAmount; i++) + for (uint64_t i = 0; i < amount; i++) { - if (page_table_traverse(table, &traverse, (uintptr_t)virtAddr + i * PAGE_SIZE, PML_NONE) == ERR) + if (page_table_traverse(table, &traverse, addr + i * PAGE_SIZE, PML_NONE) == ERR) { continue; } @@ -1024,57 +1005,57 @@ static inline bool page_table_is_pinned(page_table_t* table, const void* virtAdd * `PML_PRESENT | PML_USER | PML_OWNED` flags set. * * @param table The page table. - * @param virtAddr The starting virtual address. - * @param pageAmount The number of pages to check. + * @param addr The starting virtual address. + * @param amount The number of pages to check. * @param flags The flags to check for. * @return The number of pages with the specified flags set. */ -static inline uint64_t page_table_count_pages_with_flags(page_table_t* table, void* virtAddr, size_t pageAmount, +static inline uint64_t page_table_count_pages_with_flags(page_table_t* table, void* addr, size_t amount, pml_flags_t flags) { uint64_t count = 0; - while (pageAmount > 0) + while (amount > 0) { - pml_index_t idx4 = PML_ADDR_TO_INDEX((uintptr_t)virtAddr, PML4); + pml_index_t idx4 = PML_ADDR_TO_INDEX((uintptr_t)addr, PML4); pml_entry_t* entry4 = &table->pml4->entries[idx4]; if (!entry4->present) { - uint64_t skipPages = MIN(pageAmount, (PML_INDEX_TO_ADDR(idx4 + 1, PML4) - (uintptr_t)virtAddr) / PAGE_SIZE); - virtAddr = (void*)((uintptr_t)virtAddr + skipPages * PAGE_SIZE); - pageAmount -= skipPages; + uint64_t skipPages = MIN(amount, (PML_INDEX_TO_ADDR(idx4 + 1, PML4) - (uintptr_t)addr) / PAGE_SIZE); + addr = (void*)((uintptr_t)addr + skipPages * PAGE_SIZE); + amount -= skipPages; continue; } - pml_t* pml3 = (pml_t*)pml_accessible_addr(*entry4); - pml_index_t idx3 = PML_ADDR_TO_INDEX((uintptr_t)virtAddr, PML3); + pml_t* pml3 = PFN_TO_VIRT(entry4->pfn); + pml_index_t idx3 = PML_ADDR_TO_INDEX((uintptr_t)addr, PML3); pml_entry_t* entry3 = &pml3->entries[idx3]; if (!entry3->present) { - uint64_t skipPages = MIN(pageAmount, (PML_INDEX_TO_ADDR(idx3 + 1, PML3) - (uintptr_t)virtAddr) / PAGE_SIZE); - virtAddr = (void*)((uintptr_t)virtAddr + skipPages * PAGE_SIZE); - pageAmount -= skipPages; + uint64_t skipPages = MIN(amount, (PML_INDEX_TO_ADDR(idx3 + 1, PML3) - (uintptr_t)addr) / PAGE_SIZE); + addr = (void*)((uintptr_t)addr + skipPages * PAGE_SIZE); + amount -= skipPages; continue; } - pml_t* pml2 = (pml_t*)pml_accessible_addr(*entry3); - pml_index_t idx2 = PML_ADDR_TO_INDEX((uintptr_t)virtAddr, PML2); + pml_t* pml2 = PFN_TO_VIRT(entry3->pfn); + pml_index_t idx2 = PML_ADDR_TO_INDEX((uintptr_t)addr, PML2); pml_entry_t* entry2 = &pml2->entries[idx2]; if (!entry2->present) { - uint64_t skipPages = MIN(pageAmount, (PML_INDEX_TO_ADDR(idx2 + 1, PML2) - (uintptr_t)virtAddr) / PAGE_SIZE); - virtAddr = (void*)((uintptr_t)virtAddr + skipPages * PAGE_SIZE); - pageAmount -= skipPages; + uint64_t skipPages = MIN(amount, (PML_INDEX_TO_ADDR(idx2 + 1, PML2) - (uintptr_t)addr) / PAGE_SIZE); + addr = (void*)((uintptr_t)addr + skipPages * PAGE_SIZE); + amount -= skipPages; continue; } - pml_t* pml1 = (pml_t*)pml_accessible_addr(*entry2); - pml_index_t idx1 = PML_ADDR_TO_INDEX((uintptr_t)virtAddr, PML1); + pml_t* pml1 = PFN_TO_VIRT(entry2->pfn); + pml_index_t idx1 = PML_ADDR_TO_INDEX((uintptr_t)addr, PML1); - for (; idx1 < PML_INDEX_AMOUNT && pageAmount > 0; - idx1++, virtAddr = (void*)((uintptr_t)virtAddr + PAGE_SIZE), pageAmount--) + for (; idx1 < PML_INDEX_AMOUNT && amount > 0; + idx1++, addr = (void*)((uintptr_t)addr + PAGE_SIZE), amount--) { pml_entry_t* entry1 = &pml1->entries[idx1]; if (!entry1->present) diff --git a/include/kernel/mem/paging_types.h b/include/kernel/mem/paging_types.h index 74205f767..ecf15b86a 100644 --- a/include/kernel/mem/paging_types.h +++ b/include/kernel/mem/paging_types.h @@ -31,6 +31,61 @@ * @{ */ +/** + * @brief Page Frame Number type. + */ +typedef size_t pfn_t; + +/** + * @brief Physical address type. + */ +typedef uintptr_t phys_addr_t; + +/** + * @brief Invalid physical address. + */ +#define PHYS_ADDR_INVALID ((phys_addr_t)(UINTPTR_MAX)) + +/** + * @brief Convert a PFN to its physical address. + * + * @param _pfn The Page Frame Number. + * @return The physical address of the page. + */ +#define PFN_TO_PHYS(_pfn) ((phys_addr_t)(_pfn) * PAGE_SIZE) + +/** + * @brief Convert a physical address to its PFN. + * + * @param _addr The physical address. + * @return The Page Frame Number of the page. + */ +#define PHYS_TO_PFN(_addr) ((pfn_t)(_addr) / PAGE_SIZE) + +/** + * @brief Convert a PFN to its identity mapped higher half virtual address. + * + * @param _pfn The Page Frame Number. + * @return The higher half virtual address of the page. + */ +#ifndef _BOOT_ +#define PFN_TO_VIRT(_pfn) ((void*)PML_LOWER_TO_HIGHER((uintptr_t)(_pfn) * PAGE_SIZE)) +#else +#define PFN_TO_VIRT(_pfn) ((void*)PFN_TO_PHYS(_pfn)) +#endif + +/** + * @brief Convert a identity mapped higher half virtual address to its PFN. + * + * @param _addr The higher half virtual address. + * @return The Page Frame Number of the page. + */ +#ifndef _BOOT_ +#define VIRT_TO_PFN(_addr) ((pfn_t)(PML_HIGHER_TO_LOWER(_addr) / PAGE_SIZE)) +#else +#define VIRT_TO_PFN(_pfn) PHYS_TO_PFN(_pfn) +#endif + /** * @brief One entry in a page table. * @typedef pml_entry_t @@ -77,7 +132,7 @@ typedef struct * Check the virtual memory manager for more information. (Defined by PatchworkOS) */ uint64_t lowCallbackId : 1; - uint64_t addr : 40; ///< The address contained in the entry, note that this is shifted right by 12 bits. + pfn_t pfn : 40; ///< The physical frame number (physical address >> 12). /** * The high bits of the callback ID associated with this page. * @@ -272,7 +327,7 @@ typedef enum * @return The address in the lower half of the address space. */ #define PML_ENSURE_LOWER_HALF(addr) \ - ((uintptr_t)(addr) >= PML_HIGHER_HALF_START ? PML_HIGHER_TO_LOWER(addr) : (uintptr_t)(addr)) + ((phys_addr_t)(addr) >= PML_HIGHER_HALF_START ? PML_HIGHER_TO_LOWER(addr) : (phys_addr_t)(addr)) /** * @brief Ensures that the given address is in the higher half of the address space. @@ -353,14 +408,14 @@ typedef struct * * Used to allow both the kernel and bootloader to provide their own page allocation functions. */ -typedef uint64_t (*pml_alloc_pages_t)(void**, size_t); +typedef uint64_t (*pml_alloc_pages_t)(pfn_t*, size_t); /** * @brief Generic page free function type. * * Used to allow both the kernel and bootloader to provide their own page free functions. */ -typedef void (*pml_free_pages_t)(void**, size_t); +typedef void (*pml_free_pages_t)(pfn_t*, size_t); /** * @brief Size of the page buffer used to batch page allocations and frees. diff --git a/include/kernel/mem/pmm.h b/include/kernel/mem/pmm.h index 39d77d15f..2fce5021f 100644 --- a/include/kernel/mem/pmm.h +++ b/include/kernel/mem/pmm.h @@ -11,9 +11,6 @@ * * The Physical Memory Manager (PMM) is responsible for allocating and freeing physical memory pages. * - * @note All physical pages are identity mapped to the higher half of the address space, as such, the PMM will always - * return valid and usable higher half addresses. - * * ## The Free Page Stack * * For most allocations, the PMM uses a fast `O(1)` stack-based allocator to manage free pages, with the limitation that @@ -38,7 +35,7 @@ /** * @brief The type used for page reference counts. */ -typedef uint16_t pmm_ref_t; +typedef uint16_t page_ref_t; /** * @brief Invalid page reference count. @@ -48,7 +45,7 @@ typedef uint16_t pmm_ref_t; /** * @brief Maximum number of free pages that can be stored in a free page. */ -#define FREE_PAGE_MAX (PAGE_SIZE / sizeof(void*) - 1) +#define FREE_PAGE_MAX (PAGE_SIZE / sizeof(pfn_t) - 1) /** * @brief Stored in free pages to form a stack of free pages. @@ -57,7 +54,7 @@ typedef uint16_t pmm_ref_t; typedef struct page_stack { struct page_stack* next; - void* pages[FREE_PAGE_MAX]; + pfn_t pages[FREE_PAGE_MAX]; } page_stack_t; static_assert(sizeof(page_stack_t) == PAGE_SIZE, "page_stack_t must be exactly one page in size"); @@ -73,39 +70,39 @@ void pmm_init(void); * Will by default use the free stack allocator, but if no pages are available there, it will fall back to the bitmap * allocator. * - * @return Pointer to the allocated page, or `NULL` if no memory is available. + * @return The PFN of the allocated page, or `ERR` if no memory is available. */ -void* pmm_alloc(void); +pfn_t pmm_alloc(void); /** * @brief Allocate multiple pages of physical memory. * * Usefull for reducing overhead from locking when allocating many pages. * - * @param addresses Array to store the allocated page addresses. + * @param pfns Array to store the allocated page PFNs. * @param count Number of pages to allocate. * @return On success, `0`. On failure, `ERR` and no pages are allocated. */ -uint64_t pmm_alloc_pages(void** addresses, size_t count); +uint64_t pmm_alloc_pages(pfn_t* pfns, size_t count); /** * @brief Allocate a contiguous region of physical memory using the bitmap. * * @param count Number of pages to allocate. - * @param maxAddr Maximum address to allocate up to (exclusive). - * @param alignment Alignment of the region. - * @return Pointer to the first allocated page, or `NULL` if no memory is available. + * @param maxPfn Maximum PFN to allocate up to (exclusive). + * @param alignPfn Alignment of the region in pages. + * @return The PFN of the first allocated page, or `ERR` if no memory is available. */ -void* pmm_alloc_bitmap(size_t count, uintptr_t maxAddr, uint64_t alignment); +pfn_t pmm_alloc_bitmap(size_t count, pfn_t maxPfn, pfn_t alignPfn); /** * @brief Free a single page of physical memory. * * The page will only be reclaimed if its reference count reaches zero. * - * @param address Pointer to the page to free. + * @param pfn The PFN of the page to free. */ -void pmm_free(void* address); +void pmm_free(pfn_t pfn); /** * @brief Free multiple pages of physical memory. @@ -114,30 +111,30 @@ void pmm_free(void* address); * * The pages will only be reclaimed if its reference count reaches zero. * - * @param addresses Array of pointers to the pages to free. + * @param pfns Array of PFNs to free. * @param count Number of pages to free. */ -void pmm_free_pages(void** addresses, size_t count); +void pmm_free_pages(pfn_t* pfns, size_t count); /** * @brief Free a contiguous region of physical memory. * * The pages will only be reclaimed if its reference count reaches zero. * - * @param address Pointer to the first page of the region to free. + * @param pfn The PFN of the first page of the region to free. * @param count Number of pages to free. */ -void pmm_free_region(void* address, size_t count); +void pmm_free_region(pfn_t pfn, size_t count); /** * @brief Increment the reference count of a physical page. * * Will fail if the page is not allocated. * - * @param address Address of the physical page. + * @param pfn The PFN of the physical page. * @return On success, the new reference count. On failure, `ERR`. */ -uint64_t pmm_ref_inc(void* address); +uint64_t pmm_ref_inc(pfn_t pfn); /** * @brief Get the total number of physical pages. diff --git a/include/kernel/mem/pool.h b/include/kernel/mem/pool.h index 966e55cdc..ac2be2323 100644 --- a/include/kernel/mem/pool.h +++ b/include/kernel/mem/pool.h @@ -15,8 +15,8 @@ * performance-critical structures and as a even more specialized alternative to the Object Cache. * * In addition to its performance advantages, since the pool uses an array to store its objects, it is possible for - * certain structures to avoid storing full pointers to objects allocated from a pool instead using a `pool_idx_t` and - * thus saving memory or allowing better caching. + * certain structures to avoid storing full pointers to objects allocated from a pool instead using a `pool_idx_t` thus + * saving memory or allowing better caching. * * @{ */ diff --git a/include/kernel/mem/space.h b/include/kernel/mem/space.h index 864134a88..764862f07 100644 --- a/include/kernel/mem/space.h +++ b/include/kernel/mem/space.h @@ -214,7 +214,7 @@ uint64_t space_check_access(space_t* space, const void* addr, size_t length); typedef struct { void* virtAddr; - void* physAddr; + phys_addr_t physAddr; size_t pageAmount; pml_flags_t flags; } space_mapping_t; @@ -234,7 +234,7 @@ typedef struct * @param space The target address space. * @param mapping Will be filled with parsed information about the mapping. * @param virtAddr The virtual address the mapping will apply to. Can be `NULL` to let the kernel choose an address. - * @param physAddr The physical address to map from. Can be `NULL`. + * @param physAddr The physical address to map from. Can be `PHYS_ADDR_INVALID`. * @param length The length of the virtual memory region to modify, in bytes. * @param alignment The required alignment for the virtual memory region in bytes. * @param flags The page table flags for the mapping. @@ -244,7 +244,7 @@ typedef struct * - `EFAULT`: The addresses are outside the allowed range. * - `ENOMEM`: Not enough memory. */ -uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* virtAddr, void* physAddr, size_t length, +uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* virtAddr, phys_addr_t physAddr, size_t length, size_t alignment, pml_flags_t flags); /** diff --git a/include/kernel/mem/vmm.h b/include/kernel/mem/vmm.h index de9d5c381..02d3cc577 100644 --- a/include/kernel/mem/vmm.h +++ b/include/kernel/mem/vmm.h @@ -44,7 +44,7 @@ * at `VMM_KERNEL_HEAP_MIN` and grows up towards `VMM_KERNEL_HEAP_MAX`. This section takes up 2 indices in the * page table and is mapped identically for all processes. * - * Fourthly (is fourthly really a word?), we have the identity mapped physical memory. All physical memory will be + * Fourthly, we have the identity mapped physical memory. All physical memory will be * mapped here by simply taking the original physical address and adding `0xFFFF800000000000` to it. This means that the * physical address `0x123456` will be mapped to the virtual address `0xFFFF800000123456`. This section takes up all * remaining indices below the kernel heap to the end of the higher half and is mapped identically for all processes. @@ -182,7 +182,7 @@ void* vmm_alloc(space_t* space, void* virtAddr, size_t length, size_t alignment, * * @param space The target address space, if `NULL`, the kernel space is used. * @param virtAddr The desired virtual address to map to, if `NULL`, the kernel chooses an available address. - * @param physAddr The physical address to map from. Must not be `NULL`. + * @param physAddr The physical address to map from. Must not be `PHYS_ADDR_INVALID`. * @param length The length of the memory region to map, in bytes. * @param flags The page table flags for the mapping, must have `PML_PRESENT` set. * @param func The callback function to call when the mapped memory is unmapped or the address space is freed. If @@ -195,7 +195,7 @@ void* vmm_alloc(space_t* space, void* virtAddr, size_t length, size_t alignment, * - `ENOMEM`: Not enough memory. * - Other values from `space_mapping_start()`. */ -void* vmm_map(space_t* space, void* virtAddr, void* physAddr, size_t length, pml_flags_t flags, +void* vmm_map(space_t* space, void* virtAddr, phys_addr_t physAddr, size_t length, pml_flags_t flags, space_callback_func_t func, void* data); /** @@ -207,8 +207,8 @@ void* vmm_map(space_t* space, void* virtAddr, void* physAddr, size_t length, pml * * @param space The target address space, if `NULL`, the kernel space is used. * @param virtAddr The desired virtual address to map to, if `NULL`, the kernel chooses an available address. - * @param pages An array of physical page addresses to map. - * @param pageAmount The number of physical pages in the `pages` array, must not be zero. + * @param pfns An array of page frame numbers to map from. + * @param amount The number of pages to map. * @param flags The page table flags for the mapping, must have `PML_PRESENT` set. * @param func The callback function to call when the mapped memory is unmapped or the address space is freed. If * `NULL`, then no callback will be called. @@ -220,7 +220,7 @@ void* vmm_map(space_t* space, void* virtAddr, void* physAddr, size_t length, pml * - `ENOMEM`: Not enough memory. * - Other values from `space_mapping_start()`. */ -void* vmm_map_pages(space_t* space, void* virtAddr, void** pages, size_t pageAmount, pml_flags_t flags, +void* vmm_map_pages(space_t* space, void* virtAddr, pfn_t* pfns, size_t amount, pml_flags_t flags, space_callback_func_t func, void* data); /** diff --git a/include/kernel/sync/async.h b/include/kernel/sync/async.h index 04413271f..5b5cfa67f 100644 --- a/include/kernel/sync/async.h +++ b/include/kernel/sync/async.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -112,6 +113,7 @@ typedef struct async { rings_t rings; ///< Asynchronous rings information. irp_pool_t* irps; ///< Pool of preallocated IRPs. + mem_desc_pool_t* descs; ///< Pool of preallocated memory descriptors. void* userAddr; ///< Userspace address of the rings. void* kernelAddr; ///< Kernel address of the rings. size_t pageAmount; ///< Amount of pages mapped for the rings. diff --git a/include/kernel/sync/irp.h b/include/kernel/sync/irp.h index 8ad36dd8d..b421528f5 100644 --- a/include/kernel/sync/irp.h +++ b/include/kernel/sync/irp.h @@ -3,20 +3,17 @@ #include #include #include +#include #include #include +#include #include #include #include #include #include -#include -typedef struct async async_t; -typedef struct process process_t; -typedef struct file file_t; -typedef struct pathname pathname_t; typedef struct irp irp_t; @@ -274,7 +271,8 @@ typedef struct ALIGNED(64) irp struct { file_t* from; - pathname_t* path; + mem_desc_t* path; + size_t length; } open; uint64_t _args[IRP_ARGS_MAX]; }; @@ -332,7 +330,7 @@ void irp_pool_free(irp_pool_t* pool); */ static inline irp_pool_t* irp_pool_get(irp_t* irp) { - return (irp_pool_t*)((uintptr_t)irp - (irp->index * sizeof(irp_t)) - offsetof(irp_pool_t, irps)); + return CONTAINER_OF(irp, irp_pool_t, irps[irp->index]); } /** diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/rings.h index 3e4fa5deb..fb838ca11 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/rings.h @@ -84,6 +84,7 @@ typedef struct sqe { fd_t from; char* path; + size_t length; } open; uint64_t _args[SEQ_MAX_ARGS]; }; diff --git a/src/boot/main.c b/src/boot/main.c index a6299472e..6c00b1450 100644 --- a/src/boot/main.c +++ b/src/boot/main.c @@ -194,7 +194,7 @@ static EFI_STATUS gop_init(boot_gop_t* buffer) } #endif - buffer->physAddr = (uint32_t*)gop->Mode->FrameBufferBase; + buffer->physAddr = gop->Mode->FrameBufferBase; buffer->virtAddr = (uint32_t*)PML_LOWER_TO_HIGHER(gop->Mode->FrameBufferBase); buffer->size = gop->Mode->FrameBufferSize; buffer->width = gop->Mode->Info->HorizontalResolution; @@ -284,7 +284,7 @@ _NORETURN static void panic_halt(panic_code_t code) { for (size_t x = 0; x < gop->width; x++) { - gop->physAddr[x + (y * gop->stride)] = color; + ((uint32_t*)gop->physAddr)[x + (y * gop->stride)] = color; } } } @@ -404,18 +404,21 @@ static EFI_STATUS mem_allocator_init(void) /** * @brief Allocate pages from the basic allocator. * - * @param pages Output pointer to store the allocated pages. + * @param pfns Pointer to store allocated page PFNs. * @param amount Number of pages to allocate. * @return Always returns `0`. */ -static uint64_t basic_allocator_alloc_pages(void** pages, size_t amount) +static uint64_t basic_allocator_alloc_pages(pfn_t* pfns, size_t amount) { if (basicAllocator.pagesAllocated + amount > basicAllocator.maxPages) { panic_halt(PANIC_ALLOCATOR_EXHAUSTED); } - *pages = (void*)(uintptr_t)(basicAllocator.buffer + (basicAllocator.pagesAllocated * PAGE_SIZE)); + for (size_t i = 0; i < amount; i++) + { + pfns[i] = PHYS_TO_PFN(basicAllocator.buffer + ((basicAllocator.pagesAllocated + i) * PAGE_SIZE)); + } basicAllocator.pagesAllocated += amount; return 0; @@ -465,7 +468,7 @@ static void mem_page_table_init(page_table_t* table, boot_memory_map_t* map, boo panic_halt(PANIC_HIGHER_HALF_INVALID); } - if (page_table_map(table, (void*)desc->VirtualStart, (void*)desc->PhysicalStart, desc->NumberOfPages, + if (page_table_map(table, (void*)desc->VirtualStart, desc->PhysicalStart, desc->NumberOfPages, PML_WRITE | PML_PRESENT, PML_CALLBACK_NONE) == ERR) { panic_halt(PANIC_HIGHER_HALF_MAP); @@ -483,7 +486,7 @@ static void mem_page_table_init(page_table_t* table, boot_memory_map_t* map, boo panic_halt(PANIC_KERNEL_MAP); } - if (page_table_map(table, (void*)gop->virtAddr, (void*)gop->physAddr, BYTES_TO_PAGES(gop->size), + if (page_table_map(table, (void*)gop->virtAddr, gop->physAddr, BYTES_TO_PAGES(gop->size), PML_WRITE | PML_PRESENT | PML_WRITE_THROUGH, PML_CALLBACK_NONE) == ERR) { panic_halt(PANIC_GOP_MAP); @@ -876,7 +879,7 @@ static EFI_STATUS kernel_load(boot_kernel_t* kernel, EFI_FILE* rootHandle) Print(L" Loading kernel...\n"); elf64_load_segments(&kernel->elf, kernelPhys, minVaddr); - kernel->physAddr = (void*)kernelPhys; + kernel->physAddr = kernelPhys; Print(L" Entry Code: "); for (size_t i = 0; i < 16; i++) diff --git a/src/kernel/cpu/simd.c b/src/kernel/cpu/simd.c index ec80441c6..c31d9d243 100644 --- a/src/kernel/cpu/simd.c +++ b/src/kernel/cpu/simd.c @@ -122,12 +122,12 @@ PERCPU_DEFINE_CTOR(static void, pcpu_simd) uint64_t simd_ctx_init(simd_ctx_t* ctx) { - ctx->buffer = pmm_alloc(); - if (ctx->buffer == NULL) + pfn_t pfn = pmm_alloc(); + if (pfn == ERR) { return ERR; } - + ctx->buffer = PFN_TO_VIRT(pfn); memcpy(ctx->buffer, initCtx, PAGE_SIZE); return 0; @@ -135,7 +135,8 @@ uint64_t simd_ctx_init(simd_ctx_t* ctx) void simd_ctx_deinit(simd_ctx_t* ctx) { - pmm_free(ctx->buffer); + pfn_t pfn = VIRT_TO_PFN(ctx->buffer); + pmm_free(pfn); } void simd_ctx_save(simd_ctx_t* ctx) diff --git a/src/kernel/init/boot_info.c b/src/kernel/init/boot_info.c index 5c9daa7e8..9dad33d18 100644 --- a/src/kernel/init/boot_info.c +++ b/src/kernel/init/boot_info.c @@ -67,7 +67,7 @@ static void boot_dir_to_higher_half(boot_dir_t* dir) void boot_info_to_higher_half(void) { - bootInfo->gop.physAddr = (void*)PML_ENSURE_HIGHER_HALF(bootInfo->gop.physAddr); + bootInfo->gop.physAddr = PML_ENSURE_HIGHER_HALF(bootInfo->gop.physAddr); bootInfo->gop.virtAddr = (void*)PML_ENSURE_HIGHER_HALF(bootInfo->gop.virtAddr); bootInfo->rsdp = (void*)PML_ENSURE_HIGHER_HALF(bootInfo->rsdp); @@ -77,7 +77,7 @@ void boot_info_to_higher_half(void) bootInfo->disk.root = (boot_dir_t*)PML_ENSURE_HIGHER_HALF(bootInfo->disk.root); boot_dir_to_higher_half(bootInfo->disk.root); - bootInfo->kernel.physAddr = (void*)PML_ENSURE_HIGHER_HALF(bootInfo->kernel.physAddr); + bootInfo->kernel.physAddr = PML_ENSURE_HIGHER_HALF(bootInfo->kernel.physAddr); bootInfo->memory.map.descriptors = (EFI_MEMORY_DESCRIPTOR*)PML_ENSURE_HIGHER_HALF(bootInfo->memory.map.descriptors); @@ -109,7 +109,7 @@ void boot_info_free(void) // Clear the memory to deliberately cause corruption if the memory is actually being used. memset((void*)desc->VirtualStart, 0xCC, desc->NumberOfPages * PAGE_SIZE); #endif - pmm_free_region((void*)desc->VirtualStart, desc->NumberOfPages); + pmm_free_region(VIRT_TO_PFN(desc->VirtualStart), desc->NumberOfPages); } } diff --git a/src/kernel/mem/mdl.c b/src/kernel/mem/mdl.c deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/kernel/mem/mem_desc.c b/src/kernel/mem/mem_desc.c new file mode 100644 index 000000000..26eb7598e --- /dev/null +++ b/src/kernel/mem/mem_desc.c @@ -0,0 +1,58 @@ +#include + +#include +#include + +mem_desc_pool_t* mem_desc_pool_new(size_t size) +{ + size_t poolSize = sizeof(mem_desc_pool_t) + (size * sizeof(mem_desc_t)); + mem_desc_pool_t* pool = malloc(poolSize); + if (pool == NULL) + { + errno = ENOMEM; + return NULL; + } + + for (size_t i = 0; i < size; i++) + { + pool->descs[i].index = i; + } + pool_init(&pool->pool, pool->descs, size, sizeof(mem_desc_t), offsetof(mem_desc_t, next)); + return pool; +} + +void mem_desc_pool_free(mem_desc_pool_t* pool) +{ + free(pool); +} + +uint64_t mem_desc_add(mem_desc_t* desc, void* addr, size_t size) +{ + mem_seg_t* seg = NULL; + if (desc->amount < MEM_SEGS_SMALL_MAX) + { + seg = &desc->small[desc->amount++]; + } + else if (desc->amount - MEM_SEGS_SMALL_MAX < desc->capacity) + { + seg = &desc->large[desc->amount++ - MEM_SEGS_SMALL_MAX]; + } + else + { + mem_seg_t* newLarge = realloc( + desc->large, (desc->capacity + 4) * sizeof(mem_seg_t)); + if (newLarge == NULL) + { + return ERR; + } + + desc->large = newLarge; + desc->capacity += 4; + seg = &desc->large[desc->amount++ - MEM_SEGS_SMALL_MAX]; + } + + seg->page = (void*)ROUND_DOWN(addr, PAGE_SIZE); + seg->offset = (uint32_t)((uintptr_t)addr - (uintptr_t)seg->page); + seg->length = (uint32_t)size; + return 0; +} \ No newline at end of file diff --git a/src/kernel/mem/pmm.c b/src/kernel/mem/pmm.c index fe1684063..4f358f121 100644 --- a/src/kernel/mem/pmm.c +++ b/src/kernel/mem/pmm.c @@ -35,14 +35,14 @@ static const char* efiMemTypeToString[] = { "persistent", }; -static pmm_ref_t* refs = NULL; +static page_ref_t* refs = NULL; static page_stack_t* stack = NULL; static size_t location = FREE_PAGE_MAX; BITMAP_CREATE_ONE(bitmap, CONFIG_PMM_BITMAP_MAX_ADDR / PAGE_SIZE); -static uintptr_t highest = 0; +static pfn_t highest = 0; static size_t total = 0; static size_t avail = 0; @@ -68,14 +68,14 @@ static bool pmm_is_mem_avail(EFI_MEMORY_TYPE type) } } -static inline pmm_ref_t* pmm_ref_get(void* address) +static inline page_ref_t* pmm_ref_get(pfn_t pfn) { - return &refs[PML_HIGHER_TO_LOWER(address) / PAGE_SIZE]; + return &refs[pfn]; } -static inline void pmm_ref_set(void* address, size_t count, pmm_ref_t value) +static inline void pmm_ref_set(pfn_t pfn, size_t count, page_ref_t value) { - pmm_ref_t* ref = pmm_ref_get(address); + page_ref_t* ref = pmm_ref_get(pfn); for (size_t i = 0; i < count; i++) { ref[i] = value; @@ -84,14 +84,14 @@ static inline void pmm_ref_set(void* address, size_t count, pmm_ref_t value) static inline size_t pmm_refs_size(void) { - return (PML_HIGHER_TO_LOWER(highest) / PAGE_SIZE) * sizeof(pmm_ref_t); + return highest * sizeof(page_ref_t); } -static inline void pmm_stack_push(void* addr) +static inline void pmm_stack_push(pfn_t pfn) { if (stack == NULL || location == 0) { - page_stack_t* page = addr; + page_stack_t* page = PFN_TO_VIRT(pfn); page->next = stack; stack = page; location = FREE_PAGE_MAX; @@ -99,52 +99,49 @@ static inline void pmm_stack_push(void* addr) return; } - stack->pages[--location] = addr; + stack->pages[--location] = pfn; } -static inline void* pmm_stack_pop(void) +static inline pfn_t pmm_stack_pop(void) { if (location == FREE_PAGE_MAX) { if (stack == NULL) { - return NULL; + return ERR; } - void* addr = stack; + page_stack_t* page = stack; stack = stack->next; location = (stack == NULL) ? FREE_PAGE_MAX : 0; - return addr; + return VIRT_TO_PFN(page); } return stack->pages[location++]; } -static inline void* pmm_bitmap_set(size_t count, uintptr_t maxAddr, size_t alignment) +static inline pfn_t pmm_bitmap_set(size_t count, pfn_t maxPfn, pfn_t alignPfn) { - alignment = MAX(ROUND_UP(alignment, PAGE_SIZE), PAGE_SIZE); - maxAddr = MIN(maxAddr, CONFIG_PMM_BITMAP_MAX_ADDR); + alignPfn = MAX(alignPfn, 1); + maxPfn = MIN(maxPfn, CONFIG_PMM_BITMAP_MAX_ADDR / PAGE_SIZE); - size_t index = bitmap_find_clear_region_and_set(&bitmap, 0, maxAddr / PAGE_SIZE, count, alignment / PAGE_SIZE); + size_t index = bitmap_find_clear_region_and_set(&bitmap, 0, maxPfn, count, alignPfn); if (index == bitmap.length) { - return NULL; + return ERR; } - return (void*)PML_LOWER_TO_HIGHER(index * PAGE_SIZE); + return (pfn_t)index; } -static inline void pmm_bitmap_clear(void* addr, size_t pageAmount) +static inline void pmm_bitmap_clear(pfn_t pfn, size_t pageAmount) { - addr = (void*)ROUND_DOWN(addr, PAGE_SIZE); - - size_t index = PML_HIGHER_TO_LOWER(addr) / PAGE_SIZE; - bitmap_clear_range(&bitmap, index, pageAmount); + bitmap_clear_range(&bitmap, pfn, pfn + pageAmount); } -static void pmm_free_unlocked(void* address) +static void pmm_free_unlocked(pfn_t pfn) { - pmm_ref_t* ref = pmm_ref_get(address); + page_ref_t* ref = pmm_ref_get(pfn); assert(*ref > 0); (*ref)--; if (*ref > 0) @@ -152,27 +149,23 @@ static void pmm_free_unlocked(void* address) return; } - if (address >= (void*)PML_LOWER_TO_HIGHER(CONFIG_PMM_BITMAP_MAX_ADDR)) - { - pmm_stack_push(address); - } - else if (address >= (void*)PML_LOWER_TO_HIGHER(0)) + if (pfn >= CONFIG_PMM_BITMAP_MAX_ADDR / PAGE_SIZE) { - pmm_bitmap_clear(address, 1); + pmm_stack_push(pfn); } else { - panic(NULL, "Attempt to free lower half address %p", address); + pmm_bitmap_clear(pfn, 1); } avail++; } -static void pmm_free_region_unlocked(void* address, size_t count) +static void pmm_free_region_unlocked(pfn_t pfn, size_t count) { for (size_t i = 0; i < count; i++) { - pmm_free_unlocked((void*)((uintptr_t)address + (i * PAGE_SIZE))); + pmm_free_unlocked(pfn + i); } } @@ -188,11 +181,12 @@ static void pmm_detect_memory(const boot_memory_map_t* map) { total += desc->NumberOfPages; } - highest = MAX(highest, (uintptr_t)desc->VirtualStart + (desc->NumberOfPages * PAGE_SIZE)); + + pfn_t endPfn = VIRT_TO_PFN(desc->VirtualStart) + desc->NumberOfPages; + highest = MAX(highest, endPfn); } LOG_INFO("page amount %llu\n", total); - LOG_INFO("highest address %p\n", highest); } static void pmm_init_refs(const boot_memory_map_t* map) @@ -204,7 +198,7 @@ static void pmm_init_refs(const boot_memory_map_t* map) const EFI_MEMORY_DESCRIPTOR* desc = BOOT_MEMORY_MAP_GET_DESCRIPTOR(map, i); if (desc->Type == EfiConventionalMemory && desc->NumberOfPages >= pages) { - refs = (pmm_ref_t*)desc->VirtualStart; + refs = (page_ref_t*)desc->VirtualStart; memset(refs, -1, pages * PAGE_SIZE); LOG_INFO("pmm ref [%p-%p]\n", refs, (uintptr_t)refs + pages * PAGE_SIZE); return; @@ -216,18 +210,21 @@ static void pmm_init_refs(const boot_memory_map_t* map) static void pmm_load_memory(const boot_memory_map_t* map) { + pfn_t refsPfn = VIRT_TO_PFN(refs); + size_t refPages = BYTES_TO_PAGES(pmm_refs_size()); + for (size_t i = 0; i < map->length; i++) { const EFI_MEMORY_DESCRIPTOR* desc = BOOT_MEMORY_MAP_GET_DESCRIPTOR(map, i); - uintptr_t address = desc->VirtualStart; + pfn_t pfn = VIRT_TO_PFN(desc->VirtualStart); size_t pages = desc->NumberOfPages; - if (address == (uintptr_t)refs) + + if (pfn == refsPfn) { // Skip the refs array. - size_t refPages = BYTES_TO_PAGES(pmm_refs_size()); assert(pages >= refPages); - address += refPages * PAGE_SIZE; + pfn += refPages; pages -= refPages; } @@ -235,14 +232,14 @@ static void pmm_load_memory(const boot_memory_map_t* map) { #ifndef NDEBUG // Clear the memory to deliberately cause corruption if the memory is actually being used. - memset((void*)address, 0xCC, pages * PAGE_SIZE); + memset(PFN_TO_VIRT(pfn), 0xCC, pages * PAGE_SIZE); #endif - pmm_ref_set((void*)address, pages, 1); - pmm_free_region_unlocked((void*)address, pages); + pmm_ref_set(pfn, pages, 1); + pmm_free_region_unlocked(pfn, pages); } else { - LOG_INFO("reserve [%p-%p] pages=%d type=%s\n", address, (uintptr_t)address + (pages * PAGE_SIZE), pages, + LOG_INFO("reserve [%p-%p] pages=%d type=%s\n", PFN_TO_VIRT(pfn), PFN_TO_VIRT(pfn + pages), pages, efiMemTypeToString[desc->Type]); } } @@ -261,55 +258,55 @@ void pmm_init(void) pmm_load_memory(map); } -void* pmm_alloc(void) +pfn_t pmm_alloc(void) { lock_acquire(&lock); - void* addr = pmm_stack_pop(); - if (addr == NULL) + pfn_t pfn = pmm_stack_pop(); + if (pfn == ERR) { - addr = pmm_bitmap_set(1, CONFIG_PMM_BITMAP_MAX_ADDR, PAGE_SIZE); + pfn = pmm_bitmap_set(1, CONFIG_PMM_BITMAP_MAX_ADDR / PAGE_SIZE, 1); } - if (addr != NULL) + if (pfn != ERR) { - pmm_ref_t* ref = pmm_ref_get(addr); + page_ref_t* ref = pmm_ref_get(pfn); assert(*ref == 0); *ref = 1; avail--; } lock_release(&lock); - if (addr == NULL) + if (pfn == ERR) { LOG_WARN("out of memory in pmm_alloc()\n"); } - return addr; + return pfn; } -uint64_t pmm_alloc_pages(void** addresses, size_t count) +uint64_t pmm_alloc_pages(pfn_t* pfns, size_t count) { lock_acquire(&lock); for (size_t i = 0; i < count; i++) { - addresses[i] = pmm_stack_pop(); - if (addresses[i] == NULL) + pfns[i] = pmm_stack_pop(); + if (pfns[i] == ERR) { - addresses[i] = pmm_bitmap_set(1, CONFIG_PMM_BITMAP_MAX_ADDR, PAGE_SIZE); + pfns[i] = pmm_bitmap_set(1, CONFIG_PMM_BITMAP_MAX_ADDR / PAGE_SIZE, 1); } - if (addresses[i] == NULL) + if (pfns[i] == ERR) { LOG_WARN("out of memory in pmm_alloc_pages()\n"); for (size_t j = 0; j < i; j++) { - if (addresses[j] >= (void*)PML_LOWER_TO_HIGHER(CONFIG_PMM_BITMAP_MAX_ADDR)) + if (pfns[j] >= CONFIG_PMM_BITMAP_MAX_ADDR / PAGE_SIZE) { - pmm_stack_push(addresses[j]); + pmm_stack_push(pfns[j]); } else { - pmm_bitmap_clear(addresses[j], 1); + pmm_bitmap_clear(pfns[j], 1); } } lock_release(&lock); @@ -319,7 +316,7 @@ uint64_t pmm_alloc_pages(void** addresses, size_t count) for (size_t i = 0; i < count; i++) { - pmm_ref_t* ref = pmm_ref_get(addresses[i]); + page_ref_t* ref = pmm_ref_get(pfns[i]); assert(*ref == 0); *ref = 1; } @@ -329,15 +326,15 @@ uint64_t pmm_alloc_pages(void** addresses, size_t count) return 0; } -void* pmm_alloc_bitmap(size_t count, uintptr_t maxAddr, uint64_t alignment) +pfn_t pmm_alloc_bitmap(size_t count, pfn_t maxPfn, pfn_t alignPfn) { lock_acquire(&lock); - void* addr = pmm_bitmap_set(count, maxAddr, alignment); - if (addr != NULL) + pfn_t pfn = pmm_bitmap_set(count, maxPfn, alignPfn); + if (pfn != ERR) { for (size_t i = 0; i < count; i++) { - pmm_ref_t* ref = pmm_ref_get((void*)((uintptr_t)addr + (i * PAGE_SIZE))); + page_ref_t* ref = pmm_ref_get(pfn + i); assert(*ref == 0); *ref = 1; } @@ -345,42 +342,42 @@ void* pmm_alloc_bitmap(size_t count, uintptr_t maxAddr, uint64_t alignment) } lock_release(&lock); - if (addr == NULL) + if (pfn == ERR) { LOG_WARN("out of memory in pmm_alloc_bitmap()\n"); } - return addr; + return pfn; } -void pmm_free(void* address) +void pmm_free(pfn_t pfn) { lock_acquire(&lock); - pmm_free_unlocked(address); + pmm_free_unlocked(pfn); lock_release(&lock); } -void pmm_free_pages(void** addresses, size_t count) +void pmm_free_pages(pfn_t* pfns, size_t count) { lock_acquire(&lock); for (size_t i = 0; i < count; i++) { - pmm_free_unlocked(addresses[i]); + pmm_free_unlocked(pfns[i]); } lock_release(&lock); } -void pmm_free_region(void* address, size_t count) +void pmm_free_region(pfn_t pfn, size_t count) { lock_acquire(&lock); - pmm_free_region_unlocked(address, count); + pmm_free_region_unlocked(pfn, count); lock_release(&lock); } -uint64_t pmm_ref_inc(void* address) +uint64_t pmm_ref_inc(pfn_t pfn) { lock_acquire(&lock); - pmm_ref_t* ref = pmm_ref_get(address); + page_ref_t* ref = pmm_ref_get(pfn); assert(*ref != PAGE_REF_MAX); (*ref)++; uint64_t ret = *ref; diff --git a/src/kernel/mem/space.c b/src/kernel/mem/space.c index 652c05c36..cc68367f6 100644 --- a/src/kernel/mem/space.c +++ b/src/kernel/mem/space.c @@ -17,20 +17,20 @@ #include #include -static uint64_t space_pmm_bitmap_alloc_pages(void** pages, size_t pageAmount) +static uint64_t space_pmm_bitmap_alloc_pages(pfn_t* pfns, size_t pageAmount) { for (size_t i = 0; i < pageAmount; i++) { - void* page = pmm_alloc_bitmap(1, UINT32_MAX, 0); - if (page == NULL) + pfn_t pfn = pmm_alloc_bitmap(1, UINT32_MAX, 0); + if (pfn == ERR) { for (size_t j = 0; j < i; j++) { - pmm_free(pages[j]); + pmm_free(pfns[j]); } return ERR; } - pages[i] = page; + pfns[i] = pfn; } return 0; } @@ -172,16 +172,16 @@ static uint64_t space_populate_user_region(space_t* space, const void* buffer, s continue; } - void* page = pmm_alloc(); - if (page == NULL) + pfn_t pfn = pmm_alloc(); + if (pfn == ERR) { return ERR; } - if (page_table_map(&space->pageTable, (void*)addr, page, 1, PML_PRESENT | PML_USER | PML_WRITE | PML_OWNED, + if (page_table_map(&space->pageTable, (void*)addr, PFN_TO_PHYS(pfn), 1, PML_PRESENT | PML_USER | PML_WRITE | PML_OWNED, PML_CALLBACK_NONE) == ERR) { - pmm_free(page); + pmm_free(pfn); return ERR; } } @@ -189,14 +189,14 @@ static uint64_t space_populate_user_region(space_t* space, const void* buffer, s return 0; } -static void space_pin_depth_dec(space_t* space, const void* address, uint64_t pageAmount) +static void space_pin_depth_dec(space_t* space, const void* address, uint64_t amount) { address = (void*)ROUND_DOWN((uintptr_t)address, PAGE_SIZE); page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - for (uint64_t i = 0; i < pageAmount; i++) + for (uint64_t i = 0; i < amount; i++) { - uintptr_t addr = (uintptr_t)address + (i * PAGE_SIZE); + const void* addr = address + (i * PAGE_SIZE); if (page_table_traverse(&space->pageTable, &traverse, addr, PML_NONE) == ERR) { continue; @@ -207,7 +207,7 @@ static void space_pin_depth_dec(space_t* space, const void* address, uint64_t pa continue; } - map_key_t key = map_key_uint64(addr); + map_key_t key = map_key_uint64((uintptr_t)addr); map_entry_t* entry = map_get(&space->pinnedPages, &key); if (entry == NULL) // Not pinned more then once { @@ -226,14 +226,14 @@ static void space_pin_depth_dec(space_t* space, const void* address, uint64_t pa } } -static inline uint64_t space_pin_depth_inc(space_t* space, const void* address, uint64_t pageAmount) +static inline uint64_t space_pin_depth_inc(space_t* space, const void* address, uint64_t amount) { address = (void*)ROUND_DOWN((uintptr_t)address, PAGE_SIZE); page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; - for (uint64_t i = 0; i < pageAmount; i++) + for (uint64_t i = 0; i < amount; i++) { - uintptr_t addr = (uintptr_t)address + (i * PAGE_SIZE); + const void* addr = address + (i * PAGE_SIZE); if (page_table_traverse(&space->pageTable, &traverse, addr, PML_NONE) == ERR) { continue; @@ -250,7 +250,7 @@ static inline uint64_t space_pin_depth_inc(space_t* space, const void* address, continue; } - map_key_t key = map_key_uint64(addr); + map_key_t key = map_key_uint64((uintptr_t)addr); map_entry_t* entry = map_get(&space->pinnedPages, &key); if (entry != NULL) // Already pinned more than once { @@ -491,7 +491,7 @@ static void* space_find_free_region(space_t* space, uint64_t pageAmount, uint64_ return NULL; } -uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* virtAddr, void* physAddr, size_t length, +uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* virtAddr, phys_addr_t physAddr, size_t length, size_t alignment, pml_flags_t flags) { if (space == NULL || mapping == NULL || length == 0) @@ -508,7 +508,7 @@ uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* vir } uintptr_t physOverflow = (uintptr_t)physAddr + length; - if (physAddr != NULL && physOverflow < (uintptr_t)physAddr) + if (physAddr != PHYS_ADDR_INVALID && physOverflow < (uintptr_t)physAddr) { errno = EOVERFLOW; return ERR; @@ -554,13 +554,13 @@ uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* vir } mapping->virtAddr = virtAddr; - if (physAddr != NULL) + if (physAddr != PHYS_ADDR_INVALID) { - mapping->physAddr = (void*)PML_ENSURE_LOWER_HALF(ROUND_DOWN(physAddr, PAGE_SIZE)); + mapping->physAddr = PML_ENSURE_LOWER_HALF(ROUND_DOWN(physAddr, PAGE_SIZE)); } else { - mapping->physAddr = NULL; + mapping->physAddr = PHYS_ADDR_INVALID; } mapping->flags = flags; diff --git a/src/kernel/mem/vmm.c b/src/kernel/mem/vmm.c index a4b4b62ad..d3fc7f1d2 100644 --- a/src/kernel/mem/vmm.c +++ b/src/kernel/mem/vmm.c @@ -83,7 +83,7 @@ void vmm_init(void) panic(NULL, "Memory descriptor %d has invalid physical address %p", i, desc->PhysicalStart); } - if (page_table_map(&kernelSpace.pageTable, (void*)desc->VirtualStart, (void*)desc->PhysicalStart, + if (page_table_map(&kernelSpace.pageTable, (void*)desc->VirtualStart, desc->PhysicalStart, desc->NumberOfPages, PML_WRITE | PML_GLOBAL | PML_PRESENT, PML_CALLBACK_NONE) == ERR) { panic(NULL, "Failed to map memory descriptor %d (phys=%p-%p virt=%p)", i, desc->PhysicalStart, @@ -106,7 +106,7 @@ void vmm_init(void) LOG_INFO("GOP virt=[%p-%p] phys=[%p-%p]\n", gop->virtAddr, gop->virtAddr + gop->size, gop->physAddr, gop->physAddr + gop->size); - if (page_table_map(&kernelSpace.pageTable, (void*)gop->virtAddr, (void*)gop->physAddr, BYTES_TO_PAGES(gop->size), + if (page_table_map(&kernelSpace.pageTable, (void*)gop->virtAddr, gop->physAddr, BYTES_TO_PAGES(gop->size), PML_WRITE | PML_GLOBAL | PML_PRESENT, PML_CALLBACK_NONE) == ERR) { panic(NULL, "Failed to map GOP memory"); @@ -165,7 +165,7 @@ void* vmm_alloc(space_t* space, void* virtAddr, size_t length, size_t alignment, } space_mapping_t mapping; - if (space_mapping_start(space, &mapping, virtAddr, NULL, length, alignment, pmlFlags | PML_OWNED) == ERR) + if (space_mapping_start(space, &mapping, virtAddr, PHYS_ADDR_INVALID, length, alignment, pmlFlags | PML_OWNED) == ERR) { return NULL; } @@ -189,11 +189,11 @@ void* vmm_alloc(space_t* space, void* virtAddr, size_t length, size_t alignment, uint64_t remainingPages = mapping.pageAmount; while (remainingPages != 0) { - uintptr_t currentVirtAddr = (uintptr_t)mapping.virtAddr + (mapping.pageAmount - remainingPages) * PAGE_SIZE; + void* currentVirtAddr = mapping.virtAddr + (mapping.pageAmount - remainingPages) * PAGE_SIZE; - void* addresses[maxBatchSize]; + pfn_t pages[maxBatchSize]; uint64_t batchSize = MIN(remainingPages, maxBatchSize); - if (pmm_alloc_pages(addresses, batchSize) == ERR) + if (pmm_alloc_pages(pages, batchSize) == ERR) { // Page table will free the previously allocated pages as they are owned by the Page table. vmm_page_table_unmap_with_shootdown(space, mapping.virtAddr, mapping.pageAmount - remainingPages); @@ -204,11 +204,11 @@ void* vmm_alloc(space_t* space, void* virtAddr, size_t length, size_t alignment, { for (uint64_t i = 0; i < batchSize; i++) { - memset(addresses[i], 0, PAGE_SIZE); + memset(PFN_TO_VIRT(pages[i]), 0, PAGE_SIZE); } } - if (page_table_map_pages(&space->pageTable, (void*)currentVirtAddr, addresses, batchSize, mapping.flags, + if (page_table_map_pages(&space->pageTable, currentVirtAddr, pages, batchSize, mapping.flags, PML_CALLBACK_NONE) == ERR) { // Page table will free the previously allocated pages as they are owned by the Page table. @@ -222,10 +222,10 @@ void* vmm_alloc(space_t* space, void* virtAddr, size_t length, size_t alignment, return space_mapping_end(space, &mapping, EOK); } -void* vmm_map(space_t* space, void* virtAddr, void* physAddr, size_t length, pml_flags_t flags, +void* vmm_map(space_t* space, void* virtAddr, phys_addr_t physAddr, size_t length, pml_flags_t flags, space_callback_func_t func, void* data) { - if (physAddr == NULL || length == 0 || !(flags & PML_PRESENT)) + if (physAddr == PHYS_ADDR_INVALID || length == 0 || !(flags & PML_PRESENT)) { errno = EINVAL; return NULL; @@ -277,10 +277,10 @@ void* vmm_map(space_t* space, void* virtAddr, void* physAddr, size_t length, pml return space_mapping_end(space, &mapping, EOK); } -void* vmm_map_pages(space_t* space, void* virtAddr, void** pages, size_t pageAmount, pml_flags_t flags, +void* vmm_map_pages(space_t* space, void* virtAddr, pfn_t* pfns, size_t amount, pml_flags_t flags, space_callback_func_t func, void* data) { - if (pages == NULL || pageAmount == 0 || !(flags & PML_PRESENT)) + if (pfns == NULL || amount == 0 || !(flags & PML_PRESENT)) { errno = EINVAL; return NULL; @@ -292,7 +292,7 @@ void* vmm_map_pages(space_t* space, void* virtAddr, void** pages, size_t pageAmo } space_mapping_t mapping; - if (space_mapping_start(space, &mapping, virtAddr, NULL, pageAmount * PAGE_SIZE, 1, flags) == ERR) + if (space_mapping_start(space, &mapping, virtAddr, PHYS_ADDR_INVALID, amount * PAGE_SIZE, 1, flags) == ERR) { return NULL; } @@ -305,7 +305,7 @@ void* vmm_map_pages(space_t* space, void* virtAddr, void** pages, size_t pageAmo pml_callback_id_t callbackId = PML_CALLBACK_NONE; if (func != NULL) { - callbackId = space_alloc_callback(space, pageAmount, func, data); + callbackId = space_alloc_callback(space, amount, func, data); if (callbackId == PML_MAX_CALLBACK) { return space_mapping_end(space, &mapping, ENOSPC); @@ -317,7 +317,7 @@ void* vmm_map_pages(space_t* space, void* virtAddr, void** pages, size_t pageAmo vmm_page_table_unmap_with_shootdown(space, mapping.virtAddr, mapping.pageAmount); } - if (page_table_map_pages(&space->pageTable, mapping.virtAddr, pages, mapping.pageAmount, mapping.flags, + if (page_table_map_pages(&space->pageTable, mapping.virtAddr, pfns, mapping.pageAmount, mapping.flags, callbackId) == ERR) { if (callbackId != PML_CALLBACK_NONE) @@ -346,7 +346,7 @@ void* vmm_unmap(space_t* space, void* virtAddr, size_t length) } space_mapping_t mapping; - if (space_mapping_start(space, &mapping, virtAddr, NULL, length, 1, PML_NONE) == ERR) + if (space_mapping_start(space, &mapping, virtAddr, PHYS_ADDR_INVALID, length, 1, PML_NONE) == ERR) { return NULL; } @@ -413,7 +413,7 @@ void* vmm_protect(space_t* space, void* virtAddr, size_t length, pml_flags_t fla } space_mapping_t mapping; - if (space_mapping_start(space, &mapping, virtAddr, NULL, length, 1, flags) == ERR) + if (space_mapping_start(space, &mapping, virtAddr, PHYS_ADDR_INVALID, length, 1, flags) == ERR) { return NULL; } diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index 490cc49fa..708a8bed6 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -52,7 +53,7 @@ static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, ri return ERR; } - void* pages[CONFIG_MAX_ASYNC_PAGES]; + pfn_t pages[CONFIG_MAX_ASYNC_PAGES]; if (pmm_alloc_pages(pages, pageAmount) == ERR) { errno = ENOMEM; @@ -61,7 +62,7 @@ static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, ri for (size_t i = 0; i < pageAmount; i++) { - memset(pages[i], 0, PAGE_SIZE); + memset(PFN_TO_VIRT(pages[i]), 0, PAGE_SIZE); } // PML_OWNED means that the pages will be freed when unmapped. @@ -87,6 +88,16 @@ static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, ri return ERR; } + mem_desc_pool_t* descs = mem_desc_pool_new(centries); + if (descs == NULL) + { + irp_pool_free(irps); + mem_desc_pool_free(descs); + vmm_unmap(space, userAddr, pageAmount * PAGE_SIZE); + vmm_unmap(NULL, kernelAddr, pageAmount * PAGE_SIZE); + return ERR; + } + rings_shared_t* shared = (rings_shared_t*)kernelAddr; atomic_init(&shared->shead, 0); atomic_init(&shared->stail, 0); @@ -116,6 +127,7 @@ static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, ri kernelRings->cmask = centries - 1; ctx->irps = irps; + ctx->descs = descs; ctx->userAddr = userAddr; ctx->kernelAddr = kernelAddr; ctx->pageAmount = pageAmount; @@ -130,6 +142,9 @@ static inline uint64_t async_unmap(async_t* ctx) irp_pool_free(ctx->irps); ctx->irps = NULL; + mem_desc_pool_free(ctx->descs); + ctx->descs = NULL; + vmm_unmap(ctx->space, ctx->userAddr, ctx->pageAmount * PAGE_SIZE); vmm_unmap(NULL, ctx->kernelAddr, ctx->pageAmount * PAGE_SIZE); @@ -269,7 +284,7 @@ static void async_dispatch(irp_t* irp) { case VERB_NOP: break; - case VERB_OPEN: + /*case VERB_OPEN: { file_t* from = NULL; if (irp->sqe.open.from != FD_NONE) @@ -283,15 +298,9 @@ static void async_dispatch(irp_t* irp) } irp->open.from = from; - irp->open.path = malloc(sizeof(pathname_t)); - if (irp->open.path == NULL) - { - UNREF(from); - irp->err = ENOMEM; - break; - } + irp->open.path = } - break; + break;*/ default: break; } diff --git a/src/modules/acpi/acpi.c b/src/modules/acpi/acpi.c index 29e135a4f..27dc4f7bc 100644 --- a/src/modules/acpi/acpi.c +++ b/src/modules/acpi/acpi.c @@ -69,7 +69,7 @@ void acpi_reclaim_memory(const boot_memory_map_t* map) if (desc->Type == EfiACPIReclaimMemory) { - pmm_free_region((void*)PML_LOWER_TO_HIGHER(desc->PhysicalStart), desc->NumberOfPages); + pmm_free_region(PHYS_TO_PFN(desc->PhysicalStart), desc->NumberOfPages); LOG_INFO("reclaim memory [%p-%p]\n", desc->PhysicalStart, ((uintptr_t)desc->PhysicalStart) + desc->NumberOfPages * PAGE_SIZE); } diff --git a/src/modules/acpi/aml/runtime/field_unit.c b/src/modules/acpi/aml/runtime/field_unit.c index 3e3fd8587..9ca0b9587 100644 --- a/src/modules/acpi/aml/runtime/field_unit.c +++ b/src/modules/acpi/aml/runtime/field_unit.c @@ -1,3 +1,4 @@ +#include #include #include @@ -31,7 +32,7 @@ static void* aml_ensure_mem_is_mapped(uint64_t address, aml_bit_size_t accessSiz for (uint64_t page = 0; page < (crossesBoundary ? 2 : 1); page++) { - void* physAddr = (void*)((uintptr_t)address + page * PAGE_SIZE); + phys_addr_t physAddr = (phys_addr_t)address + page * PAGE_SIZE; void* virtAddt = (void*)PML_LOWER_TO_HIGHER(physAddr); if (vmm_map(NULL, virtAddt, physAddr, PAGE_SIZE, PML_GLOBAL | PML_WRITE | PML_PRESENT, NULL, NULL) == NULL) { diff --git a/src/modules/drivers/apic/ioapic.c b/src/modules/drivers/apic/ioapic.c index 6df70f682..4851f57aa 100644 --- a/src/modules/drivers/apic/ioapic.c +++ b/src/modules/drivers/apic/ioapic.c @@ -106,9 +106,8 @@ uint64_t ioapic_all_init(void) continue; } - void* physAddr = (void*)(uint64_t)ioapic->ioApicAddress; - void* virtAddr = (void*)PML_LOWER_TO_HIGHER(physAddr); - if (vmm_map(NULL, virtAddr, physAddr, PAGE_SIZE, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == NULL) + void* virtAddr = (void*)PML_LOWER_TO_HIGHER(ioapic->ioApicAddress); + if (vmm_map(NULL, virtAddr, ioapic->ioApicAddress, PAGE_SIZE, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == NULL) { LOG_ERR("failed to map io apic\n"); return ERR; diff --git a/src/modules/drivers/apic/lapic.c b/src/modules/drivers/apic/lapic.c index d0648678b..b59393458 100644 --- a/src/modules/drivers/apic/lapic.c +++ b/src/modules/drivers/apic/lapic.c @@ -99,7 +99,7 @@ uint64_t lapic_global_init(void) } lapicBase = PML_LOWER_TO_HIGHER(madt->localInterruptControllerAddress); - if (vmm_map(NULL, (void*)lapicBase, (void*)(uintptr_t)madt->localInterruptControllerAddress, PAGE_SIZE, + if (vmm_map(NULL, (void*)lapicBase, madt->localInterruptControllerAddress, PAGE_SIZE, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == NULL) { LOG_ERR("failed to map local apic\n"); diff --git a/src/modules/drivers/gop/gop.c b/src/modules/drivers/gop/gop.c index a090c6919..7614a7a36 100644 --- a/src/modules/drivers/gop/gop.c +++ b/src/modules/drivers/gop/gop.c @@ -67,14 +67,14 @@ static void* gop_mmap(fb_t* fb, void* addr, size_t length, size_t* offset, pml_f process_t* process = process_current(); uintptr_t physAddr = (uintptr_t)gop.physAddr + *offset; - uintptr_t endAddr = physAddr + length; - if (endAddr > (uintptr_t)gop.physAddr + (gop.stride * gop.height * sizeof(uint32_t))) + phys_addr_t endAddr = physAddr + length; + if (endAddr > gop.physAddr + (gop.stride * gop.height * sizeof(uint32_t))) { errno = EINVAL; return NULL; } - return vmm_map(&process->space, addr, (void*)physAddr, length, flags, NULL, NULL); + return vmm_map(&process->space, addr, physAddr, length, flags, NULL, NULL); } static fb_ops_t ops = { diff --git a/src/modules/drivers/hpet/hpet.c b/src/modules/drivers/hpet/hpet.c index 2d66a8c57..46c0e0d00 100644 --- a/src/modules/drivers/hpet/hpet.c +++ b/src/modules/drivers/hpet/hpet.c @@ -218,7 +218,7 @@ static uint64_t hpet_init(void) } address = (uintptr_t)PML_LOWER_TO_HIGHER(hpet->address); - if (vmm_map(NULL, (void*)address, (void*)hpet->address, PAGE_SIZE, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, + if (vmm_map(NULL, (void*)address, hpet->address, PAGE_SIZE, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == NULL) { LOG_ERR("failed to map HPET memory at %p\n", hpet->address); diff --git a/src/modules/drivers/pci/config.c b/src/modules/drivers/pci/config.c index 64df4c2a7..9f3fd8aa9 100644 --- a/src/modules/drivers/pci/config.c +++ b/src/modules/drivers/pci/config.c @@ -43,7 +43,7 @@ static uint64_t pci_config_init(void) uint64_t length = busCount * 256 * 4096; void* virtAddr = (void*)PML_LOWER_TO_HIGHER(entry->base); - if (vmm_map(NULL, virtAddr, (void*)entry->base, length, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == + if (vmm_map(NULL, virtAddr, entry->base, length, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == NULL) { LOG_ERR("failed to map PCI-e configuration space at %p\n", entry->base); diff --git a/src/modules/ipc/pipe/pipe.c b/src/modules/ipc/pipe/pipe.c index b8a635d73..c62c06730 100644 --- a/src/modules/ipc/pipe/pipe.c +++ b/src/modules/ipc/pipe/pipe.c @@ -59,7 +59,7 @@ static uint64_t pipe_open(file_t* file) { return ERR; } - data->buffer = pmm_alloc(); + data->buffer = malloc(PAGE_SIZE); if (data->buffer == NULL) { free(data); @@ -84,7 +84,7 @@ static uint64_t pipe_open2(file_t* files[2]) { return ERR; } - data->buffer = pmm_alloc(); + data->buffer = malloc(PAGE_SIZE); if (data->buffer == NULL) { free(data); @@ -122,7 +122,7 @@ static void pipe_close(file_t* file) { lock_release(&data->lock); wait_queue_deinit(&data->waitQueue); - pmm_free(data->buffer); + free(data->buffer); free(data); return; } diff --git a/src/modules/ipc/shmem/shmem.c b/src/modules/ipc/shmem/shmem.c index a89bda5f3..4dcb6a03e 100644 --- a/src/modules/ipc/shmem/shmem.c +++ b/src/modules/ipc/shmem/shmem.c @@ -44,7 +44,7 @@ typedef struct { ref_t ref; uint64_t pageAmount; - void** pages; + pfn_t* pages; lock_t lock; } shmem_object_t; @@ -100,7 +100,7 @@ static void shmem_vmm_callback(void* data) static void* shmem_object_allocate_pages(shmem_object_t* shmem, uint64_t pageAmount, space_t* space, void* address, pml_flags_t flags) { - shmem->pages = malloc(sizeof(void*) * pageAmount); + shmem->pages = malloc(sizeof(pfn_t) * pageAmount); if (shmem->pages == NULL) { return NULL; @@ -110,7 +110,7 @@ static void* shmem_object_allocate_pages(shmem_object_t* shmem, uint64_t pageAmo for (uint64_t i = 0; i < pageAmount; i++) { shmem->pages[i] = pmm_alloc(); - if (shmem->pages[i] == NULL) + if (shmem->pages[i] == ERR) { for (uint64_t j = 0; j < i; j++) { @@ -197,27 +197,25 @@ static void* shmem_mmap(file_t* file, void* address, size_t length, size_t* offs assert(shmem->pages == NULL); return shmem_object_allocate_pages(shmem, pageAmount, space, address, flags); } - else - { - assert(shmem->pages != NULL); - if (*offset >= shmem->pageAmount * PAGE_SIZE) - { - errno = EINVAL; - return NULL; - } + assert(shmem->pages != NULL); - if (*offset % PAGE_SIZE != 0) - { - errno = EINVAL; - return NULL; - } + if (*offset >= shmem->pageAmount * PAGE_SIZE) + { + errno = EINVAL; + return NULL; + } - uint64_t pageOffset = *offset / PAGE_SIZE; - uint64_t availablePages = shmem->pageAmount - pageOffset; - return vmm_map_pages(space, address, &shmem->pages[pageOffset], MIN(pageAmount, availablePages), flags, - shmem_vmm_callback, REF(shmem)); + if (*offset % PAGE_SIZE != 0) + { + errno = EINVAL; + return NULL; } + + uint64_t pageOffset = *offset / PAGE_SIZE; + uint64_t availablePages = shmem->pageAmount - pageOffset; + return vmm_map_pages(space, address, &shmem->pages[pageOffset], MIN(pageAmount, availablePages), flags, + shmem_vmm_callback, REF(shmem)); } static file_ops_t fileOps = { diff --git a/src/modules/smp/trampoline.c b/src/modules/smp/trampoline.c index e99b13e74..89da26f43 100644 --- a/src/modules/smp/trampoline.c +++ b/src/modules/smp/trampoline.c @@ -23,21 +23,23 @@ static atomic_bool cpuReadyFlag = ATOMIC_VAR_INIT(false); void trampoline_init(void) { - backupBuffer = pmm_alloc(); - if (backupBuffer == NULL) + pfn_t pfn = pmm_alloc(); + if (pfn == ERR) { panic(NULL, "Failed to allocate memory for trampoline backup"); } + backupBuffer = PFN_TO_VIRT(pfn); - trampolineStack = pmm_alloc(); - if (trampolineStack == NULL) + pfn = pmm_alloc(); + if (pfn == ERR) { panic(NULL, "Failed to allocate memory for trampoline stack"); } + trampolineStack = PFN_TO_VIRT(pfn); assert(TRAMPOLINE_SIZE < PAGE_SIZE); - if (vmm_map(NULL, (void*)TRAMPOLINE_BASE_ADDR, (void*)TRAMPOLINE_BASE_ADDR, PAGE_SIZE, PML_WRITE | PML_PRESENT, + if (vmm_map(NULL, (void*)TRAMPOLINE_BASE_ADDR, TRAMPOLINE_BASE_ADDR, PAGE_SIZE, PML_WRITE | PML_PRESENT, NULL, NULL) == NULL) { panic(NULL, "Failed to map trampoline"); @@ -63,9 +65,9 @@ void trampoline_deinit(void) vmm_unmap(NULL, (void*)TRAMPOLINE_BASE_ADDR, PAGE_SIZE); - pmm_free(backupBuffer); + pmm_free(VIRT_TO_PFN(backupBuffer)); backupBuffer = NULL; - pmm_free(trampolineStack); + pmm_free(VIRT_TO_PFN(trampolineStack)); trampolineStack = NULL; LOG_DEBUG("trampoline deinitialized\n"); From e2e5944061be1ea8148d4f802a16088c9302be8e Mon Sep 17 00:00:00 2001 From: KN Date: Tue, 20 Jan 2026 20:25:31 +0100 Subject: [PATCH 14/23] feat(kernel:irp): add enter and leave callbacks --- include/kernel/mem/mem_desc.h | 105 +++++++++++++------- include/kernel/mem/paging.h | 6 +- include/kernel/mem/paging_types.h | 8 +- include/kernel/mem/pmm.h | 41 +++++--- include/kernel/mem/space.h | 15 ++- include/kernel/sync/irp.h | 75 ++++++++++++--- include/libstd/sys/rings.h | 17 +++- src/kernel/mem/mem_desc.c | 155 ++++++++++++++++++++++++++++-- src/kernel/mem/pmm.c | 109 +++++++++++---------- src/kernel/mem/space.c | 27 +++++- src/kernel/mem/vmm.c | 7 +- src/kernel/sync/async.c | 43 ++------- src/kernel/sync/irp.c | 34 ++++++- src/modules/drivers/apic/ioapic.c | 3 +- src/modules/drivers/hpet/hpet.c | 4 +- src/modules/drivers/pci/config.c | 3 +- src/modules/smp/trampoline.c | 4 +- 17 files changed, 470 insertions(+), 186 deletions(-) diff --git a/include/kernel/mem/mem_desc.h b/include/kernel/mem/mem_desc.h index 290660cf6..6248e07ea 100644 --- a/include/kernel/mem/mem_desc.h +++ b/include/kernel/mem/mem_desc.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -28,8 +29,8 @@ typedef struct process process_t; */ typedef struct mem_seg { - void* page; ///< Pointer to the first page of the segment in the higher half. - uint32_t length; ///< Length of the segment in bytes. + pfn_t pfn; ///< Page frame number. + uint32_t size; ///< Size of the segment in bytes. uint32_t offset; ///< Offset in bytes within the first page. } mem_seg_t; @@ -88,46 +89,84 @@ static inline mem_desc_pool_t* mem_desc_pool_get(mem_desc_t* desc) * @brief Allocate a new Memory Descriptor from a pool. * * @param pool Pointer to the Memory Descriptor pool. - * @return On success, a pointer to the allocated Memory Descriptor. On failure, `NULL` and `errno` is set. + * @return On success, a pointer to the allocated Memory Descriptor. On failure, `NULL` and `errno` is set to: + * - `EINVAL`: Invalid parameters. + * - `ENOSPC`: No space available in the pool. */ -static inline mem_desc_t* mem_desc_new(mem_desc_pool_t* pool) -{ - pool_idx_t idx = pool_alloc(&pool->pool); - if (idx == POOL_IDX_MAX) - { - errno = ENOSPC; - return NULL; - } - - mem_desc_t* desc = &pool->descs[idx]; - desc->next = POOL_IDX_MAX; - desc->amount = 0; - desc->capacity = 0; - desc->size = 0; - desc->large = NULL; - return desc; -} +mem_desc_t* mem_desc_new(mem_desc_pool_t* pool); /** * @brief Free a Memory Descriptor back to its pool. * * @param desc Pointer to the Memory Descriptor to free. */ -static inline void mem_desc_free(mem_desc_t* desc) -{ - mem_desc_pool_t* pool = mem_desc_pool_get(desc); - pool_free(&pool->pool, desc->next); -} +void mem_desc_free(mem_desc_t* desc); -static inline size_t mem_desc_size(mem_desc_t* desc) -{ - return desc->size; -} +/** + * @brief Add a physical memory region to the Memory Descriptor. + * + * @param desc Pointer to the Memory Descriptor. + * @param phys The physical address of the memory region. + * @param size The size of the memory region in bytes. + * @return On success, `0`. On failure, `ERR` and `errno` is set to: + * - `EINVAL`: Invalid parameters. + * - `EFAULT`: The physical address is not allocated. + * - `ENOMEM`: Not enough memory. + */ +uint64_t mem_desc_add(mem_desc_t* desc, phys_addr_t phys, size_t size); + +/** + * @brief Add a user space memory region to the Memory Descriptor. + * + * @param desc Pointer to the Memory Descriptor. + * @param process The process the user space memory region belongs to. + * @param addr The virtual address of the user space memory region. + * @param size The size of the user space memory region in bytes. + * @return On success, `0`. On failure, `ERR` and `errno` is set to: + * - See `mem_desc_add()` for possible error codes. + */ +uint64_t mem_desc_add_user(mem_desc_t* desc, process_t* process, const void* addr, size_t size); -uint64_t mem_desc_add(mem_desc_t* desc, void* addr, size_t size); +/** + * @brief Read from a Memory Descriptor into a buffer. + * + * @param desc The Memory Descriptor to read from. + * @param buffer The buffer to read into. + * @param count Number of bytes to read. + * @param offset Offset within the Memory Descriptor to start reading from. + * @return The number of bytes read. + */ +uint64_t mem_desc_read(mem_desc_t* desc, void* buffer, size_t count, size_t offset); -static inline uint64_t mem_desc_populate_user(mem_desc_t* desc, process_t* process, void* addr, size_t size) -{ -} +/** + * @brief Write to a Memory Descriptor from a buffer. + * + * @param desc The Memory Descriptor to write to. + * @param buffer The buffer to write from. + * @param count Number of bytes to write. + * @param offset Offset within the Memory Descriptor to start writing to. + * @return The number of bytes written. + */ +uint64_t mem_desc_write(mem_desc_t* desc, const void* buffer, size_t count, size_t offset); + +/** + * @brief Copy data between two Memory Descriptors. + * + * @param dest The destination Memory Descriptor. + * @param destOffset Offset within the destination Memory Descriptor to start writing to. + * @param src The source Memory Descriptor. + * @param srcOffset Offset within the source Memory Descriptor to start reading from. + * @param count Number of bytes to copy. + * @return The number of bytes copied. + */ +uint64_t mem_desc_copy(mem_desc_t* dest, size_t destOffset, mem_desc_t* src, size_t srcOffset, size_t count); + +/** + * @brief Iterate over objects within a Memory Descriptor. + * + * @param _element The iterator variable. + * @param _desc Pointer to the Memory Descriptor. + */ +#define MEM_DESC_FOR_EACH(_element, _desc) /** @} */ \ No newline at end of file diff --git a/include/kernel/mem/paging.h b/include/kernel/mem/paging.h index 0d559f253..64d4b1a82 100644 --- a/include/kernel/mem/paging.h +++ b/include/kernel/mem/paging.h @@ -644,8 +644,7 @@ static inline void page_table_clear(page_table_t* table, void* addr, size_t amou * @param amount The number of pages to check. * @param callbacks An array of size `PML_MAX_CALLBACK` that will be filled with the occurrences of each callback ID. */ -static inline void page_table_collect_callbacks(page_table_t* table, void* addr, size_t amount, - uint64_t* callbacks) +static inline void page_table_collect_callbacks(page_table_t* table, void* addr, size_t amount, uint64_t* callbacks) { page_table_traverse_t traverse = PAGE_TABLE_TRAVERSE_CREATE; @@ -1054,8 +1053,7 @@ static inline uint64_t page_table_count_pages_with_flags(page_table_t* table, vo pml_t* pml1 = PFN_TO_VIRT(entry2->pfn); pml_index_t idx1 = PML_ADDR_TO_INDEX((uintptr_t)addr, PML1); - for (; idx1 < PML_INDEX_AMOUNT && amount > 0; - idx1++, addr = (void*)((uintptr_t)addr + PAGE_SIZE), amount--) + for (; idx1 < PML_INDEX_AMOUNT && amount > 0; idx1++, addr = (void*)((uintptr_t)addr + PAGE_SIZE), amount--) { pml_entry_t* entry1 = &pml1->entries[idx1]; if (!entry1->present) diff --git a/include/kernel/mem/paging_types.h b/include/kernel/mem/paging_types.h index ecf15b86a..932ac5b90 100644 --- a/include/kernel/mem/paging_types.h +++ b/include/kernel/mem/paging_types.h @@ -48,7 +48,7 @@ typedef uintptr_t phys_addr_t; /** * @brief Convert a PFN to its physical address. - * + * * @param _pfn The Page Frame Number. * @return The physical address of the page. */ @@ -56,7 +56,7 @@ typedef uintptr_t phys_addr_t; /** * @brief Convert a physical address to its PFN. - * + * * @param _addr The physical address. * @return The Page Frame Number of the page. */ @@ -64,7 +64,7 @@ typedef uintptr_t phys_addr_t; /** * @brief Convert a PFN to its identity mapped higher half virtual address. - * + * * @param _pfn The Page Frame Number. * @return The higher half virtual address of the page. */ @@ -76,7 +76,7 @@ typedef uintptr_t phys_addr_t; /** * @brief Convert a identity mapped higher half virtual address to its PFN. - * + * * @param _addr The higher half virtual address. * @return The Page Frame Number of the page. */ diff --git a/include/kernel/mem/pmm.h b/include/kernel/mem/pmm.h index 2fce5021f..ab65d3a07 100644 --- a/include/kernel/mem/pmm.h +++ b/include/kernel/mem/pmm.h @@ -33,14 +33,15 @@ */ /** - * @brief The type used for page reference counts. - */ -typedef uint16_t page_ref_t; - -/** - * @brief Invalid page reference count. + * @brief Page metadata structure. + * @struct page_t + * + * Used internally by the PMM. */ -#define PAGE_REF_MAX UINT16_MAX +typedef struct +{ + uint16_t ref; +} page_t; /** * @brief Maximum number of free pages that can be stored in a free page. @@ -70,7 +71,7 @@ void pmm_init(void); * Will by default use the free stack allocator, but if no pages are available there, it will fall back to the bitmap * allocator. * - * @return The PFN of the allocated page, or `ERR` if no memory is available. + * @return On success, the PFN of the allocated page. On failure, `ERR`. */ pfn_t pmm_alloc(void); @@ -91,7 +92,7 @@ uint64_t pmm_alloc_pages(pfn_t* pfns, size_t count); * @param count Number of pages to allocate. * @param maxPfn Maximum PFN to allocate up to (exclusive). * @param alignPfn Alignment of the region in pages. - * @return The PFN of the first allocated page, or `ERR` if no memory is available. + * @return On success, the PFN of the first page of the allocated region. On failure, `ERR`. */ pfn_t pmm_alloc_bitmap(size_t count, pfn_t maxPfn, pfn_t alignPfn); @@ -127,14 +128,28 @@ void pmm_free_pages(pfn_t* pfns, size_t count); void pmm_free_region(pfn_t pfn, size_t count); /** - * @brief Increment the reference count of a physical page. + * @brief Increment the reference count of a physical region. * - * Will fail if the page is not allocated. + * Will fail if any of the pages are not allocated. * - * @param pfn The PFN of the physical page. + * @param pfn The PFN of the first physical page. + * @param count Number of pages to increment the reference count of. * @return On success, the new reference count. On failure, `ERR`. */ -uint64_t pmm_ref_inc(pfn_t pfn); +uint64_t pmm_ref_inc(pfn_t pfn, size_t count); + +/** + * @brief Decrement the reference count of a physical region. + * + * If the reference count reaches zero, the pages will be freed. + * + * @param pfn The PFN of the first physical page. + * @param count Number of pages to decrement the reference count of. + */ +static inline void pmm_ref_dec(pfn_t pfn, size_t count) +{ + pmm_free_region(pfn, count); +} /** * @brief Get the total number of physical pages. diff --git a/include/kernel/mem/space.h b/include/kernel/mem/space.h index 764862f07..467864169 100644 --- a/include/kernel/mem/space.h +++ b/include/kernel/mem/space.h @@ -244,8 +244,8 @@ typedef struct * - `EFAULT`: The addresses are outside the allowed range. * - `ENOMEM`: Not enough memory. */ -uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* virtAddr, phys_addr_t physAddr, size_t length, - size_t alignment, pml_flags_t flags); +uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* virtAddr, phys_addr_t physAddr, + size_t length, size_t alignment, pml_flags_t flags); /** * @brief Allocate a callback. @@ -308,4 +308,15 @@ bool space_is_mapped(space_t* space, const void* virtAddr, size_t length); */ uint64_t space_user_page_count(space_t* space); +/** + * @brief Translate a virtual address to a physical address in the address space. + * + * @param space The target address space. + * @param virtAddr The virtual address to translate. + * @return On success, `0`. On failure, `ERR` and `errno` is set to: + * - `EINVAL`: Invalid parameters. + * - `EFAULT`: The virtual address is not mapped. + */ +phys_addr_t space_virt_to_phys(space_t* space, const void* virtAddr); + /** @} */ diff --git a/include/kernel/sync/irp.h b/include/kernel/sync/irp.h index b421528f5..edbb56608 100644 --- a/include/kernel/sync/irp.h +++ b/include/kernel/sync/irp.h @@ -1,19 +1,18 @@ #pragma once #include +#include #include #include -#include #include #include -#include #include #include #include #include #include - +#include typedef struct irp irp_t; @@ -28,7 +27,8 @@ typedef struct irp irp_t; * The IRP system is designed to be generic enough to be used by any system in the kernel, however it is primarily used * by the asynchronous rings system. * - * @warning The IRP system is not thread-safe, it is the responsibility of the caller to ensure proper synchronization. + * @warning While the cancellation or completion of an IRP is thread safe, the setup of an IRP is not (as in pushing + * layers to it). As such, its up to the caller to ensure that only one thread is manipulating it during setup. * * ## Completion * @@ -55,6 +55,7 @@ typedef struct irp irp_t; * } * * int result = fun_a(); + * // Do stuff with the result * ``` * * When the code is executed, `fun_a()` would be called, which calls `fun_b()`, which in turn calls `fun_c()`. At this @@ -114,8 +115,15 @@ typedef struct irp irp_t; * fun_b(irp); * } * - * irp_t* irp = irp_new(pool); - * // We could add our own complete here to handle the final result. + * void my_completion(irp_t* irp) + * { + * // Do stuff with the result in irp->result. + * irp_free(irp); + * } + * + * irp_t* irp = irp_new(pool, NULL); + * // We can set arguments here if we want. + * irp_push(irp, my_completion, NULL); // Our completion to handle cleanup. * fun_a_do(irp); * // Continue executing even if fun_c() cannot complete immediately. * ``` @@ -130,6 +138,32 @@ typedef struct irp irp_t; * `cqe_t` to its Rings, before passing the IRP to the VFS which may pass it to a filesystem. Each layer pushing its own * completion to handle its part of the operation. * + * Finally, it is also possible to use the `irp_dispatch()` function. This function allows us to dispatch the IRP to a + * appropriate handler depending on the IRPs specified verb. For example: + * + * ``` + * void my_completion(irp_t* irp) + * { + * // Do stuff with the result in irp->result. + * irp_free(irp); + * } + * + * irp_t* irp = irp_new(pool, NULL); + * + * // Set our desired verb and arguments. + * irp->verb = VERB_OPEN; + * irp->open.from = from; + * irp->open.path = path; + * irp->open.length = strlen(path); + * + * // Our completion to receive the result. + * irp_push(irp, my_completion, NULL); + * + * // Finally, dispatch the IRP to the appropriate handler. + * irp_dispatch(irp); + * // Continue executing even if the operation cannot complete immediately. + * ``` + * * ## Cancellation * * The current owner of a IRP is responsible for handling cancellation. The current owner being the last subsystem to @@ -206,6 +240,8 @@ typedef struct irp irp_t; * * @see kernel_sync_async for the asynchronous rings system. * @see [Wikipedia](https://en.wikipedia.org/wiki/I/O_request_packet) for more information about IRPs. + * @see [Microsoft _IRP](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_irp) for information + * on how Windows NT implements IRPs. * @{ */ @@ -272,7 +308,6 @@ typedef struct ALIGNED(64) irp { file_t* from; mem_desc_t* path; - size_t length; } open; uint64_t _args[IRP_ARGS_MAX]; }; @@ -358,14 +393,21 @@ void irp_timeouts_check(void); * The pool that the IRP was allocated from, and its context, can be retrieved using the `irp_pool_get()` * function. * + * @note If a SQE is provided then the IRP will be considered a user IRP, causing the `irp_handler_t::enter` and + * `irp_handler_t::leave` callbacks to be invoked on the IRP when its dispatched and freed respectively. Otherwise, the + * caller is responsible for the lifecycle and arguments of the IRP. + * * @param pool Pointer to the IRP pool. - * @return On success, a pointer to the allocated IRP. On failure, `NULL`. + * @param sqe The Submission Queue Entry associated with the IRP, if `NULL` the IRP will be a kernel IRP. + * @return On success, a pointer to the allocated IRP. On failure, `NULL` and `errno` is set. */ -irp_t* irp_new(irp_pool_t* pool); +irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe); /** * @brief Free a IRP back to its pool. * + * If the IRP is a user IRP, the `irp_handler_t::leave` callback will be invoked before freeing the IRP. + * * @param irp Pointer to the IRP to free. */ void irp_free(irp_t* irp); @@ -511,6 +553,9 @@ uint64_t irp_cancel(irp_t* irp); * * If `irp->err != EINPROGRESS` the IRP is immediately completed. * + * If the IRP is a user IRP and it has not yet been entered, the `irp_handler_t::enter` callback for the verb is + * invoked. + * * @param irp Pointer to the IRP to dispatch. */ void irp_dispatch(irp_t* irp); @@ -527,11 +572,15 @@ void irp_table_init(void); typedef struct { verb_t verb; - void (*handler)(irp_t* irp); + void (*enter)(irp_t* irp); ///< Will be called on user IRPs to process arguments. + void (*leave)(irp_t* irp); ///< Will be called on user IRPs to cleanup resources. + void (*handler)(irp_t* irp); ///< The handler function for the verb. } irp_handler_t; /** * @brief Linker defined start of the IRP handlers table. + * + * After `irp_table_init()` has sorted the IRP table, the table can be indexed by verb. */ extern irp_handler_t _irp_table_start[]; @@ -544,11 +593,15 @@ extern irp_handler_t _irp_table_end[]; * @brief Macro to register a IRP handler to a verb using the `._irp_table` section. * * @param _verb The verb to register the handler for. + * @param _enter The enter function. + * @param _leave The leave function. * @param _handler The handler function. */ -#define IRP_REGISTER(_verb, _handler) \ +#define IRP_REGISTER(_verb, _enter, _leave, _handler) \ static irp_handler_t __irp_##_verb __attribute__((section("._irp_table"), used)) = { \ .verb = (_verb), \ + .enter = (_enter), \ + .leave = (_leave), \ .handler = (_handler), \ }; diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/rings.h index fb838ca11..acdd280d9 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/rings.h @@ -59,6 +59,12 @@ typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. /// applies within one `enter()` call. #define SQE_HARDLINK \ (1 << (SQE_FLAGS_SHIFT + 1)) ///< Like `SQE_LINK`) but will process the next SQE even if this one fails. +#ifdef _KERNEL_ +#define SQE_KERNEL \ + (1 << (SQE_FLAGS_SHIFT + 2)) ///< The operation was created by the kernel, used internally by the kernel. +#define SQE_KERNEL_ENTERED \ + (1 << (SQE_FLAGS_SHIFT + 3)) ///< The operations enter callback has been called, used internally by the kernel. +#endif /** * @brief Asynchronous submission queue entry (SQE). @@ -148,11 +154,14 @@ typedef uint64_t rings_id_t; */ typedef struct ALIGNED(64) rings_shared { - atomic_uint32_t shead; ///< Submission head index, updated by the kernel. - atomic_uint32_t ctail; ///< Completion tail index, updated by the kernel. - atomic_uint32_t stail ALIGNED(64); ///< Submission tail index, updated by userspace. - atomic_uint32_t chead; ///< Completion head index, updated by userspace. + atomic_uint32_t shead; ///< Submission head index, updated by the kernel. + atomic_uint32_t ctail; ///< Completion tail index, updated by the kernel. + uint8_t _padding0[64 - sizeof(atomic_uint32_t) * 2]; + atomic_uint32_t stail; ///< Submission tail index, updated by userspace. + atomic_uint32_t chead; ///< Completion head index, updated by userspace. + uint8_t _padding1[64 - sizeof(atomic_uint32_t) * 2]; atomic_uint64_t regs[SEQ_REGS_MAX] ALIGNED(64); ///< General purpose registers. + uint8_t _reserved[8]; } rings_shared_t; /** diff --git a/src/kernel/mem/mem_desc.c b/src/kernel/mem/mem_desc.c index 26eb7598e..68f32bfdb 100644 --- a/src/kernel/mem/mem_desc.c +++ b/src/kernel/mem/mem_desc.c @@ -1,7 +1,26 @@ #include +#include +#include +#include -#include #include +#include +#include + +static inline mem_seg_t* mem_desc_get_seg(mem_desc_t* desc, size_t index) +{ + if (index >= desc->amount) + { + return NULL; + } + + if (index < MEM_SEGS_SMALL_MAX) + { + return &desc->small[index]; + } + + return &desc->large[index - MEM_SEGS_SMALL_MAX]; +} mem_desc_pool_t* mem_desc_pool_new(size_t size) { @@ -26,8 +45,67 @@ void mem_desc_pool_free(mem_desc_pool_t* pool) free(pool); } -uint64_t mem_desc_add(mem_desc_t* desc, void* addr, size_t size) +mem_desc_t* mem_desc_new(mem_desc_pool_t* pool) { + if (pool == NULL) + { + errno = EINVAL; + return NULL; + } + + pool_idx_t idx = pool_alloc(&pool->pool); + if (idx == POOL_IDX_MAX) + { + errno = ENOSPC; + return NULL; + } + + mem_desc_t* desc = &pool->descs[idx]; + desc->next = POOL_IDX_MAX; + desc->amount = 0; + desc->capacity = 0; + desc->size = 0; + desc->large = NULL; + return desc; +} + +void mem_desc_free(mem_desc_t* desc) +{ + if (desc == NULL) + { + return; + } + + size_t i = 0; + for (; i < desc->amount && i < MEM_SEGS_SMALL_MAX; i++) + { + pmm_ref_dec(desc->small[i].pfn, BYTES_TO_PAGES(desc->small[i].offset + desc->small[i].size)); + } + + for (; i < desc->amount; i++) + { + pmm_ref_dec(desc->large[i - MEM_SEGS_SMALL_MAX].pfn, + BYTES_TO_PAGES(desc->large[i - MEM_SEGS_SMALL_MAX].offset + desc->large[i - MEM_SEGS_SMALL_MAX].size)); + } + + if (desc->large != NULL) + { + free(desc->large); + } + desc->large = NULL; + + mem_desc_pool_t* pool = mem_desc_pool_get(desc); + pool_free(&pool->pool, desc->index); +} + +uint64_t mem_desc_add(mem_desc_t* desc, phys_addr_t phys, size_t size) +{ + if (desc == NULL) + { + errno = EINVAL; + return ERR; + } + mem_seg_t* seg = NULL; if (desc->amount < MEM_SEGS_SMALL_MAX) { @@ -39,10 +117,10 @@ uint64_t mem_desc_add(mem_desc_t* desc, void* addr, size_t size) } else { - mem_seg_t* newLarge = realloc( - desc->large, (desc->capacity + 4) * sizeof(mem_seg_t)); + mem_seg_t* newLarge = realloc(desc->large, (desc->capacity + 4) * sizeof(mem_seg_t)); if (newLarge == NULL) { + errno = ENOMEM; return ERR; } @@ -51,8 +129,71 @@ uint64_t mem_desc_add(mem_desc_t* desc, void* addr, size_t size) seg = &desc->large[desc->amount++ - MEM_SEGS_SMALL_MAX]; } - seg->page = (void*)ROUND_DOWN(addr, PAGE_SIZE); - seg->offset = (uint32_t)((uintptr_t)addr - (uintptr_t)seg->page); - seg->length = (uint32_t)size; + pfn_t pfn = PHYS_TO_PFN(phys); + uint32_t offset = phys % PAGE_SIZE; + if (pmm_ref_inc(pfn, BYTES_TO_PAGES(offset + size)) == ERR) + { + errno = EFAULT; + return ERR; + } + + seg->pfn = pfn; + seg->size = size; + seg->offset = offset; + desc->size += size; + return 0; +} + +uint64_t mem_desc_add_user(mem_desc_t* desc, process_t* process, const void* addr, size_t size) +{ + const uint8_t* ptr = addr; + size_t remaining = size; + + while (remaining > 0) + { + phys_addr_t phys = space_virt_to_phys(&process->space, ptr); + if (phys == ERR) + { + return ERR; + } + + size_t offset = phys % PAGE_SIZE; + size_t len = MIN(remaining, PAGE_SIZE - offset); + + if (mem_desc_add(desc, phys, len) == ERR) + { + return ERR; + } + + ptr += len; + remaining -= len; + } + return 0; +} + +uint64_t mem_desc_read(mem_desc_t* desc, void* buffer, size_t count, size_t offset) +{ + if (desc == NULL || buffer == NULL || offset + count > desc->size) + { + return 0; + } + +} + +uint64_t mem_desc_write(mem_desc_t* desc, const void* buffer, size_t count, size_t offset) +{ + if (desc == NULL || buffer == NULL || offset + count > desc->size) + { + return 0; + } +} + + uint64_t mem_desc_copy(mem_desc_t* dest, size_t destOffset, mem_desc_t* src, size_t srcOffset, + size_t count) +{ + if (dest == NULL || src == NULL || destOffset + count > dest->size || srcOffset + count > src->size) + { + return 0; + } } \ No newline at end of file diff --git a/src/kernel/mem/pmm.c b/src/kernel/mem/pmm.c index 4f358f121..628132c28 100644 --- a/src/kernel/mem/pmm.c +++ b/src/kernel/mem/pmm.c @@ -35,7 +35,7 @@ static const char* efiMemTypeToString[] = { "persistent", }; -static page_ref_t* refs = NULL; +static page_t* pages = NULL; static page_stack_t* stack = NULL; static size_t location = FREE_PAGE_MAX; @@ -68,25 +68,6 @@ static bool pmm_is_mem_avail(EFI_MEMORY_TYPE type) } } -static inline page_ref_t* pmm_ref_get(pfn_t pfn) -{ - return &refs[pfn]; -} - -static inline void pmm_ref_set(pfn_t pfn, size_t count, page_ref_t value) -{ - page_ref_t* ref = pmm_ref_get(pfn); - for (size_t i = 0; i < count; i++) - { - ref[i] = value; - } -} - -static inline size_t pmm_refs_size(void) -{ - return highest * sizeof(page_ref_t); -} - static inline void pmm_stack_push(pfn_t pfn) { if (stack == NULL || location == 0) @@ -141,10 +122,9 @@ static inline void pmm_bitmap_clear(pfn_t pfn, size_t pageAmount) static void pmm_free_unlocked(pfn_t pfn) { - page_ref_t* ref = pmm_ref_get(pfn); - assert(*ref > 0); - (*ref)--; - if (*ref > 0) + page_t* page = &pages[pfn]; + assert(page->ref > 0); + if (--page->ref > 0) { return; } @@ -191,16 +171,14 @@ static void pmm_detect_memory(const boot_memory_map_t* map) static void pmm_init_refs(const boot_memory_map_t* map) { - size_t size = pmm_refs_size(); - size_t pages = BYTES_TO_PAGES(size); + size_t size = highest * sizeof(page_t); for (size_t i = 0; i < map->length; i++) { const EFI_MEMORY_DESCRIPTOR* desc = BOOT_MEMORY_MAP_GET_DESCRIPTOR(map, i); - if (desc->Type == EfiConventionalMemory && desc->NumberOfPages >= pages) + if (desc->Type == EfiConventionalMemory && desc->NumberOfPages >= BYTES_TO_PAGES(size)) { - refs = (page_ref_t*)desc->VirtualStart; - memset(refs, -1, pages * PAGE_SIZE); - LOG_INFO("pmm ref [%p-%p]\n", refs, (uintptr_t)refs + pages * PAGE_SIZE); + pages = (page_t*)desc->VirtualStart; + LOG_INFO("pages [%p-%p]\n", pages, (uintptr_t)pages + size); return; } } @@ -210,35 +188,45 @@ static void pmm_init_refs(const boot_memory_map_t* map) static void pmm_load_memory(const boot_memory_map_t* map) { - pfn_t refsPfn = VIRT_TO_PFN(refs); - size_t refPages = BYTES_TO_PAGES(pmm_refs_size()); + pfn_t pagesPfn = VIRT_TO_PFN(pages); for (size_t i = 0; i < map->length; i++) { const EFI_MEMORY_DESCRIPTOR* desc = BOOT_MEMORY_MAP_GET_DESCRIPTOR(map, i); pfn_t pfn = VIRT_TO_PFN(desc->VirtualStart); - size_t pages = desc->NumberOfPages; + size_t amount = desc->NumberOfPages; - if (pfn == refsPfn) + if (pfn == pagesPfn) { - // Skip the refs array. - assert(pages >= refPages); - pfn += refPages; - pages -= refPages; + // Skip the pages array. + size_t toSkip = BYTES_TO_PAGES(highest * sizeof(page_t)); + pfn += toSkip; + amount -= toSkip; } if (pmm_is_mem_avail(desc->Type)) { #ifndef NDEBUG // Clear the memory to deliberately cause corruption if the memory is actually being used. - memset(PFN_TO_VIRT(pfn), 0xCC, pages * PAGE_SIZE); + memset(PFN_TO_VIRT(pfn), 0xCC, amount * PAGE_SIZE); #endif - pmm_ref_set(pfn, pages, 1); - pmm_free_region_unlocked(pfn, pages); + for (size_t j = 0; j < amount; j++) + { + page_t* page = &pages[pfn + j]; + page->ref = 1; + } + + pmm_free_region_unlocked(pfn, amount); } else { + for (size_t j = 0; j < amount; j++) + { + page_t* page = &pages[pfn + j]; + page->ref = UINT16_MAX; + } + LOG_INFO("reserve [%p-%p] pages=%d type=%s\n", PFN_TO_VIRT(pfn), PFN_TO_VIRT(pfn + pages), pages, efiMemTypeToString[desc->Type]); } @@ -269,9 +257,9 @@ pfn_t pmm_alloc(void) if (pfn != ERR) { - page_ref_t* ref = pmm_ref_get(pfn); - assert(*ref == 0); - *ref = 1; + page_t* page = &pages[pfn]; + assert(page->ref == 0); + page->ref = 1; avail--; } lock_release(&lock); @@ -316,9 +304,9 @@ uint64_t pmm_alloc_pages(pfn_t* pfns, size_t count) for (size_t i = 0; i < count; i++) { - page_ref_t* ref = pmm_ref_get(pfns[i]); - assert(*ref == 0); - *ref = 1; + page_t* page = &pages[pfns[i]]; + assert(page->ref == 0); + page->ref = 1; } avail -= count; @@ -334,9 +322,9 @@ pfn_t pmm_alloc_bitmap(size_t count, pfn_t maxPfn, pfn_t alignPfn) { for (size_t i = 0; i < count; i++) { - page_ref_t* ref = pmm_ref_get(pfn + i); - assert(*ref == 0); - *ref = 1; + page_t* page = &pages[pfn + i]; + assert(page->ref == 0); + page->ref = 1; } avail -= count; } @@ -374,13 +362,24 @@ void pmm_free_region(pfn_t pfn, size_t count) lock_release(&lock); } -uint64_t pmm_ref_inc(pfn_t pfn) +uint64_t pmm_ref_inc(pfn_t pfn, size_t count) { lock_acquire(&lock); - page_ref_t* ref = pmm_ref_get(pfn); - assert(*ref != PAGE_REF_MAX); - (*ref)++; - uint64_t ret = *ref; + for (size_t i = 0; i < count; i++) + { + page_t* page = &pages[pfn + i]; + if (page->ref == 0 || page->ref == UINT16_MAX) + { + for (size_t j = 0; j < i; j++) + { + pages[pfn + j].ref--; + } + lock_release(&lock); + return ERR; + } + page->ref++; + } + uint64_t ret = pages[pfn].ref; lock_release(&lock); return ret; } diff --git a/src/kernel/mem/space.c b/src/kernel/mem/space.c index cc68367f6..a7808c3f0 100644 --- a/src/kernel/mem/space.c +++ b/src/kernel/mem/space.c @@ -178,8 +178,8 @@ static uint64_t space_populate_user_region(space_t* space, const void* buffer, s return ERR; } - if (page_table_map(&space->pageTable, (void*)addr, PFN_TO_PHYS(pfn), 1, PML_PRESENT | PML_USER | PML_WRITE | PML_OWNED, - PML_CALLBACK_NONE) == ERR) + if (page_table_map(&space->pageTable, (void*)addr, PFN_TO_PHYS(pfn), 1, + PML_PRESENT | PML_USER | PML_WRITE | PML_OWNED, PML_CALLBACK_NONE) == ERR) { pmm_free(pfn); return ERR; @@ -491,8 +491,8 @@ static void* space_find_free_region(space_t* space, uint64_t pageAmount, uint64_ return NULL; } -uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* virtAddr, phys_addr_t physAddr, size_t length, - size_t alignment, pml_flags_t flags) +uint64_t space_mapping_start(space_t* space, space_mapping_t* mapping, void* virtAddr, phys_addr_t physAddr, + size_t length, size_t alignment, pml_flags_t flags) { if (space == NULL || mapping == NULL || length == 0) { @@ -666,3 +666,22 @@ uint64_t space_user_page_count(space_t* space) return page_table_count_pages_with_flags(&space->pageTable, (void*)VMM_USER_SPACE_MIN, BYTES_TO_PAGES(VMM_USER_SPACE_MAX - VMM_USER_SPACE_MIN), PML_PRESENT | PML_USER | PML_OWNED); } + +phys_addr_t space_virt_to_phys(space_t* space, const void* virtAddr) +{ + if (space == NULL) + { + errno = EINVAL; + return ERR; + } + + phys_addr_t physAddr; + LOCK_SCOPE(&space->lock); + if (page_table_get_phys_addr(&space->pageTable, (void*)virtAddr, &physAddr) == ERR) + { + errno = EFAULT; + return ERR; + } + + return physAddr; +} \ No newline at end of file diff --git a/src/kernel/mem/vmm.c b/src/kernel/mem/vmm.c index d3fc7f1d2..b277e0053 100644 --- a/src/kernel/mem/vmm.c +++ b/src/kernel/mem/vmm.c @@ -83,8 +83,8 @@ void vmm_init(void) panic(NULL, "Memory descriptor %d has invalid physical address %p", i, desc->PhysicalStart); } - if (page_table_map(&kernelSpace.pageTable, (void*)desc->VirtualStart, desc->PhysicalStart, - desc->NumberOfPages, PML_WRITE | PML_GLOBAL | PML_PRESENT, PML_CALLBACK_NONE) == ERR) + if (page_table_map(&kernelSpace.pageTable, (void*)desc->VirtualStart, desc->PhysicalStart, desc->NumberOfPages, + PML_WRITE | PML_GLOBAL | PML_PRESENT, PML_CALLBACK_NONE) == ERR) { panic(NULL, "Failed to map memory descriptor %d (phys=%p-%p virt=%p)", i, desc->PhysicalStart, desc->PhysicalStart + desc->NumberOfPages * PAGE_SIZE, desc->VirtualStart); @@ -165,7 +165,8 @@ void* vmm_alloc(space_t* space, void* virtAddr, size_t length, size_t alignment, } space_mapping_t mapping; - if (space_mapping_start(space, &mapping, virtAddr, PHYS_ADDR_INVALID, length, alignment, pmlFlags | PML_OWNED) == ERR) + if (space_mapping_start(space, &mapping, virtAddr, PHYS_ADDR_INVALID, length, alignment, pmlFlags | PML_OWNED) == + ERR) { return NULL; } diff --git a/src/kernel/sync/async.c b/src/kernel/sync/async.c index 708a8bed6..1b96a622d 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/async.c @@ -1,4 +1,3 @@ -#include <_internal/fd_t.h> #include #include #include @@ -280,31 +279,6 @@ static void async_dispatch(irp_t* irp) irp->sqe._args[i] = atomic_load_explicit(&ctx->rings.shared->regs[reg], memory_order_acquire); } - switch (irp->verb) - { - case VERB_NOP: - break; - /*case VERB_OPEN: - { - file_t* from = NULL; - if (irp->sqe.open.from != FD_NONE) - { - from = file_table_get(&ctx->process->fileTable, irp->sqe.open.from); - if (from == NULL) - { - irp->err = EBADF; - break; - } - } - - irp->open.from = from; - irp->open.path = - } - break;*/ - default: - break; - } - irp_push(irp, async_complete, NULL); irp_dispatch(irp); } @@ -317,13 +291,6 @@ typedef struct static uint64_t async_sqe_pop(async_t* ctx, async_notify_ctx_t* notify) { - irp_t* irp = irp_new(ctx->irps); - if (irp == NULL) - { - errno = ENOSPC; - return ERR; - } - if (atomic_load(&ctx->irps->pool.used) == 1) { process_t* process = process_current(); @@ -337,11 +304,17 @@ static uint64_t async_sqe_pop(async_t* ctx, async_notify_ctx_t* notify) if (shead == stail) { - irp_free(irp); + errno = EAGAIN; + return ERR; + } + + sqe_t* sqe = &rings->squeue[shead & rings->smask]; + irp_t* irp = irp_new(ctx->irps, sqe); + if (irp == NULL) + { return ERR; } - irp->sqe = rings->squeue[shead & rings->smask]; atomic_store_explicit(&rings->shared->shead, shead + 1, memory_order_release); if (notify->link != NULL) diff --git a/src/kernel/sync/irp.c b/src/kernel/sync/irp.c index 2aaac9602..6a6e75974 100644 --- a/src/kernel/sync/irp.c +++ b/src/kernel/sync/irp.c @@ -1,6 +1,6 @@ -#include <_internal/clock_t.h> #include #include +#include #include #include #include @@ -75,7 +75,7 @@ void irp_pool_free(irp_pool_t* pool) free(pool); } -irp_t* irp_new(irp_pool_t* pool) +irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) { pool_idx_t idx = pool_alloc(&pool->pool); if (idx == POOL_IDX_MAX) @@ -89,8 +89,19 @@ irp_t* irp_new(irp_pool_t* pool) irp->next = POOL_IDX_MAX; irp->err = EINPROGRESS; irp->result = 0; - irp->sqe = (sqe_t){0}; atomic_store_explicit(&irp->cancel, NULL, memory_order_relaxed); + + if (sqe == NULL) + { + irp->sqe = (sqe_t){0}; + irp->flags |= SQE_KERNEL; + } + else + { + irp->sqe = *sqe; + irp->sqe.flags &= ~(SQE_KERNEL | SQE_KERNEL_ENTERED); + } + irp->next = POOL_IDX_MAX; irp->cpu = CPU_ID_INVALID; for (size_t j = 0; j < IRP_LOC_MAX; j++) @@ -103,6 +114,12 @@ irp_t* irp_new(irp_pool_t* pool) void irp_free(irp_t* irp) { + if (irp->flags & SQE_KERNEL_ENTERED && irp->verb < VERB_MAX && _irp_table_start[irp->verb].leave != NULL) + { + assert(!(irp->flags & SQE_KERNEL)); + _irp_table_start[irp->verb].leave(irp); + } + irp_pool_t* pool = irp_pool_get(irp); pool_free(&pool->pool, irp->index); } @@ -238,6 +255,15 @@ void irp_dispatch(irp_t* irp) return; } + if (!(irp->flags & SQE_KERNEL) && !(irp->flags & SQE_KERNEL_ENTERED)) + { + if (_irp_table_start[irp->verb].enter != NULL) + { + _irp_table_start[irp->verb].enter(irp); + } + irp->flags |= SQE_KERNEL_ENTERED; + } + _irp_table_start[irp->verb].handler(irp); } @@ -279,4 +305,4 @@ void nop_do(irp_t* irp) } } -IRP_REGISTER(VERB_NOP, nop_do); \ No newline at end of file +IRP_REGISTER(VERB_NOP, NULL, NULL, nop_do); \ No newline at end of file diff --git a/src/modules/drivers/apic/ioapic.c b/src/modules/drivers/apic/ioapic.c index 4851f57aa..12fcfd3f7 100644 --- a/src/modules/drivers/apic/ioapic.c +++ b/src/modules/drivers/apic/ioapic.c @@ -107,7 +107,8 @@ uint64_t ioapic_all_init(void) } void* virtAddr = (void*)PML_LOWER_TO_HIGHER(ioapic->ioApicAddress); - if (vmm_map(NULL, virtAddr, ioapic->ioApicAddress, PAGE_SIZE, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == NULL) + if (vmm_map(NULL, virtAddr, ioapic->ioApicAddress, PAGE_SIZE, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, + NULL) == NULL) { LOG_ERR("failed to map io apic\n"); return ERR; diff --git a/src/modules/drivers/hpet/hpet.c b/src/modules/drivers/hpet/hpet.c index 46c0e0d00..ade98a5eb 100644 --- a/src/modules/drivers/hpet/hpet.c +++ b/src/modules/drivers/hpet/hpet.c @@ -218,8 +218,8 @@ static uint64_t hpet_init(void) } address = (uintptr_t)PML_LOWER_TO_HIGHER(hpet->address); - if (vmm_map(NULL, (void*)address, hpet->address, PAGE_SIZE, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, - NULL) == NULL) + if (vmm_map(NULL, (void*)address, hpet->address, PAGE_SIZE, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == + NULL) { LOG_ERR("failed to map HPET memory at %p\n", hpet->address); return ERR; diff --git a/src/modules/drivers/pci/config.c b/src/modules/drivers/pci/config.c index 9f3fd8aa9..821e29c3c 100644 --- a/src/modules/drivers/pci/config.c +++ b/src/modules/drivers/pci/config.c @@ -43,8 +43,7 @@ static uint64_t pci_config_init(void) uint64_t length = busCount * 256 * 4096; void* virtAddr = (void*)PML_LOWER_TO_HIGHER(entry->base); - if (vmm_map(NULL, virtAddr, entry->base, length, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == - NULL) + if (vmm_map(NULL, virtAddr, entry->base, length, PML_WRITE | PML_GLOBAL | PML_PRESENT, NULL, NULL) == NULL) { LOG_ERR("failed to map PCI-e configuration space at %p\n", entry->base); return ERR; diff --git a/src/modules/smp/trampoline.c b/src/modules/smp/trampoline.c index 89da26f43..f9d8adfe2 100644 --- a/src/modules/smp/trampoline.c +++ b/src/modules/smp/trampoline.c @@ -39,8 +39,8 @@ void trampoline_init(void) assert(TRAMPOLINE_SIZE < PAGE_SIZE); - if (vmm_map(NULL, (void*)TRAMPOLINE_BASE_ADDR, TRAMPOLINE_BASE_ADDR, PAGE_SIZE, PML_WRITE | PML_PRESENT, - NULL, NULL) == NULL) + if (vmm_map(NULL, (void*)TRAMPOLINE_BASE_ADDR, TRAMPOLINE_BASE_ADDR, PAGE_SIZE, PML_WRITE | PML_PRESENT, NULL, + NULL) == NULL) { panic(NULL, "Failed to map trampoline"); } From 609889a77d8cb91aded3beee8a3b8d75814effa9 Mon Sep 17 00:00:00 2001 From: KN Date: Tue, 20 Jan 2026 22:54:19 +0100 Subject: [PATCH 15/23] feat(kernel:mem_desc): implement memory descriptors; cleanup ring names --- include/kernel/config.h | 43 ++-- include/kernel/mem/mem_desc.h | 80 ++++-- include/kernel/proc/process.h | 4 +- include/kernel/sync/irp.h | 8 +- include/kernel/sync/{async.h => ring.h} | 58 ++--- include/libstd/sys/{rings.h => uring.h} | 159 ++++++------ meta/doxy/Doxyfile | 2 +- src/kernel/mem/mem_desc.c | 114 +++++++-- src/kernel/proc/process.c | 10 +- src/kernel/sync/{async.c => ring.c} | 230 +++++++++--------- src/libstd/user/common/syscalls.h | 14 +- src/libstd/user/functions/async/setup.c | 13 - .../user/functions/{async => uring}/enter.c | 4 +- src/libstd/user/functions/uring/setup.c | 13 + .../functions/{async => uring}/teardown.c | 4 +- .../utils/{ringstest => ringtest}/main.c | 32 +-- .../ringstest.mk => ringtest/ringtest.mk} | 0 17 files changed, 446 insertions(+), 342 deletions(-) rename include/kernel/sync/{async.h => ring.h} (71%) rename include/libstd/sys/{rings.h => uring.h} (59%) rename src/kernel/sync/{async.c => ring.c} (51%) delete mode 100644 src/libstd/user/functions/async/setup.c rename src/libstd/user/functions/{async => uring}/enter.c (69%) create mode 100644 src/libstd/user/functions/uring/setup.c rename src/libstd/user/functions/{async => uring}/teardown.c (75%) rename src/programs/utils/{ringstest => ringtest}/main.c (52%) rename src/programs/utils/{ringstest/ringstest.mk => ringtest/ringtest.mk} (100%) diff --git a/include/kernel/config.h b/include/kernel/config.h index 6cd05e59f..69e96c824 100644 --- a/include/kernel/config.h +++ b/include/kernel/config.h @@ -1,8 +1,14 @@ #pragma once +/** + * @brief Kernel configuration. + * @defgroup kernel_config Configuration + * + * @{ + */ + /** * @brief Interrupt stack configuration. - * @ingroup kernel * @def CONFIG_INTERRUPT_STACK_PAGES * * The `CONFIG_INTERRUPT_STACK_PAGES` constant defines the amount of pages that are allocated for the per-CPU interrupt, @@ -15,7 +21,6 @@ /** * @brief Kernel stack configuration. - * @ingroup kernel * @def CONFIG_MAX_KERNEL_STACK_PAGES * * The `CONFIG_MAX_KERNEL_STACK_PAGES` constant defines the maximum amount of pages that are allowed to be allocated for @@ -27,7 +32,6 @@ /** * @brief User stack configuration. - * @ingroup kernel * @def CONFIG_MAX_USER_STACK_PAGES * * The `CONFIG_MAX_USER_STACK_PAGES` constant defines the maximum amount of pages that are allowed to be allocated for a @@ -38,7 +42,6 @@ /** * @brief Maximum file descriptor configuration. - * @ingroup kernel * @def CONFIG_MAX_FD * * The `CONFIG_MAX_FD` constant defines the maximum amount of file descriptors that a process is allowed to have open. @@ -48,7 +51,6 @@ /** * @brief Serial logging configuration. - * @ingroup kernel * @def CONFIG_LOG_SERIAL * * The `CONFIG_LOG_SERIAL` constant defines if to output logged strings via serial. @@ -58,7 +60,6 @@ /** * @brief Maximum note queue configuration. - * @ingroup kernel * @def CONFIG_MAX_NOTES * * The `CONFIG_MAX_NOTES` constant defines the maximum length of a threads note queue. @@ -68,7 +69,6 @@ /** * @brief Maximum argument vector configuration. - * @ingroup kernel * @def CONFIG_MAX_ARGC * * The `CONFIG_MAX_ARGC` constant defines the maximum amount of arguments that can be passed to a process via its @@ -79,7 +79,6 @@ /** * @brief Minimum timer timeout configuration. - * @ingroup kernel * @def CONFIG_MIN_TIMER_TIMEOUT * * The `CONFIG_MIN_TIMER_TIMEOUT` constant defines the minimum timeout that can be set for timers. @@ -89,7 +88,6 @@ /** * @brief Time slice configuration. - * @ingroup kernel * @def CONFIG_TIME_SLICE * * The `CONFIG_TIME_SLICE` constant defines the default time slice given to threads when they are scheduled. @@ -99,7 +97,6 @@ /** * @brief Cache hot threshold configuration. - * @ingroup kernel * @def CONFIG_CACHE_HOT_THRESHOLD * * The `CONFIG_CACHE_HOT_THRESHOLD` constant defines the threshold below which a time duration is considered "cache @@ -110,7 +107,6 @@ /** * @brief Maximum mutex slow spin configuration. - * @ingroup kernel * @def CONFIG_MUTEX_MAX_SLOW_SPIN * * The `CONFIG_MUTEX_MAX_SLOW_SPIN` constant defines the maximum number of iterations a thread will spin before blocking @@ -121,7 +117,6 @@ /** * @brief Maximum screen lines configuration. - * @ingroup kernel * @def CONFIG_SCREEN_MAX_LINES * * The `CONFIG_SCREEN_MAX_LINES` constant defines the maximum number of lines that the logging system will display. @@ -131,7 +126,6 @@ /** * @brief Maximum bitmap allocator address. - * @ingroup kernel * @def CONFIG_PMM_BITMAP_MAX_ADDR * * The `CONFIG_PMM_BITMAP_MAX_ADDR` constant defines the maximum address below which pages will be handled by the bitmap @@ -142,7 +136,6 @@ /** * @brief Process reaper interval configuration. - * @ingroup kernel * @def CONFIG_PROCESS_REAPER_INTERVAL * * The `CONFIG_PROCESS_REAPER_INTERVAL` constant defines the minimum interval at which the process reaper runs to clean @@ -153,7 +146,6 @@ /** * @brief Maximum environment variables configuration. - * @ingroup kernel * @def CONFIG_MAX_ENV_VARS * * The `CONFIG_MAX_ENV_VARS` constant defines the maximum number of environment variables that a process can have. @@ -163,7 +155,6 @@ /** * @brief Kernel log buffer size configuration. - * @ingroup kernel * @def CONFIG_LOG_KLOG_BUFFER_SIZE * * The `CONFIG_LOG_KLOG_BUFFER_SIZE` constant defines the size of the buffer used for the `/dev/klog` file. @@ -173,7 +164,6 @@ /** * @brief Per-CPU data size configuration. - * @ingroup kernel * @def CONFIG_PERCPU_SIZE * * The `CONFIG_PERCPU_SIZE` constant defines the size allocated for per-CPU data. @@ -183,7 +173,6 @@ /** * @brief Maximum wait queues configuration. - * @ingroup kernel * @def CONFIG_MAX_WAIT_QUEUES * * The `CONFIG_MAX_WAIT_QUEUES` constant defines the maximum amount of wait queues that a thread can wait on @@ -193,22 +182,22 @@ #define CONFIG_MAX_WAIT_QUEUES 64 /** - * @brief Maximum asynchronous rings configuration. - * @ingroup kernel - * @def CONFIG_MAX_ASYNC_RINGS + * @brief Maximum rings configuration. + * @def CONFIG_MAX_RINGS * - * The `CONFIG_MAX_ASYNC_RINGS` constant defines the maximum amount of asynchronous rings that each process can have. + * The `CONFIG_MAX_RINGS` constant defines the maximum amount of asynchronous rings that each process can have. * */ -#define CONFIG_MAX_ASYNC_RINGS 8 +#define CONFIG_MAX_RINGS 8 /** * @brief Maximum async ring pages configuration. - * @ingroup kernel - * @def CONFIG_MAX_ASYNC_PAGES + * @def CONFIG_MAX_RINGS_PAGES * - * The `CONFIG_MAX_ASYNC_PAGES` constant defines the maximum amount of pages that can be allocated for a async rings + * The `CONFIG_MAX_RINGS_PAGES` constant defines the maximum amount of pages that can be allocated for a async rings * buffer. * */ -#define CONFIG_MAX_ASYNC_PAGES 1024 \ No newline at end of file +#define CONFIG_MAX_RINGS_PAGES 1024 + +/** @} */ \ No newline at end of file diff --git a/include/kernel/mem/mem_desc.h b/include/kernel/mem/mem_desc.h index 6248e07ea..f7f8410b1 100644 --- a/include/kernel/mem/mem_desc.h +++ b/include/kernel/mem/mem_desc.h @@ -127,9 +127,18 @@ uint64_t mem_desc_add(mem_desc_t* desc, phys_addr_t phys, size_t size); */ uint64_t mem_desc_add_user(mem_desc_t* desc, process_t* process, const void* addr, size_t size); +/** + * @brief Get a Memory Segment from a Memory Descriptor. + * + * @param desc The Memory Descriptor to get the segment from. + * @param index The index of the segment to get. + * @return On success, a pointer to the Memory Segment. On failure, `NULL`. + */ +mem_seg_t* mem_desc_get_seg(mem_desc_t* desc, size_t index); + /** * @brief Read from a Memory Descriptor into a buffer. - * + * * @param desc The Memory Descriptor to read from. * @param buffer The buffer to read into. * @param count Number of bytes to read. @@ -140,33 +149,74 @@ uint64_t mem_desc_read(mem_desc_t* desc, void* buffer, size_t count, size_t offs /** * @brief Write to a Memory Descriptor from a buffer. - * + * * @param desc The Memory Descriptor to write to. * @param buffer The buffer to write from. * @param count Number of bytes to write. * @param offset Offset within the Memory Descriptor to start writing to. - * @return The number of bytes written. + * @return The number of bytes written. */ uint64_t mem_desc_write(mem_desc_t* desc, const void* buffer, size_t count, size_t offset); /** - * @brief Copy data between two Memory Descriptors. - * - * @param dest The destination Memory Descriptor. - * @param destOffset Offset within the destination Memory Descriptor to start writing to. - * @param src The source Memory Descriptor. - * @param srcOffset Offset within the source Memory Descriptor to start reading from. - * @param count Number of bytes to copy. - * @return The number of bytes copied. + * @brief Memory Descriptor Iterator structure. + * @struct mem_desc_iter_t + */ +typedef struct +{ + mem_desc_t* desc; + size_t segIndex; + size_t segOffset; +} mem_desc_iter_t; + +/** + * @brief Create a Memory Descriptor Iterator initializer. + * + * @param _desc Pointer to the Memory Descriptor to iterate over. + * @return Memory Descriptor Iterator initializer. */ -uint64_t mem_desc_copy(mem_desc_t* dest, size_t destOffset, mem_desc_t* src, size_t srcOffset, size_t count); +#define MEM_DESC_ITER_CREATE(_desc) \ + { \ + .desc = (_desc), \ + .segIndex = 0, \ + .segOffset = 0, \ + } + +/** + * @brief Get the next byte from a Memory Descriptor Iterator. + * + * @param iter Pointer to the Memory Descriptor Iterator. + * @param byte Pointer to store the retrieved byte. + * @return `true` if a byte was retrieved, `false` if the end of the Memory Descriptor was reached. + */ +static inline bool mem_desc_iter_next(mem_desc_iter_t* iter, uint8_t* byte) +{ + mem_seg_t* seg = mem_desc_get_seg(iter->desc, iter->segIndex); + if (seg == NULL) + { + return false; + } + + uint8_t* addr = PFN_TO_VIRT(seg->pfn) + seg->offset + iter->segOffset; + *byte = *(addr); + + iter->segOffset++; + if (iter->segOffset >= seg->size) + { + iter->segIndex++; + iter->segOffset = 0; + } + + return true; +} /** - * @brief Iterate over objects within a Memory Descriptor. + * @brief Iterate over bytes within a Memory Descriptor. * - * @param _element The iterator variable. + * @param _byte The iterator variable. * @param _desc Pointer to the Memory Descriptor. */ -#define MEM_DESC_FOR_EACH(_element, _desc) +#define MEM_DESC_FOR_EACH(_byte, _desc) \ + for (mem_desc_iter_t _iter = MEM_DESC_ITER_CREATE(_desc); mem_desc_iter_next(&_iter, (_byte));) /** @} */ \ No newline at end of file diff --git a/include/kernel/proc/process.h b/include/kernel/proc/process.h index fe92b1d49..95cc9bd7f 100644 --- a/include/kernel/proc/process.h +++ b/include/kernel/proc/process.h @@ -12,9 +12,9 @@ #include #include #include -#include #include #include +#include #include #include @@ -88,7 +88,7 @@ typedef struct process file_table_t fileTable; futex_ctx_t futexCtx; perf_process_ctx_t perf; - async_t async[CONFIG_MAX_ASYNC_RINGS]; + ring_ctx_t rings[CONFIG_MAX_RINGS]; note_handler_t noteHandler; wait_queue_t suspendQueue; wait_queue_t dyingQueue; diff --git a/include/kernel/sync/irp.h b/include/kernel/sync/irp.h index edbb56608..cf433d0a3 100644 --- a/include/kernel/sync/irp.h +++ b/include/kernel/sync/irp.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include typedef struct irp irp_t; @@ -25,7 +25,7 @@ typedef struct irp irp_t; * structure used internally by the kernel for asynchronous operations. * * The IRP system is designed to be generic enough to be used by any system in the kernel, however it is primarily used - * by the asynchronous rings system. + * by the ring system. * * @warning While the cancellation or completion of an IRP is thread safe, the setup of an IRP is not (as in pushing * layers to it). As such, its up to the caller to ensure that only one thread is manipulating it during setup. @@ -134,7 +134,7 @@ typedef struct irp irp_t; * Each time a completion is called via `irp_complete()`, the next completion on the stack is called until the stack is * empty, at which point the IRP is considered fully completed. * - * A real world example of this would be the Async Rings system allocating a IRP, pushing a completion which will add a + * A real world example of this would be the ring system allocating a IRP, pushing a completion which will add a * `cqe_t` to its Rings, before passing the IRP to the VFS which may pass it to a filesystem. Each layer pushing its own * completion to handle its part of the operation. * @@ -238,7 +238,7 @@ typedef struct irp irp_t; * - `ETIMEDOUT`: Operation timed out. * - `EINPROGRESS`: Operation is in a timeout queue. * - * @see kernel_sync_async for the asynchronous rings system. + * @see kernel_sync_ring for the ring system. * @see [Wikipedia](https://en.wikipedia.org/wiki/I/O_request_packet) for more information about IRPs. * @see [Microsoft _IRP](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_irp) for information * on how Windows NT implements IRPs. diff --git a/include/kernel/sync/async.h b/include/kernel/sync/ring.h similarity index 71% rename from include/kernel/sync/async.h rename to include/kernel/sync/ring.h index 5b5cfa67f..3187964a7 100644 --- a/include/kernel/sync/async.h +++ b/include/kernel/sync/ring.h @@ -9,43 +9,43 @@ #include #include -#include +#include /** * @brief Programmable submission/completion interface. - * @defgroup kernel_sync_async Asynchronous Rings + * @defgroup kernel_sync_ring Kernel-Side Ring Interface * @ingroup kernel_sync * - * @todo The rings system is primarily a design document for now as it remains very work in progress and subject to + * @todo The ring system is primarily a design document for now as it remains very work in progress and subject to * change, currently being mostly unimplemented. * - * The Asynchronous rings provide the core of all interfaces in PatchworkOS, all implemented in an interface + * The ring provide the core of all interfaces in PatchworkOS, all implemented in an interface * inspired by `io_uring()` from Linux. * * Synchronous operations are implemented on top of this API in userspace. * - * @see libstd_sys_rings for the userspace interface to the asynchronous rings. + * @see libstd_sys_uring for the userspace interface to the asynchronous ring. * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`. * @see [Manpages](https://man7.org/linux/man-pages/man7/io_uring.7.html) for more information about `io_uring`. * * ## Syncronization * - * The rings structure is designed to be safe under the assumption that there is a single producer (one user-space + * The ring structure is designed to be safe under the assumption that there is a single producer (one user-space * thread) and a single consumer (the kernel). * - * If a rings structure needs multiple producers (needs to be accessed by multiple threads) it is the responsibility of + * If a ring structure needs multiple producers (needs to be accessed by multiple threads) it is the responsibility of * the caller to ensure proper synchronization. * * @note The reason for this limitation is optimization for the common case, as the syncronization logic for multiple * producers would add significant overhead. * - * Regarding the rings structure itself, the structure can only be torndown as long as nothing is using it and there are + * Regarding the ring structure itself, the structure can only be torndown as long as nothing is using it and there are * no pending operations. * * ## Registers * * Operations performed on a ring can load arguments from, and save their results to, seven 64-bit general purpose - * registers. All registers are stored in the shared area of the rings structure, as such they can be inspected and + * registers. All registers are stored in the shared area of the ring structure, as such they can be inspected and * modified by user space. * * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for each argument and @@ -95,47 +95,47 @@ */ /** - * @brief Async context flags. - * @enum async_flags_t + * @brief Ring context flags. + * @enum ring_ctx_flags_t */ typedef enum { - ASYNC_NONE = 0, ///< No flags set. - ASYNC_BUSY = 1 << 0, ///< Context is currently being used, used for fast locking. - ASYNC_MAPPED = 1 << 1, ///< Context rings are mapped. -} async_flags_t; + RING_CTX_NONE = 0, ///< No flags set. + RING_CTX_BUSY = 1 << 0, ///< Context is currently being used, used for fast locking. + RING_CTX_MAPPED = 1 << 1, ///< Context is currently mapped into userspace. +} ring_ctx_flags_t; /** - * @brief The kernel-side asynchronous context structure. - * @struct async_t + * @brief The kernel-side ring context structure. + * @struct ring_ctx_t */ -typedef struct async +typedef struct ring_ctx { - rings_t rings; ///< Asynchronous rings information. + ring_t ring; ///< The kernel-side ring structure. irp_pool_t* irps; ///< Pool of preallocated IRPs. mem_desc_pool_t* descs; ///< Pool of preallocated memory descriptors. - void* userAddr; ///< Userspace address of the rings. - void* kernelAddr; ///< Kernel address of the rings. - size_t pageAmount; ///< Amount of pages mapped for the rings. + void* userAddr; ///< Userspace address of the ring. + void* kernelAddr; ///< Kernel address of the ring. + size_t pageAmount; ///< Amount of pages mapped for the ring. space_t* space; ///< Pointer to the owning address space. wait_queue_t waitQueue; ///< Wait queue for completions. process_t* process; ///< Holds a reference to the owner process while there are pending requests. - _Atomic(async_flags_t) flags; -} async_t; + _Atomic(ring_ctx_flags_t) flags; +} ring_ctx_t; /** - * @brief Initialize a async context. + * @brief Initialize a ring context. * * @param ctx Pointer to the context to initialize. */ -void async_init(async_t* ctx); +void ring_ctx_init(ring_ctx_t* ctx); /** - * @brief Deinitialize a async context. + * @brief Deinitialize a ring context. * * @param ctx Pointer to the context to deinitialize. */ -void async_deinit(async_t* ctx); +void ring_ctx_deinit(ring_ctx_t* ctx); /** * @brief Notify the context of new SQEs. @@ -145,6 +145,6 @@ void async_deinit(async_t* ctx); * @param wait The minimum number of CQEs to wait for. * @return On success, the number of SQEs processed. On failure, `ERR` and `errno` is set. */ -uint64_t async_notify(async_t* ctx, size_t amount, size_t wait); +uint64_t ring_ctx_notify(ring_ctx_t* ctx, size_t amount, size_t wait); /** @} */ \ No newline at end of file diff --git a/include/libstd/sys/rings.h b/include/libstd/sys/uring.h similarity index 59% rename from include/libstd/sys/rings.h rename to include/libstd/sys/uring.h index acdd280d9..4463d858a 100644 --- a/include/libstd/sys/rings.h +++ b/include/libstd/sys/uring.h @@ -1,5 +1,5 @@ -#ifndef _SYS_RINGS_H -#define _SYS_RINGS_H 1 +#ifndef _SYS_URING_H +#define _SYS_URING_H 1 #include #include @@ -19,7 +19,7 @@ extern "C" #include "_internal/fd_t.h" /** - * @addtogroup kernel_sync_async + * @addtogroup kernel_sync_ring * @{ */ @@ -29,7 +29,7 @@ typedef uint32_t verb_t; ///< Verb type. #define VERB_OPEN 1 ///< Open file verb. #define VERB_MAX 1 ///< Maximum verb. -#define SEQ_MAX_ARGS 5 ///< Maximum number of arguments for a rings operation. +#define SQE_MAX_ARGS 5 ///< Maximum number of arguments for a ring operation. typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. @@ -41,31 +41,40 @@ typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. #define SQE_REG5 (5) ///< The sixth register. #define SQE_REG6 (6) ///< The seventh register. #define SQE_REG_NONE (7) ///< No register. -#define SEQ_REGS_MAX (7) ///< The maximum number of registers. +#define SQE_REGS_MAX (7) ///< The maximum number of registers. #define SQE_REG_SHIFT (3) ///< The bitshift for each register specifier in a `sqe_flags_t`. #define SQE_REG_MASK (0b111) ///< The bitmask for a register specifier in a `sqe_flags_t`. -#define SQE_LOAD0 (0) ///< The offset to specify which register to load into the first argument. -#define SQE_LOAD1 \ - (SQE_LOAD0 + SQE_REG_SHIFT) ///< The offset to specify which register to load into the second argument. -#define SQE_LOAD2 (SQE_LOAD1 + SQE_REG_SHIFT) ///< The offset to specify which register to load into the third argument. -#define SQE_LOAD3 \ - (SQE_LOAD2 + SQE_REG_SHIFT) ///< The offset to specify which register to load into the fourth argument. -#define SQE_LOAD4 (SQE_LOAD3 + SQE_REG_SHIFT) ///< The offset to specify which register to load into the fifth argument. +#define SQE_LOAD0 (0) ///< The offset to specify the register to load into the first argument. +#define SQE_LOAD1 (SQE_LOAD0 + SQE_REG_SHIFT) ///< The offset to specify the register to load into the second argument. +#define SQE_LOAD2 (SQE_LOAD1 + SQE_REG_SHIFT) ///< The offset to specify the register to load into the third argument. +#define SQE_LOAD3 (SQE_LOAD2 + SQE_REG_SHIFT) ///< The offset to specify the register to load into the fourth argument. +#define SQE_LOAD4 (SQE_LOAD3 + SQE_REG_SHIFT) ///< The offset to specify the register to load into the fifth argument. #define SQE_SAVE (SQE_LOAD4 + SQE_REG_SHIFT) ///< The offset to specify the register to save the result into. -#define SQE_FLAGS_SHIFT (SQE_SAVE + SQE_REG_SHIFT) ///< The bitshift for where bit flags start in a `sqe_flags_t`. -#define SQE_LINK \ - (1 << (SQE_FLAGS_SHIFT)) ///< Only process the next SQE when this one completes successfully) only - /// applies within one `enter()` call. -#define SQE_HARDLINK \ - (1 << (SQE_FLAGS_SHIFT + 1)) ///< Like `SQE_LINK`) but will process the next SQE even if this one fails. + +#define _SEQ_FLAGS (SQE_SAVE + SQE_REG_SHIFT) ///< The bitshift for where bit flags start in a `sqe_flags_t`. + #ifdef _KERNEL_ -#define SQE_KERNEL \ - (1 << (SQE_FLAGS_SHIFT + 2)) ///< The operation was created by the kernel, used internally by the kernel. -#define SQE_KERNEL_ENTERED \ - (1 << (SQE_FLAGS_SHIFT + 3)) ///< The operations enter callback has been called, used internally by the kernel. +/** + * The operation was created by the kernel, used internally by the kernel. + */ +#define SQE_KERNEL (1 << (_SEQ_FLAGS)) + +/** + * The operations enter callback has been called, used internally by the kernel. + */ +#define SQE_KERNEL_ENTERED (1 << (_SEQ_FLAGS + 1)) #endif +/** + * Only process the next SQE when this one completes successfully) only applies within one `enter()` call. + */ +#define SQE_LINK (1 << (_SEQ_FLAGS + 2)) +/** + * Like `SQE_LINK` but will process the next SQE even if this one fails. + */ +#define SQE_HARDLINK (1 << (_SEQ_FLAGS + 3)) + /** * @brief Asynchronous submission queue entry (SQE). * @struct sqe_t @@ -92,7 +101,7 @@ typedef struct sqe char* path; size_t length; } open; - uint64_t _args[SEQ_MAX_ARGS]; + uint64_t _args[SQE_MAX_ARGS]; }; } sqe_t; @@ -139,20 +148,20 @@ static_assert(sizeof(cqe_t) == 32, "cqe_t is not 32 bytes"); #endif /** - * @brief Rings ID type. + * @brief User-Rings ID type. */ -typedef uint64_t rings_id_t; +typedef uint64_t ring_id_t; /** - * @brief Shared asynchronous rings structure. - * @struct rings_shared_t + * @brief Shared ring control structure. + * @struct ring_ctrl_t * * Used as the intermediate between userspace and the kernel. * * @note The structure is aligned in such a way to reduce false sharing. * */ -typedef struct ALIGNED(64) rings_shared +typedef struct ALIGNED(64) ring_ctrl { atomic_uint32_t shead; ///< Submission head index, updated by the kernel. atomic_uint32_t ctail; ///< Completion tail index, updated by the kernel. @@ -160,41 +169,41 @@ typedef struct ALIGNED(64) rings_shared atomic_uint32_t stail; ///< Submission tail index, updated by userspace. atomic_uint32_t chead; ///< Completion head index, updated by userspace. uint8_t _padding1[64 - sizeof(atomic_uint32_t) * 2]; - atomic_uint64_t regs[SEQ_REGS_MAX] ALIGNED(64); ///< General purpose registers. + atomic_uint64_t regs[SQE_REGS_MAX] ALIGNED(64); ///< General purpose registers. uint8_t _reserved[8]; -} rings_shared_t; - -/** - * @brief Asynchronous rings structure. - * @struct rings_t - * - * The kernel and userspace will have their own instances of this structure. - */ -typedef struct rings -{ - rings_shared_t* shared; ///< Pointer to the shared structure. - rings_id_t id; ///< The ID of the rings. - sqe_t* squeue; ///< Pointer to the submission queue. - size_t sentries; ///< Number of entries in the submission queue. - size_t smask; ///< Bitmask for submission queue (sentries - 1). - cqe_t* cqueue; ///< Pointer to the completion queue. - size_t centries; ///< Number of entries in the completion queue. - size_t cmask; ///< Bitmask for completion queue (centries - 1). -} rings_t; +} ring_ctrl_t; /** * @} - * @brief User-side asynchronous rings interface. - * @defgroup libstd_sys_rings User Asynchronous Rings + * @brief User-side asynchronous ring interface. + * @defgroup libstd_sys_uring User-Side Ring Interface * @ingroup libstd * - * The rings interface acts as the interface for all asynchronous operations in the kernel. + * The ring interface acts as the interface for all asynchronous operations in the kernel. * - * @see kernel_sync_async for more information about the asynchronous rings system. + * @see kernel_sync_ring for more information about the asynchronous ring system. * * @{ */ +/** + * @brief User asynchronous ring structure. + * @struct ring_t + * + * The kernel and userspace will have their own instances of this structure. + */ +typedef struct ring +{ + ring_ctrl_t* ctrl; ///< Pointer to the shared control structure. + ring_id_t id; ///< The ID of the ring. + sqe_t* squeue; ///< Pointer to the submission queue. + size_t sentries; ///< Number of entries in the submission queue. + size_t smask; ///< Bitmask for submission queue (sentries - 1). + cqe_t* cqueue; ///< Pointer to the completion queue. + size_t centries; ///< Number of entries in the completion queue. + size_t cmask; ///< Bitmask for completion queue (centries - 1). +} ring_t; + /** * @brief Dont wait for any submissions to complete. */ @@ -206,58 +215,58 @@ typedef struct rings #define WAIT_ONE 0x1 /** - * @brief System call to initialize the asynchronous rings. + * @brief System call to initialize the asynchronous ring. * * This system call will populate the given structure with the necessary pointers and metadata for the submission and - * completion rings. + * completion ring. * - * @param rings Pointer to the structure to populate. - * @param address Desired address to allocate the rings, or `NULL` to let the kernel choose. + * @param ring Pointer to the ring structure to populate. + * @param address Desired address to allocate the ring, or `NULL` to let the kernel choose. * @param sentries Number of entires to allocate for the submission queue, must be a power of two. * @param centries Number of entries to allocate for the completion queue, must be a power of two. * @return On success, the ring ID. On failure, `ERR` and `errno` is set. */ -rings_id_t setup(rings_t* rings, void* address, size_t sentries, size_t centries); +ring_id_t setup(ring_t* ring, void* address, size_t sentries, size_t centries); /** - * @brief System call to deinitialize the asynchronous rings. + * @brief System call to deinitialize the asynchronous ring. * - * @param id The ID of the rings to deinitialize. + * @param id The ID of the ring to deinitialize. * @return On success, `0`. On failure, `ERR` and `errno` is set. */ -uint64_t teardown(rings_id_t id); +uint64_t teardown(ring_id_t id); /** * @brief System call to notify the kernel of new submission queue entries (SQEs). * - * @param id The ID of the rings to notify. + * @param id The ID of the ring to notify. * @param amount The number of SQEs that the kernel should process. * @param wait The minimum number of completion queue entries (CQEs) to wait for. * @return On success, the number of SQEs successfully processed. On failure, `ERR` and `errno` is set. */ -uint64_t enter(rings_id_t id, size_t amount, size_t wait); +uint64_t enter(ring_id_t id, size_t amount, size_t wait); /** * @brief Pushes a submission queue entry (SQE) to the submission queue. * * After pushing SQEs, `enter()` must be called to notify the kernel of the new entries. * - * @param rings Pointer to the asynchronous rings structure. + * @param ring Pointer to the asynchronous ring structure. * @param sqe Pointer to the SQE to push. * @return `true` if the SQE was pushed, `false` if the submission queue is full. */ -static inline bool sqe_push(rings_t* rings, sqe_t* sqe) +static inline bool sqe_push(ring_t* ring, sqe_t* sqe) { - uint32_t tail = atomic_load_explicit(&rings->shared->stail, memory_order_relaxed); - uint32_t head = atomic_load_explicit(&rings->shared->shead, memory_order_acquire); + uint32_t tail = atomic_load_explicit(&ring->ctrl->stail, memory_order_relaxed); + uint32_t head = atomic_load_explicit(&ring->ctrl->shead, memory_order_acquire); - if ((tail - head) >= rings->sentries) + if ((tail - head) >= ring->sentries) { return false; } - rings->squeue[tail & rings->smask] = *sqe; - atomic_store_explicit(&rings->shared->stail, tail + 1, memory_order_release); + ring->squeue[tail & ring->smask] = *sqe; + atomic_store_explicit(&ring->ctrl->stail, tail + 1, memory_order_release); return true; } @@ -265,22 +274,22 @@ static inline bool sqe_push(rings_t* rings, sqe_t* sqe) /** * @brief Pops a completion queue entry (CQE) from the completion queue. * - * @param rings Pointer to the asynchronous rings structure. + * @param ring Pointer to the asynchronous ring structure. * @param cqe Pointer to the CQE to pop. * @return `true` if a CQE was popped, `false` if the completion queue is empty. */ -static inline bool cqe_pop(rings_t* rings, cqe_t* cqe) +static inline bool cqe_pop(ring_t* ring, cqe_t* cqe) { - uint32_t head = atomic_load_explicit(&rings->shared->chead, memory_order_relaxed); - uint32_t tail = atomic_load_explicit(&rings->shared->ctail, memory_order_acquire); + uint32_t head = atomic_load_explicit(&ring->ctrl->chead, memory_order_relaxed); + uint32_t tail = atomic_load_explicit(&ring->ctrl->ctail, memory_order_acquire); if (head == tail) { return false; } - *cqe = rings->cqueue[head & rings->cmask]; - atomic_store_explicit(&rings->shared->chead, head + 1, memory_order_release); + *cqe = ring->cqueue[head & ring->cmask]; + atomic_store_explicit(&ring->ctrl->chead, head + 1, memory_order_release); return true; } diff --git a/meta/doxy/Doxyfile b/meta/doxy/Doxyfile index b9a782793..afeb3f8cd 100644 --- a/meta/doxy/Doxyfile +++ b/meta/doxy/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = "PatchworkOS" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "de3685e6-dirty" +PROJECT_NUMBER = "30cbaaa8-dirty" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewers a diff --git a/src/kernel/mem/mem_desc.c b/src/kernel/mem/mem_desc.c index 68f32bfdb..700624b24 100644 --- a/src/kernel/mem/mem_desc.c +++ b/src/kernel/mem/mem_desc.c @@ -7,21 +7,6 @@ #include #include -static inline mem_seg_t* mem_desc_get_seg(mem_desc_t* desc, size_t index) -{ - if (index >= desc->amount) - { - return NULL; - } - - if (index < MEM_SEGS_SMALL_MAX) - { - return &desc->small[index]; - } - - return &desc->large[index - MEM_SEGS_SMALL_MAX]; -} - mem_desc_pool_t* mem_desc_pool_new(size_t size) { size_t poolSize = sizeof(mem_desc_pool_t) + (size * sizeof(mem_desc_t)); @@ -100,7 +85,13 @@ void mem_desc_free(mem_desc_t* desc) uint64_t mem_desc_add(mem_desc_t* desc, phys_addr_t phys, size_t size) { - if (desc == NULL) + if (size > UINT32_MAX) + { + errno = EOVERFLOW; + return ERR; + } + + if (desc == NULL || desc->size + size < desc->size) { errno = EINVAL; return ERR; @@ -109,11 +100,11 @@ uint64_t mem_desc_add(mem_desc_t* desc, phys_addr_t phys, size_t size) mem_seg_t* seg = NULL; if (desc->amount < MEM_SEGS_SMALL_MAX) { - seg = &desc->small[desc->amount++]; + seg = &desc->small[desc->amount]; } else if (desc->amount - MEM_SEGS_SMALL_MAX < desc->capacity) { - seg = &desc->large[desc->amount++ - MEM_SEGS_SMALL_MAX]; + seg = &desc->large[desc->amount - MEM_SEGS_SMALL_MAX]; } else { @@ -126,7 +117,7 @@ uint64_t mem_desc_add(mem_desc_t* desc, phys_addr_t phys, size_t size) desc->large = newLarge; desc->capacity += 4; - seg = &desc->large[desc->amount++ - MEM_SEGS_SMALL_MAX]; + seg = &desc->large[desc->amount - MEM_SEGS_SMALL_MAX]; } pfn_t pfn = PHYS_TO_PFN(phys); @@ -141,6 +132,8 @@ uint64_t mem_desc_add(mem_desc_t* desc, phys_addr_t phys, size_t size) seg->size = size; seg->offset = offset; desc->size += size; + + desc->amount++; return 0; } @@ -172,28 +165,95 @@ uint64_t mem_desc_add_user(mem_desc_t* desc, process_t* process, const void* add return 0; } -uint64_t mem_desc_read(mem_desc_t* desc, void* buffer, size_t count, size_t offset) +mem_seg_t* mem_desc_get_seg(mem_desc_t* desc, size_t index) { - if (desc == NULL || buffer == NULL || offset + count > desc->size) + if (index >= desc->amount) { - return 0; + return NULL; } + if (index < MEM_SEGS_SMALL_MAX) + { + return &desc->small[index]; + } + + return &desc->large[index - MEM_SEGS_SMALL_MAX]; } -uint64_t mem_desc_write(mem_desc_t* desc, const void* buffer, size_t count, size_t offset) +uint64_t mem_desc_read(mem_desc_t* desc, void* buffer, size_t count, size_t offset) { - if (desc == NULL || buffer == NULL || offset + count > desc->size) + if (desc == NULL || buffer == NULL || offset > desc->size || count > desc->size - offset) { return 0; } + + size_t start = 0; + size_t i = 0; + for (; i < desc->amount; i++) + { + mem_seg_t* seg = mem_desc_get_seg(desc, i); + if (start + seg->size > offset) + { + break; + } + start += seg->size; + } + + uint8_t* ptr = buffer; + size_t remaining = count; + + size_t segOffset = offset - start; + while (remaining > 0 && i < desc->amount) + { + mem_seg_t* seg = mem_desc_get_seg(desc, i); + size_t toRead = MIN(remaining, seg->size - segOffset); + void* addr = PFN_TO_VIRT(seg->pfn) + seg->offset + segOffset; + memcpy(ptr, addr, toRead); + + ptr += toRead; + remaining -= toRead; + segOffset = 0; + i++; + } + + return count - remaining; } - uint64_t mem_desc_copy(mem_desc_t* dest, size_t destOffset, mem_desc_t* src, size_t srcOffset, - size_t count) +uint64_t mem_desc_write(mem_desc_t* desc, const void* buffer, size_t count, size_t offset) { - if (dest == NULL || src == NULL || destOffset + count > dest->size || srcOffset + count > src->size) + if (desc == NULL || buffer == NULL || offset > desc->size || count > desc->size - offset) { return 0; } + + size_t start = 0; + size_t i = 0; + for (; i < desc->amount; i++) + { + mem_seg_t* seg = mem_desc_get_seg(desc, i); + if (start + seg->size > offset) + { + break; + } + start += seg->size; + } + + const uint8_t* ptr = buffer; + size_t remaining = count; + + size_t segOffset = offset - start; + while (remaining > 0 && i < desc->amount) + { + mem_seg_t* seg = mem_desc_get_seg(desc, i); + size_t toWrite = MIN(remaining, seg->size - segOffset); + void* addr = PFN_TO_VIRT(seg->pfn) + seg->offset + segOffset; + memcpy(addr, ptr, toWrite); + + ptr += toWrite; + remaining -= toWrite; + segOffset = 0; + i++; + } + + return count - remaining; } \ No newline at end of file diff --git a/src/kernel/proc/process.c b/src/kernel/proc/process.c index 700dd20a9..876320606 100644 --- a/src/kernel/proc/process.c +++ b/src/kernel/proc/process.c @@ -19,9 +19,9 @@ #include #include #include -#include #include #include +#include #include #include @@ -111,9 +111,9 @@ static void process_free(process_t* process) } space_deinit(&process->space); futex_ctx_deinit(&process->futexCtx); - for (uint64_t i = 0; i < CONFIG_MAX_ASYNC_RINGS; i++) + for (uint64_t i = 0; i < ARRAY_SIZE(process->rings); i++) { - async_deinit(&process->async[i]); + ring_ctx_deinit(&process->rings[i]); } wait_queue_deinit(&process->dyingQueue); wait_queue_deinit(&process->suspendQueue); @@ -154,9 +154,9 @@ process_t* process_new(priority_t priority, group_member_t* group, namespace_t* file_table_init(&process->fileTable); futex_ctx_init(&process->futexCtx); perf_process_ctx_init(&process->perf); - for (uint64_t i = 0; i < CONFIG_MAX_ASYNC_RINGS; i++) + for (uint64_t i = 0; i < ARRAY_SIZE(process->rings); i++) { - async_init(&process->async[i]); + ring_ctx_init(&process->rings[i]); } note_handler_init(&process->noteHandler); wait_queue_init(&process->suspendQueue); diff --git a/src/kernel/sync/async.c b/src/kernel/sync/ring.c similarity index 51% rename from src/kernel/sync/async.c rename to src/kernel/sync/ring.c index 1b96a622d..f08899728 100644 --- a/src/kernel/sync/async.c +++ b/src/kernel/sync/ring.c @@ -9,18 +9,18 @@ #include #include #include -#include #include +#include #include #include -#include +#include #include -static inline uint64_t async_acquire(async_t* ctx) +static inline uint64_t ring_ctx_acquire(ring_ctx_t* ctx) { - async_flags_t expected = atomic_load(&ctx->flags); - if (!(expected & ASYNC_BUSY) && atomic_compare_exchange_strong(&ctx->flags, &expected, expected | ASYNC_BUSY)) + ring_ctx_flags_t expected = atomic_load(&ctx->flags); + if (!(expected & RING_CTX_BUSY) && atomic_compare_exchange_strong(&ctx->flags, &expected, expected | RING_CTX_BUSY)) { return 0; } @@ -28,19 +28,18 @@ static inline uint64_t async_acquire(async_t* ctx) return ERR; } -static inline void async_release(async_t* ctx) +static inline void ring_ctx_release(ring_ctx_t* ctx) { - atomic_fetch_and(&ctx->flags, ~ASYNC_BUSY); + atomic_fetch_and(&ctx->flags, ~RING_CTX_BUSY); } -static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, rings_t* userRings, void* address, +static inline uint64_t ring_ctx_map(ring_ctx_t* ctx, space_t* space, ring_id_t id, ring_t* userRing, void* address, size_t sentries, size_t centries) { - rings_t* kernelRings = &ctx->rings; + ring_t* kernelRing = &ctx->ring; - size_t pageAmount = - BYTES_TO_PAGES(sizeof(rings_shared_t) + (sentries * sizeof(sqe_t)) + (centries * sizeof(cqe_t))); - if (pageAmount >= CONFIG_MAX_ASYNC_PAGES) + size_t pageAmount = BYTES_TO_PAGES(sizeof(ring_ctrl_t) + (sentries * sizeof(sqe_t)) + (centries * sizeof(cqe_t))); + if (pageAmount >= CONFIG_MAX_RINGS_PAGES) { errno = ENOMEM; return ERR; @@ -52,7 +51,7 @@ static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, ri return ERR; } - pfn_t pages[CONFIG_MAX_ASYNC_PAGES]; + pfn_t pages[CONFIG_MAX_RINGS_PAGES]; if (pmm_alloc_pages(pages, pageAmount) == ERR) { errno = ENOMEM; @@ -97,33 +96,33 @@ static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, ri return ERR; } - rings_shared_t* shared = (rings_shared_t*)kernelAddr; - atomic_init(&shared->shead, 0); - atomic_init(&shared->stail, 0); - atomic_init(&shared->ctail, 0); - atomic_init(&shared->chead, 0); - for (size_t i = 0; i < SEQ_REGS_MAX; i++) + ring_ctrl_t* ctrl = (ring_ctrl_t*)kernelAddr; + atomic_init(&ctrl->shead, 0); + atomic_init(&ctrl->stail, 0); + atomic_init(&ctrl->ctail, 0); + atomic_init(&ctrl->chead, 0); + for (size_t i = 0; i < SQE_REGS_MAX; i++) { - atomic_init(&shared->regs[i], 0); + atomic_init(&ctrl->regs[i], 0); } - userRings->shared = userAddr; - userRings->id = id; - userRings->squeue = (sqe_t*)((uintptr_t)userAddr + sizeof(rings_shared_t)); - userRings->sentries = sentries; - userRings->smask = sentries - 1; - userRings->cqueue = (cqe_t*)((uintptr_t)userAddr + sizeof(rings_shared_t) + (sentries * sizeof(sqe_t))); - userRings->centries = centries; - userRings->cmask = centries - 1; + userRing->ctrl = userAddr; + userRing->id = id; + userRing->squeue = (sqe_t*)((uintptr_t)userAddr + sizeof(ring_ctrl_t)); + userRing->sentries = sentries; + userRing->smask = sentries - 1; + userRing->cqueue = (cqe_t*)((uintptr_t)userAddr + sizeof(ring_ctrl_t) + (sentries * sizeof(sqe_t))); + userRing->centries = centries; + userRing->cmask = centries - 1; - kernelRings->shared = kernelAddr; - kernelRings->id = id; - kernelRings->squeue = (sqe_t*)((uintptr_t)kernelAddr + sizeof(rings_shared_t)); - kernelRings->sentries = sentries; - kernelRings->smask = sentries - 1; - kernelRings->cqueue = (cqe_t*)((uintptr_t)kernelAddr + sizeof(rings_shared_t) + (sentries * sizeof(sqe_t))); - kernelRings->centries = centries; - kernelRings->cmask = centries - 1; + kernelRing->ctrl = kernelAddr; + kernelRing->id = id; + kernelRing->squeue = (sqe_t*)((uintptr_t)kernelAddr + sizeof(ring_ctrl_t)); + kernelRing->sentries = sentries; + kernelRing->smask = sentries - 1; + kernelRing->cqueue = (cqe_t*)((uintptr_t)kernelAddr + sizeof(ring_ctrl_t) + (sentries * sizeof(sqe_t))); + kernelRing->centries = centries; + kernelRing->cmask = centries - 1; ctx->irps = irps; ctx->descs = descs; @@ -132,11 +131,11 @@ static inline uint64_t async_map(async_t* ctx, space_t* space, rings_id_t id, ri ctx->pageAmount = pageAmount; ctx->space = space; - atomic_fetch_or(&ctx->flags, ASYNC_MAPPED); + atomic_fetch_or(&ctx->flags, RING_CTX_MAPPED); return 0; } -static inline uint64_t async_unmap(async_t* ctx) +static inline uint64_t ring_ctx_unmap(ring_ctx_t* ctx) { irp_pool_free(ctx->irps); ctx->irps = NULL; @@ -147,90 +146,90 @@ static inline uint64_t async_unmap(async_t* ctx) vmm_unmap(ctx->space, ctx->userAddr, ctx->pageAmount * PAGE_SIZE); vmm_unmap(NULL, ctx->kernelAddr, ctx->pageAmount * PAGE_SIZE); - atomic_fetch_and(&ctx->flags, ~ASYNC_MAPPED); + atomic_fetch_and(&ctx->flags, ~RING_CTX_MAPPED); return 0; } -static inline uint64_t async_avail_cqes(async_t* ctx) +static inline uint64_t ring_ctx_avail_cqes(ring_ctx_t* ctx) { - rings_t* rings = &ctx->rings; - uint32_t ctail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); - uint32_t chead = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); + ring_t* ring = &ctx->ring; + uint32_t ctail = atomic_load_explicit(&ring->ctrl->ctail, memory_order_relaxed); + uint32_t chead = atomic_load_explicit(&ring->ctrl->chead, memory_order_acquire); return ctail - chead; } -void async_init(async_t* ctx) +void ring_ctx_init(ring_ctx_t* ctx) { if (ctx == NULL) { return; } - ctx->rings = (rings_t){0}; + ctx->ring = (ring_t){0}; ctx->irps = NULL; ctx->userAddr = NULL; ctx->kernelAddr = NULL; ctx->pageAmount = 0; ctx->space = NULL; wait_queue_init(&ctx->waitQueue); - atomic_init(&ctx->flags, ASYNC_NONE); + atomic_init(&ctx->flags, RING_CTX_NONE); } -void async_deinit(async_t* ctx) +void ring_ctx_deinit(ring_ctx_t* ctx) { if (ctx == NULL) { return; } - if (async_acquire(ctx) == ERR) + if (ring_ctx_acquire(ctx) == ERR) { panic(NULL, "failed to acquire async context for deinitialization"); } - if (atomic_load(&ctx->flags) & ASYNC_MAPPED) + if (atomic_load(&ctx->flags) & RING_CTX_MAPPED) { - if (async_unmap(ctx) == ERR) + if (ring_ctx_unmap(ctx) == ERR) { panic(NULL, "failed to deinitialize async context"); } } - async_release(ctx); + ring_ctx_release(ctx); wait_queue_deinit(&ctx->waitQueue); } -static void async_dispatch(irp_t* irp); +static void ring_ctx_dispatch(irp_t* irp); -static void async_complete(irp_t* irp, void* _ptr) +static void ring_ctx_complete(irp_t* irp, void* _ptr) { UNUSED(_ptr); - async_t* ctx = irp_get_ctx(irp); + ring_ctx_t* ctx = irp_get_ctx(irp); + ring_t* ring = &ctx->ring; sqe_flags_t reg = (irp->flags >> SQE_SAVE) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - atomic_store_explicit(&ctx->rings.shared->regs[reg], irp->result, memory_order_release); + atomic_store_explicit(&ring->ctrl->regs[reg], irp->result, memory_order_release); } - rings_t* rings = &ctx->rings; - uint32_t tail = atomic_load_explicit(&rings->shared->ctail, memory_order_relaxed); - uint32_t head = atomic_load_explicit(&rings->shared->chead, memory_order_acquire); + uint32_t tail = atomic_load_explicit(&ring->ctrl->ctail, memory_order_relaxed); + uint32_t head = atomic_load_explicit(&ring->ctrl->chead, memory_order_acquire); - if ((tail - head) >= rings->centries) + if ((tail - head) >= ring->centries) { /// @todo Handle overflow properly. panic(NULL, "Async completion queue overflow"); } - cqe_t* cqe = &rings->cqueue[tail & rings->cmask]; + cqe_t* cqe = &ring->cqueue[tail & ring->cmask]; cqe->verb = irp->verb; cqe->error = irp->err; cqe->data = irp->data; cqe->_result = irp->result; - atomic_store_explicit(&rings->shared->ctail, tail + 1, memory_order_release); + atomic_store_explicit(&ring->ctrl->ctail, tail + 1, memory_order_release); wait_unblock(&ctx->waitQueue, WAIT_ALL, EOK); if (irp->err != EOK && !(irp->flags & SQE_HARDLINK)) @@ -251,7 +250,7 @@ static void async_complete(irp_t* irp, void* _ptr) irp_t* next = irp_next(irp); if (next != NULL) { - async_dispatch(next); + ring_ctx_dispatch(next); } } @@ -264,11 +263,12 @@ static void async_complete(irp_t* irp, void* _ptr) } } -static void async_dispatch(irp_t* irp) +static void ring_ctx_dispatch(irp_t* irp) { - async_t* ctx = irp_get_ctx(irp); + ring_ctx_t* ctx = irp_get_ctx(irp); + ring_t* ring = &ctx->ring; - for (uint64_t i = 0; i < SEQ_MAX_ARGS; i++) + for (uint64_t i = 0; i < SQE_MAX_ARGS; i++) { sqe_flags_t reg = (irp->flags >> (i * SQE_REG_SHIFT)) & SQE_REG_MASK; if (reg == SQE_REG_NONE) @@ -276,10 +276,10 @@ static void async_dispatch(irp_t* irp) continue; } - irp->sqe._args[i] = atomic_load_explicit(&ctx->rings.shared->regs[reg], memory_order_acquire); + irp->sqe._args[i] = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } - irp_push(irp, async_complete, NULL); + irp_push(irp, ring_ctx_complete, NULL); irp_dispatch(irp); } @@ -287,20 +287,18 @@ typedef struct { list_t irps; irp_t* link; -} async_notify_ctx_t; +} ring_ctx_notify_ctx_t; -static uint64_t async_sqe_pop(async_t* ctx, async_notify_ctx_t* notify) +static uint64_t ring_ctx_sqe_pop(ring_ctx_t* ctx, ring_ctx_notify_ctx_t* notify) { if (atomic_load(&ctx->irps->pool.used) == 1) { - process_t* process = process_current(); - assert(&process->async[0] <= ctx && ctx <= &process->async[CONFIG_MAX_ASYNC_RINGS - 1]); - ctx->process = REF(process); + ctx->process = REF(process_current()); } - rings_t* rings = &ctx->rings; - uint32_t stail = atomic_load_explicit(&rings->shared->stail, memory_order_acquire); - uint32_t shead = atomic_load_explicit(&rings->shared->shead, memory_order_relaxed); + ring_t* ring = &ctx->ring; + uint32_t stail = atomic_load_explicit(&ring->ctrl->stail, memory_order_acquire); + uint32_t shead = atomic_load_explicit(&ring->ctrl->shead, memory_order_relaxed); if (shead == stail) { @@ -308,14 +306,14 @@ static uint64_t async_sqe_pop(async_t* ctx, async_notify_ctx_t* notify) return ERR; } - sqe_t* sqe = &rings->squeue[shead & rings->smask]; + sqe_t* sqe = &ring->squeue[shead & ring->smask]; irp_t* irp = irp_new(ctx->irps, sqe); if (irp == NULL) { return ERR; } - atomic_store_explicit(&rings->shared->shead, shead + 1, memory_order_release); + atomic_store_explicit(&ring->ctrl->shead, shead + 1, memory_order_release); if (notify->link != NULL) { @@ -335,36 +333,36 @@ static uint64_t async_sqe_pop(async_t* ctx, async_notify_ctx_t* notify) return 0; } -uint64_t async_notify(async_t* ctx, size_t amount, size_t wait) +uint64_t ring_ctx_notify(ring_ctx_t* ctx, size_t amount, size_t wait) { if (amount == 0) { return 0; } - if (async_acquire(ctx) == ERR) + if (ring_ctx_acquire(ctx) == ERR) { errno = EBUSY; return ERR; } - if (!(atomic_load(&ctx->flags) & ASYNC_MAPPED)) + if (!(atomic_load(&ctx->flags) & RING_CTX_MAPPED)) { - async_release(ctx); + ring_ctx_release(ctx); errno = EINVAL; return ERR; } size_t processed = 0; - async_notify_ctx_t notify = { + ring_ctx_notify_ctx_t notify = { .irps = LIST_CREATE(notify.irps), .link = NULL, }; while (processed < amount) { - if (async_sqe_pop(ctx, ¬ify) == ERR) + if (ring_ctx_sqe_pop(ctx, ¬ify) == ERR) { break; } @@ -374,28 +372,28 @@ uint64_t async_notify(async_t* ctx, size_t amount, size_t wait) while (!list_is_empty(¬ify.irps)) { irp_t* irp = CONTAINER_OF(list_pop_front(¬ify.irps), irp_t, entry); - async_dispatch(irp); + ring_ctx_dispatch(irp); } if (wait == 0) { - async_release(ctx); + ring_ctx_release(ctx); return processed; } - if (WAIT_BLOCK(&ctx->waitQueue, async_avail_cqes(ctx) >= wait) == ERR) + if (WAIT_BLOCK(&ctx->waitQueue, ring_ctx_avail_cqes(ctx) >= wait) == ERR) { - async_release(ctx); + ring_ctx_release(ctx); return processed > 0 ? processed : ERR; } - async_release(ctx); + ring_ctx_release(ctx); return processed; } -SYSCALL_DEFINE(SYS_SETUP, rings_id_t, rings_t* userRings, void* address, size_t sentries, size_t centries) +SYSCALL_DEFINE(SYS_SETUP, ring_id_t, ring_t* userRing, void* address, size_t sentries, size_t centries) { - if (userRings == NULL || sentries == 0 || centries == 0 || !IS_POW2(sentries) || !IS_POW2(centries)) + if (userRing == NULL || sentries == 0 || centries == 0 || !IS_POW2(sentries) || !IS_POW2(centries)) { errno = EINVAL; return ERR; @@ -404,14 +402,14 @@ SYSCALL_DEFINE(SYS_SETUP, rings_id_t, rings_t* userRings, void* address, size_t process_t* process = process_current(); space_t* space = &process->space; - async_t* ctx = NULL; - rings_id_t id = 0; - for (id = 0; id < CONFIG_MAX_ASYNC_RINGS; id++) + ring_ctx_t* ctx = NULL; + ring_id_t id = 0; + for (id = 0; id < ARRAY_SIZE(process->rings); id++) { - async_flags_t expected = ASYNC_NONE; - if (atomic_compare_exchange_strong(&process->async[id].flags, &expected, ASYNC_BUSY)) + ring_ctx_flags_t expected = RING_CTX_NONE; + if (atomic_compare_exchange_strong(&process->rings[id].flags, &expected, RING_CTX_BUSY)) { - ctx = &process->async[id]; + ctx = &process->rings[id]; break; } } @@ -422,67 +420,65 @@ SYSCALL_DEFINE(SYS_SETUP, rings_id_t, rings_t* userRings, void* address, size_t return ERR; } - if (async_map(ctx, space, id, userRings, address, sentries, centries) == ERR) + if (ring_ctx_map(ctx, space, id, userRing, address, sentries, centries) == ERR) { - async_release(ctx); + ring_ctx_release(ctx); return ERR; } - async_release(ctx); + ring_ctx_release(ctx); return id; } -SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, rings_id_t id) +SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, ring_id_t id) { - if (id >= CONFIG_MAX_ASYNC_RINGS) + process_t* process = process_current(); + if (id >= ARRAY_SIZE(process->rings)) { errno = EINVAL; return ERR; } - process_t* process = process_current(); - async_t* ctx = &process->async[id]; - - if (async_acquire(ctx) == ERR) + ring_ctx_t* ctx = &process->rings[id]; + if (ring_ctx_acquire(ctx) == ERR) { errno = EBUSY; return ERR; } - if (!(atomic_load(&ctx->flags) & ASYNC_MAPPED)) + if (!(atomic_load(&ctx->flags) & RING_CTX_MAPPED)) { - async_release(ctx); + ring_ctx_release(ctx); errno = EINVAL; return ERR; } if (ctx->irps != NULL && atomic_load(&ctx->irps->pool.used) != 0) { - async_release(ctx); + ring_ctx_release(ctx); errno = EBUSY; return ERR; } - if (async_unmap(ctx) == ERR) + if (ring_ctx_unmap(ctx) == ERR) { - async_release(ctx); + ring_ctx_release(ctx); return ERR; } - async_release(ctx); + ring_ctx_release(ctx); return 0; } -SYSCALL_DEFINE(SYS_ENTER, uint64_t, rings_id_t id, size_t amount, size_t wait) +SYSCALL_DEFINE(SYS_ENTER, uint64_t, ring_id_t id, size_t amount, size_t wait) { - if (id >= CONFIG_MAX_ASYNC_RINGS) + process_t* process = process_current(); + if (id >= ARRAY_SIZE(process->rings)) { errno = EINVAL; return ERR; } - process_t* process = process_current(); - async_t* ctx = &process->async[id]; - - return async_notify(ctx, amount, wait); + ring_ctx_t* ctx = &process->rings[id]; + return ring_ctx_notify(ctx, amount, wait); } \ No newline at end of file diff --git a/src/libstd/user/common/syscalls.h b/src/libstd/user/common/syscalls.h index 000baca15..7831d6d25 100644 --- a/src/libstd/user/common/syscalls.h +++ b/src/libstd/user/common/syscalls.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #define _SYSCALL0(retType, num) \ @@ -286,17 +286,17 @@ static inline uint64_t _syscall_arch_prctl(arch_prctl_t code, uintptr_t addr) return _SYSCALL2(uint64_t, SYS_ARCH_PRCTL, arch_prctl_t, code, uintptr_t, addr); } -static inline uint64_t _syscall_setup(rings_t* rings, void* address, size_t sentries, size_t centries) +static inline uint64_t _syscall_setup(ring_t* ring, void* address, size_t sentries, size_t centries) { - return _SYSCALL4(uint64_t, SYS_SETUP, rings_t*, rings, void*, address, size_t, sentries, size_t, centries); + return _SYSCALL4(uint64_t, SYS_SETUP, ring_t*, ring, void*, address, size_t, sentries, size_t, centries); } -static inline uint64_t _syscall_teardown(rings_id_t id) +static inline uint64_t _syscall_teardown(ring_id_t id) { - return _SYSCALL1(uint64_t, SYS_TEARDOWN, rings_id_t, id); + return _SYSCALL1(uint64_t, SYS_TEARDOWN, ring_id_t, id); } -static inline uint64_t _syscall_enter(rings_id_t id, size_t amount, size_t wait) +static inline uint64_t _syscall_enter(ring_id_t id, size_t amount, size_t wait) { - return _SYSCALL3(uint64_t, SYS_ENTER, rings_id_t, id, size_t, amount, size_t, wait); + return _SYSCALL3(uint64_t, SYS_ENTER, ring_id_t, id, size_t, amount, size_t, wait); } \ No newline at end of file diff --git a/src/libstd/user/functions/async/setup.c b/src/libstd/user/functions/async/setup.c deleted file mode 100644 index 6dcaea1d8..000000000 --- a/src/libstd/user/functions/async/setup.c +++ /dev/null @@ -1,13 +0,0 @@ -#include - -#include "user/common/syscalls.h" - -rings_id_t setup(rings_t* rings, void* address, size_t sentries, size_t centries) -{ - rings_id_t result = _syscall_setup(rings, address, sentries, centries); - if (result == ERR) - { - errno = _syscall_errno(); - } - return result; -} \ No newline at end of file diff --git a/src/libstd/user/functions/async/enter.c b/src/libstd/user/functions/uring/enter.c similarity index 69% rename from src/libstd/user/functions/async/enter.c rename to src/libstd/user/functions/uring/enter.c index 865335dba..5c7d42f84 100644 --- a/src/libstd/user/functions/async/enter.c +++ b/src/libstd/user/functions/uring/enter.c @@ -1,8 +1,8 @@ -#include +#include #include "user/common/syscalls.h" -uint64_t enter(rings_id_t id, size_t amount, size_t wait) +uint64_t enter(ring_id_t id, size_t amount, size_t wait) { uint64_t result = _syscall_enter(id, amount, wait); if (result == ERR) diff --git a/src/libstd/user/functions/uring/setup.c b/src/libstd/user/functions/uring/setup.c new file mode 100644 index 000000000..6d1da7b32 --- /dev/null +++ b/src/libstd/user/functions/uring/setup.c @@ -0,0 +1,13 @@ +#include + +#include "user/common/syscalls.h" + +ring_id_t setup(ring_t* ring, void* address, size_t sentries, size_t centries) +{ + ring_id_t result = _syscall_setup(ring, address, sentries, centries); + if (result == ERR) + { + errno = _syscall_errno(); + } + return result; +} \ No newline at end of file diff --git a/src/libstd/user/functions/async/teardown.c b/src/libstd/user/functions/uring/teardown.c similarity index 75% rename from src/libstd/user/functions/async/teardown.c rename to src/libstd/user/functions/uring/teardown.c index 0213e3ea4..6e2cc241d 100644 --- a/src/libstd/user/functions/async/teardown.c +++ b/src/libstd/user/functions/uring/teardown.c @@ -1,8 +1,8 @@ -#include +#include #include "user/common/syscalls.h" -uint64_t teardown(rings_id_t id) +uint64_t teardown(ring_id_t id) { uint64_t result = _syscall_teardown(id); if (result == ERR) diff --git a/src/programs/utils/ringstest/main.c b/src/programs/utils/ringtest/main.c similarity index 52% rename from src/programs/utils/ringstest/main.c rename to src/programs/utils/ringtest/main.c index cc33ea78b..82aaa2243 100644 --- a/src/programs/utils/ringstest/main.c +++ b/src/programs/utils/ringtest/main.c @@ -1,41 +1,41 @@ #include #include #include -#include +#include #define SENTRIES 64 #define CENTRIES 128 int main() { - printf("setting up rings test...\n"); - rings_t rings; - rings_id_t id = setup(&rings, NULL, SENTRIES, CENTRIES); + printf("setting up ring test...\n"); + ring_t ring; + ring_id_t id = setup(&ring, NULL, SENTRIES, CENTRIES); if (id == ERR) { - printf("failed to set up rings\n"); + printf("failed to set up ring\n"); return errno; } - memset(&rings.shared->regs, -1, sizeof(rings.shared->regs)); + memset(&ring.ctrl->regs, -1, sizeof(ring.ctrl->regs)); - printf("pushing nop sqe to rings %llu...\n", id); + printf("pushing nop sqe to ring %llu...\n", id); sqe_t sqe = SQE_CREATE(VERB_NOP, SQE_HARDLINK | (SQE_REG0 << SQE_SAVE), CLOCKS_PER_SEC, 0x1234); - sqe_push(&rings, &sqe); + sqe_push(&ring, &sqe); - printf("pushing nop sqe to rings %llu...\n", id); + printf("pushing nop sqe to ring %llu...\n", id); sqe = (sqe_t)SQE_CREATE(VERB_NOP, SQE_LINK, CLOCKS_PER_SEC, 0x5678); - sqe_push(&rings, &sqe); + sqe_push(&ring, &sqe); - printf("entering rings...\n"); + printf("entering ring...\n"); if (enter(id, 2, 2) == ERR) { - printf("failed to enter rings\n"); + printf("failed to enter ring\n"); return errno; } cqe_t cqe; - while (cqe_pop(&rings, &cqe)) + while (cqe_pop(&ring, &cqe)) { printf("cqe:\n"); @@ -46,12 +46,12 @@ int main() } printf("registers:\n"); - for (uint64_t i = 0; i < SEQ_REGS_MAX; i++) + for (uint64_t i = 0; i < SQE_REGS_MAX; i++) { - printf("reg[%llu]: %llu\n", i, rings.shared->regs[i]); + printf("reg[%llu]: %llu\n", i, ring.ctrl->regs[i]); } - printf("tearing down rings...\n"); + printf("tearing down ring...\n"); teardown(id); return 0; } diff --git a/src/programs/utils/ringstest/ringstest.mk b/src/programs/utils/ringtest/ringtest.mk similarity index 100% rename from src/programs/utils/ringstest/ringstest.mk rename to src/programs/utils/ringtest/ringtest.mk From a01160b0fb0d100edf09225556c57959ea188101 Mon Sep 17 00:00:00 2001 From: KN Date: Wed, 21 Jan 2026 10:14:48 +0100 Subject: [PATCH 16/23] refactor: reorganize ring and IRP subsystems into an I/O subsystem --- include/kernel/cpu/{io.h => port.h} | 24 +-- include/kernel/{sync/ring.h => io/io.h} | 56 +++---- include/kernel/{sync => io}/irp.h | 23 +-- include/kernel/proc/process.h | 4 +- include/libstd/sys/{uring.h => ioring.h} | 97 +++++------- include/modules/acpi/aml/encoding/name.h | 6 +- include/modules/acpi/aml/encoding/term.h | 2 +- include/modules/acpi/aml/namespace.h | 4 +- include/modules/acpi/aml/object.h | 14 +- include/modules/acpi/aml/to_string.h | 2 +- include/modules/acpi/devices.h | 2 +- src/kernel/cpu/interrupt.c | 2 +- src/kernel/cpu/{io.c => port.c} | 8 +- src/kernel/drivers/com.c | 6 +- src/kernel/drivers/pic.c | 32 ++-- src/kernel/{sync/ring.c => io/io.c} | 144 +++++++++--------- src/kernel/{sync => io}/irp.c | 2 +- src/kernel/log/panic.c | 4 +- src/kernel/mem/pmm.c | 2 +- src/kernel/proc/process.c | 6 +- src/libstd/user/common/syscalls.h | 14 +- .../user/functions/{uring => ioring}/enter.c | 4 +- src/libstd/user/functions/ioring/setup.c | 13 ++ .../functions/{uring => ioring}/teardown.c | 4 +- src/libstd/user/functions/uring/setup.c | 13 -- src/modules/acpi/aml/encoding/data.c | 2 +- src/modules/acpi/aml/encoding/name.c | 6 +- src/modules/acpi/aml/encoding/named.c | 56 +++---- .../acpi/aml/encoding/namespace_modifier.c | 8 +- src/modules/acpi/aml/encoding/term.c | 2 +- src/modules/acpi/aml/namespace.c | 6 +- src/modules/acpi/aml/object.c | 6 +- src/modules/acpi/aml/patch_up.c | 2 +- src/modules/acpi/aml/runtime/convert.c | 20 +-- src/modules/acpi/aml/runtime/field_unit.c | 16 +- src/modules/acpi/aml/to_string.c | 2 +- src/modules/acpi/devices.c | 4 +- src/modules/drivers/ps2/ps2.c | 18 +-- src/modules/drivers/ps2/ps2.h | 4 +- src/modules/drivers/rtc/rtc.c | 8 +- src/programs/utils/ringtest/main.c | 10 +- 41 files changed, 322 insertions(+), 336 deletions(-) rename include/kernel/cpu/{io.h => port.h} (84%) rename include/kernel/{sync/ring.h => io/io.h} (70%) rename include/kernel/{sync => io}/irp.h (96%) rename include/libstd/sys/{uring.h => ioring.h} (77%) rename src/kernel/cpu/{io.c => port.c} (79%) rename src/kernel/{sync/ring.c => io/io.c} (71%) rename src/kernel/{sync => io}/irp.c (99%) rename src/libstd/user/functions/{uring => ioring}/enter.c (69%) create mode 100644 src/libstd/user/functions/ioring/setup.c rename src/libstd/user/functions/{uring => ioring}/teardown.c (75%) delete mode 100644 src/libstd/user/functions/uring/setup.c diff --git a/include/kernel/cpu/io.h b/include/kernel/cpu/port.h similarity index 84% rename from include/kernel/cpu/io.h rename to include/kernel/cpu/port.h index e1e97cb39..027a954bf 100644 --- a/include/kernel/cpu/io.h +++ b/include/kernel/cpu/port.h @@ -5,7 +5,7 @@ /** * @brief I/O port operations and reservations - * @defgroup kernel_cpu_io Port I/O + * @defgroup kernel_cpu_port Port I/O * @ingroup kernel_cpu * * The CPU can communicate with certain hardware through I/O ports, these ports are accessed using special opcodes. @@ -13,8 +13,8 @@ * ## Reserving I/O Ports * * To avoid conflicts between different subsystems or drivers trying to use the same I/O ports, we provide a simple - * reservation mechanism. Before a range of I/O ports is used, it should be reserved using `io_reserve()`. Once the - * ports are no longer needed, they should be released using `io_release()`. + * reservation mechanism. Before a range of I/O ports is used, it should be reserved using `port_reserve()`. Once the + * ports are no longer needed, they should be released using `port_release()`. * * There is no strict enforcement of I/O port reservations at the hardware level, so we have no choice but to hope that * everyone is on their best behaviour. @@ -30,7 +30,7 @@ typedef uint16_t port_t; /** * @brief Maximum I/O port number */ -#define IO_PORT_MAX UINT16_MAX +#define PORT_MAX UINT16_MAX /** * @brief Find and reserve a range of I/O ports if available. @@ -52,7 +52,7 @@ typedef uint16_t port_t; * - `EOVERFLOW`: The requested range overflows. * - `ENOSPC`: No suitable range of I/O ports available. */ -uint64_t io_reserve(port_t* out, port_t minBase, port_t maxBase, uint64_t alignment, uint64_t length, +uint64_t port_reserve(port_t* out, port_t minBase, port_t maxBase, uint64_t alignment, uint64_t length, const char* owner); /** @@ -61,7 +61,7 @@ uint64_t io_reserve(port_t* out, port_t minBase, port_t maxBase, uint64_t alignm * @param base The base I/O port address of the reserved range. * @param length The amount of contiguous I/O ports to release. */ -void io_release(port_t base, uint64_t length); +void port_release(port_t base, uint64_t length); /** * @brief Write an 8-bit value to an I/O port. @@ -69,7 +69,7 @@ void io_release(port_t base, uint64_t length); * @param port The I/O port to write to. * @param val The value to write. */ -static inline void io_out8(port_t port, uint8_t val) +static inline void out8(port_t port, uint8_t val) { ASM("outb %0, %1" : : "a"(val), "Nd"(port) : "memory"); } @@ -80,7 +80,7 @@ static inline void io_out8(port_t port, uint8_t val) * @param port The I/O port to read from. * @return The value read from the port. */ -static inline uint8_t io_in8(port_t port) +static inline uint8_t in8(port_t port) { uint8_t ret; ASM("inb %1, %0" : "=a"(ret) : "Nd"(port) : "memory"); @@ -93,7 +93,7 @@ static inline uint8_t io_in8(port_t port) * @param port The I/O port to write to. * @param val The value to write. */ -static inline void io_out16(port_t port, uint16_t val) +static inline void out16(port_t port, uint16_t val) { ASM("outw %0, %1" : : "a"(val), "Nd"(port) : "memory"); } @@ -104,7 +104,7 @@ static inline void io_out16(port_t port, uint16_t val) * @param port The I/O port to read from. * @return The value read from the port. */ -static inline uint16_t io_in16(port_t port) +static inline uint16_t in16(port_t port) { uint16_t ret; ASM("inw %1, %0" : "=a"(ret) : "Nd"(port) : "memory"); @@ -117,7 +117,7 @@ static inline uint16_t io_in16(port_t port) * @param port The I/O port to write to. * @param val The value to write. */ -static inline uint32_t io_in32(port_t port) +static inline uint32_t in32(port_t port) { uint32_t ret; ASM("inl %1, %0" : "=a"(ret) : "Nd"(port) : "memory"); @@ -130,7 +130,7 @@ static inline uint32_t io_in32(port_t port) * @param port The I/O port to read from. * @return The value read from the port. */ -static inline void io_out32(port_t port, uint32_t val) +static inline void out32(port_t port, uint32_t val) { ASM("outl %0, %1" : : "a"(val), "Nd"(port) : "memory"); } diff --git a/include/kernel/sync/ring.h b/include/kernel/io/io.h similarity index 70% rename from include/kernel/sync/ring.h rename to include/kernel/io/io.h index 3187964a7..c616a119a 100644 --- a/include/kernel/sync/ring.h +++ b/include/kernel/io/io.h @@ -5,54 +5,54 @@ #include #include #include -#include +#include #include #include -#include +#include /** * @brief Programmable submission/completion interface. - * @defgroup kernel_sync_ring Kernel-Side Ring Interface - * @ingroup kernel_sync + * @defgroup kernel_io Kernel-side I/O Ring Interface + * @ingroup kernel * - * @todo The ring system is primarily a design document for now as it remains very work in progress and subject to + * @todo The I/O ring system is primarily a design document for now as it remains very work in progress and subject to * change, currently being mostly unimplemented. * - * The ring provide the core of all interfaces in PatchworkOS, all implemented in an interface + * The I/O ring provides the core of all interfaces in PatchworkOS, all implemented in an interface * inspired by `io_uring()` from Linux. * * Synchronous operations are implemented on top of this API in userspace. * - * @see libstd_sys_uring for the userspace interface to the asynchronous ring. + * @see libstd_sys_ioring for the userspace interface to the asynchronous ring. * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`. * @see [Manpages](https://man7.org/linux/man-pages/man7/io_uring.7.html) for more information about `io_uring`. * * ## Syncronization * - * The ring structure is designed to be safe under the assumption that there is a single producer (one user-space + * The I/O ring structure is designed to be safe under the assumption that there is a single producer (one user-space * thread) and a single consumer (the kernel). * - * If a ring structure needs multiple producers (needs to be accessed by multiple threads) it is the responsibility of + * If an I/O ring needs multiple producers (needs to be accessed by multiple threads) it is the responsibility of * the caller to ensure proper synchronization. * * @note The reason for this limitation is optimization for the common case, as the syncronization logic for multiple * producers would add significant overhead. * - * Regarding the ring structure itself, the structure can only be torndown as long as nothing is using it and there are + * Regarding the I/O ring structure itself, the structure can only be torndown as long as nothing is using it and there are * no pending operations. * * ## Registers * - * Operations performed on a ring can load arguments from, and save their results to, seven 64-bit general purpose - * registers. All registers are stored in the shared area of the ring structure, as such they can be inspected and + * Operations performed on a I/O ring can load arguments from, and save their results to, seven 64-bit general purpose + * registers. All registers are stored in the shared control area of the I/O ring structure (`ioring_ctrl_t`), as such they can be inspected and * modified by user space. * * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for each argument and * one for the result. Each specifier is stored as three bits, with a `SQE_REG_NONE` value indicating no-op and any * other value representing the n-th register. The offset of the specifier specifies its meaning, for example, bits * `0-2` specify the register to load into the first argument, bits `3-5` specify the register to load into the second - * argument, and so on until bits `15-18` which specify the register to save the result into. + * argument, and so on until bits `15-17` which specify the register to save the result into. * * This system, when combined with `SQE_LINK`, allows for multiple operations to be performed at once, for example, it * would be possible to open a file, read from it, seek to a new position, write to it, and finally close the file, with @@ -96,22 +96,22 @@ /** * @brief Ring context flags. - * @enum ring_ctx_flags_t + * @enum io_ctx_flags_t */ typedef enum { - RING_CTX_NONE = 0, ///< No flags set. - RING_CTX_BUSY = 1 << 0, ///< Context is currently being used, used for fast locking. - RING_CTX_MAPPED = 1 << 1, ///< Context is currently mapped into userspace. -} ring_ctx_flags_t; + IO_CTX_NONE = 0, ///< No flags set. + IO_CTX_BUSY = 1 << 0, ///< Context is currently being used, used for fast locking. + IO_CTX_MAPPED = 1 << 1, ///< Context is currently mapped into userspace. +} io_ctx_flags_t; /** * @brief The kernel-side ring context structure. - * @struct ring_ctx_t + * @struct io_ctx_t */ -typedef struct ring_ctx +typedef struct io_ctx { - ring_t ring; ///< The kernel-side ring structure. + ioring_t ring; ///< The kernel-side ring structure. irp_pool_t* irps; ///< Pool of preallocated IRPs. mem_desc_pool_t* descs; ///< Pool of preallocated memory descriptors. void* userAddr; ///< Userspace address of the ring. @@ -120,22 +120,22 @@ typedef struct ring_ctx space_t* space; ///< Pointer to the owning address space. wait_queue_t waitQueue; ///< Wait queue for completions. process_t* process; ///< Holds a reference to the owner process while there are pending requests. - _Atomic(ring_ctx_flags_t) flags; -} ring_ctx_t; + _Atomic(io_ctx_flags_t) flags; +} io_ctx_t; /** - * @brief Initialize a ring context. + * @brief Initialize a I/O context. * * @param ctx Pointer to the context to initialize. */ -void ring_ctx_init(ring_ctx_t* ctx); +void io_ctx_init(io_ctx_t* ctx); /** - * @brief Deinitialize a ring context. + * @brief Deinitialize a I/O context. * * @param ctx Pointer to the context to deinitialize. */ -void ring_ctx_deinit(ring_ctx_t* ctx); +void io_ctx_deinit(io_ctx_t* ctx); /** * @brief Notify the context of new SQEs. @@ -145,6 +145,6 @@ void ring_ctx_deinit(ring_ctx_t* ctx); * @param wait The minimum number of CQEs to wait for. * @return On success, the number of SQEs processed. On failure, `ERR` and `errno` is set. */ -uint64_t ring_ctx_notify(ring_ctx_t* ctx, size_t amount, size_t wait); +uint64_t io_ctx_notify(io_ctx_t* ctx, size_t amount, size_t wait); /** @} */ \ No newline at end of file diff --git a/include/kernel/sync/irp.h b/include/kernel/io/irp.h similarity index 96% rename from include/kernel/sync/irp.h rename to include/kernel/io/irp.h index cf433d0a3..645d2e5f6 100644 --- a/include/kernel/sync/irp.h +++ b/include/kernel/io/irp.h @@ -11,15 +11,15 @@ #include #include #include -#include +#include #include typedef struct irp irp_t; /** * @brief I/O Request Packet. - * @defgroup kernel_sync_irp I/O Request Packet - * @ingroup kernel_sync + * @defgroup kernel_io_irp I/O Request Packet + * @ingroup kernel_io * * The I/O Request Packet is a lock-less, self-contained, layered, completion-based request that act as the primary * structure used internally by the kernel for asynchronous operations. @@ -34,6 +34,7 @@ typedef struct irp irp_t; * * The IRP system is designed around the concept of layered completions as it may take more than one subsystem within * the kernel to complete a IRP. + * the kernel to complete an IRP. * * Consider a traditional synchronous set of functions: * @@ -134,8 +135,8 @@ typedef struct irp irp_t; * Each time a completion is called via `irp_complete()`, the next completion on the stack is called until the stack is * empty, at which point the IRP is considered fully completed. * - * A real world example of this would be the ring system allocating a IRP, pushing a completion which will add a - * `cqe_t` to its Rings, before passing the IRP to the VFS which may pass it to a filesystem. Each layer pushing its own + * A real world example of this would be the ring system allocating an IRP, pushing a completion which will add a + * `cqe_t` to its rings, before passing the IRP to the VFS which may pass it to a filesystem. Each layer pushing its own * completion to handle its part of the operation. * * Finally, it is also possible to use the `irp_dispatch()` function. This function allows us to dispatch the IRP to a @@ -174,12 +175,12 @@ typedef struct irp irp_t; * though the synchronous `fun_c()` returned an error code instead of the data. * * The owner implements cancellation by calling `irp_set_cancel()` to set a cancellation callback when it pushes its - * completion. When a IRP is to be cancelled or timedout the cancellation callback will be invoked and atomically + * completion. When an IRP is to be cancelled or timed out the cancellation callback will be invoked and atomically * exchanged with a `IRP_CANCELLED` sentinel value. At which point the owner should cleanup the IRP and call * `irp_complete()`. * * It is not possible for the IRP system to perform this atomic exchange for completions. As such, to avoid race - * conditions while completing a IRP, it is vital that the owner of the IRP atomically exchanges the cancellation + * conditions while completing an IRP, it is vital that the owner of the IRP atomically exchanges the cancellation * callback with the `IRP_CANCELLED` sentinel value. For the sake of convenience, the `irp_claim()` function is provided * to perform this operation. * @@ -238,7 +239,7 @@ typedef struct irp irp_t; * - `ETIMEDOUT`: Operation timed out. * - `EINPROGRESS`: Operation is in a timeout queue. * - * @see kernel_sync_ring for the ring system. + * @see kernel_io for the ring system. * @see [Wikipedia](https://en.wikipedia.org/wiki/I/O_request_packet) for more information about IRPs. * @see [Microsoft _IRP](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_irp) for information * on how Windows NT implements IRPs. @@ -247,7 +248,7 @@ typedef struct irp irp_t; #define IRP_LOC_MAX 8 ///< The maximum number of locations in a IRP. -#define IRP_ARGS_MAX 5 ///< The maximum number of arguments in a IRP. +#define IRP_ARGS_MAX SQE_MAX_ARGS ///< The maximum number of arguments in an IRP. /** * @brief IRP completion callback type. @@ -292,7 +293,7 @@ typedef struct ALIGNED(64) irp { list_entry_t entry; ///< Used to store the IRP in various lists. list_entry_t timeoutEntry; ///< Used to store the IRP in the timeout queue. - _Atomic(irp_cancel_t) cancel; ///< Cancellation callback, must be atomic to ensure a IRP is only cancelled once. + _Atomic(irp_cancel_t) cancel; ///< Cancellation callback, must be atomic to ensure an IRP is only cancelled once. union { struct { @@ -444,7 +445,7 @@ static inline bool irp_claim(irp_t* irp) } /** - * @brief Retrieve the context of the IRP pool that a IRP was allocated from. + * @brief Retrieve the context of the IRP pool that an IRP was allocated from. * * @param irp Pointer to the IRP. * @return Pointer to the context. diff --git a/include/kernel/proc/process.h b/include/kernel/proc/process.h index 95cc9bd7f..977b907e8 100644 --- a/include/kernel/proc/process.h +++ b/include/kernel/proc/process.h @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include @@ -88,7 +88,7 @@ typedef struct process file_table_t fileTable; futex_ctx_t futexCtx; perf_process_ctx_t perf; - ring_ctx_t rings[CONFIG_MAX_RINGS]; + io_ctx_t rings[CONFIG_MAX_RINGS]; note_handler_t noteHandler; wait_queue_t suspendQueue; wait_queue_t dyingQueue; diff --git a/include/libstd/sys/uring.h b/include/libstd/sys/ioring.h similarity index 77% rename from include/libstd/sys/uring.h rename to include/libstd/sys/ioring.h index 4463d858a..3dda9083f 100644 --- a/include/libstd/sys/uring.h +++ b/include/libstd/sys/ioring.h @@ -1,5 +1,5 @@ -#ifndef _SYS_URING_H -#define _SYS_URING_H 1 +#ifndef _SYS_IORING_H +#define _SYS_IORING_H 1 #include #include @@ -19,7 +19,7 @@ extern "C" #include "_internal/fd_t.h" /** - * @addtogroup kernel_sync_ring + * @addtogroup kernel_io * @{ */ @@ -52,28 +52,28 @@ typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. #define SQE_LOAD4 (SQE_LOAD3 + SQE_REG_SHIFT) ///< The offset to specify the register to load into the fifth argument. #define SQE_SAVE (SQE_LOAD4 + SQE_REG_SHIFT) ///< The offset to specify the register to save the result into. -#define _SEQ_FLAGS (SQE_SAVE + SQE_REG_SHIFT) ///< The bitshift for where bit flags start in a `sqe_flags_t`. +#define _SQE_FLAGS (SQE_SAVE + SQE_REG_SHIFT) ///< The bitshift for where bit flags start in a `sqe_flags_t`. #ifdef _KERNEL_ /** * The operation was created by the kernel, used internally by the kernel. */ -#define SQE_KERNEL (1 << (_SEQ_FLAGS)) +#define SQE_KERNEL (1 << (_SQE_FLAGS)) /** * The operations enter callback has been called, used internally by the kernel. */ -#define SQE_KERNEL_ENTERED (1 << (_SEQ_FLAGS + 1)) +#define SQE_KERNEL_ENTERED (1 << (_SQE_FLAGS + 1)) #endif /** * Only process the next SQE when this one completes successfully) only applies within one `enter()` call. */ -#define SQE_LINK (1 << (_SEQ_FLAGS + 2)) +#define SQE_LINK (1 << (_SQE_FLAGS + 2)) /** * Like `SQE_LINK` but will process the next SQE even if this one fails. */ -#define SQE_HARDLINK (1 << (_SEQ_FLAGS + 3)) +#define SQE_HARDLINK (1 << (_SQE_FLAGS + 3)) /** * @brief Asynchronous submission queue entry (SQE). @@ -82,7 +82,7 @@ typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. * @warning It is the responsibility of userspace to ensure that any pointers * passed to the kernel remain valid until the operation is complete. * - * @see kernel_sync_async for more information on the possible operations. + * @see kernel_io for more information on the possible operations. */ typedef struct sqe { @@ -129,7 +129,7 @@ static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); * @brief Asynchronous completion queue entry (CQE). * @struct cqe_t * - * @see kernel_sync_async for more information on the possible operations. + * @see kernel_io for more information on the possible operations. */ typedef struct ALIGNED(32) cqe { @@ -147,21 +147,16 @@ typedef struct ALIGNED(32) cqe static_assert(sizeof(cqe_t) == 32, "cqe_t is not 32 bytes"); #endif -/** - * @brief User-Rings ID type. - */ -typedef uint64_t ring_id_t; - /** * @brief Shared ring control structure. - * @struct ring_ctrl_t + * @struct ioring_ctrl_t * * Used as the intermediate between userspace and the kernel. * * @note The structure is aligned in such a way to reduce false sharing. * */ -typedef struct ALIGNED(64) ring_ctrl +typedef struct ALIGNED(64) ioring_ctrl { atomic_uint32_t shead; ///< Submission head index, updated by the kernel. atomic_uint32_t ctail; ///< Completion tail index, updated by the kernel. @@ -171,51 +166,43 @@ typedef struct ALIGNED(64) ring_ctrl uint8_t _padding1[64 - sizeof(atomic_uint32_t) * 2]; atomic_uint64_t regs[SQE_REGS_MAX] ALIGNED(64); ///< General purpose registers. uint8_t _reserved[8]; -} ring_ctrl_t; +} ioring_ctrl_t; /** * @} - * @brief User-side asynchronous ring interface. - * @defgroup libstd_sys_uring User-Side Ring Interface + * @brief Programmable submission/completion interface. + * @defgroup libstd_sys_ioring User-side I/O Ring Interface * @ingroup libstd * * The ring interface acts as the interface for all asynchronous operations in the kernel. * - * @see kernel_sync_ring for more information about the asynchronous ring system. + * @see kernel_io for more information about the I/O ring system. * * @{ */ +typedef uint64_t io_id_t; ///< I/O ring ID type. + /** - * @brief User asynchronous ring structure. - * @struct ring_t + * @brief User I/O ring structure. + * @struct ioring_t * * The kernel and userspace will have their own instances of this structure. */ -typedef struct ring +typedef struct ioring { - ring_ctrl_t* ctrl; ///< Pointer to the shared control structure. - ring_id_t id; ///< The ID of the ring. - sqe_t* squeue; ///< Pointer to the submission queue. - size_t sentries; ///< Number of entries in the submission queue. - size_t smask; ///< Bitmask for submission queue (sentries - 1). - cqe_t* cqueue; ///< Pointer to the completion queue. - size_t centries; ///< Number of entries in the completion queue. - size_t cmask; ///< Bitmask for completion queue (centries - 1). -} ring_t; - -/** - * @brief Dont wait for any submissions to complete. - */ -#define WAIT_NONE 0x0 - -/** - * @brief Wait for at least one submission to complete. - */ -#define WAIT_ONE 0x1 + ioring_ctrl_t* ctrl; ///< Pointer to the shared control structure. + io_id_t id; ///< The ID of the ring. + sqe_t* squeue; ///< Pointer to the submission queue. + size_t sentries; ///< Number of entries in the submission queue. + size_t smask; ///< Bitmask for submission queue (sentries - 1). + cqe_t* cqueue; ///< Pointer to the completion queue. + size_t centries; ///< Number of entries in the completion queue. + size_t cmask; ///< Bitmask for completion queue (centries - 1). +} ioring_t; /** - * @brief System call to initialize the asynchronous ring. + * @brief System call to initialize the I/O ring. * * This system call will populate the given structure with the necessary pointers and metadata for the submission and * completion ring. @@ -224,38 +211,38 @@ typedef struct ring * @param address Desired address to allocate the ring, or `NULL` to let the kernel choose. * @param sentries Number of entires to allocate for the submission queue, must be a power of two. * @param centries Number of entries to allocate for the completion queue, must be a power of two. - * @return On success, the ring ID. On failure, `ERR` and `errno` is set. + * @return On success, the ID of the new I/O ring. On failure, `ERR` and `errno` is set. */ -ring_id_t setup(ring_t* ring, void* address, size_t sentries, size_t centries); +io_id_t setup(ioring_t* ring, void* address, size_t sentries, size_t centries); /** - * @brief System call to deinitialize the asynchronous ring. + * @brief System call to deinitialize the I/O ring. * - * @param id The ID of the ring to deinitialize. + * @param id The ID of the I/O ring to teardown. * @return On success, `0`. On failure, `ERR` and `errno` is set. */ -uint64_t teardown(ring_id_t id); +uint64_t teardown(io_id_t id); /** * @brief System call to notify the kernel of new submission queue entries (SQEs). * - * @param id The ID of the ring to notify. + * @param id The ID of the I/O ring to notify. * @param amount The number of SQEs that the kernel should process. * @param wait The minimum number of completion queue entries (CQEs) to wait for. * @return On success, the number of SQEs successfully processed. On failure, `ERR` and `errno` is set. */ -uint64_t enter(ring_id_t id, size_t amount, size_t wait); +uint64_t enter(io_id_t id, size_t amount, size_t wait); /** * @brief Pushes a submission queue entry (SQE) to the submission queue. * * After pushing SQEs, `enter()` must be called to notify the kernel of the new entries. * - * @param ring Pointer to the asynchronous ring structure. + * @param ring Pointer to the I/O ring structure. * @param sqe Pointer to the SQE to push. * @return `true` if the SQE was pushed, `false` if the submission queue is full. */ -static inline bool sqe_push(ring_t* ring, sqe_t* sqe) +static inline bool sqe_push(ioring_t* ring, sqe_t* sqe) { uint32_t tail = atomic_load_explicit(&ring->ctrl->stail, memory_order_relaxed); uint32_t head = atomic_load_explicit(&ring->ctrl->shead, memory_order_acquire); @@ -274,11 +261,11 @@ static inline bool sqe_push(ring_t* ring, sqe_t* sqe) /** * @brief Pops a completion queue entry (CQE) from the completion queue. * - * @param ring Pointer to the asynchronous ring structure. + * @param ring Pointer to the I/O ring structure. * @param cqe Pointer to the CQE to pop. * @return `true` if a CQE was popped, `false` if the completion queue is empty. */ -static inline bool cqe_pop(ring_t* ring, cqe_t* cqe) +static inline bool cqe_pop(ioring_t* ring, cqe_t* cqe) { uint32_t head = atomic_load_explicit(&ring->ctrl->chead, memory_order_relaxed); uint32_t tail = atomic_load_explicit(&ring->ctrl->ctail, memory_order_acquire); diff --git a/include/modules/acpi/aml/encoding/name.h b/include/modules/acpi/aml/encoding/name.h index ce763e2e1..f9dc99ff6 100644 --- a/include/modules/acpi/aml/encoding/name.h +++ b/include/modules/acpi/aml/encoding/name.h @@ -81,14 +81,14 @@ typedef struct /** * @brief A NameString structure. - * @struct aml_name_string_t + * @struct aml_name_stioring_t */ typedef struct { aml_root_char_t rootChar; aml_prefix_path_t prefixPath; aml_name_path_t namePath; -} aml_name_string_t; +} aml_name_stioring_t; /** * @brief Reads the next data as a SegCount structure from the AML bytecode stream. @@ -192,7 +192,7 @@ uint64_t aml_root_char_read(aml_term_list_ctx_t* ctx, aml_root_char_t* out); * @param out Pointer to destination where the NameString will be stored. * @return On success, `0`. On failure, `ERR` and `errno` is set. */ -uint64_t aml_name_string_read(aml_term_list_ctx_t* ctx, aml_name_string_t* out); +uint64_t aml_name_string_read(aml_term_list_ctx_t* ctx, aml_name_stioring_t* out); /** * @brief Reads the next data as a NameString structure from the AML bytecode stream and resolves it to a object. diff --git a/include/modules/acpi/aml/encoding/term.h b/include/modules/acpi/aml/encoding/term.h index 445f9a161..b15f1af06 100644 --- a/include/modules/acpi/aml/encoding/term.h +++ b/include/modules/acpi/aml/encoding/term.h @@ -69,7 +69,7 @@ uint64_t aml_term_arg_read_integer(aml_term_list_ctx_t* ctx, aml_uint_t* out); * @param ctx The context of the TermList that this structure is part of. * @return On success, the string. On failure, `NULL` and `errno` is set. */ -aml_string_t* aml_term_arg_read_string(aml_term_list_ctx_t* ctx); +aml_stioring_t* aml_term_arg_read_string(aml_term_list_ctx_t* ctx); /** * @brief Wrapper around `aml_term_arg_read()` that converts the result to a buffer. diff --git a/include/modules/acpi/aml/namespace.h b/include/modules/acpi/aml/namespace.h index 4900133c8..e6d82c3cf 100644 --- a/include/modules/acpi/aml/namespace.h +++ b/include/modules/acpi/aml/namespace.h @@ -207,7 +207,7 @@ aml_object_t* aml_namespace_find(aml_overlay_t* overlay, aml_object_t* start, ui * @return The object reference or `NULL` if it could not be found. */ aml_object_t* aml_namespace_find_by_name_string(aml_overlay_t* overlay, aml_object_t* start, - const aml_name_string_t* nameString); + const aml_name_stioring_t* nameString); /** * @brief Find an object in the namespace heirarchy by a path string. @@ -255,7 +255,7 @@ uint64_t aml_namespace_add_child(aml_overlay_t* overlay, aml_object_t* parent, a * @return On success, `0`. On failure, `ERR` and `errno` is set. */ uint64_t aml_namespace_add_by_name_string(aml_overlay_t* overlay, aml_object_t* start, - const aml_name_string_t* nameString, aml_object_t* object); + const aml_name_stioring_t* nameString, aml_object_t* object); /** * @brief Remove an object from the namespace heirarchy it was added to. diff --git a/include/modules/acpi/aml/object.h b/include/modules/acpi/aml/object.h index d81ca3844..d02096001 100644 --- a/include/modules/acpi/aml/object.h +++ b/include/modules/acpi/aml/object.h @@ -15,7 +15,7 @@ typedef struct acpi_device_cfg acpi_device_cfg_t; typedef struct aml_state aml_state_t; typedef struct aml_object aml_object_t; typedef struct aml_opregion aml_opregion_t; -typedef struct aml_string aml_string_t; +typedef struct aml_string aml_stioring_t; typedef struct aml_method aml_method_t; /** @@ -384,7 +384,7 @@ typedef struct aml_processor /** * @brief Data for a string object. - * @struct aml_string_t + * @struct aml_stioring_t */ typedef struct aml_string { @@ -392,7 +392,7 @@ typedef struct aml_string char* content; uint64_t length; char smallString[AML_SMALL_STRING_SIZE + 1]; ///< Used for small object optimization. -} aml_string_t; +} aml_stioring_t; /** * @brief Data for an alias object. @@ -411,7 +411,7 @@ typedef struct aml_alias typedef struct aml_unresolved { AML_OBJECT_COMMON_HEADER; - aml_name_string_t nameString; ///< The NameString representing the path to the target object. + aml_name_stioring_t nameString; ///< The NameString representing the path to the target object. aml_object_t* from; ///< The object to start the search from when resolving the reference. aml_patch_up_resolve_callback_t callback; ///< The callback to call when a matching object is found. } aml_unresolved_t; @@ -464,7 +464,7 @@ typedef struct aml_object aml_package_t package; aml_power_resource_t powerResource; aml_processor_t processor; - aml_string_t string; + aml_stioring_t string; aml_alias_t alias; aml_unresolved_t unresolved; @@ -773,7 +773,7 @@ uint64_t aml_string_set(aml_object_t* object, const char* str); * @param newLength The new length of the string, not including the null terminator. * @return On success, the new length of the string. On failure, `ERR` and `errno` is set. */ -uint64_t aml_string_resize(aml_string_t* string, uint64_t newLength); +uint64_t aml_string_resize(aml_stioring_t* string, uint64_t newLength); /** * @brief Set a object as a thermal zone. @@ -815,7 +815,7 @@ aml_object_t* aml_alias_traverse(aml_alias_t* alias); * @param callback Pointer to a callback function that will be called when a matching object is found * @return On success, `0`. On failure, `ERR` and `errno` is set. */ -uint64_t aml_unresolved_set(aml_object_t* object, const aml_name_string_t* nameString, aml_object_t* from, +uint64_t aml_unresolved_set(aml_object_t* object, const aml_name_stioring_t* nameString, aml_object_t* from, aml_patch_up_resolve_callback_t callback); /** diff --git a/include/modules/acpi/aml/to_string.h b/include/modules/acpi/aml/to_string.h index 3092148b8..836e5abbe 100644 --- a/include/modules/acpi/aml/to_string.h +++ b/include/modules/acpi/aml/to_string.h @@ -65,6 +65,6 @@ const char* aml_object_to_string(aml_object_t* object); * @param nameString ACPI AML NameString. * @return String representation of the NameString or "Unknown" if it is invalid. */ -const char* aml_name_string_to_string(const aml_name_string_t* nameString); +const char* aml_name_stioring_to_string(const aml_name_stioring_t* nameString); /** @} */ diff --git a/include/modules/acpi/devices.h b/include/modules/acpi/devices.h index fb066bf1e..efc8e10f4 100644 --- a/include/modules/acpi/devices.h +++ b/include/modules/acpi/devices.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/src/kernel/cpu/interrupt.c b/src/kernel/cpu/interrupt.c index cf57120b5..b59a28cad 100644 --- a/src/kernel/cpu/interrupt.c +++ b/src/kernel/cpu/interrupt.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include diff --git a/src/kernel/cpu/io.c b/src/kernel/cpu/port.c similarity index 79% rename from src/kernel/cpu/io.c rename to src/kernel/cpu/port.c index 33db453df..eaea7a092 100644 --- a/src/kernel/cpu/io.c +++ b/src/kernel/cpu/port.c @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -9,10 +9,10 @@ #include #include -static BITMAP_CREATE(ports, IO_PORT_MAX + 1); +static BITMAP_CREATE(ports, PORT_MAX + 1); static lock_t lock = LOCK_CREATE(); -uint64_t io_reserve(port_t* out, port_t minBase, port_t maxBase, uint64_t alignment, uint64_t length, const char* owner) +uint64_t port_reserve(port_t* out, port_t minBase, port_t maxBase, uint64_t alignment, uint64_t length, const char* owner) { UNUSED(owner); @@ -40,7 +40,7 @@ uint64_t io_reserve(port_t* out, port_t minBase, port_t maxBase, uint64_t alignm return 0; } -void io_release(port_t base, uint64_t length) +void port_release(port_t base, uint64_t length) { if (length == 0 || base + length < base || base + length > ports.length) { diff --git a/src/kernel/drivers/com.c b/src/kernel/drivers/com.c index eb9893d04..bd3678d96 100644 --- a/src/kernel/drivers/com.c +++ b/src/kernel/drivers/com.c @@ -1,4 +1,4 @@ -#include +#include #include void com_init(com_port_t port) @@ -31,10 +31,10 @@ void com_write(com_port_t port, uint8_t value) uint8_t com_reg_read(com_port_t port, com_reg_t reg) { - return io_in8(port + reg); + return in8(port + reg); } void com_reg_write(com_port_t port, com_reg_t reg, uint8_t value) { - io_out8(port + reg, value); + out8(port + reg, value); } diff --git a/src/kernel/drivers/pic.c b/src/kernel/drivers/pic.c index 91a3f2dbc..48be515b2 100644 --- a/src/kernel/drivers/pic.c +++ b/src/kernel/drivers/pic.c @@ -2,12 +2,12 @@ #include #include -#include +#include #include static void pic_wait(void) { - io_out8(0x80, 0); + out8(0x80, 0); } void pic_disable(void) @@ -15,39 +15,39 @@ void pic_disable(void) // We initialize the PIC before we then mask all interrupts. // Probably not needed but it ensures that the PIC is in a known state before we disable it. - uint8_t a1 = io_in8(PIC1_DATA); + uint8_t a1 = in8(PIC1_DATA); pic_wait(); - uint8_t a2 = io_in8(PIC2_DATA); + uint8_t a2 = in8(PIC2_DATA); pic_wait(); - io_out8(PIC1_COMMAND, ICW1_INIT | ICW1_ICW4); + out8(PIC1_COMMAND, ICW1_INIT | ICW1_ICW4); pic_wait(); - io_out8(PIC2_COMMAND, ICW1_INIT | ICW1_ICW4); + out8(PIC2_COMMAND, ICW1_INIT | ICW1_ICW4); pic_wait(); - io_out8(PIC1_DATA, VECTOR_EXTERNAL_START); + out8(PIC1_DATA, VECTOR_EXTERNAL_START); pic_wait(); - io_out8(PIC2_DATA, VECTOR_EXTERNAL_START + 0x8); + out8(PIC2_DATA, VECTOR_EXTERNAL_START + 0x8); pic_wait(); - io_out8(PIC1_DATA, 4); + out8(PIC1_DATA, 4); pic_wait(); - io_out8(PIC2_DATA, 2); + out8(PIC2_DATA, 2); pic_wait(); - io_out8(PIC1_DATA, ICW4_8086); + out8(PIC1_DATA, ICW4_8086); pic_wait(); - io_out8(PIC2_DATA, ICW4_8086); + out8(PIC2_DATA, ICW4_8086); pic_wait(); - io_out8(PIC1_DATA, a1); + out8(PIC1_DATA, a1); pic_wait(); - io_out8(PIC2_DATA, a2); + out8(PIC2_DATA, a2); pic_wait(); // Mask all interrupts. - io_out8(PIC1_DATA, 0xFF); - io_out8(PIC2_DATA, 0xFF); + out8(PIC1_DATA, 0xFF); + out8(PIC2_DATA, 0xFF); LOG_INFO("pic disabled\n"); } diff --git a/src/kernel/sync/ring.c b/src/kernel/io/io.c similarity index 71% rename from src/kernel/sync/ring.c rename to src/kernel/io/io.c index f08899728..9f1ec6e1e 100644 --- a/src/kernel/sync/ring.c +++ b/src/kernel/io/io.c @@ -9,18 +9,18 @@ #include #include #include -#include -#include +#include +#include #include #include -#include +#include #include -static inline uint64_t ring_ctx_acquire(ring_ctx_t* ctx) +static inline uint64_t io_ctx_acquire(io_ctx_t* ctx) { - ring_ctx_flags_t expected = atomic_load(&ctx->flags); - if (!(expected & RING_CTX_BUSY) && atomic_compare_exchange_strong(&ctx->flags, &expected, expected | RING_CTX_BUSY)) + io_ctx_flags_t expected = atomic_load(&ctx->flags); + if (!(expected & IO_CTX_BUSY) && atomic_compare_exchange_strong(&ctx->flags, &expected, expected | IO_CTX_BUSY)) { return 0; } @@ -28,17 +28,17 @@ static inline uint64_t ring_ctx_acquire(ring_ctx_t* ctx) return ERR; } -static inline void ring_ctx_release(ring_ctx_t* ctx) +static inline void io_ctx_release(io_ctx_t* ctx) { - atomic_fetch_and(&ctx->flags, ~RING_CTX_BUSY); + atomic_fetch_and(&ctx->flags, ~IO_CTX_BUSY); } -static inline uint64_t ring_ctx_map(ring_ctx_t* ctx, space_t* space, ring_id_t id, ring_t* userRing, void* address, +static inline uint64_t io_ctx_map(io_ctx_t* ctx, space_t* space, io_id_t id, ioring_t* userRing, void* address, size_t sentries, size_t centries) { - ring_t* kernelRing = &ctx->ring; + ioring_t* kernelRing = &ctx->ring; - size_t pageAmount = BYTES_TO_PAGES(sizeof(ring_ctrl_t) + (sentries * sizeof(sqe_t)) + (centries * sizeof(cqe_t))); + size_t pageAmount = BYTES_TO_PAGES(sizeof(ioring_ctrl_t) + (sentries * sizeof(sqe_t)) + (centries * sizeof(cqe_t))); if (pageAmount >= CONFIG_MAX_RINGS_PAGES) { errno = ENOMEM; @@ -96,7 +96,7 @@ static inline uint64_t ring_ctx_map(ring_ctx_t* ctx, space_t* space, ring_id_t i return ERR; } - ring_ctrl_t* ctrl = (ring_ctrl_t*)kernelAddr; + ioring_ctrl_t* ctrl = (ioring_ctrl_t*)kernelAddr; atomic_init(&ctrl->shead, 0); atomic_init(&ctrl->stail, 0); atomic_init(&ctrl->ctail, 0); @@ -108,19 +108,19 @@ static inline uint64_t ring_ctx_map(ring_ctx_t* ctx, space_t* space, ring_id_t i userRing->ctrl = userAddr; userRing->id = id; - userRing->squeue = (sqe_t*)((uintptr_t)userAddr + sizeof(ring_ctrl_t)); + userRing->squeue = (sqe_t*)((uintptr_t)userAddr + sizeof(ioring_ctrl_t)); userRing->sentries = sentries; userRing->smask = sentries - 1; - userRing->cqueue = (cqe_t*)((uintptr_t)userAddr + sizeof(ring_ctrl_t) + (sentries * sizeof(sqe_t))); + userRing->cqueue = (cqe_t*)((uintptr_t)userAddr + sizeof(ioring_ctrl_t) + (sentries * sizeof(sqe_t))); userRing->centries = centries; userRing->cmask = centries - 1; kernelRing->ctrl = kernelAddr; kernelRing->id = id; - kernelRing->squeue = (sqe_t*)((uintptr_t)kernelAddr + sizeof(ring_ctrl_t)); + kernelRing->squeue = (sqe_t*)((uintptr_t)kernelAddr + sizeof(ioring_ctrl_t)); kernelRing->sentries = sentries; kernelRing->smask = sentries - 1; - kernelRing->cqueue = (cqe_t*)((uintptr_t)kernelAddr + sizeof(ring_ctrl_t) + (sentries * sizeof(sqe_t))); + kernelRing->cqueue = (cqe_t*)((uintptr_t)kernelAddr + sizeof(ioring_ctrl_t) + (sentries * sizeof(sqe_t))); kernelRing->centries = centries; kernelRing->cmask = centries - 1; @@ -131,11 +131,11 @@ static inline uint64_t ring_ctx_map(ring_ctx_t* ctx, space_t* space, ring_id_t i ctx->pageAmount = pageAmount; ctx->space = space; - atomic_fetch_or(&ctx->flags, RING_CTX_MAPPED); + atomic_fetch_or(&ctx->flags, IO_CTX_MAPPED); return 0; } -static inline uint64_t ring_ctx_unmap(ring_ctx_t* ctx) +static inline uint64_t io_ctx_unmap(io_ctx_t* ctx) { irp_pool_free(ctx->irps); ctx->irps = NULL; @@ -146,67 +146,67 @@ static inline uint64_t ring_ctx_unmap(ring_ctx_t* ctx) vmm_unmap(ctx->space, ctx->userAddr, ctx->pageAmount * PAGE_SIZE); vmm_unmap(NULL, ctx->kernelAddr, ctx->pageAmount * PAGE_SIZE); - atomic_fetch_and(&ctx->flags, ~RING_CTX_MAPPED); + atomic_fetch_and(&ctx->flags, ~IO_CTX_MAPPED); return 0; } -static inline uint64_t ring_ctx_avail_cqes(ring_ctx_t* ctx) +static inline uint64_t io_ctx_avail_cqes(io_ctx_t* ctx) { - ring_t* ring = &ctx->ring; + ioring_t* ring = &ctx->ring; uint32_t ctail = atomic_load_explicit(&ring->ctrl->ctail, memory_order_relaxed); uint32_t chead = atomic_load_explicit(&ring->ctrl->chead, memory_order_acquire); return ctail - chead; } -void ring_ctx_init(ring_ctx_t* ctx) +void io_ctx_init(io_ctx_t* ctx) { if (ctx == NULL) { return; } - ctx->ring = (ring_t){0}; + ctx->ring = (ioring_t){0}; ctx->irps = NULL; ctx->userAddr = NULL; ctx->kernelAddr = NULL; ctx->pageAmount = 0; ctx->space = NULL; wait_queue_init(&ctx->waitQueue); - atomic_init(&ctx->flags, RING_CTX_NONE); + atomic_init(&ctx->flags, IO_CTX_NONE); } -void ring_ctx_deinit(ring_ctx_t* ctx) +void io_ctx_deinit(io_ctx_t* ctx) { if (ctx == NULL) { return; } - if (ring_ctx_acquire(ctx) == ERR) + if (io_ctx_acquire(ctx) == ERR) { panic(NULL, "failed to acquire async context for deinitialization"); } - if (atomic_load(&ctx->flags) & RING_CTX_MAPPED) + if (atomic_load(&ctx->flags) & IO_CTX_MAPPED) { - if (ring_ctx_unmap(ctx) == ERR) + if (io_ctx_unmap(ctx) == ERR) { panic(NULL, "failed to deinitialize async context"); } } - ring_ctx_release(ctx); + io_ctx_release(ctx); wait_queue_deinit(&ctx->waitQueue); } -static void ring_ctx_dispatch(irp_t* irp); +static void io_ctx_dispatch(irp_t* irp); -static void ring_ctx_complete(irp_t* irp, void* _ptr) +static void io_ctx_complete(irp_t* irp, void* _ptr) { UNUSED(_ptr); - ring_ctx_t* ctx = irp_get_ctx(irp); - ring_t* ring = &ctx->ring; + io_ctx_t* ctx = irp_get_ctx(irp); + ioring_t* ring = &ctx->ring; sqe_flags_t reg = (irp->flags >> SQE_SAVE) & SQE_REG_MASK; if (reg != SQE_REG_NONE) @@ -250,7 +250,7 @@ static void ring_ctx_complete(irp_t* irp, void* _ptr) irp_t* next = irp_next(irp); if (next != NULL) { - ring_ctx_dispatch(next); + io_ctx_dispatch(next); } } @@ -263,10 +263,10 @@ static void ring_ctx_complete(irp_t* irp, void* _ptr) } } -static void ring_ctx_dispatch(irp_t* irp) +static void io_ctx_dispatch(irp_t* irp) { - ring_ctx_t* ctx = irp_get_ctx(irp); - ring_t* ring = &ctx->ring; + io_ctx_t* ctx = irp_get_ctx(irp); + ioring_t* ring = &ctx->ring; for (uint64_t i = 0; i < SQE_MAX_ARGS; i++) { @@ -279,7 +279,7 @@ static void ring_ctx_dispatch(irp_t* irp) irp->sqe._args[i] = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } - irp_push(irp, ring_ctx_complete, NULL); + irp_push(irp, io_ctx_complete, NULL); irp_dispatch(irp); } @@ -287,16 +287,16 @@ typedef struct { list_t irps; irp_t* link; -} ring_ctx_notify_ctx_t; +} io_ctx_notify_ctx_t; -static uint64_t ring_ctx_sqe_pop(ring_ctx_t* ctx, ring_ctx_notify_ctx_t* notify) +static uint64_t io_ctx_sqe_pop(io_ctx_t* ctx, io_ctx_notify_ctx_t* notify) { if (atomic_load(&ctx->irps->pool.used) == 1) { ctx->process = REF(process_current()); } - ring_t* ring = &ctx->ring; + ioring_t* ring = &ctx->ring; uint32_t stail = atomic_load_explicit(&ring->ctrl->stail, memory_order_acquire); uint32_t shead = atomic_load_explicit(&ring->ctrl->shead, memory_order_relaxed); @@ -333,36 +333,36 @@ static uint64_t ring_ctx_sqe_pop(ring_ctx_t* ctx, ring_ctx_notify_ctx_t* notify) return 0; } -uint64_t ring_ctx_notify(ring_ctx_t* ctx, size_t amount, size_t wait) +uint64_t io_ctx_notify(io_ctx_t* ctx, size_t amount, size_t wait) { if (amount == 0) { return 0; } - if (ring_ctx_acquire(ctx) == ERR) + if (io_ctx_acquire(ctx) == ERR) { errno = EBUSY; return ERR; } - if (!(atomic_load(&ctx->flags) & RING_CTX_MAPPED)) + if (!(atomic_load(&ctx->flags) & IO_CTX_MAPPED)) { - ring_ctx_release(ctx); + io_ctx_release(ctx); errno = EINVAL; return ERR; } size_t processed = 0; - ring_ctx_notify_ctx_t notify = { + io_ctx_notify_ctx_t notify = { .irps = LIST_CREATE(notify.irps), .link = NULL, }; while (processed < amount) { - if (ring_ctx_sqe_pop(ctx, ¬ify) == ERR) + if (io_ctx_sqe_pop(ctx, ¬ify) == ERR) { break; } @@ -372,26 +372,26 @@ uint64_t ring_ctx_notify(ring_ctx_t* ctx, size_t amount, size_t wait) while (!list_is_empty(¬ify.irps)) { irp_t* irp = CONTAINER_OF(list_pop_front(¬ify.irps), irp_t, entry); - ring_ctx_dispatch(irp); + io_ctx_dispatch(irp); } if (wait == 0) { - ring_ctx_release(ctx); + io_ctx_release(ctx); return processed; } - if (WAIT_BLOCK(&ctx->waitQueue, ring_ctx_avail_cqes(ctx) >= wait) == ERR) + if (WAIT_BLOCK(&ctx->waitQueue, io_ctx_avail_cqes(ctx) >= wait) == ERR) { - ring_ctx_release(ctx); + io_ctx_release(ctx); return processed > 0 ? processed : ERR; } - ring_ctx_release(ctx); + io_ctx_release(ctx); return processed; } -SYSCALL_DEFINE(SYS_SETUP, ring_id_t, ring_t* userRing, void* address, size_t sentries, size_t centries) +SYSCALL_DEFINE(SYS_SETUP, io_id_t, ioring_t* userRing, void* address, size_t sentries, size_t centries) { if (userRing == NULL || sentries == 0 || centries == 0 || !IS_POW2(sentries) || !IS_POW2(centries)) { @@ -402,12 +402,12 @@ SYSCALL_DEFINE(SYS_SETUP, ring_id_t, ring_t* userRing, void* address, size_t sen process_t* process = process_current(); space_t* space = &process->space; - ring_ctx_t* ctx = NULL; - ring_id_t id = 0; + io_ctx_t* ctx = NULL; + io_id_t id = 0; for (id = 0; id < ARRAY_SIZE(process->rings); id++) { - ring_ctx_flags_t expected = RING_CTX_NONE; - if (atomic_compare_exchange_strong(&process->rings[id].flags, &expected, RING_CTX_BUSY)) + io_ctx_flags_t expected = IO_CTX_NONE; + if (atomic_compare_exchange_strong(&process->rings[id].flags, &expected, IO_CTX_BUSY)) { ctx = &process->rings[id]; break; @@ -420,17 +420,17 @@ SYSCALL_DEFINE(SYS_SETUP, ring_id_t, ring_t* userRing, void* address, size_t sen return ERR; } - if (ring_ctx_map(ctx, space, id, userRing, address, sentries, centries) == ERR) + if (io_ctx_map(ctx, space, id, userRing, address, sentries, centries) == ERR) { - ring_ctx_release(ctx); + io_ctx_release(ctx); return ERR; } - ring_ctx_release(ctx); + io_ctx_release(ctx); return id; } -SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, ring_id_t id) +SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, io_id_t id) { process_t* process = process_current(); if (id >= ARRAY_SIZE(process->rings)) @@ -439,38 +439,38 @@ SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, ring_id_t id) return ERR; } - ring_ctx_t* ctx = &process->rings[id]; - if (ring_ctx_acquire(ctx) == ERR) + io_ctx_t* ctx = &process->rings[id]; + if (io_ctx_acquire(ctx) == ERR) { errno = EBUSY; return ERR; } - if (!(atomic_load(&ctx->flags) & RING_CTX_MAPPED)) + if (!(atomic_load(&ctx->flags) & IO_CTX_MAPPED)) { - ring_ctx_release(ctx); + io_ctx_release(ctx); errno = EINVAL; return ERR; } if (ctx->irps != NULL && atomic_load(&ctx->irps->pool.used) != 0) { - ring_ctx_release(ctx); + io_ctx_release(ctx); errno = EBUSY; return ERR; } - if (ring_ctx_unmap(ctx) == ERR) + if (io_ctx_unmap(ctx) == ERR) { - ring_ctx_release(ctx); + io_ctx_release(ctx); return ERR; } - ring_ctx_release(ctx); + io_ctx_release(ctx); return 0; } -SYSCALL_DEFINE(SYS_ENTER, uint64_t, ring_id_t id, size_t amount, size_t wait) +SYSCALL_DEFINE(SYS_ENTER, uint64_t, io_id_t id, size_t amount, size_t wait) { process_t* process = process_current(); if (id >= ARRAY_SIZE(process->rings)) @@ -479,6 +479,6 @@ SYSCALL_DEFINE(SYS_ENTER, uint64_t, ring_id_t id, size_t amount, size_t wait) return ERR; } - ring_ctx_t* ctx = &process->rings[id]; - return ring_ctx_notify(ctx, amount, wait); + io_ctx_t* ctx = &process->rings[id]; + return io_ctx_notify(ctx, amount, wait); } \ No newline at end of file diff --git a/src/kernel/sync/irp.c b/src/kernel/io/irp.c similarity index 99% rename from src/kernel/sync/irp.c rename to src/kernel/io/irp.c index 6a6e75974..600446d4e 100644 --- a/src/kernel/sync/irp.c +++ b/src/kernel/io/irp.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/kernel/log/panic.c b/src/kernel/log/panic.c index 2b9b7190f..a748bf2c0 100644 --- a/src/kernel/log/panic.c +++ b/src/kernel/log/panic.c @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include #include @@ -457,7 +457,7 @@ void panic(const interrupt_frame_t* frame, const char* format, ...) LOG_PANIC("!!! Please restart your machine !!!\n"); #ifdef QEMU_EXIT_ON_PANIC - io_out8(QEMU_EXIT_ON_PANIC_PORT, -1); + out8(QEMU_EXIT_ON_PANIC_PORT, -1); #endif while (true) diff --git a/src/kernel/mem/pmm.c b/src/kernel/mem/pmm.c index 628132c28..6be21b83c 100644 --- a/src/kernel/mem/pmm.c +++ b/src/kernel/mem/pmm.c @@ -178,7 +178,7 @@ static void pmm_init_refs(const boot_memory_map_t* map) if (desc->Type == EfiConventionalMemory && desc->NumberOfPages >= BYTES_TO_PAGES(size)) { pages = (page_t*)desc->VirtualStart; - LOG_INFO("pages [%p-%p]\n", pages, (uintptr_t)pages + size); + LOG_INFO("pages [%p-%p]\n", pages, (uintptr_t)pages + size); return; } } diff --git a/src/kernel/proc/process.c b/src/kernel/proc/process.c index 876320606..2c392b4bb 100644 --- a/src/kernel/proc/process.c +++ b/src/kernel/proc/process.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include @@ -113,7 +113,7 @@ static void process_free(process_t* process) futex_ctx_deinit(&process->futexCtx); for (uint64_t i = 0; i < ARRAY_SIZE(process->rings); i++) { - ring_ctx_deinit(&process->rings[i]); + io_ctx_deinit(&process->rings[i]); } wait_queue_deinit(&process->dyingQueue); wait_queue_deinit(&process->suspendQueue); @@ -156,7 +156,7 @@ process_t* process_new(priority_t priority, group_member_t* group, namespace_t* perf_process_ctx_init(&process->perf); for (uint64_t i = 0; i < ARRAY_SIZE(process->rings); i++) { - ring_ctx_init(&process->rings[i]); + io_ctx_init(&process->rings[i]); } note_handler_init(&process->noteHandler); wait_queue_init(&process->suspendQueue); diff --git a/src/libstd/user/common/syscalls.h b/src/libstd/user/common/syscalls.h index 7831d6d25..b90fdc84d 100644 --- a/src/libstd/user/common/syscalls.h +++ b/src/libstd/user/common/syscalls.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #define _SYSCALL0(retType, num) \ @@ -286,17 +286,17 @@ static inline uint64_t _syscall_arch_prctl(arch_prctl_t code, uintptr_t addr) return _SYSCALL2(uint64_t, SYS_ARCH_PRCTL, arch_prctl_t, code, uintptr_t, addr); } -static inline uint64_t _syscall_setup(ring_t* ring, void* address, size_t sentries, size_t centries) +static inline uint64_t _syscall_setup(ioring_t* ring, void* address, size_t sentries, size_t centries) { - return _SYSCALL4(uint64_t, SYS_SETUP, ring_t*, ring, void*, address, size_t, sentries, size_t, centries); + return _SYSCALL4(uint64_t, SYS_SETUP, ioring_t*, ring, void*, address, size_t, sentries, size_t, centries); } -static inline uint64_t _syscall_teardown(ring_id_t id) +static inline uint64_t _syscall_teardown(io_id_t id) { - return _SYSCALL1(uint64_t, SYS_TEARDOWN, ring_id_t, id); + return _SYSCALL1(uint64_t, SYS_TEARDOWN, io_id_t, id); } -static inline uint64_t _syscall_enter(ring_id_t id, size_t amount, size_t wait) +static inline uint64_t _syscall_enter(io_id_t id, size_t amount, size_t wait) { - return _SYSCALL3(uint64_t, SYS_ENTER, ring_id_t, id, size_t, amount, size_t, wait); + return _SYSCALL3(uint64_t, SYS_ENTER, io_id_t, id, size_t, amount, size_t, wait); } \ No newline at end of file diff --git a/src/libstd/user/functions/uring/enter.c b/src/libstd/user/functions/ioring/enter.c similarity index 69% rename from src/libstd/user/functions/uring/enter.c rename to src/libstd/user/functions/ioring/enter.c index 5c7d42f84..b4e2838ed 100644 --- a/src/libstd/user/functions/uring/enter.c +++ b/src/libstd/user/functions/ioring/enter.c @@ -1,8 +1,8 @@ -#include +#include #include "user/common/syscalls.h" -uint64_t enter(ring_id_t id, size_t amount, size_t wait) +uint64_t enter(io_id_t id, size_t amount, size_t wait) { uint64_t result = _syscall_enter(id, amount, wait); if (result == ERR) diff --git a/src/libstd/user/functions/ioring/setup.c b/src/libstd/user/functions/ioring/setup.c new file mode 100644 index 000000000..3cc9a0bd4 --- /dev/null +++ b/src/libstd/user/functions/ioring/setup.c @@ -0,0 +1,13 @@ +#include + +#include "user/common/syscalls.h" + +io_id_t setup(ioring_t* ring, void* address, size_t sentries, size_t centries) +{ + io_id_t result = _syscall_setup(ring, address, sentries, centries); + if (result == ERR) + { + errno = _syscall_errno(); + } + return result; +} \ No newline at end of file diff --git a/src/libstd/user/functions/uring/teardown.c b/src/libstd/user/functions/ioring/teardown.c similarity index 75% rename from src/libstd/user/functions/uring/teardown.c rename to src/libstd/user/functions/ioring/teardown.c index 6e2cc241d..bbba4d2e6 100644 --- a/src/libstd/user/functions/uring/teardown.c +++ b/src/libstd/user/functions/ioring/teardown.c @@ -1,8 +1,8 @@ -#include +#include #include "user/common/syscalls.h" -uint64_t teardown(ring_id_t id) +uint64_t teardown(io_id_t id) { uint64_t result = _syscall_teardown(id); if (result == ERR) diff --git a/src/libstd/user/functions/uring/setup.c b/src/libstd/user/functions/uring/setup.c deleted file mode 100644 index 6d1da7b32..000000000 --- a/src/libstd/user/functions/uring/setup.c +++ /dev/null @@ -1,13 +0,0 @@ -#include - -#include "user/common/syscalls.h" - -ring_id_t setup(ring_t* ring, void* address, size_t sentries, size_t centries) -{ - ring_id_t result = _syscall_setup(ring, address, sentries, centries); - if (result == ERR) - { - errno = _syscall_errno(); - } - return result; -} \ No newline at end of file diff --git a/src/modules/acpi/aml/encoding/data.c b/src/modules/acpi/aml/encoding/data.c index cfe7281a0..3a8c43c59 100644 --- a/src/modules/acpi/aml/encoding/data.c +++ b/src/modules/acpi/aml/encoding/data.c @@ -349,7 +349,7 @@ uint64_t aml_package_element_read(aml_term_list_ctx_t* ctx, aml_object_t* out) if (token.props->type == AML_TOKEN_TYPE_NAME) { - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); diff --git a/src/modules/acpi/aml/encoding/name.c b/src/modules/acpi/aml/encoding/name.c index 3acb5ba64..c3c34510c 100644 --- a/src/modules/acpi/aml/encoding/name.c +++ b/src/modules/acpi/aml/encoding/name.c @@ -186,12 +186,12 @@ uint64_t aml_root_char_read(aml_term_list_ctx_t* ctx, aml_root_char_t* out) return 0; } -uint64_t aml_name_string_read(aml_term_list_ctx_t* ctx, aml_name_string_t* out) +uint64_t aml_name_string_read(aml_term_list_ctx_t* ctx, aml_name_stioring_t* out) { aml_token_t token; aml_token_peek(ctx, &token); - aml_name_string_t nameString = {0}; + aml_name_stioring_t nameString = {0}; // Starts with either rootchar or prefixpath. switch (token.num) { @@ -226,7 +226,7 @@ uint64_t aml_name_string_read(aml_term_list_ctx_t* ctx, aml_name_string_t* out) aml_object_t* aml_name_string_read_and_resolve(aml_term_list_ctx_t* ctx) { - aml_name_string_t nameStringLocal; + aml_name_stioring_t nameStringLocal; if (aml_name_string_read(ctx, &nameStringLocal) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); diff --git a/src/modules/acpi/aml/encoding/named.c b/src/modules/acpi/aml/encoding/named.c index db20a0452..2cbfa7934 100644 --- a/src/modules/acpi/aml/encoding/named.c +++ b/src/modules/acpi/aml/encoding/named.c @@ -73,7 +73,7 @@ uint64_t aml_def_opregion_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -104,7 +104,7 @@ uint64_t aml_def_opregion_read(aml_term_list_ctx_t* ctx) aml_object_t* newObject = aml_object_new(); if (newObject == NULL) { - AML_DEBUG_ERROR(ctx, "Failed to create object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to create object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } UNREF_DEFER(newObject); @@ -112,7 +112,7 @@ uint64_t aml_def_opregion_read(aml_term_list_ctx_t* ctx) if (aml_operation_region_set(newObject, regionSpace, regionOffset, regionLen) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, newObject) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -554,7 +554,7 @@ uint64_t aml_def_method_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -580,7 +580,7 @@ uint64_t aml_def_method_read(aml_term_list_ctx_t* ctx) if (aml_method_set(newObject, methodFlags, ctx->current, end, NULL) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, newObject) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -607,7 +607,7 @@ uint64_t aml_def_device_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -626,7 +626,7 @@ uint64_t aml_def_device_read(aml_term_list_ctx_t* ctx) if (aml_device_set(device) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, device) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -668,7 +668,7 @@ uint64_t aml_def_mutex_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -692,7 +692,7 @@ uint64_t aml_def_mutex_read(aml_term_list_ctx_t* ctx) if (aml_mutex_set(newObject, syncFlags) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, newObject) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -746,7 +746,7 @@ uint64_t aml_def_processor_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -786,7 +786,7 @@ uint64_t aml_def_processor_read(aml_term_list_ctx_t* ctx) if (aml_processor_set(processor, procId, pblkAddr, pblkLen) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, processor) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -859,7 +859,7 @@ uint64_t aml_def_create_bit_field_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -876,7 +876,7 @@ uint64_t aml_def_create_bit_field_read(aml_term_list_ctx_t* ctx) if (aml_buffer_field_set(newObject, sourceBuff, bitIndex, 1) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, newObject) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -909,7 +909,7 @@ static inline uint64_t aml_def_create_field_read_helper(aml_term_list_ctx_t* ctx return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -926,7 +926,7 @@ static inline uint64_t aml_def_create_field_read_helper(aml_term_list_ctx_t* ctx if (aml_buffer_field_set(newObject, sourceBuff, byteIndex * 8, fieldWidth) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, newObject) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -961,7 +961,7 @@ uint64_t aml_def_event_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -978,7 +978,7 @@ uint64_t aml_def_event_read(aml_term_list_ctx_t* ctx) if (aml_event_set(newObject) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, newObject) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -1002,7 +1002,7 @@ uint64_t aml_def_thermal_zone_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -1021,7 +1021,7 @@ uint64_t aml_def_thermal_zone_read(aml_term_list_ctx_t* ctx) if (aml_thermal_zone_set(thermalZone) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, thermalZone) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -1072,7 +1072,7 @@ uint64_t aml_def_power_res_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -1105,7 +1105,7 @@ uint64_t aml_def_power_res_read(aml_term_list_ctx_t* ctx) if (aml_power_resource_set(powerResource, systemLevel, resourceOrder) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, powerResource) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -1162,7 +1162,7 @@ uint64_t aml_def_create_field_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -1179,7 +1179,7 @@ uint64_t aml_def_create_field_read(aml_term_list_ctx_t* ctx) if (aml_buffer_field_set(newObject, sourceBuff, bitIndex, numBits) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, newObject) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -1194,14 +1194,14 @@ uint64_t aml_def_data_region_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t regionName; + aml_name_stioring_t regionName; if (aml_name_string_read(ctx, ®ionName) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read RegionName"); return ERR; } - aml_string_t* signature = aml_term_arg_read_string(ctx); + aml_stioring_t* signature = aml_term_arg_read_string(ctx); if (signature == NULL) { AML_DEBUG_ERROR(ctx, "Failed to read Signature"); @@ -1209,7 +1209,7 @@ uint64_t aml_def_data_region_read(aml_term_list_ctx_t* ctx) } UNREF_DEFER(signature); - aml_string_t* oemId = aml_term_arg_read_string(ctx); + aml_stioring_t* oemId = aml_term_arg_read_string(ctx); if (oemId == NULL) { AML_DEBUG_ERROR(ctx, "Failed to read OemId"); @@ -1217,7 +1217,7 @@ uint64_t aml_def_data_region_read(aml_term_list_ctx_t* ctx) } UNREF_DEFER(oemId); - aml_string_t* oemTableId = aml_term_arg_read_string(ctx); + aml_stioring_t* oemTableId = aml_term_arg_read_string(ctx); if (oemTableId == NULL) { AML_DEBUG_ERROR(ctx, "Failed to read OemTableId"); @@ -1285,7 +1285,7 @@ uint64_t aml_def_data_region_read(aml_term_list_ctx_t* ctx) if (aml_operation_region_set(newObject, AML_REGION_SYSTEM_MEMORY, (uint64_t)table, table->length) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, ®ionName, newObject) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(®ionName)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(®ionName)); return ERR; } diff --git a/src/modules/acpi/aml/encoding/namespace_modifier.c b/src/modules/acpi/aml/encoding/namespace_modifier.c index 948937820..4ad434484 100644 --- a/src/modules/acpi/aml/encoding/namespace_modifier.c +++ b/src/modules/acpi/aml/encoding/namespace_modifier.c @@ -31,7 +31,7 @@ uint64_t aml_def_alias_read(aml_term_list_ctx_t* ctx) } UNREF_DEFER(source); - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read or resolve target NameString"); @@ -48,7 +48,7 @@ uint64_t aml_def_alias_read(aml_term_list_ctx_t* ctx) if (aml_alias_set(newObject, source) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, newObject) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add alias object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add alias object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } @@ -63,7 +63,7 @@ uint64_t aml_def_name_read(aml_term_list_ctx_t* ctx) return ERR; } - aml_name_string_t nameString; + aml_name_stioring_t nameString; if (aml_name_string_read(ctx, &nameString) == ERR) { AML_DEBUG_ERROR(ctx, "Failed to read NameString"); @@ -80,7 +80,7 @@ uint64_t aml_def_name_read(aml_term_list_ctx_t* ctx) if (aml_data_ref_object_read(ctx, newObject) == ERR || aml_namespace_add_by_name_string(&ctx->state->overlay, ctx->scope, &nameString, newObject) == ERR) { - AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_string_to_string(&nameString)); + AML_DEBUG_ERROR(ctx, "Failed to add object '%s'", aml_name_stioring_to_string(&nameString)); return ERR; } diff --git a/src/modules/acpi/aml/encoding/term.c b/src/modules/acpi/aml/encoding/term.c index 509b5f15d..7af7c3122 100644 --- a/src/modules/acpi/aml/encoding/term.c +++ b/src/modules/acpi/aml/encoding/term.c @@ -82,7 +82,7 @@ uint64_t aml_term_arg_read_integer(aml_term_list_ctx_t* ctx, aml_uint_t* out) return 0; } -aml_string_t* aml_term_arg_read_string(aml_term_list_ctx_t* ctx) +aml_stioring_t* aml_term_arg_read_string(aml_term_list_ctx_t* ctx) { aml_object_t* temp = aml_term_arg_read(ctx, AML_STRING); if (temp == NULL) diff --git a/src/modules/acpi/aml/namespace.c b/src/modules/acpi/aml/namespace.c index efbc48596..441f3283e 100644 --- a/src/modules/acpi/aml/namespace.c +++ b/src/modules/acpi/aml/namespace.c @@ -235,7 +235,7 @@ aml_object_t* aml_namespace_find(aml_overlay_t* overlay, aml_object_t* start, ui } aml_object_t* aml_namespace_find_by_name_string(aml_overlay_t* overlay, aml_object_t* start, - const aml_name_string_t* nameString) + const aml_name_stioring_t* nameString) { if (nameString == NULL) { @@ -447,7 +447,7 @@ uint64_t aml_namespace_add_child(aml_overlay_t* overlay, aml_object_t* parent, a } uint64_t aml_namespace_add_by_name_string(aml_overlay_t* overlay, aml_object_t* start, - const aml_name_string_t* nameString, aml_object_t* object) + const aml_name_stioring_t* nameString, aml_object_t* object) { if (nameString == NULL || nameString->namePath.segmentCount == 0) { @@ -470,7 +470,7 @@ uint64_t aml_namespace_add_by_name_string(aml_overlay_t* overlay, aml_object_t* return aml_namespace_add_child(overlay, parent, targetName, object); } - aml_name_string_t parentNameString = *nameString; + aml_name_stioring_t parentNameString = *nameString; parentNameString.namePath.segmentCount--; aml_object_t* parent = aml_namespace_find_by_name_string(overlay, start, &parentNameString); diff --git a/src/modules/acpi/aml/object.c b/src/modules/acpi/aml/object.c index 13ecfc2ac..d708389c9 100644 --- a/src/modules/acpi/aml/object.c +++ b/src/modules/acpi/aml/object.c @@ -227,7 +227,7 @@ void aml_object_clear(aml_object_t* object) UNREF(object->unresolved.from); } object->unresolved.from = NULL; - object->unresolved.nameString = (aml_name_string_t){0}; + object->unresolved.nameString = (aml_name_stioring_t){0}; object->unresolved.callback = NULL; break; case AML_PREDEFINED_SCOPE: @@ -1062,7 +1062,7 @@ uint64_t aml_string_set(aml_object_t* object, const char* str) return 0; } -uint64_t aml_string_resize(aml_string_t* string, uint64_t newLength) +uint64_t aml_string_resize(aml_stioring_t* string, uint64_t newLength) { if (string == NULL) { @@ -1169,7 +1169,7 @@ aml_object_t* aml_alias_traverse(aml_alias_t* alias) return current; } -uint64_t aml_unresolved_set(aml_object_t* object, const aml_name_string_t* nameString, aml_object_t* from, +uint64_t aml_unresolved_set(aml_object_t* object, const aml_name_stioring_t* nameString, aml_object_t* from, aml_patch_up_resolve_callback_t callback) { if (object == NULL || nameString == NULL || callback == NULL) diff --git a/src/modules/acpi/aml/patch_up.c b/src/modules/acpi/aml/patch_up.c index b10bd1f92..4ccfd578b 100644 --- a/src/modules/acpi/aml/patch_up.c +++ b/src/modules/acpi/aml/patch_up.c @@ -74,7 +74,7 @@ uint64_t aml_patch_up_resolve_all(void) aml_namespace_find_by_name_string(&state.overlay, entry->unresolved->from, &entry->unresolved->nameString); if (match == NULL) { - LOG_DEBUG("Still could not resolve '%s'\n", aml_name_string_to_string(&entry->unresolved->nameString)); + LOG_DEBUG("Still could not resolve '%s'\n", aml_name_stioring_to_string(&entry->unresolved->nameString)); errno = EOK; continue; } diff --git a/src/modules/acpi/aml/runtime/convert.c b/src/modules/acpi/aml/runtime/convert.c index abf212830..e95dff7c0 100644 --- a/src/modules/acpi/aml/runtime/convert.c +++ b/src/modules/acpi/aml/runtime/convert.c @@ -261,10 +261,10 @@ static aml_convert_entry_t packageConverters[AML_TYPE_AMOUNT] = { {AML_PACKAGE, AML_DEBUG_OBJECT, aml_package_to_debug_object}, }; -static uint64_t aml_string_to_integer(aml_state_t* state, aml_object_t* string, aml_object_t* dest) +static uint64_t aml_stioring_to_integer(aml_state_t* state, aml_object_t* string, aml_object_t* dest) { UNUSED(state); - aml_string_t* stringData = &string->string; + aml_stioring_t* stringData = &string->string; uint64_t value = 0; uint64_t maxChars = MIN(stringData->length, aml_integer_byte_size() * 2); // Two hex chars per byte @@ -287,10 +287,10 @@ static uint64_t aml_string_to_integer(aml_state_t* state, aml_object_t* string, return aml_integer_set(dest, value); } -static uint64_t aml_string_to_buffer(aml_state_t* state, aml_object_t* string, aml_object_t* dest) +static uint64_t aml_stioring_to_buffer(aml_state_t* state, aml_object_t* string, aml_object_t* dest) { UNUSED(state); - aml_string_t* stringData = &string->string; + aml_stioring_t* stringData = &string->string; // Regarding zero-length strings the spec says "... the string is treated as a buffer, with each // ASCII string character copied to one buffer byte, including the null @@ -325,7 +325,7 @@ static uint64_t aml_string_to_buffer(aml_state_t* state, aml_object_t* string, a return 0; } -static uint64_t aml_string_to_debug_object(aml_state_t* state, aml_object_t* string, aml_object_t* dest) +static uint64_t aml_stioring_to_debug_object(aml_state_t* state, aml_object_t* string, aml_object_t* dest) { UNUSED(state); UNUSED(dest); @@ -340,9 +340,9 @@ static uint64_t aml_string_to_debug_object(aml_state_t* state, aml_object_t* str } static aml_convert_entry_t stringConverters[AML_TYPE_AMOUNT] = { - {AML_STRING, AML_INTEGER, aml_string_to_integer}, - {AML_STRING, AML_BUFFER, aml_string_to_buffer}, - {AML_STRING, AML_DEBUG_OBJECT, aml_string_to_debug_object}, + {AML_STRING, AML_INTEGER, aml_stioring_to_integer}, + {AML_STRING, AML_BUFFER, aml_stioring_to_buffer}, + {AML_STRING, AML_DEBUG_OBJECT, aml_stioring_to_debug_object}, }; static aml_convert_entry_t* aml_converters_get(aml_type_t srcType) @@ -639,7 +639,7 @@ uint64_t aml_convert_to_buffer(aml_state_t* state, aml_object_t* src, aml_object } else if (src->type == AML_STRING) { - if (aml_string_to_buffer(state, src, temp) == ERR) + if (aml_stioring_to_buffer(state, src, temp) == ERR) { return ERR; } @@ -881,7 +881,7 @@ uint64_t aml_convert_to_integer(aml_state_t* state, aml_object_t* src, aml_objec if (src->type == AML_STRING) { - aml_string_t* stringData = &src->string; + aml_stioring_t* stringData = &src->string; if (stringData->length == 0 || stringData->content == NULL) { errno = EILSEQ; diff --git a/src/modules/acpi/aml/runtime/field_unit.c b/src/modules/acpi/aml/runtime/field_unit.c index 9ca0b9587..82b25d97f 100644 --- a/src/modules/acpi/aml/runtime/field_unit.c +++ b/src/modules/acpi/aml/runtime/field_unit.c @@ -1,6 +1,5 @@ #include #include - #include #include #include @@ -8,8 +7,7 @@ #include #include #include - -#include +#include #include #include @@ -138,13 +136,13 @@ static uint64_t aml_system_io_read(aml_state_t* state, aml_opregion_t* opregion, switch (accessSize) { case 8: - *out = io_in8(address); + *out = in8(address); break; case 16: - *out = io_in16(address); + *out = in16(address); break; case 32: - *out = io_in32(address); + *out = in32(address); break; default: LOG_ERR("unable to read opregion with access size %u\n", accessSize); @@ -163,13 +161,13 @@ static uint64_t aml_system_io_write(aml_state_t* state, aml_opregion_t* opregion switch (accessSize) { case 8: - io_out8(address, (uint8_t)value); + out8(address, (uint8_t)value); break; case 16: - io_out16(address, (uint16_t)value); + out16(address, (uint16_t)value); break; case 32: - io_out32(address, (uint32_t)value); + out32(address, (uint32_t)value); break; default: LOG_ERR("unable to write opregion with access size %u\n", accessSize); diff --git a/src/modules/acpi/aml/to_string.c b/src/modules/acpi/aml/to_string.c index a7510ab35..46b38c4f4 100644 --- a/src/modules/acpi/aml/to_string.c +++ b/src/modules/acpi/aml/to_string.c @@ -251,7 +251,7 @@ const char* aml_object_to_string(aml_object_t* object) } } -const char* aml_name_string_to_string(const aml_name_string_t* nameString) +const char* aml_name_stioring_to_string(const aml_name_stioring_t* nameString) { static char buffer[256]; memset(buffer, 0, sizeof(buffer)); diff --git a/src/modules/acpi/devices.c b/src/modules/acpi/devices.c index 2c4c38170..b0752bbc1 100644 --- a/src/modules/acpi/devices.c +++ b/src/modules/acpi/devices.c @@ -336,7 +336,7 @@ static void acpi_device_cfg_free(acpi_device_cfg_t* cfg) for (uint64_t i = 0; i < cfg->ioCount; i++) { - io_release(cfg->ios[i].base, cfg->ios[i].length); + port_release(cfg->ios[i].base, cfg->ios[i].length); } free(cfg->irqs); @@ -451,7 +451,7 @@ static uint64_t acpi_device_configure(const char* name) cfg->ios = newIos; port_t base; - if (io_reserve(&base, desc->minBase, desc->maxBase, desc->alignment, desc->length, name) == ERR) + if (port_reserve(&base, desc->minBase, desc->maxBase, desc->alignment, desc->length, name) == ERR) { LOG_ERR("failed to reserve IO ports for ACPI device '%s' due to '%s'\n", name, strerror(errno)); goto error; diff --git a/src/modules/drivers/ps2/ps2.c b/src/modules/drivers/ps2/ps2.c index 843771955..eb363637e 100644 --- a/src/modules/drivers/ps2/ps2.c +++ b/src/modules/drivers/ps2/ps2.c @@ -325,9 +325,9 @@ static uint64_t ps2_devices_test(void) void ps2_drain(void) { clock_wait(PS2_SMALL_DELAY); - while ((io_in8(statusPort)) & PS2_STATUS_OUT_FULL) + while ((in8(statusPort)) & PS2_STATUS_OUT_FULL) { - io_in8(dataPort); + in8(dataPort); clock_wait(PS2_SMALL_DELAY); } } @@ -335,7 +335,7 @@ void ps2_drain(void) uint64_t ps2_wait_until_set(ps2_status_bits_t status) { uint64_t startTime = clock_uptime(); - while ((io_in8(statusPort) & status) == 0) + while ((in8(statusPort) & status) == 0) { if ((clock_uptime() - startTime) > PS2_WAIT_TIMEOUT) { @@ -350,7 +350,7 @@ uint64_t ps2_wait_until_set(ps2_status_bits_t status) uint64_t ps2_wait_until_clear(ps2_status_bits_t status) { uint64_t startTime = clock_uptime(); - while ((io_in8(statusPort) & status) != 0) + while ((in8(statusPort) & status) != 0) { if ((clock_uptime() - startTime) > PS2_WAIT_TIMEOUT) { @@ -369,18 +369,18 @@ uint64_t ps2_read(void) errno = ETIMEDOUT; return ERR; } - return io_in8(dataPort); + return in8(dataPort); } uint64_t ps2_read_no_wait(void) { - if (!(io_in8(statusPort) & PS2_STATUS_OUT_FULL)) + if (!(in8(statusPort) & PS2_STATUS_OUT_FULL)) { errno = EAGAIN; return ERR; } - return io_in8(dataPort); + return in8(dataPort); } uint64_t ps2_write(uint8_t data) @@ -390,7 +390,7 @@ uint64_t ps2_write(uint8_t data) errno = ETIMEDOUT; return ERR; } - io_out8(dataPort, data); + out8(dataPort, data); return 0; } @@ -401,7 +401,7 @@ uint64_t ps2_cmd(ps2_cmd_t command) errno = ETIMEDOUT; return ERR; } - io_out8(commandPort, command); + out8(commandPort, command); return 0; } diff --git a/src/modules/drivers/ps2/ps2.h b/src/modules/drivers/ps2/ps2.h index f1f73b0a7..544c915fe 100644 --- a/src/modules/drivers/ps2/ps2.h +++ b/src/modules/drivers/ps2/ps2.h @@ -1,8 +1,8 @@ #pragma once -#include - +#include #include + #include #include diff --git a/src/modules/drivers/rtc/rtc.c b/src/modules/drivers/rtc/rtc.c index 658acf25e..c8f3c923a 100644 --- a/src/modules/drivers/rtc/rtc.c +++ b/src/modules/drivers/rtc/rtc.c @@ -1,11 +1,11 @@ -#include +#include #include #include #include #include #include - #include + #include /** @@ -28,8 +28,8 @@ static lock_t lock = LOCK_CREATE(); static uint8_t rtc_read(uint8_t reg) { - io_out8(addressPort, reg | 0x80); // Force NMI disable - return io_in8(dataPort); + out8(addressPort, reg | 0x80); // Force NMI disable + return in8(dataPort); } static int rtc_update_in_progress(void) diff --git a/src/programs/utils/ringtest/main.c b/src/programs/utils/ringtest/main.c index 82aaa2243..7f897974f 100644 --- a/src/programs/utils/ringtest/main.c +++ b/src/programs/utils/ringtest/main.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #define SENTRIES 64 #define CENTRIES 128 @@ -9,8 +9,8 @@ int main() { printf("setting up ring test...\n"); - ring_t ring; - ring_id_t id = setup(&ring, NULL, SENTRIES, CENTRIES); + ioring_t ring; + io_id_t id = isetup(&ring, NULL, SENTRIES, CENTRIES); if (id == ERR) { printf("failed to set up ring\n"); @@ -19,11 +19,11 @@ int main() memset(&ring.ctrl->regs, -1, sizeof(ring.ctrl->regs)); - printf("pushing nop sqe to ring %llu...\n", id); + printf("pushing nop sqe to ring %llu...\n", ring.id); sqe_t sqe = SQE_CREATE(VERB_NOP, SQE_HARDLINK | (SQE_REG0 << SQE_SAVE), CLOCKS_PER_SEC, 0x1234); sqe_push(&ring, &sqe); - printf("pushing nop sqe to ring %llu...\n", id); + printf("pushing nop sqe to ring %llu...\n", ring.id); sqe = (sqe_t)SQE_CREATE(VERB_NOP, SQE_LINK, CLOCKS_PER_SEC, 0x5678); sqe_push(&ring, &sqe); From bf997a223057b2bf1012185e071c1c0f27952a39 Mon Sep 17 00:00:00 2001 From: KN Date: Wed, 21 Jan 2026 13:51:47 +0100 Subject: [PATCH 17/23] refactor: move sys/io.h to sys/fs.h; add ioring archetype system --- include/kernel/fs/dentry.h | 2 +- include/kernel/fs/devfs.h | 2 +- include/kernel/fs/file.h | 2 +- include/kernel/fs/filesystem.h | 2 +- include/kernel/fs/inode.h | 2 +- include/kernel/fs/key.h | 2 +- include/kernel/fs/namespace.h | 2 +- include/kernel/fs/netfs.h | 2 +- include/kernel/fs/path.h | 2 +- include/kernel/fs/superblock.h | 2 +- include/kernel/fs/sysfs.h | 2 +- include/kernel/fs/tmpfs.h | 2 +- include/kernel/fs/vfs.h | 2 +- include/kernel/io/io.h | 115 ++++++++++++++---- include/kernel/io/irp.h | 45 +++++-- include/kernel/ipc/note.h | 2 +- include/kernel/log/log.h | 2 +- include/kernel/module/module.h | 2 +- include/kernel/proc/process.h | 2 +- include/kernel/sync/rcu.h | 2 +- include/kernel/utils/map.h | 2 +- include/libpatchwork/cmd.h | 2 +- include/libpatchwork/event.h | 2 +- include/libpatchwork/surface.h | 2 +- include/libstd/_internal/fd_t.h | 24 ++-- include/libstd/sys/{io.h => fs.h} | 4 +- include/libstd/sys/ioring.h | 60 ++++++--- include/modules/acpi/aml/object.h | 2 +- include/modules/acpi/devices.h | 4 +- lib/OVMFbin/OVMF_VARS-pure-efi.fd | Bin 131072 -> 131072 bytes meta/doxy/Doxyfile | 2 +- src/boxes/apps/terminal/terminal.c | 2 +- src/boxes/apps/terminal/terminal.h | 2 +- src/boxes/core/dwm/client.h | 2 +- src/boxes/core/dwm/dwm.c | 2 +- src/boxes/core/dwm/dwm.h | 2 +- src/boxes/core/dwm/screen.c | 2 +- src/boxes/core/dwm/surface.c | 2 +- src/boxes/core/dwm/surface.h | 2 +- src/boxes/core/taskbar/start_menu.c | 2 +- src/boxes/core/taskbar/taskbar.h | 2 +- src/kernel/cpu/interrupt.c | 2 +- src/kernel/cpu/port.c | 3 +- src/kernel/drivers/abstract/kbd.c | 2 +- src/kernel/drivers/perf.c | 2 +- src/kernel/fs/devfs.c | 2 +- src/kernel/fs/filesystem.c | 2 +- src/kernel/fs/namespace.c | 2 +- src/kernel/fs/netfs.c | 2 +- src/kernel/fs/procfs.c | 2 +- src/kernel/fs/sysfs.c | 2 +- src/kernel/fs/tmpfs.c | 2 +- src/kernel/fs/vfs.c | 2 +- src/kernel/io/io.c | 12 +- src/kernel/io/irp.c | 10 +- src/kernel/log/log.c | 2 +- src/kernel/log/panic.c | 2 +- src/kernel/module/module.c | 2 +- src/kernel/proc/process.c | 4 +- src/libpatchwork/config.c | 2 +- src/libpatchwork/font.c | 2 +- src/libpatchwork/internal.h | 2 +- src/libstd/common/heap.c | 2 +- src/libstd/common/init.c | 2 +- src/libstd/user/common/file.c | 2 +- src/libstd/user/common/std_streams.c | 2 +- src/libstd/user/common/syscalls.h | 4 +- src/libstd/user/functions/{io => fs}/bind.c | 2 +- src/libstd/user/functions/{io => fs}/chdir.c | 2 +- src/libstd/user/functions/{io => fs}/claim.c | 2 +- src/libstd/user/functions/{io => fs}/close.c | 2 +- src/libstd/user/functions/{io => fs}/dup.c | 2 +- src/libstd/user/functions/{io => fs}/dup2.c | 2 +- .../user/functions/{io => fs}/getdents.c | 2 +- src/libstd/user/functions/{io => fs}/ioctl.c | 2 +- src/libstd/user/functions/{io => fs}/link.c | 2 +- src/libstd/user/functions/{io => fs}/mkdir.c | 2 +- src/libstd/user/functions/{io => fs}/mount.c | 2 +- src/libstd/user/functions/{io => fs}/open.c | 2 +- src/libstd/user/functions/{io => fs}/open2.c | 2 +- src/libstd/user/functions/{io => fs}/openat.c | 2 +- src/libstd/user/functions/{io => fs}/poll.c | 2 +- src/libstd/user/functions/{io => fs}/poll1.c | 2 +- src/libstd/user/functions/{io => fs}/read.c | 2 +- .../user/functions/{io => fs}/readdir.c | 2 +- .../user/functions/{io => fs}/readfile.c | 2 +- .../user/functions/{io => fs}/readlink.c | 2 +- src/libstd/user/functions/{io => fs}/rmdir.c | 2 +- src/libstd/user/functions/{io => fs}/scan.c | 2 +- .../user/functions/{io => fs}/scanfile.c | 2 +- src/libstd/user/functions/{io => fs}/seek.c | 2 +- src/libstd/user/functions/{io => fs}/share.c | 2 +- .../user/functions/{io => fs}/sharefile.c | 2 +- src/libstd/user/functions/{io => fs}/sread.c | 2 +- .../user/functions/{io => fs}/sreadfile.c | 2 +- src/libstd/user/functions/{io => fs}/stat.c | 2 +- src/libstd/user/functions/{io => fs}/swrite.c | 2 +- .../user/functions/{io => fs}/swritefile.c | 2 +- .../user/functions/{io => fs}/symlink.c | 2 +- src/libstd/user/functions/{io => fs}/unlink.c | 2 +- .../user/functions/{io => fs}/unmount.c | 2 +- src/libstd/user/functions/{io => fs}/vscan.c | 2 +- .../user/functions/{io => fs}/vscanfile.c | 2 +- src/libstd/user/functions/{io => fs}/write.c | 2 +- .../user/functions/{io => fs}/writefile.c | 2 +- src/libstd/user/functions/proc/arch_prctl.c | 2 +- src/libstd/user/functions/proc/futex.c | 2 +- src/libstd/user/functions/proc/getpid.c | 2 +- src/libstd/user/functions/proc/gettid.c | 2 +- src/libstd/user/functions/proc/kill.c | 2 +- src/libstd/user/functions/proc/mmap.c | 2 +- src/libstd/user/functions/proc/mprotect.c | 2 +- src/libstd/user/functions/proc/munmap.c | 2 +- src/libstd/user/functions/proc/nanosleep.c | 2 +- src/libstd/user/functions/proc/spawn.c | 2 +- src/libstd/user/functions/proc/uptime.c | 2 +- src/libstd/user/functions/stdio/fseek.c | 2 +- src/libstd/user/functions/stdio/remove.c | 2 +- src/libstd/user/functions/stdlib/system.c | 2 +- src/libstd/user/user.c | 2 +- src/modules/acpi/aml/runtime/field_unit.c | 8 +- src/modules/drivers/ps2/ps2.h | 2 +- src/modules/drivers/rtc/rtc.c | 2 +- src/modules/ipc/pipe/pipe.c | 2 +- src/modules/net/local/local.c | 2 +- src/modules/net/local/local.h | 2 +- src/modules/net/local/local_conn.h | 2 +- src/modules/net/local/local_listen.h | 2 +- src/programs/core/boxd/main.c | 2 +- src/programs/core/boxd/manifest.c | 2 +- src/programs/core/boxspawn/main.c | 2 +- src/programs/core/init/main.c | 2 +- src/programs/core/shell/builtin.c | 2 +- src/programs/core/shell/interactive.c | 2 +- src/programs/core/shell/main.c | 2 +- src/programs/core/shell/pipeline.c | 2 +- src/programs/core/shell/pipeline.h | 2 +- src/programs/utils/benchmark/main.c | 2 +- src/programs/utils/cat/main.c | 2 +- src/programs/utils/echo/main.c | 2 +- src/programs/utils/link/main.c | 2 +- src/programs/utils/ls/main.c | 2 +- src/programs/utils/mv/main.c | 2 +- src/programs/utils/readlink/main.c | 2 +- src/programs/utils/ringtest/main.c | 2 +- src/programs/utils/rm/main.c | 2 +- src/programs/utils/stat/main.c | 2 +- src/programs/utils/symlink/main.c | 2 +- src/programs/utils/tail/main.c | 2 +- src/programs/utils/top/main.c | 2 +- src/programs/utils/touch/main.c | 2 +- 151 files changed, 344 insertions(+), 225 deletions(-) rename include/libstd/sys/{io.h => fs.h} (99%) rename src/libstd/user/functions/{io => fs}/bind.c (91%) rename src/libstd/user/functions/{io => fs}/chdir.c (92%) rename src/libstd/user/functions/{io => fs}/claim.c (91%) rename src/libstd/user/functions/{io => fs}/close.c (92%) rename src/libstd/user/functions/{io => fs}/dup.c (92%) rename src/libstd/user/functions/{io => fs}/dup2.c (93%) rename src/libstd/user/functions/{io => fs}/getdents.c (93%) rename src/libstd/user/functions/{io => fs}/ioctl.c (94%) rename src/libstd/user/functions/{io => fs}/link.c (93%) rename src/libstd/user/functions/{io => fs}/mkdir.c (93%) rename src/libstd/user/functions/{io => fs}/mount.c (92%) rename src/libstd/user/functions/{io => fs}/open.c (92%) rename src/libstd/user/functions/{io => fs}/open2.c (93%) rename src/libstd/user/functions/{io => fs}/openat.c (92%) rename src/libstd/user/functions/{io => fs}/poll.c (93%) rename src/libstd/user/functions/{io => fs}/poll1.c (94%) rename src/libstd/user/functions/{io => fs}/read.c (93%) rename src/libstd/user/functions/{io => fs}/readdir.c (98%) rename src/libstd/user/functions/{io => fs}/readfile.c (94%) rename src/libstd/user/functions/{io => fs}/readlink.c (93%) rename src/libstd/user/functions/{io => fs}/rmdir.c (87%) rename src/libstd/user/functions/{io => fs}/scan.c (92%) rename src/libstd/user/functions/{io => fs}/scanfile.c (92%) rename src/libstd/user/functions/{io => fs}/seek.c (93%) rename src/libstd/user/functions/{io => fs}/share.c (92%) rename src/libstd/user/functions/{io => fs}/sharefile.c (93%) rename src/libstd/user/functions/{io => fs}/sread.c (98%) rename src/libstd/user/functions/{io => fs}/sreadfile.c (89%) rename src/libstd/user/functions/{io => fs}/stat.c (93%) rename src/libstd/user/functions/{io => fs}/swrite.c (87%) rename src/libstd/user/functions/{io => fs}/swritefile.c (92%) rename src/libstd/user/functions/{io => fs}/symlink.c (93%) rename src/libstd/user/functions/{io => fs}/unlink.c (91%) rename src/libstd/user/functions/{io => fs}/unmount.c (91%) rename src/libstd/user/functions/{io => fs}/vscan.c (97%) rename src/libstd/user/functions/{io => fs}/vscanfile.c (93%) rename src/libstd/user/functions/{io => fs}/write.c (93%) rename src/libstd/user/functions/{io => fs}/writefile.c (94%) diff --git a/include/kernel/fs/dentry.h b/include/kernel/fs/dentry.h index 737451408..f7f4e623f 100644 --- a/include/kernel/fs/dentry.h +++ b/include/kernel/fs/dentry.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include typedef struct dentry dentry_t; diff --git a/include/kernel/fs/devfs.h b/include/kernel/fs/devfs.h index d8aefc46e..bb1c6deba 100644 --- a/include/kernel/fs/devfs.h +++ b/include/kernel/fs/devfs.h @@ -2,7 +2,7 @@ #include #include -#include +#include typedef struct file file_t; typedef struct file_ops file_ops_t; diff --git a/include/kernel/fs/file.h b/include/kernel/fs/file.h index fd4587f8a..1c10cf572 100644 --- a/include/kernel/fs/file.h +++ b/include/kernel/fs/file.h @@ -6,7 +6,7 @@ #include #include -#include +#include #include typedef struct wait_queue wait_queue_t; diff --git a/include/kernel/fs/filesystem.h b/include/kernel/fs/filesystem.h index ed1f503c7..fe3431b15 100644 --- a/include/kernel/fs/filesystem.h +++ b/include/kernel/fs/filesystem.h @@ -11,7 +11,7 @@ #include #include -#include +#include #include #include #include diff --git a/include/kernel/fs/inode.h b/include/kernel/fs/inode.h index 1c3be5781..0f7b5771b 100644 --- a/include/kernel/fs/inode.h +++ b/include/kernel/fs/inode.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include diff --git a/include/kernel/fs/key.h b/include/kernel/fs/key.h index 8eeba9498..847ccc593 100644 --- a/include/kernel/fs/key.h +++ b/include/kernel/fs/key.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include /** diff --git a/include/kernel/fs/namespace.h b/include/kernel/fs/namespace.h index b2a714464..a5956aa23 100644 --- a/include/kernel/fs/namespace.h +++ b/include/kernel/fs/namespace.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include typedef struct namespace namespace_t; diff --git a/include/kernel/fs/netfs.h b/include/kernel/fs/netfs.h index 290fc8687..7a5bc6fce 100644 --- a/include/kernel/fs/netfs.h +++ b/include/kernel/fs/netfs.h @@ -7,7 +7,7 @@ #include #include -#include +#include #include typedef struct netfs_family netfs_family_t; diff --git a/include/kernel/fs/path.h b/include/kernel/fs/path.h index ee04fbb7d..91c5fe4d8 100644 --- a/include/kernel/fs/path.h +++ b/include/kernel/fs/path.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include typedef struct path path_t; typedef struct mount mount_t; diff --git a/include/kernel/fs/superblock.h b/include/kernel/fs/superblock.h index 929a3b89c..cf5050838 100644 --- a/include/kernel/fs/superblock.h +++ b/include/kernel/fs/superblock.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include typedef struct filesystem filesystem_t; diff --git a/include/kernel/fs/sysfs.h b/include/kernel/fs/sysfs.h index b9f5a45a0..164a298c1 100644 --- a/include/kernel/fs/sysfs.h +++ b/include/kernel/fs/sysfs.h @@ -2,7 +2,7 @@ #include #include -#include +#include typedef struct file file_t; typedef struct file_ops file_ops_t; diff --git a/include/kernel/fs/tmpfs.h b/include/kernel/fs/tmpfs.h index ac8447b07..a137c528f 100644 --- a/include/kernel/fs/tmpfs.h +++ b/include/kernel/fs/tmpfs.h @@ -6,7 +6,7 @@ #include -#include +#include #include /** diff --git a/include/kernel/fs/vfs.h b/include/kernel/fs/vfs.h index 178da66f8..47459ae64 100644 --- a/include/kernel/fs/vfs.h +++ b/include/kernel/fs/vfs.h @@ -11,7 +11,7 @@ #include #include -#include +#include #include #include #include diff --git a/include/kernel/io/io.h b/include/kernel/io/io.h index c616a119a..2bbb2bd24 100644 --- a/include/kernel/io/io.h +++ b/include/kernel/io/io.h @@ -1,11 +1,11 @@ #pragma once #include +#include #include #include #include #include -#include #include #include @@ -19,13 +19,18 @@ * @todo The I/O ring system is primarily a design document for now as it remains very work in progress and subject to * change, currently being mostly unimplemented. * - * The I/O ring provides the core of all interfaces in PatchworkOS, all implemented in an interface - * inspired by `io_uring()` from Linux. + * The I/O ring provides the core of all interfaces in PatchworkOS, where user-space submits Submission Queue Entries + * (SQEs) and receives Completion Queue Entries (CQEs) from it, all within shared memory. Allowing for highly efficient + * and asynchronous I/O operations, especially since PatchworkOS is designed to be natively asynchronous. + * + * Each SQE specifies a verb (the operation to perform) and a set of up to `SQE_MAX_ARG` arguments, while each CQE + * returns the result of a previously submitted SQE. * * Synchronous operations are implemented on top of this API in userspace. * * @see libstd_sys_ioring for the userspace interface to the asynchronous ring. - * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`. + * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`, the inspiration for this + * system. * @see [Manpages](https://man7.org/linux/man-pages/man7/io_uring.7.html) for more information about `io_uring`. * * ## Syncronization @@ -37,16 +42,17 @@ * the caller to ensure proper synchronization. * * @note The reason for this limitation is optimization for the common case, as the syncronization logic for multiple - * producers would add significant overhead. + * producers would add significant overhead. Additionally, it is rather straight forward for user-space to protect the + * ring with a mutex should it need to. * - * Regarding the I/O ring structure itself, the structure can only be torndown as long as nothing is using it and there are - * no pending operations. + * Regarding the I/O ring structure itself, the structure can only be torndown as long as nothing is using it and there + * are no pending operations. * * ## Registers * * Operations performed on a I/O ring can load arguments from, and save their results to, seven 64-bit general purpose - * registers. All registers are stored in the shared control area of the I/O ring structure (`ioring_ctrl_t`), as such they can be inspected and - * modified by user space. + * registers. All registers are stored in the shared control area of the I/O ring structure (`ioring_ctrl_t`), as such + * they can be inspected and modified by user space. * * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for each argument and * one for the result. Each specifier is stored as three bits, with a `SQE_REG_NONE` value indicating no-op and any @@ -60,36 +66,91 @@ * * @see `sqe_flags_t` for more information about register specifiers and their formatting. * + * ## Arguments + * + * Instead of manually indexing the `_args` array each SQE stores the arguments in a union with several "argument + * archetypes". Such that several verbs can use the same archetype for their arguments. + * + * @note The kernels internal I/O Request Packet structure contains the same archetypes but with the kernel equivalents + * of the arguments, for example, a `file_t*` instead of a `fd_t`. + * + * Included below is a list of all argument archetypes. + * + * ### `.handle` + * + * Used for verbs that act on a single existing file descriptor. + * + * **Arguments:** + * - `fd`: The file descriptor to act upon. + * + * ### `.path` + * + * Used for verbs that act on a filesystem path. + * + * **Arguments:** + * - `dirfd`: The directory file descriptor to resolve the path from, or `FD_CWD` to use the current working directory. + * - `path`: Pointer to the path string, null-termination is ignored. + * - `len`: Length of the path string. + * + * ### `.rw` + * + * Used for verbs used for data transfer. + * + * **Arguments:** + * - `fd`: The file descriptor to access. + * - `buffer`: Pointer to the buffer to read into or write from. + * - `len`: Length of the buffer. + * - `off`: Offset within the file to access, or `IO_CUR` to use the current file offset. + * + * ### `.seek` + * + * Used for verbs that seek within a file. + * + * **Arguments:** + * - `fd`: The file descriptor to seek within. + * - `off`: Offset to seek to. + * - `whence`: Origin for the seek operation, `IO_SET`, `IO_END` or `IO_CUR`. + * + * ### `.poll` + * + * Used for verbs that wait for events on a file descriptor. + * + * **Arguments:** + * - `fd`: The file descriptor to poll. + * - `events`: The events to wait for (e.g., `POLLIN`, `POLLOUT`). + * + * ## Results + * + * The result of a SQE is stored in its corresponding CQE using a single 64-bit value. For convenience, the result is + * stored as a union of various types. Note that this does not actually change the stored value, just how it is + * interpreted. + * + * If a SQE fails, the error code will be stored separately from the result and the result it self may be undefined. + * Some verbs may allow partial failures in which case the result may still be valid even if an error code is present. + * + * @todo Decide if partial failures are a good idea or not. + * * ## Errors * - * The majority of errors are returned in the completion queue entries, certain errors (such as `ENOMEM`) may be + * The majority of errors are returned in the CQEs, certain errors (such as `ENOMEM`) may be * reported directly from the `enter()` call. * - * Certain error values that may be returned in a completion queue entry include: + * Error values that may be returned in a CQE include: * - `EOK`: Success. - * - `ECANCELED`: The operation was cancelled. - * - `ETIMEDOUT`: The operation timed out. - * - Other values may be returned depending on the operation. + * - `ECANCELED`: The verb was cancelled. + * - `ETIMEDOUT`: The verb timed out. + * - Other values may be returned depending on the verb. * * ## Verbs * - * A verb specifies the operation to perform. Included is a list of currently defines verbs. + * Included below is a list of all currently implemented verbs. * * ### `VERB_NOP` * - * Never completes, can be used to implement a sleep equivalent by specifying a timeout. + * A no-operation verb that does nothing but is usefull for implementing sleeping. * * @param None - * @return Always `0`. - * - * ### `VERB_OPEN` - * - * Opens a file, including regular files, directories, symlinks, etc. - * - * @param from The file descriptor to open the file relative to, or `FD_NONE` to open from the current working - * directory. - * @param path Pointer to a null-terminated string containing the path to the file to open - * @return The file descriptor of the opened file. + * @result None * * @{ */ @@ -111,7 +172,7 @@ typedef enum */ typedef struct io_ctx { - ioring_t ring; ///< The kernel-side ring structure. + ioring_t ring; ///< The kernel-side ring structure. irp_pool_t* irps; ///< Pool of preallocated IRPs. mem_desc_pool_t* descs; ///< Pool of preallocated memory descriptors. void* userAddr; ///< Userspace address of the ring. diff --git a/include/kernel/io/irp.h b/include/kernel/io/irp.h index 645d2e5f6..993b753ff 100644 --- a/include/kernel/io/irp.h +++ b/include/kernel/io/irp.h @@ -7,11 +7,12 @@ #include #include +#include #include -#include +#include +#include #include #include -#include #include typedef struct irp irp_t; @@ -248,7 +249,7 @@ typedef struct irp irp_t; #define IRP_LOC_MAX 8 ///< The maximum number of locations in a IRP. -#define IRP_ARGS_MAX SQE_MAX_ARGS ///< The maximum number of arguments in an IRP. +#define IRP_ARG_MAX SQE_ARG_MAX ///< The maximum number of arguments in an IRP. /** * @brief IRP completion callback type. @@ -288,6 +289,8 @@ typedef struct irp_loc * @note We need the ability to store both the original arguments from a SQE and the parsed arguments. For example, * opening a `fd_t` into a `file_t*`. As such, to avoid using another cache line, the SQE is stored in a union with the * parsed arguments. + * + * @see kernel_io for more information for each possible verb. */ typedef struct ALIGNED(64) irp { @@ -305,17 +308,45 @@ typedef struct ALIGNED(64) irp }; void* data; ///< Private data for the operation, will be returned in the completion entry. union { + uint64_t _args[IRP_ARG_MAX]; struct { - file_t* from; + file_t* file; + } handle; + struct + { + file_t* file; mem_desc_t* path; - } open; - uint64_t _args[IRP_ARGS_MAX]; + } path; + struct + { + file_t* file; + mem_desc_t* buffer; + size_t len; + ssize_t off; + } rw; + struct + { + file_t* file; + ssize_t off; + whence_t whence; + } seek; + struct + { + file_t* file; + events_t events; + } poll; }; }; sqe_t sqe; ///< The original SQE for this IRP. }; - uint64_t result; ///< Result of the IRP. + union { + file_t* file; + size_t count; + void* ptr; + events_t events; + uint64_t _raw; + } result; errno_t err; ///< The error code of the operation, also used to specify its current state. pool_idx_t index; ///< Index of the IRP in its pool. pool_idx_t next; ///< Index of the next IRP in a chain or in the free list. diff --git a/include/kernel/ipc/note.h b/include/kernel/ipc/note.h index d9d4bb76a..f6cc7f5e9 100644 --- a/include/kernel/ipc/note.h +++ b/include/kernel/ipc/note.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include typedef struct cpu cpu_t; diff --git a/include/kernel/log/log.h b/include/kernel/log/log.h index 1e89c8534..e880e16e7 100644 --- a/include/kernel/log/log.h +++ b/include/kernel/log/log.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include diff --git a/include/kernel/module/module.h b/include/kernel/module/module.h index f3664ff28..a61f0b68c 100644 --- a/include/kernel/module/module.h +++ b/include/kernel/module/module.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include typedef struct module module_t; diff --git a/include/kernel/proc/process.h b/include/kernel/proc/process.h index 977b907e8..84b7d8073 100644 --- a/include/kernel/proc/process.h +++ b/include/kernel/proc/process.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -14,7 +15,6 @@ #include #include #include -#include #include #include diff --git a/include/kernel/sync/rcu.h b/include/kernel/sync/rcu.h index aa250840b..126045b60 100644 --- a/include/kernel/sync/rcu.h +++ b/include/kernel/sync/rcu.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include diff --git a/include/kernel/utils/map.h b/include/kernel/utils/map.h index 255c9ce90..180d8a8b9 100644 --- a/include/kernel/utils/map.h +++ b/include/kernel/utils/map.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include typedef struct map_key map_key_t; typedef struct map_entry map_entry_t; diff --git a/include/libpatchwork/cmd.h b/include/libpatchwork/cmd.h index d41b604e9..f0344180b 100644 --- a/include/libpatchwork/cmd.h +++ b/include/libpatchwork/cmd.h @@ -8,7 +8,7 @@ #include "surface.h" #include -#include +#include #include #if defined(__cplusplus) diff --git a/include/libpatchwork/event.h b/include/libpatchwork/event.h index 650adb249..49af016b1 100644 --- a/include/libpatchwork/event.h +++ b/include/libpatchwork/event.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include #if defined(__cplusplus) diff --git a/include/libpatchwork/surface.h b/include/libpatchwork/surface.h index ff25c62b9..726e3cb64 100644 --- a/include/libpatchwork/surface.h +++ b/include/libpatchwork/surface.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include #if defined(__cplusplus) extern "C" diff --git a/include/libstd/_internal/fd_t.h b/include/libstd/_internal/fd_t.h index 267f18fea..5171bf51b 100644 --- a/include/libstd/_internal/fd_t.h +++ b/include/libstd/_internal/fd_t.h @@ -2,23 +2,17 @@ #define _INTERNAL_FD_T_H 1 /** - * @brief A file descriptor. - * @ingroup libstd - * - * The `fd_t` type represents a file descriptor, which is a index into the processes files table. We also define the - * special value `FD_NONE` which is equal to `UINT64_MAX` to represent no file descriptor. + * @addtogroup libstd * + * @{ */ -typedef __UINT64_TYPE__ fd_t; -/** - * @brief No file descriptor constant. - * @ingroup libstd - * @def FD_NONE - * - * The `FD_NONE` value represents no file descriptor. - * - */ -#define FD_NONE __UINT64_MAX__ +typedef __UINT64_TYPE__ fd_t; ///< File descriptor type. + +#define FD_NONE ((fd_t) - 1) ///< No file descriptor. + +#define FD_CWD ((fd_t) - 2) ///< Use the current working directory.) + +/** @} */ #endif diff --git a/include/libstd/sys/io.h b/include/libstd/sys/fs.h similarity index 99% rename from include/libstd/sys/io.h rename to include/libstd/sys/fs.h index 5c2fa60d4..f61499b74 100644 --- a/include/libstd/sys/io.h +++ b/include/libstd/sys/fs.h @@ -25,8 +25,8 @@ extern "C" #include "_internal/time_t.h" /** - * @brief System IO header. - * @defgroup libstd_sys_io System IO + * @brief Filesystem header. + * @defgroup libstd_sys_fs Filesystem IO * @ingroup libstd * * @{ diff --git a/include/libstd/sys/ioring.h b/include/libstd/sys/ioring.h index 3dda9083f..b0dfc97ef 100644 --- a/include/libstd/sys/ioring.h +++ b/include/libstd/sys/ioring.h @@ -17,19 +17,29 @@ extern "C" #include "_internal/clock_t.h" #include "_internal/errno_t.h" #include "_internal/fd_t.h" +#include "_internal/ssize_t.h" /** * @addtogroup kernel_io * @{ */ -typedef uint32_t verb_t; ///< Verb type. +typedef uint64_t whence_t; ///< Seek origin type. +#define IO_SET ((ssize_t) - 3) ///< Use the start of the file. +#define IO_END ((ssize_t) - 2) ///< Use the end of the file. +#define IO_CUR ((ssize_t) - 1) ///< Use the current file offset. -#define VERB_NOP 0 ///< No-op verb. -#define VERB_OPEN 1 ///< Open file verb. -#define VERB_MAX 1 ///< Maximum verb. +typedef uint64_t events_t; ///< Poll events type. +#define IO_READABLE (1 << 0) ///< File descriptor is ready to read. +#define IO_WRITABLE (1 << 1) ///< File descriptor is ready to write +#define IO_ERROR (1 << 2) ///< File descriptor caused an error. +#define IO_CLOSED (1 << 3) ///< File descriptor is closed. +#define IO_INVALID (1 << 4) ///< Invalid file descriptor. -#define SQE_MAX_ARGS 5 ///< Maximum number of arguments for a ring operation. +typedef uint32_t verb_t; ///< Verb type. +#define VERB_NOP 0 ///< No-op verb. +#define VERB_OPEN 1 ///< Open file verb. +#define VERB_MAX 1 ///< Maximum verb. typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. @@ -75,6 +85,8 @@ typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. */ #define SQE_HARDLINK (1 << (_SQE_FLAGS + 3)) +#define SQE_ARG_MAX 5 ///< Maximum number of arguments for a ring operation. + /** * @brief Asynchronous submission queue entry (SQE). * @struct sqe_t @@ -82,7 +94,7 @@ typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. * @warning It is the responsibility of userspace to ensure that any pointers * passed to the kernel remain valid until the operation is complete. * - * @see kernel_io for more information on the possible operations. + * @see kernel_io for more information for each possible verb. */ typedef struct sqe { @@ -91,17 +103,35 @@ typedef struct sqe clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. void* data; ///< Private data for the operation, will be returned in the completion entry. union { + uint64_t _args[SQE_ARG_MAX]; struct { - uint64_t none; - } nop; + fd_t fd; + } handle; struct { - fd_t from; + fd_t dirfd; char* path; - size_t length; - } open; - uint64_t _args[SQE_MAX_ARGS]; + size_t len; + } path; + struct + { + fd_t fd; + void* buffer; + size_t len; + ssize_t off; + } rw; + struct + { + fd_t fd; + ssize_t off; + whence_t whence; + } seek; + struct + { + fd_t fd; + events_t events; + } poll; }; } sqe_t; @@ -137,8 +167,10 @@ typedef struct ALIGNED(32) cqe errno_t error; ///< Error code, if not equal to `EOK` an error occurred. void* data; ///< Private data from the submission entry. union { - uint64_t nop; - fd_t open; + fd_t fd; + size_t count; + void* ptr; + events_t events; uint64_t _result; }; } cqe_t; diff --git a/include/modules/acpi/aml/object.h b/include/modules/acpi/aml/object.h index d02096001..bfc7af5c9 100644 --- a/include/modules/acpi/aml/object.h +++ b/include/modules/acpi/aml/object.h @@ -411,7 +411,7 @@ typedef struct aml_alias typedef struct aml_unresolved { AML_OBJECT_COMMON_HEADER; - aml_name_stioring_t nameString; ///< The NameString representing the path to the target object. + aml_name_stioring_t nameString; ///< The NameString representing the path to the target object. aml_object_t* from; ///< The object to start the search from when resolving the reference. aml_patch_up_resolve_callback_t callback; ///< The callback to call when a matching object is found. } aml_unresolved_t; diff --git a/include/modules/acpi/devices.h b/include/modules/acpi/devices.h index efc8e10f4..877993323 100644 --- a/include/modules/acpi/devices.h +++ b/include/modules/acpi/devices.h @@ -1,11 +1,11 @@ #pragma once -#include #include +#include #include #include -#include +#include /** * @brief Device and Power Management diff --git a/lib/OVMFbin/OVMF_VARS-pure-efi.fd b/lib/OVMFbin/OVMF_VARS-pure-efi.fd index 92201fe8b73f99915753e2fc6dc7750cd50b2b4d..b728cd32916c645e3def714dd4f885908f57f930 100644 GIT binary patch delta 38 ucmZo@;Am*z*s#fFa-D(F<|{Tee3LiWxUk&QW?BF)Qi~;~H#}A_b delta 30 mcmZo@;Am*z*s#fFa*(&v<}WreeA~b1GHzJX^kLZtMgaiKiVd^? diff --git a/meta/doxy/Doxyfile b/meta/doxy/Doxyfile index afeb3f8cd..148ab564c 100644 --- a/meta/doxy/Doxyfile +++ b/meta/doxy/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = "PatchworkOS" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "30cbaaa8-dirty" +PROJECT_NUMBER = "c2e0413c-dirty" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewers a diff --git a/src/boxes/apps/terminal/terminal.c b/src/boxes/apps/terminal/terminal.c index ea6ac836a..94c3a1a1a 100644 --- a/src/boxes/apps/terminal/terminal.c +++ b/src/boxes/apps/terminal/terminal.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/boxes/apps/terminal/terminal.h b/src/boxes/apps/terminal/terminal.h index 75d78dc30..5fd3f4b0d 100644 --- a/src/boxes/apps/terminal/terminal.h +++ b/src/boxes/apps/terminal/terminal.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include diff --git a/src/boxes/core/dwm/client.h b/src/boxes/core/dwm/client.h index f466f42ea..bf332f9b7 100644 --- a/src/boxes/core/dwm/client.h +++ b/src/boxes/core/dwm/client.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include #define CLIENT_RECV_BUFFER_SIZE (sizeof(cmd_buffer_t) + 128) diff --git a/src/boxes/core/dwm/dwm.c b/src/boxes/core/dwm/dwm.c index 4bb9c3083..2f7b09d2b 100644 --- a/src/boxes/core/dwm/dwm.c +++ b/src/boxes/core/dwm/dwm.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/boxes/core/dwm/dwm.h b/src/boxes/core/dwm/dwm.h index 9a890d9fe..045dce722 100644 --- a/src/boxes/core/dwm/dwm.h +++ b/src/boxes/core/dwm/dwm.h @@ -3,7 +3,7 @@ #include "surface.h" #include -#include +#include #include /** diff --git a/src/boxes/core/dwm/screen.c b/src/boxes/core/dwm/screen.c index d65536cde..ec5043aa0 100644 --- a/src/boxes/core/dwm/screen.c +++ b/src/boxes/core/dwm/screen.c @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include static uint64_t width; diff --git a/src/boxes/core/dwm/surface.c b/src/boxes/core/dwm/surface.c index 4212b519d..fe5aad8eb 100644 --- a/src/boxes/core/dwm/surface.c +++ b/src/boxes/core/dwm/surface.c @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include static surface_id_t newId = 0; diff --git a/src/boxes/core/dwm/surface.h b/src/boxes/core/dwm/surface.h index 93de56d6e..5c5299500 100644 --- a/src/boxes/core/dwm/surface.h +++ b/src/boxes/core/dwm/surface.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include typedef struct client client_t; diff --git a/src/boxes/core/taskbar/start_menu.c b/src/boxes/core/taskbar/start_menu.c index 63a5b9353..56b48c9a6 100644 --- a/src/boxes/core/taskbar/start_menu.c +++ b/src/boxes/core/taskbar/start_menu.c @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #define START_ENTRY_MAX 16 diff --git a/src/boxes/core/taskbar/taskbar.h b/src/boxes/core/taskbar/taskbar.h index d3291f718..fe4217dbf 100644 --- a/src/boxes/core/taskbar/taskbar.h +++ b/src/boxes/core/taskbar/taskbar.h @@ -3,7 +3,7 @@ #include "start_menu.h" #include -#include +#include #include #define START_WIDTH 100 diff --git a/src/kernel/cpu/interrupt.c b/src/kernel/cpu/interrupt.c index b59a28cad..8dcb2da37 100644 --- a/src/kernel/cpu/interrupt.c +++ b/src/kernel/cpu/interrupt.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -16,7 +17,6 @@ #include #include #include -#include #include diff --git a/src/kernel/cpu/port.c b/src/kernel/cpu/port.c index eaea7a092..273abafda 100644 --- a/src/kernel/cpu/port.c +++ b/src/kernel/cpu/port.c @@ -12,7 +12,8 @@ static BITMAP_CREATE(ports, PORT_MAX + 1); static lock_t lock = LOCK_CREATE(); -uint64_t port_reserve(port_t* out, port_t minBase, port_t maxBase, uint64_t alignment, uint64_t length, const char* owner) +uint64_t port_reserve(port_t* out, port_t minBase, port_t maxBase, uint64_t alignment, uint64_t length, + const char* owner) { UNUSED(owner); diff --git a/src/kernel/drivers/abstract/kbd.c b/src/kernel/drivers/abstract/kbd.c index 30d6d6231..a608cf64c 100644 --- a/src/kernel/drivers/abstract/kbd.c +++ b/src/kernel/drivers/abstract/kbd.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/kernel/drivers/perf.c b/src/kernel/drivers/perf.c index 6ddf7a4ed..0adc9d8e3 100644 --- a/src/kernel/drivers/perf.c +++ b/src/kernel/drivers/perf.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/kernel/fs/devfs.c b/src/kernel/fs/devfs.c index 4538b2a09..4b24b9697 100644 --- a/src/kernel/fs/devfs.c +++ b/src/kernel/fs/devfs.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include static dentry_t* root = NULL; diff --git a/src/kernel/fs/filesystem.c b/src/kernel/fs/filesystem.c index 08b996a9f..19ace62d0 100644 --- a/src/kernel/fs/filesystem.c +++ b/src/kernel/fs/filesystem.c @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include static dentry_t* dir = NULL; diff --git a/src/kernel/fs/namespace.c b/src/kernel/fs/namespace.c index 4f996b51b..cc0413cdf 100644 --- a/src/kernel/fs/namespace.c +++ b/src/kernel/fs/namespace.c @@ -16,7 +16,7 @@ #include #include -#include +#include #include static map_key_t mount_key(mount_id_t parentId, dentry_id_t mountpointId) diff --git a/src/kernel/fs/netfs.c b/src/kernel/fs/netfs.c index e70025595..a86dd3241 100644 --- a/src/kernel/fs/netfs.c +++ b/src/kernel/fs/netfs.c @@ -13,7 +13,7 @@ #include #include -#include +#include #include static list_t families = LIST_CREATE(families); diff --git a/src/kernel/fs/procfs.c b/src/kernel/fs/procfs.c index e1f04bc64..a59a5f736 100644 --- a/src/kernel/fs/procfs.c +++ b/src/kernel/fs/procfs.c @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include static uint64_t procfs_revalidate_hide(dentry_t* dentry) diff --git a/src/kernel/fs/sysfs.c b/src/kernel/fs/sysfs.c index 0ae524570..b06a881b2 100644 --- a/src/kernel/fs/sysfs.c +++ b/src/kernel/fs/sysfs.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include static dentry_t* root = NULL; diff --git a/src/kernel/fs/tmpfs.c b/src/kernel/fs/tmpfs.c index 79b289651..bbfc7e06f 100644 --- a/src/kernel/fs/tmpfs.c +++ b/src/kernel/fs/tmpfs.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/kernel/fs/vfs.c b/src/kernel/fs/vfs.c index d9b334877..df7d2cba2 100644 --- a/src/kernel/fs/vfs.c +++ b/src/kernel/fs/vfs.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include static uint64_t vfs_create(path_t* path, const pathname_t* pathname, namespace_t* ns) diff --git a/src/kernel/io/io.c b/src/kernel/io/io.c index 9f1ec6e1e..be9241d41 100644 --- a/src/kernel/io/io.c +++ b/src/kernel/io/io.c @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include @@ -9,12 +11,10 @@ #include #include #include -#include -#include #include -#include #include +#include #include static inline uint64_t io_ctx_acquire(io_ctx_t* ctx) @@ -211,7 +211,7 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) sqe_flags_t reg = (irp->flags >> SQE_SAVE) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - atomic_store_explicit(&ring->ctrl->regs[reg], irp->result, memory_order_release); + atomic_store_explicit(&ring->ctrl->regs[reg], irp->result._raw, memory_order_release); } uint32_t tail = atomic_load_explicit(&ring->ctrl->ctail, memory_order_relaxed); @@ -227,7 +227,7 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) cqe->verb = irp->verb; cqe->error = irp->err; cqe->data = irp->data; - cqe->_result = irp->result; + cqe->_result = irp->result._raw; atomic_store_explicit(&ring->ctrl->ctail, tail + 1, memory_order_release); wait_unblock(&ctx->waitQueue, WAIT_ALL, EOK); @@ -268,7 +268,7 @@ static void io_ctx_dispatch(irp_t* irp) io_ctx_t* ctx = irp_get_ctx(irp); ioring_t* ring = &ctx->ring; - for (uint64_t i = 0; i < SQE_MAX_ARGS; i++) + for (uint64_t i = 0; i < SQE_ARG_MAX; i++) { sqe_flags_t reg = (irp->flags >> (i * SQE_REG_SHIFT)) & SQE_REG_MASK; if (reg == SQE_REG_NONE) diff --git a/src/kernel/io/irp.c b/src/kernel/io/irp.c index 600446d4e..71b2954ed 100644 --- a/src/kernel/io/irp.c +++ b/src/kernel/io/irp.c @@ -1,9 +1,9 @@ #include +#include #include #include #include #include -#include #include #include @@ -48,11 +48,11 @@ irp_pool_t* irp_pool_new(size_t size, void* ctx) irp->flags = 0; irp->timeout = CLOCKS_NEVER; irp->data = NULL; - for (size_t j = 0; j < IRP_ARGS_MAX; j++) + for (size_t j = 0; j < IRP_ARG_MAX; j++) { - irp->sqe._args[j] = 0; + irp->_args[j] = 0; } - irp->result = 0; + irp->result._raw = 0; irp->err = EOK; irp->index = i; irp->next = i < size - 1 ? i + 1 : POOL_IDX_MAX; @@ -88,7 +88,7 @@ irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) irp->location = IRP_LOC_MAX; irp->next = POOL_IDX_MAX; irp->err = EINPROGRESS; - irp->result = 0; + irp->result._raw = 0; atomic_store_explicit(&irp->cancel, NULL, memory_order_relaxed); if (sqe == NULL) diff --git a/src/kernel/log/log.c b/src/kernel/log/log.c index 0b536a7bf..b841999d9 100644 --- a/src/kernel/log/log.c +++ b/src/kernel/log/log.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/kernel/log/panic.c b/src/kernel/log/panic.c index a748bf2c0..5db7b214d 100644 --- a/src/kernel/log/panic.c +++ b/src/kernel/log/panic.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/kernel/module/module.c b/src/kernel/module/module.c index 11cfc5752..60e5fc870 100644 --- a/src/kernel/module/module.c +++ b/src/kernel/module/module.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include static module_info_t fakeKernelModuleInfo = { diff --git a/src/kernel/proc/process.c b/src/kernel/proc/process.c index 2c392b4bb..6b5357819 100644 --- a/src/kernel/proc/process.c +++ b/src/kernel/proc/process.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,6 @@ #include #include #include -#include #include #include @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/libpatchwork/config.c b/src/libpatchwork/config.c index 5e3aaa53a..929ce0f9d 100644 --- a/src/libpatchwork/config.c +++ b/src/libpatchwork/config.c @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include typedef struct config_pair diff --git a/src/libpatchwork/font.c b/src/libpatchwork/font.c index b4759ffea..4a052fcda 100644 --- a/src/libpatchwork/font.c +++ b/src/libpatchwork/font.c @@ -2,7 +2,7 @@ #include #include -#include +#include font_t* font_default(display_t* disp) { diff --git a/src/libpatchwork/internal.h b/src/libpatchwork/internal.h index f972c6f21..18ff13f1f 100644 --- a/src/libpatchwork/internal.h +++ b/src/libpatchwork/internal.h @@ -1,6 +1,6 @@ #include -#include +#include #include #include diff --git a/src/libstd/common/heap.c b/src/libstd/common/heap.c index 8eda4dd65..8c1be803e 100644 --- a/src/libstd/common/heap.c +++ b/src/libstd/common/heap.c @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/src/libstd/common/init.c b/src/libstd/common/init.c index 3042e1fb9..5865c3efe 100644 --- a/src/libstd/common/init.c +++ b/src/libstd/common/init.c @@ -8,7 +8,7 @@ #include "user/user.h" #endif -#include +#include void _std_init(void) { diff --git a/src/libstd/user/common/file.c b/src/libstd/user/common/file.c index c80a71007..bd438999d 100644 --- a/src/libstd/user/common/file.c +++ b/src/libstd/user/common/file.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include static list_t files; static mtx_t filesMtx; diff --git a/src/libstd/user/common/std_streams.c b/src/libstd/user/common/std_streams.c index a2f2e09ac..abb10bde1 100644 --- a/src/libstd/user/common/std_streams.c +++ b/src/libstd/user/common/std_streams.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include static uint8_t _stdin_buff[BUFSIZ]; static uint8_t _stdout_buff[BUFSIZ]; diff --git a/src/libstd/user/common/syscalls.h b/src/libstd/user/common/syscalls.h index b90fdc84d..89a8b4f7d 100644 --- a/src/libstd/user/common/syscalls.h +++ b/src/libstd/user/common/syscalls.h @@ -3,9 +3,9 @@ #include #include -#include -#include +#include #include +#include #include #define _SYSCALL0(retType, num) \ diff --git a/src/libstd/user/functions/io/bind.c b/src/libstd/user/functions/fs/bind.c similarity index 91% rename from src/libstd/user/functions/io/bind.c rename to src/libstd/user/functions/fs/bind.c index cc4c4ba15..f469b9070 100644 --- a/src/libstd/user/functions/io/bind.c +++ b/src/libstd/user/functions/fs/bind.c @@ -1,4 +1,4 @@ -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/chdir.c b/src/libstd/user/functions/fs/chdir.c similarity index 92% rename from src/libstd/user/functions/io/chdir.c rename to src/libstd/user/functions/fs/chdir.c index 84343232a..66dfdb74a 100644 --- a/src/libstd/user/functions/io/chdir.c +++ b/src/libstd/user/functions/fs/chdir.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/claim.c b/src/libstd/user/functions/fs/claim.c similarity index 91% rename from src/libstd/user/functions/io/claim.c rename to src/libstd/user/functions/fs/claim.c index eff018a30..c7d425e73 100644 --- a/src/libstd/user/functions/io/claim.c +++ b/src/libstd/user/functions/fs/claim.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/close.c b/src/libstd/user/functions/fs/close.c similarity index 92% rename from src/libstd/user/functions/io/close.c rename to src/libstd/user/functions/fs/close.c index 1d76782a1..5a577f1d3 100644 --- a/src/libstd/user/functions/io/close.c +++ b/src/libstd/user/functions/fs/close.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/dup.c b/src/libstd/user/functions/fs/dup.c similarity index 92% rename from src/libstd/user/functions/io/dup.c rename to src/libstd/user/functions/fs/dup.c index cfbb75a0f..1ac7ee1a8 100644 --- a/src/libstd/user/functions/io/dup.c +++ b/src/libstd/user/functions/fs/dup.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/dup2.c b/src/libstd/user/functions/fs/dup2.c similarity index 93% rename from src/libstd/user/functions/io/dup2.c rename to src/libstd/user/functions/fs/dup2.c index a1454645d..6323909f6 100644 --- a/src/libstd/user/functions/io/dup2.c +++ b/src/libstd/user/functions/fs/dup2.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/getdents.c b/src/libstd/user/functions/fs/getdents.c similarity index 93% rename from src/libstd/user/functions/io/getdents.c rename to src/libstd/user/functions/fs/getdents.c index 401e1578a..21713a81c 100644 --- a/src/libstd/user/functions/io/getdents.c +++ b/src/libstd/user/functions/fs/getdents.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/ioctl.c b/src/libstd/user/functions/fs/ioctl.c similarity index 94% rename from src/libstd/user/functions/io/ioctl.c rename to src/libstd/user/functions/fs/ioctl.c index b3b40968a..9fbeecfb0 100644 --- a/src/libstd/user/functions/io/ioctl.c +++ b/src/libstd/user/functions/fs/ioctl.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/link.c b/src/libstd/user/functions/fs/link.c similarity index 93% rename from src/libstd/user/functions/io/link.c rename to src/libstd/user/functions/fs/link.c index b1f9d3d4d..e02c0f6c8 100644 --- a/src/libstd/user/functions/io/link.c +++ b/src/libstd/user/functions/fs/link.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/mkdir.c b/src/libstd/user/functions/fs/mkdir.c similarity index 93% rename from src/libstd/user/functions/io/mkdir.c rename to src/libstd/user/functions/fs/mkdir.c index cfa816891..e213e68ae 100644 --- a/src/libstd/user/functions/io/mkdir.c +++ b/src/libstd/user/functions/fs/mkdir.c @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/mount.c b/src/libstd/user/functions/fs/mount.c similarity index 92% rename from src/libstd/user/functions/io/mount.c rename to src/libstd/user/functions/fs/mount.c index c43c8e6b3..6a7734fcb 100644 --- a/src/libstd/user/functions/io/mount.c +++ b/src/libstd/user/functions/fs/mount.c @@ -1,4 +1,4 @@ -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/open.c b/src/libstd/user/functions/fs/open.c similarity index 92% rename from src/libstd/user/functions/io/open.c rename to src/libstd/user/functions/fs/open.c index ad4fa54fd..28a891b95 100644 --- a/src/libstd/user/functions/io/open.c +++ b/src/libstd/user/functions/fs/open.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/open2.c b/src/libstd/user/functions/fs/open2.c similarity index 93% rename from src/libstd/user/functions/io/open2.c rename to src/libstd/user/functions/fs/open2.c index 3a9ab1e3d..8e3cf0c5f 100644 --- a/src/libstd/user/functions/io/open2.c +++ b/src/libstd/user/functions/fs/open2.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/openat.c b/src/libstd/user/functions/fs/openat.c similarity index 92% rename from src/libstd/user/functions/io/openat.c rename to src/libstd/user/functions/fs/openat.c index a30a39e3d..d5890b08d 100644 --- a/src/libstd/user/functions/io/openat.c +++ b/src/libstd/user/functions/fs/openat.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/poll.c b/src/libstd/user/functions/fs/poll.c similarity index 93% rename from src/libstd/user/functions/io/poll.c rename to src/libstd/user/functions/fs/poll.c index e1eb8df70..ef9988702 100644 --- a/src/libstd/user/functions/io/poll.c +++ b/src/libstd/user/functions/fs/poll.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/poll1.c b/src/libstd/user/functions/fs/poll1.c similarity index 94% rename from src/libstd/user/functions/io/poll1.c rename to src/libstd/user/functions/fs/poll1.c index b8988f4b1..125c5544d 100644 --- a/src/libstd/user/functions/io/poll1.c +++ b/src/libstd/user/functions/fs/poll1.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/read.c b/src/libstd/user/functions/fs/read.c similarity index 93% rename from src/libstd/user/functions/io/read.c rename to src/libstd/user/functions/fs/read.c index 5498067e7..fc530c067 100644 --- a/src/libstd/user/functions/io/read.c +++ b/src/libstd/user/functions/fs/read.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/readdir.c b/src/libstd/user/functions/fs/readdir.c similarity index 98% rename from src/libstd/user/functions/io/readdir.c rename to src/libstd/user/functions/fs/readdir.c index 268355d07..cffe3c163 100644 --- a/src/libstd/user/functions/io/readdir.c +++ b/src/libstd/user/functions/fs/readdir.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/readfile.c b/src/libstd/user/functions/fs/readfile.c similarity index 94% rename from src/libstd/user/functions/io/readfile.c rename to src/libstd/user/functions/fs/readfile.c index 23b9f962d..75a715239 100644 --- a/src/libstd/user/functions/io/readfile.c +++ b/src/libstd/user/functions/fs/readfile.c @@ -1,4 +1,4 @@ -#include +#include size_t readfile(const char* path, void* buffer, size_t count, size_t offset) { diff --git a/src/libstd/user/functions/io/readlink.c b/src/libstd/user/functions/fs/readlink.c similarity index 93% rename from src/libstd/user/functions/io/readlink.c rename to src/libstd/user/functions/fs/readlink.c index e0d6c3781..2416767dd 100644 --- a/src/libstd/user/functions/io/readlink.c +++ b/src/libstd/user/functions/fs/readlink.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/rmdir.c b/src/libstd/user/functions/fs/rmdir.c similarity index 87% rename from src/libstd/user/functions/io/rmdir.c rename to src/libstd/user/functions/fs/rmdir.c index b40239a89..bc05ae556 100644 --- a/src/libstd/user/functions/io/rmdir.c +++ b/src/libstd/user/functions/fs/rmdir.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include uint64_t rmdir(const char* path) { diff --git a/src/libstd/user/functions/io/scan.c b/src/libstd/user/functions/fs/scan.c similarity index 92% rename from src/libstd/user/functions/io/scan.c rename to src/libstd/user/functions/fs/scan.c index 78130d52c..72f4c825d 100644 --- a/src/libstd/user/functions/io/scan.c +++ b/src/libstd/user/functions/fs/scan.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include uint64_t scan(fd_t fd, const char* format, ...) { diff --git a/src/libstd/user/functions/io/scanfile.c b/src/libstd/user/functions/fs/scanfile.c similarity index 92% rename from src/libstd/user/functions/io/scanfile.c rename to src/libstd/user/functions/fs/scanfile.c index c4feff69b..c804b5dc9 100644 --- a/src/libstd/user/functions/io/scanfile.c +++ b/src/libstd/user/functions/fs/scanfile.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include uint64_t scanfile(const char* path, const char* format, ...) { diff --git a/src/libstd/user/functions/io/seek.c b/src/libstd/user/functions/fs/seek.c similarity index 93% rename from src/libstd/user/functions/io/seek.c rename to src/libstd/user/functions/fs/seek.c index 47e97471d..a7f4e89ee 100644 --- a/src/libstd/user/functions/io/seek.c +++ b/src/libstd/user/functions/fs/seek.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/share.c b/src/libstd/user/functions/fs/share.c similarity index 92% rename from src/libstd/user/functions/io/share.c rename to src/libstd/user/functions/fs/share.c index c2c4dd81a..ceaeaea5d 100644 --- a/src/libstd/user/functions/io/share.c +++ b/src/libstd/user/functions/fs/share.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/sharefile.c b/src/libstd/user/functions/fs/sharefile.c similarity index 93% rename from src/libstd/user/functions/io/sharefile.c rename to src/libstd/user/functions/fs/sharefile.c index 50a9d61e8..ebff87515 100644 --- a/src/libstd/user/functions/io/sharefile.c +++ b/src/libstd/user/functions/fs/sharefile.c @@ -1,5 +1,5 @@ #include -#include +#include uint64_t sharefile(char* key, uint64_t size, const char* path, clock_t timeout) { diff --git a/src/libstd/user/functions/io/sread.c b/src/libstd/user/functions/fs/sread.c similarity index 98% rename from src/libstd/user/functions/io/sread.c rename to src/libstd/user/functions/fs/sread.c index df4d29027..7fc60da04 100644 --- a/src/libstd/user/functions/io/sread.c +++ b/src/libstd/user/functions/fs/sread.c @@ -1,6 +1,6 @@ #include #include -#include +#include char* reads(fd_t fd) { diff --git a/src/libstd/user/functions/io/sreadfile.c b/src/libstd/user/functions/fs/sreadfile.c similarity index 89% rename from src/libstd/user/functions/io/sreadfile.c rename to src/libstd/user/functions/fs/sreadfile.c index ef17cffe4..8bc51bcf2 100644 --- a/src/libstd/user/functions/io/sreadfile.c +++ b/src/libstd/user/functions/fs/sreadfile.c @@ -1,4 +1,4 @@ -#include +#include char* readfiles(const char* path) { diff --git a/src/libstd/user/functions/io/stat.c b/src/libstd/user/functions/fs/stat.c similarity index 93% rename from src/libstd/user/functions/io/stat.c rename to src/libstd/user/functions/fs/stat.c index ab8555072..9d736ba97 100644 --- a/src/libstd/user/functions/io/stat.c +++ b/src/libstd/user/functions/fs/stat.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/swrite.c b/src/libstd/user/functions/fs/swrite.c similarity index 87% rename from src/libstd/user/functions/io/swrite.c rename to src/libstd/user/functions/fs/swrite.c index b88c1ae51..d648f82ea 100644 --- a/src/libstd/user/functions/io/swrite.c +++ b/src/libstd/user/functions/fs/swrite.c @@ -1,5 +1,5 @@ #include -#include +#include size_t writes(fd_t fd, const char* string) { diff --git a/src/libstd/user/functions/io/swritefile.c b/src/libstd/user/functions/fs/swritefile.c similarity index 92% rename from src/libstd/user/functions/io/swritefile.c rename to src/libstd/user/functions/fs/swritefile.c index 874fdebad..b02d92b28 100644 --- a/src/libstd/user/functions/io/swritefile.c +++ b/src/libstd/user/functions/fs/swritefile.c @@ -1,5 +1,5 @@ #include -#include +#include size_t writefiles(const char* path, const char* string) { diff --git a/src/libstd/user/functions/io/symlink.c b/src/libstd/user/functions/fs/symlink.c similarity index 93% rename from src/libstd/user/functions/io/symlink.c rename to src/libstd/user/functions/fs/symlink.c index 580f4fe03..07deeb250 100644 --- a/src/libstd/user/functions/io/symlink.c +++ b/src/libstd/user/functions/fs/symlink.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/unlink.c b/src/libstd/user/functions/fs/unlink.c similarity index 91% rename from src/libstd/user/functions/io/unlink.c rename to src/libstd/user/functions/fs/unlink.c index 475548bad..1de2ade46 100644 --- a/src/libstd/user/functions/io/unlink.c +++ b/src/libstd/user/functions/fs/unlink.c @@ -1,4 +1,4 @@ -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/unmount.c b/src/libstd/user/functions/fs/unmount.c similarity index 91% rename from src/libstd/user/functions/io/unmount.c rename to src/libstd/user/functions/fs/unmount.c index f171bd0c5..8b7b9c8cd 100644 --- a/src/libstd/user/functions/io/unmount.c +++ b/src/libstd/user/functions/fs/unmount.c @@ -1,4 +1,4 @@ -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/vscan.c b/src/libstd/user/functions/fs/vscan.c similarity index 97% rename from src/libstd/user/functions/io/vscan.c rename to src/libstd/user/functions/fs/vscan.c index 7f127c137..d68698df9 100644 --- a/src/libstd/user/functions/io/vscan.c +++ b/src/libstd/user/functions/fs/vscan.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #define _SCAN_GET(ctx) \ ({ \ diff --git a/src/libstd/user/functions/io/vscanfile.c b/src/libstd/user/functions/fs/vscanfile.c similarity index 93% rename from src/libstd/user/functions/io/vscanfile.c rename to src/libstd/user/functions/fs/vscanfile.c index 6d37c78a6..052a62b2c 100644 --- a/src/libstd/user/functions/io/vscanfile.c +++ b/src/libstd/user/functions/fs/vscanfile.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include uint64_t vscanfile(const char* path, const char* format, va_list args) { diff --git a/src/libstd/user/functions/io/write.c b/src/libstd/user/functions/fs/write.c similarity index 93% rename from src/libstd/user/functions/io/write.c rename to src/libstd/user/functions/fs/write.c index 593d5f533..06fbbabe7 100644 --- a/src/libstd/user/functions/io/write.c +++ b/src/libstd/user/functions/fs/write.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/io/writefile.c b/src/libstd/user/functions/fs/writefile.c similarity index 94% rename from src/libstd/user/functions/io/writefile.c rename to src/libstd/user/functions/fs/writefile.c index a1639a1fb..cce3c1841 100644 --- a/src/libstd/user/functions/io/writefile.c +++ b/src/libstd/user/functions/fs/writefile.c @@ -1,4 +1,4 @@ -#include +#include size_t writefile(const char* path, const void* buffer, size_t count, size_t offset) { diff --git a/src/libstd/user/functions/proc/arch_prctl.c b/src/libstd/user/functions/proc/arch_prctl.c index 4f5a809a9..0ec152890 100644 --- a/src/libstd/user/functions/proc/arch_prctl.c +++ b/src/libstd/user/functions/proc/arch_prctl.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/proc/futex.c b/src/libstd/user/functions/proc/futex.c index 65d665795..a733da021 100644 --- a/src/libstd/user/functions/proc/futex.c +++ b/src/libstd/user/functions/proc/futex.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/proc/getpid.c b/src/libstd/user/functions/proc/getpid.c index 6465f746e..e0e6657bc 100644 --- a/src/libstd/user/functions/proc/getpid.c +++ b/src/libstd/user/functions/proc/getpid.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/proc/gettid.c b/src/libstd/user/functions/proc/gettid.c index eb0853da6..436179752 100644 --- a/src/libstd/user/functions/proc/gettid.c +++ b/src/libstd/user/functions/proc/gettid.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/proc/kill.c b/src/libstd/user/functions/proc/kill.c index 0e779cb93..ceb6e30ba 100644 --- a/src/libstd/user/functions/proc/kill.c +++ b/src/libstd/user/functions/proc/kill.c @@ -1,4 +1,4 @@ -#include +#include #include uint64_t kill(pid_t pid) diff --git a/src/libstd/user/functions/proc/mmap.c b/src/libstd/user/functions/proc/mmap.c index 6788e393f..e4e176636 100644 --- a/src/libstd/user/functions/proc/mmap.c +++ b/src/libstd/user/functions/proc/mmap.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/proc/mprotect.c b/src/libstd/user/functions/proc/mprotect.c index f806cefdf..b187f8b96 100644 --- a/src/libstd/user/functions/proc/mprotect.c +++ b/src/libstd/user/functions/proc/mprotect.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/proc/munmap.c b/src/libstd/user/functions/proc/munmap.c index 58628a21b..84fe0b6a2 100644 --- a/src/libstd/user/functions/proc/munmap.c +++ b/src/libstd/user/functions/proc/munmap.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/proc/nanosleep.c b/src/libstd/user/functions/proc/nanosleep.c index 54ba3e9d9..9f5867897 100644 --- a/src/libstd/user/functions/proc/nanosleep.c +++ b/src/libstd/user/functions/proc/nanosleep.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/proc/spawn.c b/src/libstd/user/functions/proc/spawn.c index b762f5e7d..5c674ce89 100644 --- a/src/libstd/user/functions/proc/spawn.c +++ b/src/libstd/user/functions/proc/spawn.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/proc/uptime.c b/src/libstd/user/functions/proc/uptime.c index 367edc36a..9f1fee05e 100644 --- a/src/libstd/user/functions/proc/uptime.c +++ b/src/libstd/user/functions/proc/uptime.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/stdio/fseek.c b/src/libstd/user/functions/stdio/fseek.c index 41c92cad6..e7340a709 100644 --- a/src/libstd/user/functions/stdio/fseek.c +++ b/src/libstd/user/functions/stdio/fseek.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/file.h" diff --git a/src/libstd/user/functions/stdio/remove.c b/src/libstd/user/functions/stdio/remove.c index 62678bda8..9491b20ba 100644 --- a/src/libstd/user/functions/stdio/remove.c +++ b/src/libstd/user/functions/stdio/remove.c @@ -1,5 +1,5 @@ #include -#include +#include #include "user/common/syscalls.h" diff --git a/src/libstd/user/functions/stdlib/system.c b/src/libstd/user/functions/stdlib/system.c index 2db9b3981..6c4760027 100644 --- a/src/libstd/user/functions/stdlib/system.c +++ b/src/libstd/user/functions/stdlib/system.c @@ -1,6 +1,6 @@ #include #include -#include +#include #include int system(const char* command) diff --git a/src/libstd/user/user.c b/src/libstd/user/user.c index 41739f1a2..f9d01d3bc 100644 --- a/src/libstd/user/user.c +++ b/src/libstd/user/user.c @@ -7,7 +7,7 @@ #include "user/common/file.h" #include "user/common/note.h" -#include +#include #include static void _populate_std_descriptors(void) diff --git a/src/modules/acpi/aml/runtime/field_unit.c b/src/modules/acpi/aml/runtime/field_unit.c index 82b25d97f..8a21c920a 100644 --- a/src/modules/acpi/aml/runtime/field_unit.c +++ b/src/modules/acpi/aml/runtime/field_unit.c @@ -1,15 +1,15 @@ +#include +#include #include -#include +#include #include #include #include +#include #include #include #include #include -#include -#include -#include #include #include diff --git a/src/modules/drivers/ps2/ps2.h b/src/modules/drivers/ps2/ps2.h index 544c915fe..8e357cb4c 100644 --- a/src/modules/drivers/ps2/ps2.h +++ b/src/modules/drivers/ps2/ps2.h @@ -1,7 +1,7 @@ #pragma once -#include #include +#include #include #include diff --git a/src/modules/drivers/rtc/rtc.c b/src/modules/drivers/rtc/rtc.c index c8f3c923a..d9c0797f6 100644 --- a/src/modules/drivers/rtc/rtc.c +++ b/src/modules/drivers/rtc/rtc.c @@ -2,9 +2,9 @@ #include #include #include +#include #include #include -#include #include diff --git a/src/modules/ipc/pipe/pipe.c b/src/modules/ipc/pipe/pipe.c index c62c06730..a6991061b 100644 --- a/src/modules/ipc/pipe/pipe.c +++ b/src/modules/ipc/pipe/pipe.c @@ -11,7 +11,7 @@ #include #include -#include +#include #include /** diff --git a/src/modules/net/local/local.c b/src/modules/net/local/local.c index b8e696148..81cdc4eeb 100644 --- a/src/modules/net/local/local.c +++ b/src/modules/net/local/local.c @@ -15,7 +15,7 @@ #include #include -#include +#include #include static local_listen_t* local_socket_get_listen(local_socket_t* data) diff --git a/src/modules/net/local/local.h b/src/modules/net/local/local.h index 15fe7ff17..5988d7c14 100644 --- a/src/modules/net/local/local.h +++ b/src/modules/net/local/local.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include typedef struct local_listen local_listen_t; diff --git a/src/modules/net/local/local_conn.h b/src/modules/net/local/local_conn.h index 8c1e387fa..6c0fe4fbd 100644 --- a/src/modules/net/local/local_conn.h +++ b/src/modules/net/local/local_conn.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include typedef struct local_listen local_listen_t; diff --git a/src/modules/net/local/local_listen.h b/src/modules/net/local/local_listen.h index 950a306f7..226eff1e2 100644 --- a/src/modules/net/local/local_listen.h +++ b/src/modules/net/local/local_listen.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include typedef struct socket_family socket_family_t; diff --git a/src/programs/core/boxd/main.c b/src/programs/core/boxd/main.c index 58d6ed433..414dad2b7 100644 --- a/src/programs/core/boxd/main.c +++ b/src/programs/core/boxd/main.c @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include /** diff --git a/src/programs/core/boxd/manifest.c b/src/programs/core/boxd/manifest.c index 7622d240d..99fbede53 100644 --- a/src/programs/core/boxd/manifest.c +++ b/src/programs/core/boxd/manifest.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/programs/core/boxspawn/main.c b/src/programs/core/boxspawn/main.c index 1cffc263c..9f8354600 100644 --- a/src/programs/core/boxspawn/main.c +++ b/src/programs/core/boxspawn/main.c @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/programs/core/init/main.c b/src/programs/core/init/main.c index 52a378f2b..4bbbcf240 100644 --- a/src/programs/core/init/main.c +++ b/src/programs/core/init/main.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/programs/core/shell/builtin.c b/src/programs/core/shell/builtin.c index 9deb3d7b4..359aeb5bc 100644 --- a/src/programs/core/shell/builtin.c +++ b/src/programs/core/shell/builtin.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include static uint64_t builtin_cd(uint64_t argc, const char** argv); diff --git a/src/programs/core/shell/interactive.c b/src/programs/core/shell/interactive.c index 036ec8a6c..53ad1b174 100644 --- a/src/programs/core/shell/interactive.c +++ b/src/programs/core/shell/interactive.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/programs/core/shell/main.c b/src/programs/core/shell/main.c index 596f80e08..d6166b579 100644 --- a/src/programs/core/shell/main.c +++ b/src/programs/core/shell/main.c @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include static uint64_t cmdline_read(char* buffer, uint64_t size) diff --git a/src/programs/core/shell/pipeline.c b/src/programs/core/shell/pipeline.c index 79ffd27e8..190ef8539 100644 --- a/src/programs/core/shell/pipeline.c +++ b/src/programs/core/shell/pipeline.c @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include uint64_t pipeline_init(pipeline_t* pipeline, const char* cmdline, fd_t stdin, fd_t stdout, fd_t stderr) diff --git a/src/programs/core/shell/pipeline.h b/src/programs/core/shell/pipeline.h index e95d3550d..ccfe70351 100644 --- a/src/programs/core/shell/pipeline.h +++ b/src/programs/core/shell/pipeline.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/src/programs/utils/benchmark/main.c b/src/programs/utils/benchmark/main.c index 84a9d7135..29709b884 100644 --- a/src/programs/utils/benchmark/main.c +++ b/src/programs/utils/benchmark/main.c @@ -7,7 +7,7 @@ #define GETPID_ITER 100000 #ifdef _PATCHWORK_OS_ -#include +#include #include static fd_t zeroDev; diff --git a/src/programs/utils/cat/main.c b/src/programs/utils/cat/main.c index b563ad057..e90927f58 100644 --- a/src/programs/utils/cat/main.c +++ b/src/programs/utils/cat/main.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include #define BUFFER_SIZE 1024 diff --git a/src/programs/utils/echo/main.c b/src/programs/utils/echo/main.c index e4e57ae80..119d0f12f 100644 --- a/src/programs/utils/echo/main.c +++ b/src/programs/utils/echo/main.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include int main(int argc, char** argv) { diff --git a/src/programs/utils/link/main.c b/src/programs/utils/link/main.c index 01826985a..71fc99921 100644 --- a/src/programs/utils/link/main.c +++ b/src/programs/utils/link/main.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include int main(int argc, char** argv) { diff --git a/src/programs/utils/ls/main.c b/src/programs/utils/ls/main.c index 181cfc10c..0f0571023 100644 --- a/src/programs/utils/ls/main.c +++ b/src/programs/utils/ls/main.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include static bool showAll = false; static bool showFlags = false; diff --git a/src/programs/utils/mv/main.c b/src/programs/utils/mv/main.c index e610de448..56e2deb24 100644 --- a/src/programs/utils/mv/main.c +++ b/src/programs/utils/mv/main.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include int main(int argc, char** argv) { diff --git a/src/programs/utils/readlink/main.c b/src/programs/utils/readlink/main.c index e66121d1f..a1d8c5fe2 100644 --- a/src/programs/utils/readlink/main.c +++ b/src/programs/utils/readlink/main.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include int main(int argc, char** argv) { diff --git a/src/programs/utils/ringtest/main.c b/src/programs/utils/ringtest/main.c index 7f897974f..4aff330b6 100644 --- a/src/programs/utils/ringtest/main.c +++ b/src/programs/utils/ringtest/main.c @@ -10,7 +10,7 @@ int main() { printf("setting up ring test...\n"); ioring_t ring; - io_id_t id = isetup(&ring, NULL, SENTRIES, CENTRIES); + io_id_t id = setup(&ring, NULL, SENTRIES, CENTRIES); if (id == ERR) { printf("failed to set up ring\n"); diff --git a/src/programs/utils/rm/main.c b/src/programs/utils/rm/main.c index f2dec29d2..43b08f3b5 100644 --- a/src/programs/utils/rm/main.c +++ b/src/programs/utils/rm/main.c @@ -2,7 +2,7 @@ #include #include #include -#include +#include int main(int argc, char** argv) { diff --git a/src/programs/utils/stat/main.c b/src/programs/utils/stat/main.c index 0985abc1b..cce905c82 100644 --- a/src/programs/utils/stat/main.c +++ b/src/programs/utils/stat/main.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include static const char* type_to_string(itype_t type) diff --git a/src/programs/utils/symlink/main.c b/src/programs/utils/symlink/main.c index 676f27f39..3628e6755 100644 --- a/src/programs/utils/symlink/main.c +++ b/src/programs/utils/symlink/main.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include int main(int argc, char** argv) { diff --git a/src/programs/utils/tail/main.c b/src/programs/utils/tail/main.c index 40d3dc122..7372349ff 100644 --- a/src/programs/utils/tail/main.c +++ b/src/programs/utils/tail/main.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include int main(int argc, char* argv[]) { diff --git a/src/programs/utils/top/main.c b/src/programs/utils/top/main.c index 621732acc..3f6ed7f74 100644 --- a/src/programs/utils/top/main.c +++ b/src/programs/utils/top/main.c @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/programs/utils/touch/main.c b/src/programs/utils/touch/main.c index d2ffac5f2..03d35cdd4 100644 --- a/src/programs/utils/touch/main.c +++ b/src/programs/utils/touch/main.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include int main(int argc, char** argv) { From 74bafbcc90241875368a7567cd94b802520f5c04 Mon Sep 17 00:00:00 2001 From: KN Date: Wed, 21 Jan 2026 22:17:32 +0100 Subject: [PATCH 18/23] feat(kernel:io): implement basic verb tables; rename mem_desc_t into mdl_t --- include/kernel/fs/file.h | 2 + include/kernel/fs/inode.h | 2 + include/kernel/fs/superblock.h | 1 + include/kernel/fs/vfs.h | 3 + include/kernel/io/io.h | 22 ++- include/kernel/io/irp.h | 135 ++++++++++------- include/kernel/io/verb.h | 23 +++ include/kernel/mem/mdl.h | 205 ++++++++++++++++++++++++++ include/kernel/mem/mem_desc.h | 222 ---------------------------- include/libstd/sys/ioring.h | 9 +- meta/doxy/Doxyfile | 2 +- src/kernel/fs/file.c | 18 +++ src/kernel/fs/inode.c | 1 + src/kernel/fs/superblock.c | 1 + src/kernel/fs/vfs.c | 2 +- src/kernel/io/io.c | 46 ++---- src/kernel/io/irp.c | 130 ++++++++++++++--- src/kernel/mem/mdl.c | 229 +++++++++++++++++++++++++++++ src/kernel/mem/mem_desc.c | 259 --------------------------------- 19 files changed, 709 insertions(+), 603 deletions(-) create mode 100644 include/kernel/io/verb.h create mode 100644 include/kernel/mem/mdl.h delete mode 100644 include/kernel/mem/mem_desc.h create mode 100644 src/kernel/mem/mdl.c delete mode 100644 src/kernel/mem/mem_desc.c diff --git a/include/kernel/fs/file.h b/include/kernel/fs/file.h index 1c10cf572..77e088722 100644 --- a/include/kernel/fs/file.h +++ b/include/kernel/fs/file.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -43,6 +44,7 @@ typedef struct file inode_t* inode; path_t path; const file_ops_t* ops; + const verb_table_t* verbs; void* data; } file_t; diff --git a/include/kernel/fs/inode.h b/include/kernel/fs/inode.h index 0f7b5771b..05a5176fc 100644 --- a/include/kernel/fs/inode.h +++ b/include/kernel/fs/inode.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -62,6 +63,7 @@ typedef struct inode superblock_t* superblock; const inode_ops_t* ops; const file_ops_t* fileOps; + const verb_table_t* verbs; rcu_entry_t rcu; mutex_t mutex; } inode_t; diff --git a/include/kernel/fs/superblock.h b/include/kernel/fs/superblock.h index cf5050838..6c107a0e6 100644 --- a/include/kernel/fs/superblock.h +++ b/include/kernel/fs/superblock.h @@ -39,6 +39,7 @@ typedef struct superblock dentry_t* root; ///< Root dentry of the filesystem, should not take a reference. const superblock_ops_t* ops; const dentry_ops_t* dentryOps; + const verb_table_t* defaultVerbs; filesystem_t* fs; /** * The number of mounts of this superblock. diff --git a/include/kernel/fs/vfs.h b/include/kernel/fs/vfs.h index 47459ae64..d3070e05b 100644 --- a/include/kernel/fs/vfs.h +++ b/include/kernel/fs/vfs.h @@ -24,6 +24,9 @@ * The Virtual File System (VFS) provides a single unified interface for any and all filesystems, including virtual * filesystems used to expose kernel resources to user space. * + * @todo Most of this is going to be removed when the new IRP system is fully implemented, but for now its usefull to + * keep it around during the refactor. + * * @{ */ diff --git a/include/kernel/io/io.h b/include/kernel/io/io.h index 2bbb2bd24..5fc1d42e5 100644 --- a/include/kernel/io/io.h +++ b/include/kernel/io/io.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -94,7 +94,7 @@ * * ### `.rw` * - * Used for verbs used for data transfer. + * Used for verbs that transfer data to or from a file. * * **Arguments:** * - `fd`: The file descriptor to access. @@ -147,11 +147,20 @@ * * ### `VERB_NOP` * - * A no-operation verb that does nothing but is usefull for implementing sleeping. + * A no-operation verb that does nothing but is useful for implementing sleeping. * - * @param None - * @result None + * **Arguments:** None. * + * **Result:** None. + * + * ### `VERB_READ` + * + * Reads data from a file descriptor. + * + * **Arguments:** `.rw` + + * **Result:** The number of bytes read. + * * @{ */ @@ -174,13 +183,10 @@ typedef struct io_ctx { ioring_t ring; ///< The kernel-side ring structure. irp_pool_t* irps; ///< Pool of preallocated IRPs. - mem_desc_pool_t* descs; ///< Pool of preallocated memory descriptors. void* userAddr; ///< Userspace address of the ring. void* kernelAddr; ///< Kernel address of the ring. size_t pageAmount; ///< Amount of pages mapped for the ring. - space_t* space; ///< Pointer to the owning address space. wait_queue_t waitQueue; ///< Wait queue for completions. - process_t* process; ///< Holds a reference to the owner process while there are pending requests. _Atomic(io_ctx_flags_t) flags; } io_ctx_t; diff --git a/include/kernel/io/irp.h b/include/kernel/io/irp.h index 993b753ff..a66ce1aa1 100644 --- a/include/kernel/io/irp.h +++ b/include/kernel/io/irp.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include @@ -153,10 +153,11 @@ typedef struct irp irp_t; * irp_t* irp = irp_new(pool, NULL); * * // Set our desired verb and arguments. - * irp->verb = VERB_OPEN; - * irp->open.from = from; - * irp->open.path = path; - * irp->open.length = strlen(path); + * irp->verb = VERB_READ; + * irp->rw.file = file; + * irp->rw.buffer = buffer; + * irp->rw.len = len; + * irp->rw.off = off; * * // Our completion to receive the result. * irp_push(irp, my_completion, NULL); @@ -247,7 +248,7 @@ typedef struct irp irp_t; * @{ */ -#define IRP_LOC_MAX 8 ///< The maximum number of locations in a IRP. +#define IRP_LOC_MAX 5 ///< The maximum number of locations in a IRP. #define IRP_ARG_MAX SQE_ARG_MAX ///< The maximum number of arguments in an IRP. @@ -286,10 +287,19 @@ typedef struct irp_loc * @brief I/O Request Packet structure. * @struct irp_t * + * The I/O Request Packet structure is designed to preallocate as much as possible such that in the common case there is + * no need for any allocation beyond the allocation of the IRP itself. This does require careful consideration of + * padding, alignment and field sizes to keep it within a reasonable size. + * + * @warning The `sqe` field is only valid if the IRP is a user IRP and only until the IRP is entered into the kernel via + * `irp_dispatch()`. + * * @note We need the ability to store both the original arguments from a SQE and the parsed arguments. For example, * opening a `fd_t` into a `file_t*`. As such, to avoid using another cache line, the SQE is stored in a union with the * parsed arguments. * + * @todo Consider raising `IRP_LOC_MAX` to 9 if needed, it will add another cache line tho. + * * @see kernel_io for more information for each possible verb. */ typedef struct ALIGNED(64) irp @@ -316,12 +326,12 @@ typedef struct ALIGNED(64) irp struct { file_t* file; - mem_desc_t* path; + mdl_t* path; } path; struct { file_t* file; - mem_desc_t* buffer; + mdl_t* buffer; size_t len; ssize_t off; } rw; @@ -347,12 +357,12 @@ typedef struct ALIGNED(64) irp events_t events; uint64_t _raw; } result; - errno_t err; ///< The error code of the operation, also used to specify its current state. - pool_idx_t index; ///< Index of the IRP in its pool. - pool_idx_t next; ///< Index of the next IRP in a chain or in the free list. - cpu_id_t cpu; ///< The CPU whose timeout queue the IRP is in. - uint8_t location; ///< The index of the current location in the stack. - uint8_t _reserved2[5]; + mdl_t mdl; ///< A preallocated memory descriptor list for use by the IRP. + pool_idx_t index; ///< Index of the IRP in its pool. + pool_idx_t next; ///< Index of the next IRP in a chain or in the free list. + cpu_id_t cpu; ///< The CPU whose timeout queue the IRP is in. + uint8_t err; ///< The error code of the operation, also used to specify its current state. + uint8_t location; ///< The index of the current location in the stack. irp_loc_t stack[IRP_LOC_MAX]; ///< The location stack, grows downwards. } irp_t; @@ -362,6 +372,8 @@ static_assert(offsetof(irp_t, timeout) == offsetof(irp_t, sqe.timeout), "timeout static_assert(offsetof(irp_t, data) == offsetof(irp_t, sqe.data), "data offset mismatch"); static_assert(offsetof(irp_t, _args) == offsetof(irp_t, sqe._args), "args offset mismatch"); +static_assert(sizeof(irp_t) == 256, "irp_t is not 256 bytes"); + /** * @brief Request pool structure. * @struct irp_pool @@ -369,6 +381,7 @@ static_assert(offsetof(irp_t, _args) == offsetof(irp_t, sqe._args), "args offset typedef struct irp_pool { void* ctx; + process_t* process; ///< Will only hold a reference if there is at least one allocated IRP. pool_t pool; irp_t irps[]; } irp_pool_t; @@ -377,10 +390,11 @@ typedef struct irp_pool * @brief Allocate a new IRP pool. * * @param size The amount of requests to allocate. + * @param process The process that will own the IRPs allocated from this pool. * @param ctx The context of the IRP pool. * @return On success, a pointer to the new IRP pool. On failure, `NULL` and `errno` is set. */ -irp_pool_t* irp_pool_new(size_t size, void* ctx); +irp_pool_t* irp_pool_new(size_t size, process_t* process, void* ctx); /** * @brief Free a IRP pool. @@ -389,17 +403,6 @@ irp_pool_t* irp_pool_new(size_t size, void* ctx); */ void irp_pool_free(irp_pool_t* pool); -/** - * @brief Retrieve the IRP pool that an IRP was allocated from. - * - * @param irp Pointer to the IRP. - * @return Pointer to the IRP pool. - */ -static inline irp_pool_t* irp_pool_get(irp_t* irp) -{ - return CONTAINER_OF(irp, irp_pool_t, irps[irp->index]); -} - /** * @brief Add an IRP to a per-CPU timeout queue with the timeout specified in the IRP. * @@ -422,7 +425,7 @@ void irp_timeouts_check(void); /** * @brief Allocate a new IRP from a pool. * - * The pool that the IRP was allocated from, and its context, can be retrieved using the `irp_pool_get()` + * The pool that the IRP was allocated from, and its context, can be retrieved using the `irp_get_pool()` * function. * * @note If a SQE is provided then the IRP will be considered a user IRP, causing the `irp_handler_t::enter` and @@ -444,6 +447,28 @@ irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe); */ void irp_free(irp_t* irp); +/** + * @brief Retrieve the IRP pool that an IRP was allocated from. + * + * @param irp Pointer to the IRP. + * @return Pointer to the IRP pool. + */ +static inline irp_pool_t* irp_get_pool(irp_t* irp) +{ + return CONTAINER_OF(irp, irp_pool_t, irps[irp->index]); +} + +/** + * @brief Retrieve the process that owns an IRP. + * + * @param irp Pointer to the IRP. + * @return Pointer to the process. + */ +static inline process_t* irp_get_process(irp_t* irp) +{ + return irp_get_pool(irp)->process; +} + /** * @brief Set the cancellation callback for an IRP. * @@ -483,7 +508,7 @@ static inline bool irp_claim(irp_t* irp) */ static inline void* irp_get_ctx(irp_t* irp) { - return irp_pool_get(irp)->ctx; + return irp_get_pool(irp)->ctx; } /** @@ -494,7 +519,7 @@ static inline void* irp_get_ctx(irp_t* irp) */ static inline irp_t* irp_next(irp_t* irp) { - irp_pool_t* pool = irp_pool_get(irp); + irp_pool_t* pool = irp_get_pool(irp); if (irp->next == POOL_IDX_MAX) { return NULL; @@ -572,6 +597,18 @@ static inline void irp_complete(irp_t* irp) loc->complete(irp, loc->ctx); } +/** + * @brief Helper to set an error code and complete the IRP. + * + * @param irp Pointer to the IRP. + * @param err The error code to set. + */ +static inline void irp_error(irp_t* irp, uint8_t err) +{ + irp->err = err; + irp_complete(irp); +} + /** * @brief Attempt to cancel an IRP. * @@ -580,6 +617,18 @@ static inline void irp_complete(irp_t* irp) */ uint64_t irp_cancel(irp_t* irp); +/** + * @brief Execute an IRP synchronously. + * + * This function dispatches the IRP and blocks the current thread until the operation is complete. + * + * @warning This function should only be used when the alternative of using asynchronous operations is simply not worth + * the complexity. For example, while loading modules. + * + * @param irp Pointer to the IRP to execute. + */ +void irp_run(irp_t* irp); + /** * @brief Dispatch an IRP to the appropriate handler. * @@ -604,8 +653,6 @@ void irp_table_init(void); typedef struct { verb_t verb; - void (*enter)(irp_t* irp); ///< Will be called on user IRPs to process arguments. - void (*leave)(irp_t* irp); ///< Will be called on user IRPs to cleanup resources. void (*handler)(irp_t* irp); ///< The handler function for the verb. } irp_handler_t; @@ -622,28 +669,16 @@ extern irp_handler_t _irp_table_start[]; extern irp_handler_t _irp_table_end[]; /** - * @brief Macro to register a IRP handler to a verb using the `._irp_table` section. + * @brief Macro to define an IRP handler. * - * @param _verb The verb to register the handler for. - * @param _enter The enter function. - * @param _leave The leave function. - * @param _handler The handler function. + * @param _verb The verb to define the handler for. */ -#define IRP_REGISTER(_verb, _enter, _leave, _handler) \ +#define IRP_DEFINE(_verb) \ + static void __irp_handler_##_verb(irp_t* irp); \ static irp_handler_t __irp_##_verb __attribute__((section("._irp_table"), used)) = { \ .verb = (_verb), \ - .enter = (_enter), \ - .leave = (_leave), \ - .handler = (_handler), \ - }; - -/** - * @brief Function to asynchronously do nothing. - * - * Usefull as a sleep or delay operation. - * - * @param irp Pointer to a IRP to do nothing with. - */ -void nop_do(irp_t* irp); + .handler = __irp_handler_##_verb, \ + }; \ + static void __irp_handler_##_verb(irp_t* irp) /** @} */ \ No newline at end of file diff --git a/include/kernel/io/verb.h b/include/kernel/io/verb.h new file mode 100644 index 000000000..3d7fdf244 --- /dev/null +++ b/include/kernel/io/verb.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +#include + +/** + * @brief I/O Request Packet verbs. + * @defgroup kernel_io_verb I/O Request Packet Verbs + * @ingroup kernel_io + * + * + * @{ + */ + +typedef void (*verb_handler_t)(irp_t* irp); + +typedef struct verb_table +{ + verb_handler_t handlers[VERB_MAX]; +} verb_table_t; + +/** @} */ \ No newline at end of file diff --git a/include/kernel/mem/mdl.h b/include/kernel/mem/mdl.h new file mode 100644 index 000000000..1d66e9c63 --- /dev/null +++ b/include/kernel/mem/mdl.h @@ -0,0 +1,205 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +typedef struct process process_t; + +/** + * @brief Memory Descriptor List. + * @defgroup kernel_mem_mdl Memory Descriptor List + * @ingroup kernel_mem + * + * The Memory Descriptor List (MDL) is a structure used to describe non-contiguous physical memory, allowing it be + * accessed as a single contiguous block regardless of the loaded address space. + * + * ## I/O Operations + * + * The MDL structure is primarily used to describe memory regions for I/O operations. For example, if a process + * specifies a buffer to write to but that I/O operation is later completed while a different address space is loaded, + * the kernel would be unable to access the buffer directly. + * + * Instead, the kernel can create an MDL for the buffer, which describes the physical memory pages backing that buffer, + * allowing the I/O operation to be completed regardless of the currently loaded address space. + * + * @{ + */ + +/** + * @brief Amount of memory segments statically allocated for small MDLs. + */ +#define MDL_SEGS_SMALL_MAX 2 + +/** + * @brief Memory Descriptor List Segment structure. + * @struct mdl_seg_t + */ +typedef struct mdl_seg +{ + pfn_t pfn; ///< Page frame number. + uint32_t size; ///< Size of the segment in bytes. + uint32_t offset; ///< Offset in bytes within the first page. +} mdl_seg_t; + +/** + * @brief Memory Descriptor List structure. + * @struct mdl_t + */ +typedef struct mdl +{ + struct mdl* next; ///< Pointer to the next MDL. + mdl_seg_t small[MDL_SEGS_SMALL_MAX]; ///< Statically allocated segments for small regions. + mdl_seg_t* segments; ///< Pointer to segments array. + uint32_t amount; ///< Number of memory segments. + uint32_t capacity; ///< Capacity of the `large` array. +} mdl_t; + +/** + * @brief Initialize a Memory Descriptor List. + * + * @param mdl Pointer to the MDL. + * @param prev Pointer to the previous MDL in the chain, or `NULL` if none. + */ +static inline void mdl_init(mdl_t* mdl, mdl_t* prev) +{ + if (prev != NULL) + { + prev->next = mdl; + } + mdl->next = NULL; + mdl->segments = mdl->small; + mdl->amount = 0; + mdl->capacity = MDL_SEGS_SMALL_MAX; +} + +/** + * @brief Deinitialize a Memory Descriptor List. + * + * @param mdl Pointer to the MDL. + */ +void mdl_deinit(mdl_t* mdl); + +/** + * @brief Free a Memory Descriptor List chain. + * + * Will traverse the entire chain to deinitialize and free each MDL structure using the provided `free` function. + * + * @param mdl Pointer to the first MDL in the chain. + * @param free Function to free the MDL structure itself, or `NULL` to only deinitialize. + */ +void mdl_free_chain(mdl_t* mdl, void(*free)(void*)); + +/** + * @brief Initialize a Memory Descriptor List from a memory region. + * + * @param mdl Pointer to the MDL. + * @param prev Pointer to the previous MDL in the chain, or `NULL` if none. + * @param space The address space of the region. + * @param addr The virtual address of the memory region. + * @param size The size of the memory region in bytes. + * @return On success, `0`. On failure, `ERR` and `errno` is set: + * - See `mdl_add()` for possible error codes. + */ +uint64_t mdl_from_region(mdl_t* mdl, mdl_t* prev, space_t* space, const void* addr, size_t size); + +/** + * @brief Add a memory region to the Memory Descriptor List. + * + * @param mdl Pointer to the MDL. + * @param space The address space of the user process. + * @param addr The virtual address of the memory region. + * @param size The size of the memory region in bytes. + * @return On success, `0`. On failure, `ERR` and `errno` is set to: + * - See `mdl_add()` for possible error codes. + */ +uint64_t mdl_add(mdl_t* mdl, space_t* space, const void* addr, size_t size); + +/** + * @brief Read from a Memory Descriptor List into a buffer. + * + * @param mdl The MDL to read from. + * @param buffer The buffer to read into. + * @param count Number of bytes to read. + * @param offset Offset within the MDL to start reading from. + * @return The number of bytes read. + */ +uint64_t mdl_read(mdl_t* mdl, void* buffer, size_t count, size_t offset); + +/** + * @brief Write to a Memory Descriptor List from a buffer. + * + * @param mdl The MDL to write to. + * @param buffer The buffer to write from. + * @param count Number of bytes to write. + * @param offset Offset within the MDL to start writing to. + * @return The number of bytes written. + */ +uint64_t mdl_write(mdl_t* mdl, const void* buffer, size_t count, size_t offset); + +/** + * @brief Memory Descriptor List Iterator structure. + * @struct mdl_iter_t + */ +typedef struct +{ + mdl_t* mdl; + size_t segIndex; + size_t segOffset; +} mdl_iter_t; + +/** + * @brief Create a Memory Descriptor List Iterator initializer. + * + * @param _mdl Pointer to the MDL to iterate over. + * @return MDL Iterator initializer. + */ +#define MDL_ITER_CREATE(_mdl) \ + { \ + .mdl = (_mdl), \ + .segIndex = 0, \ + .segOffset = 0, \ + } + +/** + * @brief Get the next byte from a Memory Descriptor List Iterator. + * + * @param iter Pointer to the MDL Iterator. + * @param byte Pointer to store the retrieved byte. + * @return `true` if a byte was retrieved, `false` if the end of the MDL was reached. + */ +static inline bool mdl_iter_next(mdl_iter_t* iter, uint8_t* byte) +{ + if (iter->segIndex >= iter->mdl->amount) + { + return false; + } + + mdl_seg_t* seg = &iter->mdl->segments[iter->segIndex]; + uint8_t* addr = PFN_TO_VIRT(seg->pfn) + seg->offset + iter->segOffset; + *byte = *(addr); + + iter->segOffset++; + if (iter->segOffset >= seg->size) + { + iter->segIndex++; + iter->segOffset = 0; + } + + return true; +} + +/** + * @brief Iterate over bytes within a Memory Descriptor List. + * + * @param _byte The iterator variable. + * @param _mdl Pointer to the MDL. + */ +#define MDL_FOR_EACH(_byte, _mdl) for (mdl_iter_t _iter = MDL_ITER_CREATE(_mdl); mdl_iter_next(&_iter, (_byte));) + +/** @} */ \ No newline at end of file diff --git a/include/kernel/mem/mem_desc.h b/include/kernel/mem/mem_desc.h deleted file mode 100644 index f7f8410b1..000000000 --- a/include/kernel/mem/mem_desc.h +++ /dev/null @@ -1,222 +0,0 @@ -#pragma once - -#include -#include -#include - -#include -#include -#include - -typedef struct process process_t; - -/** - * @brief Memory Descriptor. - * @defgroup kernel_mem_mem_desc Memory Descriptor - * @ingroup kernel_mem - * - * @{ - */ - -/** - * @brief Amount of memory segments statically allocated for small descriptors. - */ -#define MEM_SEGS_SMALL_MAX 2 - -/** - * @brief Memory Segment structure. - * @struct mem_seg_t - */ -typedef struct mem_seg -{ - pfn_t pfn; ///< Page frame number. - uint32_t size; ///< Size of the segment in bytes. - uint32_t offset; ///< Offset in bytes within the first page. -} mem_seg_t; - -/** - * @brief Memory Descriptor structure. - * @struct mem_desc_t - */ -typedef struct ALIGNED(64) mem_desc -{ - pool_idx_t next; ///< Index of the next descriptor or used for the pools free list. - pool_idx_t index; ///< Index of this descriptor in its pool. - uint32_t amount; ///< Number of memory segments. - uint32_t capacity; ///< Capacity of the `large` array. - size_t size; ///< Total size of the memory region in bytes. - mem_seg_t small[MEM_SEGS_SMALL_MAX]; ///< Statically allocated segments for small regions. - mem_seg_t* large; ///< Pointer to additional segments for large regions. -} mem_desc_t; - -/** - * @brief Memory Descriptor Pool structure. - * @struct mem_desc_pool_t - */ -typedef struct -{ - pool_t pool; - mem_desc_t descs[]; -} mem_desc_pool_t; - -/** - * @brief Allocate a new Memory Descriptor pool. - * - * @param size The amount of descriptors to allocate. - * @return On success, a pointer to the new Memory Descriptor pool. On failure, `NULL` and `errno` is set. - */ -mem_desc_pool_t* mem_desc_pool_new(size_t size); - -/** - * @brief Free a Memory Descriptor pool. - * - * @param pool Pointer to the Memory Descriptor pool to free. - */ -void mem_desc_pool_free(mem_desc_pool_t* pool); - -/** - * @brief Retrieve the Memory Descriptor pool that a Memory Descriptor was allocated from. - * - * @param desc Pointer to the Memory Descriptor. - * @return Pointer to the Memory Descriptor pool. - */ -static inline mem_desc_pool_t* mem_desc_pool_get(mem_desc_t* desc) -{ - return CONTAINER_OF(desc, mem_desc_pool_t, descs[desc->index]); -} - -/** - * @brief Allocate a new Memory Descriptor from a pool. - * - * @param pool Pointer to the Memory Descriptor pool. - * @return On success, a pointer to the allocated Memory Descriptor. On failure, `NULL` and `errno` is set to: - * - `EINVAL`: Invalid parameters. - * - `ENOSPC`: No space available in the pool. - */ -mem_desc_t* mem_desc_new(mem_desc_pool_t* pool); - -/** - * @brief Free a Memory Descriptor back to its pool. - * - * @param desc Pointer to the Memory Descriptor to free. - */ -void mem_desc_free(mem_desc_t* desc); - -/** - * @brief Add a physical memory region to the Memory Descriptor. - * - * @param desc Pointer to the Memory Descriptor. - * @param phys The physical address of the memory region. - * @param size The size of the memory region in bytes. - * @return On success, `0`. On failure, `ERR` and `errno` is set to: - * - `EINVAL`: Invalid parameters. - * - `EFAULT`: The physical address is not allocated. - * - `ENOMEM`: Not enough memory. - */ -uint64_t mem_desc_add(mem_desc_t* desc, phys_addr_t phys, size_t size); - -/** - * @brief Add a user space memory region to the Memory Descriptor. - * - * @param desc Pointer to the Memory Descriptor. - * @param process The process the user space memory region belongs to. - * @param addr The virtual address of the user space memory region. - * @param size The size of the user space memory region in bytes. - * @return On success, `0`. On failure, `ERR` and `errno` is set to: - * - See `mem_desc_add()` for possible error codes. - */ -uint64_t mem_desc_add_user(mem_desc_t* desc, process_t* process, const void* addr, size_t size); - -/** - * @brief Get a Memory Segment from a Memory Descriptor. - * - * @param desc The Memory Descriptor to get the segment from. - * @param index The index of the segment to get. - * @return On success, a pointer to the Memory Segment. On failure, `NULL`. - */ -mem_seg_t* mem_desc_get_seg(mem_desc_t* desc, size_t index); - -/** - * @brief Read from a Memory Descriptor into a buffer. - * - * @param desc The Memory Descriptor to read from. - * @param buffer The buffer to read into. - * @param count Number of bytes to read. - * @param offset Offset within the Memory Descriptor to start reading from. - * @return The number of bytes read. - */ -uint64_t mem_desc_read(mem_desc_t* desc, void* buffer, size_t count, size_t offset); - -/** - * @brief Write to a Memory Descriptor from a buffer. - * - * @param desc The Memory Descriptor to write to. - * @param buffer The buffer to write from. - * @param count Number of bytes to write. - * @param offset Offset within the Memory Descriptor to start writing to. - * @return The number of bytes written. - */ -uint64_t mem_desc_write(mem_desc_t* desc, const void* buffer, size_t count, size_t offset); - -/** - * @brief Memory Descriptor Iterator structure. - * @struct mem_desc_iter_t - */ -typedef struct -{ - mem_desc_t* desc; - size_t segIndex; - size_t segOffset; -} mem_desc_iter_t; - -/** - * @brief Create a Memory Descriptor Iterator initializer. - * - * @param _desc Pointer to the Memory Descriptor to iterate over. - * @return Memory Descriptor Iterator initializer. - */ -#define MEM_DESC_ITER_CREATE(_desc) \ - { \ - .desc = (_desc), \ - .segIndex = 0, \ - .segOffset = 0, \ - } - -/** - * @brief Get the next byte from a Memory Descriptor Iterator. - * - * @param iter Pointer to the Memory Descriptor Iterator. - * @param byte Pointer to store the retrieved byte. - * @return `true` if a byte was retrieved, `false` if the end of the Memory Descriptor was reached. - */ -static inline bool mem_desc_iter_next(mem_desc_iter_t* iter, uint8_t* byte) -{ - mem_seg_t* seg = mem_desc_get_seg(iter->desc, iter->segIndex); - if (seg == NULL) - { - return false; - } - - uint8_t* addr = PFN_TO_VIRT(seg->pfn) + seg->offset + iter->segOffset; - *byte = *(addr); - - iter->segOffset++; - if (iter->segOffset >= seg->size) - { - iter->segIndex++; - iter->segOffset = 0; - } - - return true; -} - -/** - * @brief Iterate over bytes within a Memory Descriptor. - * - * @param _byte The iterator variable. - * @param _desc Pointer to the Memory Descriptor. - */ -#define MEM_DESC_FOR_EACH(_byte, _desc) \ - for (mem_desc_iter_t _iter = MEM_DESC_ITER_CREATE(_desc); mem_desc_iter_next(&_iter, (_byte));) - -/** @} */ \ No newline at end of file diff --git a/include/libstd/sys/ioring.h b/include/libstd/sys/ioring.h index b0dfc97ef..e8243960f 100644 --- a/include/libstd/sys/ioring.h +++ b/include/libstd/sys/ioring.h @@ -38,8 +38,10 @@ typedef uint64_t events_t; ///< Poll events type. typedef uint32_t verb_t; ///< Verb type. #define VERB_NOP 0 ///< No-op verb. -#define VERB_OPEN 1 ///< Open file verb. -#define VERB_MAX 1 ///< Maximum verb. +#define VERB_READ 1 ///< Read verb. +// #define VERB_WRITE 2 ///< Write verb. +// #define VERB_POLL 3 ///< Poll verb. +#define VERB_MAX 2 ///< The maximum number of verbs. typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. @@ -161,7 +163,7 @@ static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); * * @see kernel_io for more information on the possible operations. */ -typedef struct ALIGNED(32) cqe +typedef struct cqe { verb_t verb; ///< Verb specifying the action that was performed. errno_t error; ///< Error code, if not equal to `EOK` an error occurred. @@ -173,6 +175,7 @@ typedef struct ALIGNED(32) cqe events_t events; uint64_t _result; }; + uint64_t _padding[1]; } cqe_t; #ifdef static_assert diff --git a/meta/doxy/Doxyfile b/meta/doxy/Doxyfile index 148ab564c..d33cab85e 100644 --- a/meta/doxy/Doxyfile +++ b/meta/doxy/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = "PatchworkOS" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "c2e0413c-dirty" +PROJECT_NUMBER = "facb5a79-dirty" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewers a diff --git a/src/kernel/fs/file.c b/src/kernel/fs/file.c index 110683ab8..ae834dc95 100644 --- a/src/kernel/fs/file.c +++ b/src/kernel/fs/file.c @@ -1,10 +1,15 @@ #include #include +#include #include #include #include +#include +#include #include +#include +#include #include #include @@ -64,6 +69,7 @@ file_t* file_new(const path_t* path, mode_t mode) file->inode = REF(path->dentry->inode); file->path = PATH_CREATE(path->mount, path->dentry); file->ops = path->dentry->inode->fileOps; + file->verbs = path->dentry->superblock->defaultVerbs; file->data = NULL; return file; } @@ -92,3 +98,15 @@ size_t file_generic_seek(file_t* file, ssize_t offset, seek_origin_t origin) file->pos = newPos; return newPos; } + +IRP_DEFINE(VERB_READ) +{ + file_t* file = irp->rw.file; + if (file->verbs == NULL || file->verbs->handlers[VERB_READ] == NULL) + { + irp_error(irp, ENOSYS); + return; + } + + file->verbs->handlers[VERB_READ](irp); +} \ No newline at end of file diff --git a/src/kernel/fs/inode.c b/src/kernel/fs/inode.c index 3ba31d3ed..efb366573 100644 --- a/src/kernel/fs/inode.c +++ b/src/kernel/fs/inode.c @@ -76,6 +76,7 @@ inode_t* inode_new(superblock_t* superblock, ino_t number, itype_t type, const i inode->superblock = REF(superblock); inode->ops = ops; inode->fileOps = fileOps; + inode->verbs = NULL; return inode; } diff --git a/src/kernel/fs/superblock.c b/src/kernel/fs/superblock.c index f25ccb264..180551824 100644 --- a/src/kernel/fs/superblock.c +++ b/src/kernel/fs/superblock.c @@ -54,6 +54,7 @@ superblock_t* superblock_new(filesystem_t* fs, const superblock_ops_t* ops, cons superblock->root = NULL; superblock->ops = ops; superblock->dentryOps = dentryOps; + superblock->defaultVerbs = NULL; superblock->fs = fs; atomic_init(&superblock->mountCount, 0); diff --git a/src/kernel/fs/vfs.c b/src/kernel/fs/vfs.c index df7d2cba2..8529618a2 100644 --- a/src/kernel/fs/vfs.c +++ b/src/kernel/fs/vfs.c @@ -1688,4 +1688,4 @@ SYSCALL_DEFINE(SYS_REMOVE, uint64_t, const char* pathString) } return vfs_remove(&pathname, process); -} \ No newline at end of file +} diff --git a/src/kernel/io/io.c b/src/kernel/io/io.c index be9241d41..ded0dc744 100644 --- a/src/kernel/io/io.c +++ b/src/kernel/io/io.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -33,7 +32,7 @@ static inline void io_ctx_release(io_ctx_t* ctx) atomic_fetch_and(&ctx->flags, ~IO_CTX_BUSY); } -static inline uint64_t io_ctx_map(io_ctx_t* ctx, space_t* space, io_id_t id, ioring_t* userRing, void* address, +static inline uint64_t io_ctx_map(io_ctx_t* ctx, process_t* process, io_id_t id, ioring_t* userRing, void* address, size_t sentries, size_t centries) { ioring_t* kernelRing = &ctx->ring; @@ -71,27 +70,18 @@ static inline uint64_t io_ctx_map(io_ctx_t* ctx, space_t* space, io_id_t id, ior return ERR; } - void* userAddr = vmm_map_pages(space, address, pages, pageAmount, PML_WRITE | PML_PRESENT | PML_USER, NULL, NULL); + void* userAddr = + vmm_map_pages(&process->space, address, pages, pageAmount, PML_WRITE | PML_PRESENT | PML_USER, NULL, NULL); if (userAddr == NULL) { vmm_unmap(NULL, kernelAddr, pageAmount * PAGE_SIZE); return ERR; } - irp_pool_t* irps = irp_pool_new(centries, ctx); + irp_pool_t* irps = irp_pool_new(centries, process, ctx); if (irps == NULL) { - vmm_unmap(space, userAddr, pageAmount * PAGE_SIZE); - vmm_unmap(NULL, kernelAddr, pageAmount * PAGE_SIZE); - return ERR; - } - - mem_desc_pool_t* descs = mem_desc_pool_new(centries); - if (descs == NULL) - { - irp_pool_free(irps); - mem_desc_pool_free(descs); - vmm_unmap(space, userAddr, pageAmount * PAGE_SIZE); + vmm_unmap(&process->space, userAddr, pageAmount * PAGE_SIZE); vmm_unmap(NULL, kernelAddr, pageAmount * PAGE_SIZE); return ERR; } @@ -125,11 +115,9 @@ static inline uint64_t io_ctx_map(io_ctx_t* ctx, space_t* space, io_id_t id, ior kernelRing->cmask = centries - 1; ctx->irps = irps; - ctx->descs = descs; ctx->userAddr = userAddr; ctx->kernelAddr = kernelAddr; ctx->pageAmount = pageAmount; - ctx->space = space; atomic_fetch_or(&ctx->flags, IO_CTX_MAPPED); return 0; @@ -137,15 +125,12 @@ static inline uint64_t io_ctx_map(io_ctx_t* ctx, space_t* space, io_id_t id, ior static inline uint64_t io_ctx_unmap(io_ctx_t* ctx) { + vmm_unmap(&ctx->irps->process->space, ctx->userAddr, ctx->pageAmount * PAGE_SIZE); + vmm_unmap(NULL, ctx->kernelAddr, ctx->pageAmount * PAGE_SIZE); + irp_pool_free(ctx->irps); ctx->irps = NULL; - mem_desc_pool_free(ctx->descs); - ctx->descs = NULL; - - vmm_unmap(ctx->space, ctx->userAddr, ctx->pageAmount * PAGE_SIZE); - vmm_unmap(NULL, ctx->kernelAddr, ctx->pageAmount * PAGE_SIZE); - atomic_fetch_and(&ctx->flags, ~IO_CTX_MAPPED); return 0; } @@ -170,7 +155,6 @@ void io_ctx_init(io_ctx_t* ctx) ctx->userAddr = NULL; ctx->kernelAddr = NULL; ctx->pageAmount = 0; - ctx->space = NULL; wait_queue_init(&ctx->waitQueue); atomic_init(&ctx->flags, IO_CTX_NONE); } @@ -255,12 +239,6 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) } irp_free(irp); - - if (atomic_load(&ctx->irps->pool.used) == 0) - { - UNREF(ctx->process); - ctx->process = NULL; - } } static void io_ctx_dispatch(irp_t* irp) @@ -291,11 +269,6 @@ typedef struct static uint64_t io_ctx_sqe_pop(io_ctx_t* ctx, io_ctx_notify_ctx_t* notify) { - if (atomic_load(&ctx->irps->pool.used) == 1) - { - ctx->process = REF(process_current()); - } - ioring_t* ring = &ctx->ring; uint32_t stail = atomic_load_explicit(&ring->ctrl->stail, memory_order_acquire); uint32_t shead = atomic_load_explicit(&ring->ctrl->shead, memory_order_relaxed); @@ -400,7 +373,6 @@ SYSCALL_DEFINE(SYS_SETUP, io_id_t, ioring_t* userRing, void* address, size_t sen } process_t* process = process_current(); - space_t* space = &process->space; io_ctx_t* ctx = NULL; io_id_t id = 0; @@ -420,7 +392,7 @@ SYSCALL_DEFINE(SYS_SETUP, io_id_t, ioring_t* userRing, void* address, size_t sen return ERR; } - if (io_ctx_map(ctx, space, id, userRing, address, sentries, centries) == ERR) + if (io_ctx_map(ctx, process, id, userRing, address, sentries, centries) == ERR) { io_ctx_release(ctx); return ERR; diff --git a/src/kernel/io/irp.c b/src/kernel/io/irp.c index 71b2954ed..5ef403f51 100644 --- a/src/kernel/io/irp.c +++ b/src/kernel/io/irp.c @@ -1,9 +1,13 @@ #include +#include #include #include #include +#include +#include #include #include +#include #include #include @@ -22,9 +26,9 @@ PERCPU_DEFINE_CTOR(irp_ctx_t, pcpu_irps) lock_init(&ctx->lock); } -irp_pool_t* irp_pool_new(size_t size, void* ctx) +irp_pool_t* irp_pool_new(size_t size, process_t* process, void* ctx) { - if (size == 0 || size >= POOL_IDX_MAX) + if (size == 0 || process == NULL || size >= POOL_IDX_MAX) { errno = EINVAL; return NULL; @@ -38,6 +42,7 @@ irp_pool_t* irp_pool_new(size_t size, void* ctx) } pool->ctx = ctx; + pool->process = process; for (pool_idx_t i = 0; i < (pool_idx_t)size; i++) { irp_t* irp = &pool->irps[i]; @@ -53,11 +58,12 @@ irp_pool_t* irp_pool_new(size_t size, void* ctx) irp->_args[j] = 0; } irp->result._raw = 0; - irp->err = EOK; + mdl_init(&irp->mdl, NULL); + irp->sqe = (sqe_t){0}; irp->index = i; - irp->next = i < size - 1 ? i + 1 : POOL_IDX_MAX; - irp->location = IRP_LOC_MAX; + irp->err = EOK; irp->cpu = CPU_ID_INVALID; + irp->location = IRP_LOC_MAX; for (size_t j = 0; j < IRP_LOC_MAX; j++) { irp->stack[j].ctx = NULL; @@ -75,6 +81,55 @@ void irp_pool_free(irp_pool_t* pool) free(pool); } +static inline void irp_enter(irp_t* irp) +{ + process_t* process = irp_get_process(irp); + + switch (irp->verb) + { + case VERB_READ: + { + file_t* file = file_table_get(&process->fileTable, irp->sqe.rw.fd); + if (file == NULL) + { + irp->err = EBADF; + return; + } + + if (mdl_from_region(&irp->mdl, NULL, &process->space, irp->sqe.rw.buffer, irp->sqe.rw.len) == ERR) + { + UNREF(file); + irp->err = EFAULT; + return; + } + + irp->rw.file = file; + irp->rw.buffer = &irp->mdl; + irp->rw.len = irp->sqe.rw.len; + irp->rw.off = irp->sqe.rw.off; + } + break; + default: + break; + } +} + +static inline void irp_leave(irp_t* irp) +{ + // MDL will be deinitialized in irp_free + + switch (irp->verb) + { + case VERB_READ: + { + UNREF(irp->rw.file); + } + break; + default: + break; + } +} + irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) { pool_idx_t idx = pool_alloc(&pool->pool); @@ -109,19 +164,34 @@ irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) irp->stack[j].ctx = NULL; irp->stack[j].complete = NULL; } + + if (atomic_load(&pool->pool.used) == 1) + { + REF(pool->process); + } + return irp; } void irp_free(irp_t* irp) { - if (irp->flags & SQE_KERNEL_ENTERED && irp->verb < VERB_MAX && _irp_table_start[irp->verb].leave != NULL) + if (irp->flags & SQE_KERNEL_ENTERED && irp->verb < VERB_MAX) { assert(!(irp->flags & SQE_KERNEL)); - _irp_table_start[irp->verb].leave(irp); + irp_leave(irp); } - irp_pool_t* pool = irp_pool_get(irp); + mdl_t* next = irp->mdl.next; + mdl_deinit(&irp->mdl); + mdl_free_chain(next, free); + + irp_pool_t* pool = irp_get_pool(irp); pool_free(&pool->pool, irp->index); + + if (atomic_load(&pool->pool.used) == 0) + { + UNREF(pool->process); + } } uint64_t irp_cancel(irp_t* irp) @@ -143,6 +213,27 @@ uint64_t irp_cancel(irp_t* irp) return handler(irp); } +static void irp_run_completion(irp_t* irp, void* ctx) +{ + UNUSED(irp); + + wait_queue_t* wait = (wait_queue_t*)ctx; + wait_unblock(wait, WAIT_ALL, EOK); +} + +void irp_run(irp_t* irp) +{ + wait_queue_t wait; + wait_queue_init(&wait); + + irp_push(irp, irp_run_completion, &wait); + irp_dispatch(irp); + + WAIT_BLOCK(&wait, irp->err != EINPROGRESS); + + wait_queue_deinit(&wait); +} + void irp_timeout_add(irp_t* irp) { if (irp->timeout == CLOCKS_NEVER) @@ -242,12 +333,6 @@ void irp_timeouts_check(void) void irp_dispatch(irp_t* irp) { - if (irp->err != EINPROGRESS) - { - irp_complete(irp); - return; - } - if (irp->verb >= VERB_MAX || _irp_table_start[irp->verb].handler == NULL) { irp->err = ENOSYS; @@ -257,13 +342,16 @@ void irp_dispatch(irp_t* irp) if (!(irp->flags & SQE_KERNEL) && !(irp->flags & SQE_KERNEL_ENTERED)) { - if (_irp_table_start[irp->verb].enter != NULL) - { - _irp_table_start[irp->verb].enter(irp); - } + irp_enter(irp); irp->flags |= SQE_KERNEL_ENTERED; } + if (irp->err != EINPROGRESS) + { + irp_complete(irp); + return; + } + _irp_table_start[irp->verb].handler(irp); } @@ -294,7 +382,7 @@ static uint64_t _nop_cancel(irp_t* irp) return 0; } -void nop_do(irp_t* irp) +IRP_DEFINE(VERB_NOP) { irp_set_cancel(irp, _nop_cancel); @@ -303,6 +391,4 @@ void nop_do(irp_t* irp) irp_timeout_add(irp); return; } -} - -IRP_REGISTER(VERB_NOP, NULL, NULL, nop_do); \ No newline at end of file +} \ No newline at end of file diff --git a/src/kernel/mem/mdl.c b/src/kernel/mem/mdl.c new file mode 100644 index 000000000..e033027fb --- /dev/null +++ b/src/kernel/mem/mdl.c @@ -0,0 +1,229 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +void mdl_deinit(mdl_t* mdl) +{ + if (mdl == NULL) + { + return; + } + + mdl->next = NULL; + + for (size_t i = 0; i < mdl->amount; i++) + { + pmm_ref_dec(mdl->segments[i].pfn, BYTES_TO_PAGES(mdl->segments[i].offset + mdl->segments[i].size)); + } + mdl->amount = 0; + + if (mdl->segments != mdl->small) + { + free(mdl->segments); + } + mdl->segments = NULL; + mdl->capacity = 0; +} + +void mdl_free_chain(mdl_t* mdl, void(*free)(void*)) +{ + while (mdl != NULL) + { + mdl_t* next = mdl->next; + mdl_deinit(mdl); + if (free != NULL) + { + free(mdl); + } + mdl = next; + } +} + +uint64_t mdl_from_region(mdl_t* mdl, mdl_t* prev, space_t* space, const void* addr, size_t size) +{ + if (mdl == NULL) + { + errno = EINVAL; + return ERR; + } + mdl_init(mdl, prev); + + if (mdl_add(mdl, space, addr, size) == ERR) + { + mdl_deinit(mdl); + return ERR; + } + + return 0; +} + +static uint64_t mdl_push(mdl_t* mdl, phys_addr_t phys, size_t size) +{ + if (size > UINT32_MAX) + { + errno = EOVERFLOW; + return ERR; + } + + if (mdl == NULL) + { + errno = EINVAL; + return ERR; + } + + if (mdl->amount == mdl->capacity) + { + uint32_t newCapacity = mdl->capacity + 4; + mdl_seg_t* newSegments; + + if (mdl->segments == mdl->small) + { + newSegments = malloc(newCapacity * sizeof(mdl_seg_t)); + if (newSegments != NULL) + { + memcpy(newSegments, mdl->small, sizeof(mdl->small)); + } + } + else + { + newSegments = realloc(mdl->segments, newCapacity * sizeof(mdl_seg_t)); + } + + if (newSegments == NULL) + { + errno = ENOMEM; + return ERR; + } + + mdl->segments = newSegments; + mdl->capacity = newCapacity; + } + + mdl_seg_t* seg = &mdl->segments[mdl->amount]; + pfn_t pfn = PHYS_TO_PFN(phys); + uint32_t offset = phys % PAGE_SIZE; + if (pmm_ref_inc(pfn, BYTES_TO_PAGES(offset + size)) == ERR) + { + errno = EFAULT; + return ERR; + } + + seg->pfn = pfn; + seg->size = size; + seg->offset = offset; + + mdl->amount++; + return 0; +} + +uint64_t mdl_add(mdl_t* mdl, space_t* space, const void* addr, size_t size) +{ + const uint8_t* ptr = addr; + size_t remaining = size; + + while (remaining > 0) + { + phys_addr_t phys = space_virt_to_phys(space, ptr); + if (phys == ERR) + { + return ERR; + } + + size_t offset = phys % PAGE_SIZE; + size_t len = MIN(remaining, PAGE_SIZE - offset); + + if (mdl_push(mdl, phys, len) == ERR) + { + return ERR; + } + + ptr += len; + remaining -= len; + } + + return 0; +} + +uint64_t mdl_read(mdl_t* mdl, void* buffer, size_t count, size_t offset) +{ + if (mdl == NULL || buffer == NULL) + { + return 0; + } + + size_t start = 0; + size_t i = 0; + for (; i < mdl->amount; i++) + { + mdl_seg_t* seg = &mdl->segments[i]; + if (start + seg->size > offset) + { + break; + } + start += seg->size; + } + + uint8_t* ptr = buffer; + size_t remaining = count; + + size_t segOffset = offset - start; + while (remaining > 0 && i < mdl->amount) + { + mdl_seg_t* seg = &mdl->segments[i]; + size_t toRead = MIN(remaining, seg->size - segOffset); + void* addr = PFN_TO_VIRT(seg->pfn) + seg->offset + segOffset; + memcpy(ptr, addr, toRead); + + ptr += toRead; + remaining -= toRead; + segOffset = 0; + i++; + } + + return count - remaining; +} + +uint64_t mdl_write(mdl_t* mdl, const void* buffer, size_t count, size_t offset) +{ + if (mdl == NULL || buffer == NULL) + { + return 0; + } + + size_t start = 0; + size_t i = 0; + for (; i < mdl->amount; i++) + { + mdl_seg_t* seg = &mdl->segments[i]; + if (start + seg->size > offset) + { + break; + } + start += seg->size; + } + + const uint8_t* ptr = buffer; + size_t remaining = count; + + size_t segOffset = offset - start; + while (remaining > 0 && i < mdl->amount) + { + mdl_seg_t* seg = &mdl->segments[i]; + size_t toWrite = MIN(remaining, seg->size - segOffset); + void* addr = PFN_TO_VIRT(seg->pfn) + seg->offset + segOffset; + memcpy(addr, ptr, toWrite); + + ptr += toWrite; + remaining -= toWrite; + segOffset = 0; + i++; + } + + return count - remaining; +} \ No newline at end of file diff --git a/src/kernel/mem/mem_desc.c b/src/kernel/mem/mem_desc.c deleted file mode 100644 index 700624b24..000000000 --- a/src/kernel/mem/mem_desc.c +++ /dev/null @@ -1,259 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include - -mem_desc_pool_t* mem_desc_pool_new(size_t size) -{ - size_t poolSize = sizeof(mem_desc_pool_t) + (size * sizeof(mem_desc_t)); - mem_desc_pool_t* pool = malloc(poolSize); - if (pool == NULL) - { - errno = ENOMEM; - return NULL; - } - - for (size_t i = 0; i < size; i++) - { - pool->descs[i].index = i; - } - pool_init(&pool->pool, pool->descs, size, sizeof(mem_desc_t), offsetof(mem_desc_t, next)); - return pool; -} - -void mem_desc_pool_free(mem_desc_pool_t* pool) -{ - free(pool); -} - -mem_desc_t* mem_desc_new(mem_desc_pool_t* pool) -{ - if (pool == NULL) - { - errno = EINVAL; - return NULL; - } - - pool_idx_t idx = pool_alloc(&pool->pool); - if (idx == POOL_IDX_MAX) - { - errno = ENOSPC; - return NULL; - } - - mem_desc_t* desc = &pool->descs[idx]; - desc->next = POOL_IDX_MAX; - desc->amount = 0; - desc->capacity = 0; - desc->size = 0; - desc->large = NULL; - return desc; -} - -void mem_desc_free(mem_desc_t* desc) -{ - if (desc == NULL) - { - return; - } - - size_t i = 0; - for (; i < desc->amount && i < MEM_SEGS_SMALL_MAX; i++) - { - pmm_ref_dec(desc->small[i].pfn, BYTES_TO_PAGES(desc->small[i].offset + desc->small[i].size)); - } - - for (; i < desc->amount; i++) - { - pmm_ref_dec(desc->large[i - MEM_SEGS_SMALL_MAX].pfn, - BYTES_TO_PAGES(desc->large[i - MEM_SEGS_SMALL_MAX].offset + desc->large[i - MEM_SEGS_SMALL_MAX].size)); - } - - if (desc->large != NULL) - { - free(desc->large); - } - desc->large = NULL; - - mem_desc_pool_t* pool = mem_desc_pool_get(desc); - pool_free(&pool->pool, desc->index); -} - -uint64_t mem_desc_add(mem_desc_t* desc, phys_addr_t phys, size_t size) -{ - if (size > UINT32_MAX) - { - errno = EOVERFLOW; - return ERR; - } - - if (desc == NULL || desc->size + size < desc->size) - { - errno = EINVAL; - return ERR; - } - - mem_seg_t* seg = NULL; - if (desc->amount < MEM_SEGS_SMALL_MAX) - { - seg = &desc->small[desc->amount]; - } - else if (desc->amount - MEM_SEGS_SMALL_MAX < desc->capacity) - { - seg = &desc->large[desc->amount - MEM_SEGS_SMALL_MAX]; - } - else - { - mem_seg_t* newLarge = realloc(desc->large, (desc->capacity + 4) * sizeof(mem_seg_t)); - if (newLarge == NULL) - { - errno = ENOMEM; - return ERR; - } - - desc->large = newLarge; - desc->capacity += 4; - seg = &desc->large[desc->amount - MEM_SEGS_SMALL_MAX]; - } - - pfn_t pfn = PHYS_TO_PFN(phys); - uint32_t offset = phys % PAGE_SIZE; - if (pmm_ref_inc(pfn, BYTES_TO_PAGES(offset + size)) == ERR) - { - errno = EFAULT; - return ERR; - } - - seg->pfn = pfn; - seg->size = size; - seg->offset = offset; - desc->size += size; - - desc->amount++; - return 0; -} - -uint64_t mem_desc_add_user(mem_desc_t* desc, process_t* process, const void* addr, size_t size) -{ - const uint8_t* ptr = addr; - size_t remaining = size; - - while (remaining > 0) - { - phys_addr_t phys = space_virt_to_phys(&process->space, ptr); - if (phys == ERR) - { - return ERR; - } - - size_t offset = phys % PAGE_SIZE; - size_t len = MIN(remaining, PAGE_SIZE - offset); - - if (mem_desc_add(desc, phys, len) == ERR) - { - return ERR; - } - - ptr += len; - remaining -= len; - } - - return 0; -} - -mem_seg_t* mem_desc_get_seg(mem_desc_t* desc, size_t index) -{ - if (index >= desc->amount) - { - return NULL; - } - - if (index < MEM_SEGS_SMALL_MAX) - { - return &desc->small[index]; - } - - return &desc->large[index - MEM_SEGS_SMALL_MAX]; -} - -uint64_t mem_desc_read(mem_desc_t* desc, void* buffer, size_t count, size_t offset) -{ - if (desc == NULL || buffer == NULL || offset > desc->size || count > desc->size - offset) - { - return 0; - } - - size_t start = 0; - size_t i = 0; - for (; i < desc->amount; i++) - { - mem_seg_t* seg = mem_desc_get_seg(desc, i); - if (start + seg->size > offset) - { - break; - } - start += seg->size; - } - - uint8_t* ptr = buffer; - size_t remaining = count; - - size_t segOffset = offset - start; - while (remaining > 0 && i < desc->amount) - { - mem_seg_t* seg = mem_desc_get_seg(desc, i); - size_t toRead = MIN(remaining, seg->size - segOffset); - void* addr = PFN_TO_VIRT(seg->pfn) + seg->offset + segOffset; - memcpy(ptr, addr, toRead); - - ptr += toRead; - remaining -= toRead; - segOffset = 0; - i++; - } - - return count - remaining; -} - -uint64_t mem_desc_write(mem_desc_t* desc, const void* buffer, size_t count, size_t offset) -{ - if (desc == NULL || buffer == NULL || offset > desc->size || count > desc->size - offset) - { - return 0; - } - - size_t start = 0; - size_t i = 0; - for (; i < desc->amount; i++) - { - mem_seg_t* seg = mem_desc_get_seg(desc, i); - if (start + seg->size > offset) - { - break; - } - start += seg->size; - } - - const uint8_t* ptr = buffer; - size_t remaining = count; - - size_t segOffset = offset - start; - while (remaining > 0 && i < desc->amount) - { - mem_seg_t* seg = mem_desc_get_seg(desc, i); - size_t toWrite = MIN(remaining, seg->size - segOffset); - void* addr = PFN_TO_VIRT(seg->pfn) + seg->offset + segOffset; - memcpy(addr, ptr, toWrite); - - ptr += toWrite; - remaining -= toWrite; - segOffset = 0; - i++; - } - - return count - remaining; -} \ No newline at end of file From 8b9472192a6df739382aaa49100cd4a1f9327eea Mon Sep 17 00:00:00 2001 From: KN Date: Thu, 22 Jan 2026 11:08:04 +0100 Subject: [PATCH 19/23] feat(kernel:io): new verb table system --- include/kernel/fs/superblock.h | 2 +- include/kernel/io/io.h | 82 ++++++---------- include/kernel/io/irp.h | 132 ++++++-------------------- include/kernel/io/verb.h | 74 ++++++++++++++- include/libstd/sys/ioring.h | 60 +++++------- meta/doxy/Doxyfile | 2 +- src/kernel/fs/file.c | 14 +-- src/kernel/fs/inode.c | 2 +- src/kernel/fs/superblock.c | 2 +- src/kernel/init/init.c | 1 - src/kernel/io/io.c | 41 ++++++-- src/kernel/io/irp.c | 166 +++++---------------------------- src/kernel/io/verb.c | 137 +++++++++++++++++++++++++++ src/kernel/mem/pmm.c | 2 +- src/programs/core/init/main.c | 2 +- 15 files changed, 349 insertions(+), 370 deletions(-) create mode 100644 src/kernel/io/verb.c diff --git a/include/kernel/fs/superblock.h b/include/kernel/fs/superblock.h index 6c107a0e6..ba4a6bc6e 100644 --- a/include/kernel/fs/superblock.h +++ b/include/kernel/fs/superblock.h @@ -39,7 +39,7 @@ typedef struct superblock dentry_t* root; ///< Root dentry of the filesystem, should not take a reference. const superblock_ops_t* ops; const dentry_ops_t* dentryOps; - const verb_table_t* defaultVerbs; + const verb_table_t* verbs; filesystem_t* fs; /** * The number of mounts of this superblock. diff --git a/include/kernel/io/io.h b/include/kernel/io/io.h index 5fc1d42e5..4c1c3bd2c 100644 --- a/include/kernel/io/io.h +++ b/include/kernel/io/io.h @@ -68,57 +68,21 @@ * * ## Arguments * - * Instead of manually indexing the `_args` array each SQE stores the arguments in a union with several "argument - * archetypes". Such that several verbs can use the same archetype for their arguments. + * Arguments within a SQE are stored in five 64-bit values, `arg1` through `arg5`. For convenience, each argument value is stored as a union with various types. + * + * To avoid nameing conflicts and to avoid having to define new arguments for each verb, we define a convention to be used for the arguments. + * + * - `arg0`: The noun or subject of the verb, for example, a `fd_t` for file operations. + * - `arg1`: The source or payload of the verb, for example, a buffer or path. + * - `arg2`: The magnitude of the operation, for example, a size or encoding. + * - `arg3`: The location or a modifier to the operation, for example, an offset or flags. + * - `arg4`: An auxiliary argument, for example, additional flags or options. + * + * It may not always be possible for a verb to follow these conventions, but they should be followed whenever reasonable. * - * @note The kernels internal I/O Request Packet structure contains the same archetypes but with the kernel equivalents + * @note The kernels internal I/O Request Packet structure uses a similar system but with the kernel equivalents * of the arguments, for example, a `file_t*` instead of a `fd_t`. * - * Included below is a list of all argument archetypes. - * - * ### `.handle` - * - * Used for verbs that act on a single existing file descriptor. - * - * **Arguments:** - * - `fd`: The file descriptor to act upon. - * - * ### `.path` - * - * Used for verbs that act on a filesystem path. - * - * **Arguments:** - * - `dirfd`: The directory file descriptor to resolve the path from, or `FD_CWD` to use the current working directory. - * - `path`: Pointer to the path string, null-termination is ignored. - * - `len`: Length of the path string. - * - * ### `.rw` - * - * Used for verbs that transfer data to or from a file. - * - * **Arguments:** - * - `fd`: The file descriptor to access. - * - `buffer`: Pointer to the buffer to read into or write from. - * - `len`: Length of the buffer. - * - `off`: Offset within the file to access, or `IO_CUR` to use the current file offset. - * - * ### `.seek` - * - * Used for verbs that seek within a file. - * - * **Arguments:** - * - `fd`: The file descriptor to seek within. - * - `off`: Offset to seek to. - * - `whence`: Origin for the seek operation, `IO_SET`, `IO_END` or `IO_CUR`. - * - * ### `.poll` - * - * Used for verbs that wait for events on a file descriptor. - * - * **Arguments:** - * - `fd`: The file descriptor to poll. - * - `events`: The events to wait for (e.g., `POLLIN`, `POLLOUT`). - * * ## Results * * The result of a SQE is stored in its corresponding CQE using a single 64-bit value. For convenience, the result is @@ -144,22 +108,30 @@ * ## Verbs * * Included below is a list of all currently implemented verbs. + * + * The arguments of each verb is specified in order as `arg0`, `arg1`, `arg2`, `arg3`, `arg4`. * * ### `VERB_NOP` * * A no-operation verb that does nothing but is useful for implementing sleeping. * - * **Arguments:** None. - * - * **Result:** None. - * + * @param arg0 Unused + * @param arg1 Unused + * @param arg2 Unused + * @param arg3 Unused + * @param arg4 Unused + * @result None + * * ### `VERB_READ` * * Reads data from a file descriptor. * - * **Arguments:** `.rw` - - * **Result:** The number of bytes read. + * @param fd The file descriptor to read from. + * @param buffer The buffer to read the data into. + * @param count The number of bytes to read. + * @param offset The offset to read from, or `IO_CUR` to use the current position. + * @param arg4 Unused + * @result The number of bytes read. * * @{ */ diff --git a/include/kernel/io/irp.h b/include/kernel/io/irp.h index a66ce1aa1..af09cb7e8 100644 --- a/include/kernel/io/irp.h +++ b/include/kernel/io/irp.h @@ -163,7 +163,7 @@ typedef struct irp irp_t; * irp_push(irp, my_completion, NULL); * * // Finally, dispatch the IRP to the appropriate handler. - * irp_dispatch(irp); + * verb_dispatch(irp); * // Continue executing even if the operation cannot complete immediately. * ``` * @@ -250,8 +250,6 @@ typedef struct irp irp_t; #define IRP_LOC_MAX 5 ///< The maximum number of locations in a IRP. -#define IRP_ARG_MAX SQE_ARG_MAX ///< The maximum number of arguments in an IRP. - /** * @brief IRP completion callback type. * @@ -317,35 +315,29 @@ typedef struct ALIGNED(64) irp clock_t deadline; ///< The time at which the IRP will be removed from a timeout queue. }; void* data; ///< Private data for the operation, will be returned in the completion entry. - union { - uint64_t _args[IRP_ARG_MAX]; - struct - { - file_t* file; - } handle; - struct - { - file_t* file; - mdl_t* path; - } path; - struct - { - file_t* file; - mdl_t* buffer; - size_t len; - ssize_t off; - } rw; - struct - { - file_t* file; - ssize_t off; - whence_t whence; - } seek; - struct - { - file_t* file; - events_t events; - } poll; + union + { + uint64_t arg0; + file_t* file; + }; + union + { + uint64_t arg1; + mdl_t* buffer; + }; + union + { + uint64_t arg2; + size_t count; + }; + union + { + uint64_t arg3; + ssize_t offset; + }; + union + { + uint64_t arg4; }; }; sqe_t sqe; ///< The original SQE for this IRP. @@ -356,7 +348,7 @@ typedef struct ALIGNED(64) irp void* ptr; events_t events; uint64_t _raw; - } result; + } res; mdl_t mdl; ///< A preallocated memory descriptor list for use by the IRP. pool_idx_t index; ///< Index of the IRP in its pool. pool_idx_t next; ///< Index of the next IRP in a chain or in the free list. @@ -370,7 +362,11 @@ static_assert(offsetof(irp_t, verb) == offsetof(irp_t, sqe.verb), "verb offset m static_assert(offsetof(irp_t, flags) == offsetof(irp_t, sqe.flags), "flags offset mismatch"); static_assert(offsetof(irp_t, timeout) == offsetof(irp_t, sqe.timeout), "timeout offset mismatch"); static_assert(offsetof(irp_t, data) == offsetof(irp_t, sqe.data), "data offset mismatch"); -static_assert(offsetof(irp_t, _args) == offsetof(irp_t, sqe._args), "args offset mismatch"); +static_assert(offsetof(irp_t, arg0) == offsetof(irp_t, sqe.arg0), "arg0 offset mismatch"); +static_assert(offsetof(irp_t, arg1) == offsetof(irp_t, sqe.arg1), "arg1 offset mismatch"); +static_assert(offsetof(irp_t, arg2) == offsetof(irp_t, sqe.arg2), "arg2 offset mismatch"); +static_assert(offsetof(irp_t, arg3) == offsetof(irp_t, sqe.arg3), "arg3 offset mismatch"); +static_assert(offsetof(irp_t, arg4) == offsetof(irp_t, sqe.arg4), "arg4 offset mismatch"); static_assert(sizeof(irp_t) == 256, "irp_t is not 256 bytes"); @@ -428,10 +424,6 @@ void irp_timeouts_check(void); * The pool that the IRP was allocated from, and its context, can be retrieved using the `irp_get_pool()` * function. * - * @note If a SQE is provided then the IRP will be considered a user IRP, causing the `irp_handler_t::enter` and - * `irp_handler_t::leave` callbacks to be invoked on the IRP when its dispatched and freed respectively. Otherwise, the - * caller is responsible for the lifecycle and arguments of the IRP. - * * @param pool Pointer to the IRP pool. * @param sqe The Submission Queue Entry associated with the IRP, if `NULL` the IRP will be a kernel IRP. * @return On success, a pointer to the allocated IRP. On failure, `NULL` and `errno` is set. @@ -617,68 +609,4 @@ static inline void irp_error(irp_t* irp, uint8_t err) */ uint64_t irp_cancel(irp_t* irp); -/** - * @brief Execute an IRP synchronously. - * - * This function dispatches the IRP and blocks the current thread until the operation is complete. - * - * @warning This function should only be used when the alternative of using asynchronous operations is simply not worth - * the complexity. For example, while loading modules. - * - * @param irp Pointer to the IRP to execute. - */ -void irp_run(irp_t* irp); - -/** - * @brief Dispatch an IRP to the appropriate handler. - * - * If `irp->err != EINPROGRESS` the IRP is immediately completed. - * - * If the IRP is a user IRP and it has not yet been entered, the `irp_handler_t::enter` callback for the verb is - * invoked. - * - * @param irp Pointer to the IRP to dispatch. - */ -void irp_dispatch(irp_t* irp); - -/** - * @brief Sort and validate the IRP handlers table. - */ -void irp_table_init(void); - -/** - * @brief IRP handler structure. - * @struct irp_handler_t - */ -typedef struct -{ - verb_t verb; - void (*handler)(irp_t* irp); ///< The handler function for the verb. -} irp_handler_t; - -/** - * @brief Linker defined start of the IRP handlers table. - * - * After `irp_table_init()` has sorted the IRP table, the table can be indexed by verb. - */ -extern irp_handler_t _irp_table_start[]; - -/** - * @brief Linker defined end of the IRP handlers table. - */ -extern irp_handler_t _irp_table_end[]; - -/** - * @brief Macro to define an IRP handler. - * - * @param _verb The verb to define the handler for. - */ -#define IRP_DEFINE(_verb) \ - static void __irp_handler_##_verb(irp_t* irp); \ - static irp_handler_t __irp_##_verb __attribute__((section("._irp_table"), used)) = { \ - .verb = (_verb), \ - .handler = __irp_handler_##_verb, \ - }; \ - static void __irp_handler_##_verb(irp_t* irp) - /** @} */ \ No newline at end of file diff --git a/include/kernel/io/verb.h b/include/kernel/io/verb.h index 3d7fdf244..c3724d163 100644 --- a/include/kernel/io/verb.h +++ b/include/kernel/io/verb.h @@ -9,15 +9,83 @@ * @defgroup kernel_io_verb I/O Request Packet Verbs * @ingroup kernel_io * - * * @{ */ -typedef void (*verb_handler_t)(irp_t* irp); +/** + * @brief Verb function type. + * + * @param irp Pointer to the IRP. + */ +typedef void (*verb_func_t)(irp_t* irp); +/** + * @brief Verb table structure. + * @struct verb_table_t + */ typedef struct verb_table { - verb_handler_t handlers[VERB_MAX]; + verb_func_t handlers[VERB_MAX]; } verb_table_t; +/** + * @brief Cleanup the arguments used by a verb. + * + * Handles both kernel IRPs and parsed user IRPs. + * + * @param irp Pointer to the IRP. + */ +void verb_args_cleanup(irp_t* irp); + +/** + * @brief Dispatch an IRP to the appropriate verb handler. + * + * If the IRP is a user IRP, the arguments will be parsed before invoking the handler. + * + * @param irp Pointer to the IRP. + */ +void verb_dispatch(irp_t* irp); + +/** + * @brief Invoke the appropriate verb handler from a verb table. + * + * @param irp Pointer to the IRP. + * @param table Pointer to the verb table. + * @return `true` if the IRP was completed, `false` otherwise. + */ +static inline bool verb_invoke(irp_t* irp, const verb_table_t* table) +{ + if (UNLIKELY(irp->verb >= VERB_MAX)) + { + irp_error(irp, EINVAL); + return true; + } + + if (table == NULL) + { + return false; + } + + verb_func_t handler = table->handlers[irp->verb]; + if (handler == NULL) + { + return false; + } + + handler(irp); + return true; +} + +/** + * @brief Execute an IRP synchronously. + * + * This function will dispatch the IRP and blocks the current thread until the operation is complete. + * + * @warning This function should only be used when the alternative of using asynchronous operations is simply not worth + * the complexity. For example, while loading modules. + * + * @param irp Pointer to the IRP to execute. + */ +void verb_run(irp_t* irp); + /** @} */ \ No newline at end of file diff --git a/include/libstd/sys/ioring.h b/include/libstd/sys/ioring.h index e8243960f..96a55aa6b 100644 --- a/include/libstd/sys/ioring.h +++ b/include/libstd/sys/ioring.h @@ -71,11 +71,6 @@ typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. * The operation was created by the kernel, used internally by the kernel. */ #define SQE_KERNEL (1 << (_SQE_FLAGS)) - -/** - * The operations enter callback has been called, used internally by the kernel. - */ -#define SQE_KERNEL_ENTERED (1 << (_SQE_FLAGS + 1)) #endif /** @@ -87,8 +82,6 @@ typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. */ #define SQE_HARDLINK (1 << (_SQE_FLAGS + 3)) -#define SQE_ARG_MAX 5 ///< Maximum number of arguments for a ring operation. - /** * @brief Asynchronous submission queue entry (SQE). * @struct sqe_t @@ -104,36 +97,29 @@ typedef struct sqe sqe_flags_t flags; ///< Submission flags. clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. void* data; ///< Private data for the operation, will be returned in the completion entry. - union { - uint64_t _args[SQE_ARG_MAX]; - struct - { - fd_t fd; - } handle; - struct - { - fd_t dirfd; - char* path; - size_t len; - } path; - struct - { - fd_t fd; - void* buffer; - size_t len; - ssize_t off; - } rw; - struct - { - fd_t fd; - ssize_t off; - whence_t whence; - } seek; - struct - { - fd_t fd; - events_t events; - } poll; + union + { + uint64_t arg0; + fd_t fd; + }; + union + { + uint64_t arg1; + void* buffer; + }; + union + { + uint64_t arg2; + size_t count; + }; + union + { + uint64_t arg3; + ssize_t offset; + }; + union + { + uint64_t arg4; }; } sqe_t; diff --git a/meta/doxy/Doxyfile b/meta/doxy/Doxyfile index d33cab85e..556ff35d1 100644 --- a/meta/doxy/Doxyfile +++ b/meta/doxy/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = "PatchworkOS" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "facb5a79-dirty" +PROJECT_NUMBER = "4fd4030b-dirty" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewers a diff --git a/src/kernel/fs/file.c b/src/kernel/fs/file.c index ae834dc95..b99946690 100644 --- a/src/kernel/fs/file.c +++ b/src/kernel/fs/file.c @@ -69,7 +69,7 @@ file_t* file_new(const path_t* path, mode_t mode) file->inode = REF(path->dentry->inode); file->path = PATH_CREATE(path->mount, path->dentry); file->ops = path->dentry->inode->fileOps; - file->verbs = path->dentry->superblock->defaultVerbs; + file->verbs = file->inode->verbs; file->data = NULL; return file; } @@ -97,16 +97,4 @@ size_t file_generic_seek(file_t* file, ssize_t offset, seek_origin_t origin) file->pos = newPos; return newPos; -} - -IRP_DEFINE(VERB_READ) -{ - file_t* file = irp->rw.file; - if (file->verbs == NULL || file->verbs->handlers[VERB_READ] == NULL) - { - irp_error(irp, ENOSYS); - return; - } - - file->verbs->handlers[VERB_READ](irp); } \ No newline at end of file diff --git a/src/kernel/fs/inode.c b/src/kernel/fs/inode.c index efb366573..eed894572 100644 --- a/src/kernel/fs/inode.c +++ b/src/kernel/fs/inode.c @@ -76,7 +76,7 @@ inode_t* inode_new(superblock_t* superblock, ino_t number, itype_t type, const i inode->superblock = REF(superblock); inode->ops = ops; inode->fileOps = fileOps; - inode->verbs = NULL; + inode->verbs = superblock->verbs; return inode; } diff --git a/src/kernel/fs/superblock.c b/src/kernel/fs/superblock.c index 180551824..63bd7f4da 100644 --- a/src/kernel/fs/superblock.c +++ b/src/kernel/fs/superblock.c @@ -54,7 +54,7 @@ superblock_t* superblock_new(filesystem_t* fs, const superblock_ops_t* ops, cons superblock->root = NULL; superblock->ops = ops; superblock->dentryOps = dentryOps; - superblock->defaultVerbs = NULL; + superblock->verbs = NULL; superblock->fs = fs; atomic_init(&superblock->mountCount, 0); diff --git a/src/kernel/init/init.c b/src/kernel/init/init.c index 10545388a..28eccd3da 100644 --- a/src/kernel/init/init.c +++ b/src/kernel/init/init.c @@ -63,7 +63,6 @@ void init_early(void) vmm_kernel_space_load(); - irp_table_init(); syscall_table_init(); _std_init(); diff --git a/src/kernel/io/io.c b/src/kernel/io/io.c index ded0dc744..b7bd1ac1e 100644 --- a/src/kernel/io/io.c +++ b/src/kernel/io/io.c @@ -195,7 +195,7 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) sqe_flags_t reg = (irp->flags >> SQE_SAVE) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - atomic_store_explicit(&ring->ctrl->regs[reg], irp->result._raw, memory_order_release); + atomic_store_explicit(&ring->ctrl->regs[reg], irp->res._raw, memory_order_release); } uint32_t tail = atomic_load_explicit(&ring->ctrl->ctail, memory_order_relaxed); @@ -211,7 +211,7 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) cqe->verb = irp->verb; cqe->error = irp->err; cqe->data = irp->data; - cqe->_result = irp->result._raw; + cqe->_result = irp->res._raw; atomic_store_explicit(&ring->ctrl->ctail, tail + 1, memory_order_release); wait_unblock(&ctx->waitQueue, WAIT_ALL, EOK); @@ -246,19 +246,40 @@ static void io_ctx_dispatch(irp_t* irp) io_ctx_t* ctx = irp_get_ctx(irp); ioring_t* ring = &ctx->ring; - for (uint64_t i = 0; i < SQE_ARG_MAX; i++) + // Ugly but the alternative is a super messy SQE structure. + + sqe_flags_t reg = (irp->flags >> SQE_LOAD0) & SQE_REG_MASK; + if (reg != SQE_REG_NONE) { - sqe_flags_t reg = (irp->flags >> (i * SQE_REG_SHIFT)) & SQE_REG_MASK; - if (reg == SQE_REG_NONE) - { - continue; - } + irp->sqe.arg0 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + } - irp->sqe._args[i] = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + reg = (irp->flags >> SQE_LOAD1) & SQE_REG_MASK; + if (reg != SQE_REG_NONE) + { + irp->sqe.arg1 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + } + + reg = (irp->flags >> SQE_LOAD2) & SQE_REG_MASK; + if (reg != SQE_REG_NONE) + { + irp->sqe.arg2 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + } + + reg = (irp->flags >> SQE_LOAD3) & SQE_REG_MASK; + if (reg != SQE_REG_NONE) + { + irp->sqe.arg3 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + } + + reg = (irp->flags >> SQE_LOAD4) & SQE_REG_MASK; + if (reg != SQE_REG_NONE) + { + irp->sqe.arg4 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } irp_push(irp, io_ctx_complete, NULL); - irp_dispatch(irp); + verb_dispatch(irp); } typedef struct diff --git a/src/kernel/io/irp.c b/src/kernel/io/irp.c index 5ef403f51..4abe30ea2 100644 --- a/src/kernel/io/irp.c +++ b/src/kernel/io/irp.c @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -53,11 +54,12 @@ irp_pool_t* irp_pool_new(size_t size, process_t* process, void* ctx) irp->flags = 0; irp->timeout = CLOCKS_NEVER; irp->data = NULL; - for (size_t j = 0; j < IRP_ARG_MAX; j++) - { - irp->_args[j] = 0; - } - irp->result._raw = 0; + irp->arg0 = 0; + irp->arg1 = 0; + irp->arg2 = 0; + irp->arg3 = 0; + irp->arg4 = 0; + irp->res._raw = 0; mdl_init(&irp->mdl, NULL); irp->sqe = (sqe_t){0}; irp->index = i; @@ -81,55 +83,6 @@ void irp_pool_free(irp_pool_t* pool) free(pool); } -static inline void irp_enter(irp_t* irp) -{ - process_t* process = irp_get_process(irp); - - switch (irp->verb) - { - case VERB_READ: - { - file_t* file = file_table_get(&process->fileTable, irp->sqe.rw.fd); - if (file == NULL) - { - irp->err = EBADF; - return; - } - - if (mdl_from_region(&irp->mdl, NULL, &process->space, irp->sqe.rw.buffer, irp->sqe.rw.len) == ERR) - { - UNREF(file); - irp->err = EFAULT; - return; - } - - irp->rw.file = file; - irp->rw.buffer = &irp->mdl; - irp->rw.len = irp->sqe.rw.len; - irp->rw.off = irp->sqe.rw.off; - } - break; - default: - break; - } -} - -static inline void irp_leave(irp_t* irp) -{ - // MDL will be deinitialized in irp_free - - switch (irp->verb) - { - case VERB_READ: - { - UNREF(irp->rw.file); - } - break; - default: - break; - } -} - irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) { pool_idx_t idx = pool_alloc(&pool->pool); @@ -143,7 +96,7 @@ irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) irp->location = IRP_LOC_MAX; irp->next = POOL_IDX_MAX; irp->err = EINPROGRESS; - irp->result._raw = 0; + irp->res._raw = 0; atomic_store_explicit(&irp->cancel, NULL, memory_order_relaxed); if (sqe == NULL) @@ -154,7 +107,7 @@ irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) else { irp->sqe = *sqe; - irp->sqe.flags &= ~(SQE_KERNEL | SQE_KERNEL_ENTERED); + irp->sqe.flags &= ~SQE_KERNEL; } irp->next = POOL_IDX_MAX; @@ -173,14 +126,22 @@ irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) return irp; } + void irp_free(irp_t* irp) { - if (irp->flags & SQE_KERNEL_ENTERED && irp->verb < VERB_MAX) + if (irp == NULL) { - assert(!(irp->flags & SQE_KERNEL)); - irp_leave(irp); + return; } + irp_timeout_remove(irp); + + assert(irp->location == IRP_LOC_MAX); + assert(irp->next == POOL_IDX_MAX); + assert(irp->cpu == CPU_ID_INVALID); + + verb_args_cleanup(irp); + mdl_t* next = irp->mdl.next; mdl_deinit(&irp->mdl); mdl_free_chain(next, free); @@ -209,31 +170,12 @@ uint64_t irp_cancel(irp_t* irp) return ERR; } + irp_timeout_remove(irp); + irp->err = ECANCELED; return handler(irp); } -static void irp_run_completion(irp_t* irp, void* ctx) -{ - UNUSED(irp); - - wait_queue_t* wait = (wait_queue_t*)ctx; - wait_unblock(wait, WAIT_ALL, EOK); -} - -void irp_run(irp_t* irp) -{ - wait_queue_t wait; - wait_queue_init(&wait); - - irp_push(irp, irp_run_completion, &wait); - irp_dispatch(irp); - - WAIT_BLOCK(&wait, irp->err != EINPROGRESS); - - wait_queue_deinit(&wait); -} - void irp_timeout_add(irp_t* irp) { if (irp->timeout == CLOCKS_NEVER) @@ -312,9 +254,9 @@ void irp_timeouts_check(void) list_remove(&irp->timeoutEntry); irp->deadline = CLOCKS_NEVER; irp->cpu = CPU_ID_INVALID; + irp_cancel_t handler = atomic_exchange(&irp->cancel, IRP_CANCELLED); lock_release(&ctx->lock); - irp_cancel_t handler = atomic_exchange(&irp->cancel, IRP_CANCELLED); if (handler == IRP_CANCELLED) { // Already cancelled @@ -329,66 +271,4 @@ void irp_timeouts_check(void) } lock_release(&ctx->lock); -} - -void irp_dispatch(irp_t* irp) -{ - if (irp->verb >= VERB_MAX || _irp_table_start[irp->verb].handler == NULL) - { - irp->err = ENOSYS; - irp_complete(irp); - return; - } - - if (!(irp->flags & SQE_KERNEL) && !(irp->flags & SQE_KERNEL_ENTERED)) - { - irp_enter(irp); - irp->flags |= SQE_KERNEL_ENTERED; - } - - if (irp->err != EINPROGRESS) - { - irp_complete(irp); - return; - } - - _irp_table_start[irp->verb].handler(irp); -} - -static int irp_handler_cmp(const void* a, const void* b) -{ - const irp_handler_t* irpA = (const irp_handler_t*)a; - const irp_handler_t* irpB = (const irp_handler_t*)b; - return (int32_t)irpA->verb - (int32_t)irpB->verb; -} - -void irp_table_init(void) -{ - const uint64_t irpsInTable = (((uint64_t)_irp_table_end - (uint64_t)_irp_table_start) / sizeof(irp_handler_t)); - assert(irpsInTable == VERB_MAX); - - LOG_INFO("sorting IRP table, total IRPs %d\n", VERB_MAX); - qsort(_irp_table_start, irpsInTable, sizeof(irp_handler_t), irp_handler_cmp); - - for (uint64_t i = 0; i < irpsInTable; i++) - { - assert(_irp_table_start[i].verb == i); - } -} - -static uint64_t _nop_cancel(irp_t* irp) -{ - irp_complete(irp); - return 0; -} - -IRP_DEFINE(VERB_NOP) -{ - irp_set_cancel(irp, _nop_cancel); - - if (irp->timeout != CLOCKS_NEVER) - { - irp_timeout_add(irp); - return; - } } \ No newline at end of file diff --git a/src/kernel/io/verb.c b/src/kernel/io/verb.c new file mode 100644 index 000000000..baf14ab74 --- /dev/null +++ b/src/kernel/io/verb.c @@ -0,0 +1,137 @@ +#include +#include +#include +#include +#include +#include + +void verb_args_cleanup(irp_t* irp) +{ + switch (irp->verb) + { + case VERB_READ: + { + UNREF(irp->file); + irp->file = NULL; + } + break; + default: + break; + } +} + +static void verb_args_user(irp_t* irp) +{ + assert(!(irp->flags & SQE_KERNEL)); + + process_t* process = irp_get_process(irp); + + switch (irp->verb) + { + case VERB_READ: + { + file_t* file = file_table_get(&process->fileTable, irp->sqe.fd); + if (file == NULL) + { + irp->err = EBADF; + return; + } + + if (mdl_from_region(&irp->mdl, NULL, &process->space, irp->sqe.buffer, irp->sqe.count) == ERR) + { + UNREF(file); + irp->err = EFAULT; + return; + } + + irp->file = file; + irp->buffer = &irp->mdl; + irp->count = irp->sqe.count; + irp->offset = irp->sqe.offset; + } + break; + default: + break; + } +} + +static uint64_t nop_cancel(irp_t* irp) +{ + irp_complete(irp); + return 0; +} + +static void verb_dispatch_file(irp_t* irp) +{ + file_t* file = irp->file; + assert(file != NULL); + + if (verb_invoke(irp, file->verbs)) + { + return; + } + + if (verb_invoke(irp, file->inode->verbs)) + { + return; + } + + if (verb_invoke(irp, file->inode->superblock->verbs)) + { + return; + } + + irp_error(irp, ENOSYS); +} + +void verb_dispatch(irp_t* irp) +{ + if (!(irp->flags & SQE_KERNEL)) + { + verb_args_user(irp); + } + + if (irp->err != EINPROGRESS) + { + irp_complete(irp); + return; + } + + if (irp->timeout != CLOCKS_NEVER) + { + irp_timeout_add(irp); + } + + switch (irp->verb) + { + case VERB_NOP: + irp_set_cancel(irp, nop_cancel); + break; + case VERB_READ: + verb_dispatch_file(irp); + break; + default: + break; + }; +} + +static void verb_run_completion(irp_t* irp, void* ctx) +{ + UNUSED(irp); + + wait_queue_t* wait = (wait_queue_t*)ctx; + wait_unblock(wait, WAIT_ALL, EOK); +} + +void verb_run(irp_t* irp) +{ + wait_queue_t wait; + wait_queue_init(&wait); + + irp_push(irp, verb_run_completion, &wait); + verb_dispatch(irp); + + WAIT_BLOCK(&wait, irp->err != EINPROGRESS); + + wait_queue_deinit(&wait); +} diff --git a/src/kernel/mem/pmm.c b/src/kernel/mem/pmm.c index 6be21b83c..f7af0fbd4 100644 --- a/src/kernel/mem/pmm.c +++ b/src/kernel/mem/pmm.c @@ -227,7 +227,7 @@ static void pmm_load_memory(const boot_memory_map_t* map) page->ref = UINT16_MAX; } - LOG_INFO("reserve [%p-%p] pages=%d type=%s\n", PFN_TO_VIRT(pfn), PFN_TO_VIRT(pfn + pages), pages, + LOG_INFO("reserve [%p-%p] pages=%d type=%s\n", PFN_TO_VIRT(pfn), PFN_TO_VIRT(pfn + amount), amount, efiMemTypeToString[desc->Type]); } } diff --git a/src/programs/core/init/main.c b/src/programs/core/init/main.c index 4bbbcf240..6034b006d 100644 --- a/src/programs/core/init/main.c +++ b/src/programs/core/init/main.c @@ -74,7 +74,7 @@ static uint64_t init_socket_addr_wait(const char* family, const char* addr) free((void*)data); - if (uptime() - start > CLOCKS_PER_SEC * 10) + if ((uptime() - start) >= CLOCKS_PER_SEC * 10) { close(addrs); return ERR; From fa40da8d3b82771f8d3640ef9bfbdf877b3af6cb Mon Sep 17 00:00:00 2001 From: KN Date: Thu, 22 Jan 2026 13:07:24 +0100 Subject: [PATCH 20/23] refactor: change inode_t to vnode_t --- include/kernel/fs/dentry.h | 38 +++--- include/kernel/fs/devfs.h | 24 ++-- include/kernel/fs/file.h | 6 +- include/kernel/fs/filesystem.h | 2 +- include/kernel/fs/inode.h | 152 +++++++++++----------- include/kernel/fs/superblock.h | 2 +- include/kernel/fs/sysfs.h | 24 ++-- include/kernel/fs/tmpfs.h | 2 +- include/kernel/fs/vfs.h | 6 +- include/kernel/fs/vnode.h | 170 +++++++++++++++++++++++++ include/kernel/io/io.h | 22 ++++ include/kernel/io/irp.h | 3 +- include/libstd/sys/fs.h | 51 +++++--- include/libstd/sys/ioring.h | 7 +- lib/OVMFbin/OVMF_VARS-pure-efi.fd | Bin 131072 -> 131072 bytes src/kernel/drivers/abstract/fb.c | 24 ++-- src/kernel/drivers/abstract/kbd.c | 18 +-- src/kernel/drivers/abstract/mouse.c | 18 +-- src/kernel/fs/dentry.c | 34 ++--- src/kernel/fs/devfs.c | 56 ++++----- src/kernel/fs/file.c | 16 +-- src/kernel/fs/filesystem.c | 52 ++++---- src/kernel/fs/inode.c | 148 ---------------------- src/kernel/fs/netfs.c | 102 +++++++-------- src/kernel/fs/path.c | 2 +- src/kernel/fs/procfs.c | 188 ++++++++++++++-------------- src/kernel/fs/sysfs.c | 56 ++++----- src/kernel/fs/tmpfs.c | 142 +++++++++++---------- src/kernel/fs/vfs.c | 126 +++++++------------ src/kernel/fs/vnode.c | 89 +++++++++++++ src/kernel/io/verb.c | 17 ++- src/kernel/module/module.c | 2 +- src/kernel/proc/group.c | 2 +- src/modules/acpi/tables.c | 2 +- src/modules/fs/9p/9p.c | 8 +- src/programs/core/init/main.c | 2 +- src/programs/core/shell/pipeline.c | 4 +- src/programs/utils/ls/main.c | 6 +- src/programs/utils/stat/main.c | 10 +- 39 files changed, 871 insertions(+), 762 deletions(-) create mode 100644 include/kernel/fs/vnode.h delete mode 100644 src/kernel/fs/inode.c create mode 100644 src/kernel/fs/vnode.c diff --git a/include/kernel/fs/dentry.h b/include/kernel/fs/dentry.h index f7f4e623f..c753e3513 100644 --- a/include/kernel/fs/dentry.h +++ b/include/kernel/fs/dentry.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include @@ -15,7 +15,7 @@ typedef struct dentry dentry_t; typedef struct dentry_ops dentry_ops_t; -typedef struct inode inode_t; +typedef struct vnode vnode_t; typedef struct superblock superblock_t; typedef struct dir_ctx dir_ctx_t; @@ -25,7 +25,7 @@ typedef struct dir_ctx dir_ctx_t; * @ingroup kernel_fs * * A dentry represents the actual name in the filesystem hierarchy. It can be either positive, meaning it has an - * associated inode, or negative, meaning it does not have an associated inode. + * associated vnode, or negative, meaning it does not have an associated vnode. * * ## Mountpoints and Root Dentries * @@ -64,31 +64,31 @@ typedef uint64_t dentry_id_t; * @param dentry The dentry to check. * @return true if the dentry is positive, false if it is negative. */ -#define DENTRY_IS_POSITIVE(dentry) ((dentry)->inode != NULL) +#define DENTRY_IS_POSITIVE(dentry) ((dentry)->vnode != NULL) /** - * @brief Check if the inode associated with a dentry is a regular file. + * @brief Check if the vnode associated with a dentry is a regular file. * * @param dentry The dentry to check. * @return true if the dentry is a regular file, false otherwise or if the dentry is negative. */ -#define DENTRY_IS_REGULAR(dentry) (DENTRY_IS_POSITIVE(dentry) && (dentry)->inode->type == INODE_REGULAR) +#define DENTRY_IS_REGULAR(dentry) (DENTRY_IS_POSITIVE(dentry) && (dentry)->vnode->type == VREG) /** - * @brief Check if the inode associated with a dentry is a directory. + * @brief Check if the vnode associated with a dentry is a directory. * * @param dentry The dentry to check. * @return true if the dentry is a directory, false otherwise or if the dentry is negative. */ -#define DENTRY_IS_DIR(dentry) (DENTRY_IS_POSITIVE(dentry) && (dentry)->inode->type == INODE_DIR) +#define DENTRY_IS_DIR(dentry) (DENTRY_IS_POSITIVE(dentry) && (dentry)->vnode->type == VDIR) /** - * @brief Check if the inode associated with a dentry is a symbolic link. + * @brief Check if the vnode associated with a dentry is a symbolic link. * * @param dentry The dentry to check. * @return true if the dentry is a symbolic link, false otherwise or if the dentry is negative. */ -#define DENTRY_IS_SYMLINK(dentry) (DENTRY_IS_POSITIVE(dentry) && (dentry)->inode->type == INODE_SYMLINK) +#define DENTRY_IS_SYMLINK(dentry) (DENTRY_IS_POSITIVE(dentry) && (dentry)->vnode->type == VSYMLINK) /** * @brief Directory context used to iterate over directory entries. @@ -104,11 +104,11 @@ typedef struct dir_ctx * * @param ctx The directory context. * @param name The name of the entry. - * @param number The inode number of the entry. - * @param type The inode type of the entry. + * @param number The vnode number of the entry. + * @param type The vnode type of the entry. * @return `true` to continue iterating, `false` to stop. */ - bool (*emit)(dir_ctx_t* ctx, const char* name, ino_t number, itype_t type); + bool (*emit)(dir_ctx_t* ctx, const char* name, vtype_t type); size_t pos; ///< The current position in the directory, can be used to skip entries. void* data; ///< Private data that the filesystem can use to conveniently pass data. size_t index; ///< An index that the filesystem can use for its own purposes. @@ -148,7 +148,7 @@ typedef struct dentry_ops * @brief Directory entry structure. * @struct dentry_t * - * A dentry structure is protected by the mutex of its inode. Note that since move and rename are not supported in favor + * A dentry structure is protected by the mutex of its vnode. Note that since move and rename are not supported in favor * of link and remove, the parent of a dentry will never change after creation which allows some optimizations. */ typedef struct dentry @@ -156,7 +156,7 @@ typedef struct dentry ref_t ref; dentry_id_t id; char name[MAX_NAME]; ///< The name of the dentry, immutable after creation. - inode_t* inode; ///< Will be `NULL` if the dentry is negative, once positive it will never be modified. + vnode_t* vnode; ///< Will be `NULL` if the dentry is negative, once positive it will never be modified. dentry_t* parent; ///< The parent dentry, will be itself if this is the root dentry, immutable after creation. list_entry_t siblingEntry; list_t children; @@ -224,14 +224,14 @@ dentry_t* dentry_rcu_get(const dentry_t* parent, const char* name, size_t length dentry_t* dentry_lookup(dentry_t* parent, const char* name, size_t length); /** - * @brief Make a dentry positive by associating it with an inode. + * @brief Make a dentry positive by associating it with an vnode. * - * This function is expected to be protected by the parent inode's mutex. + * This function is expected to be protected by the parent vnode's mutex. * * @param dentry The dentry to make positive, or `NULL` for no-op. - * @param inode The inode to associate with the dentry, or `NULL` for no-op. + * @param vnode The vnode to associate with the dentry, or `NULL` for no-op. */ -void dentry_make_positive(dentry_t* dentry, inode_t* inode); +void dentry_make_positive(dentry_t* dentry, vnode_t* vnode); /** * @brief The amount of special entries "." and ".." that `dentry_iterate_dots()` emits. diff --git a/include/kernel/fs/devfs.h b/include/kernel/fs/devfs.h index bb1c6deba..3572f11d7 100644 --- a/include/kernel/fs/devfs.h +++ b/include/kernel/fs/devfs.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include typedef struct file file_t; @@ -35,23 +35,23 @@ void devfs_init(void); * * @param parent The parent directory, if `NULL` then the root is used. * @param name The name of the new directory. - * @param inodeOps The inode operations for the new directory, can be `NULL`. - * @param private Private data to store in the inode of the new directory, can be `NULL`. + * @param vnodeOps The vnode operations for the new directory, can be `NULL`. + * @param private Private data to store in the vnode of the new directory, can be `NULL`. * @return On success, the new devfs directory. On failure, `NULL` and `errno` is set. */ -dentry_t* devfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); +dentry_t* devfs_dir_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, void* data); /** * @brief Create a new file inside a mounted devfs instance. * * @param parent The parent directory, if `NULL` then the root is used. * @param name The name of the new file. - * @param inodeOps The inode operations for the new file, can be `NULL`. + * @param vnodeOps The vnode operations for the new file, can be `NULL`. * @param fileOps The file operations for the new file, can be `NULL`. - * @param private Private data to store in the inode of the new file, can be `NULL`. + * @param private Private data to store in the vnode of the new file, can be `NULL`. * @return On success, the new devfs file. On failure, `NULL` and `errno` is set. */ -dentry_t* devfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, const file_ops_t* fileOps, +dentry_t* devfs_file_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, const file_ops_t* fileOps, void* data); /** @@ -59,11 +59,11 @@ dentry_t* devfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* * * @param parent The parent directory, if `NULL` then the root is used. * @param name The name of the new symbolic link. - * @param inodeOps The inode operations for the new symbolic link. - * @param private Private data to store in the inode of the new symbolic link, can be `NULL`. + * @param vnodeOps The vnode operations for the new symbolic link. + * @param private Private data to store in the vnode of the new symbolic link, can be `NULL`. * @return On success, the new devfs symbolic link. On failure, `NULL` and `errno` is set. */ -dentry_t* devfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); +dentry_t* devfs_symlink_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, void* data); /** * @brief Descriptor for batch file creation. @@ -72,9 +72,9 @@ dentry_t* devfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_ typedef struct devfs_file_desc { const char* name; ///< Name of the file, `NULL` marks end of array. - const inode_ops_t* inodeOps; ///< Inode operations, can be `NULL`. + const vnode_ops_t* vnodeOps; ///< Vnode operations, can be `NULL`. const file_ops_t* fileOps; ///< File operations, can be `NULL`. - void* data; ///< Private data to store in the inode of the file. + void* data; ///< Private data to store in the vnode of the file. } devfs_file_desc_t; /** diff --git a/include/kernel/fs/file.h b/include/kernel/fs/file.h index 77e088722..a778bea37 100644 --- a/include/kernel/fs/file.h +++ b/include/kernel/fs/file.h @@ -15,7 +15,7 @@ typedef struct wait_queue wait_queue_t; typedef struct file file_t; typedef struct file_ops file_ops_t; typedef struct dentry dentry_t; -typedef struct inode inode_t; +typedef struct vnode vnode_t; typedef struct poll_file poll_file_t; /** @@ -33,7 +33,7 @@ typedef struct poll_file poll_file_t; * @brief File structure. * @struct file_t * - * A file structure is protected by the mutex of its inode. + * A file structure is protected by the mutex of its vnode. * */ typedef struct file @@ -41,7 +41,7 @@ typedef struct file ref_t ref; size_t pos; mode_t mode; - inode_t* inode; + vnode_t* vnode; path_t path; const file_ops_t* ops; const verb_table_t* verbs; diff --git a/include/kernel/fs/filesystem.h b/include/kernel/fs/filesystem.h index fe3431b15..6380872d1 100644 --- a/include/kernel/fs/filesystem.h +++ b/include/kernel/fs/filesystem.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/kernel/fs/inode.h b/include/kernel/fs/inode.h index 05a5176fc..5f23f4579 100644 --- a/include/kernel/fs/inode.h +++ b/include/kernel/fs/inode.h @@ -13,203 +13,203 @@ #include #include -typedef struct inode inode_t; -typedef struct inode_ops inode_ops_t; +typedef struct vnode vnode_t; +typedef struct vnode_ops vnode_ops_t; typedef struct superblock superblock_t; typedef struct file_ops file_ops_t; typedef struct dentry dentry_t; /** * @brief Index node. - * @defgroup kernel_fs_inode Inode + * @defgroup kernel_fs_vnode Vnode * @ingroup kernel_fs * - * A inode represents the actual data and metadata of a file. It is referenced by dentries, which represent the name or - * "location" of the file but a inode can appear in multiple dentries due to hardlinks or mounts. + * A vnode represents the actual data and metadata of a file. It is referenced by dentries, which represent the name or + * "location" of the file but a vnode can appear in multiple dentries due to hardlinks or mounts. * - * @note Despite the name inodes are in no way "nodes" in any kind of tree structure, that would be the dentries. + * @note Despite the name vnodes are in no way "nodes" in any kind of tree structure, that would be the dentries. * * ## Synchronization * - * Inodes have an additional purpose within the Virtual File System (VFS) as they act as the primary means of - * synchronization. All dentries synchronize upon their inodes mutex, open files synchronize upon the mutex of the - * underlying inode and operations like create, remove, etc synchronize upon the inode mutex of the parent directory. + * Vnodes have an additional purpose within the Virtual File System (VFS) as they act as the primary means of + * synchronization. All dentries synchronize upon their vnodes mutex, open files synchronize upon the mutex of the + * underlying vnode and operations like create, remove, etc synchronize upon the vnode mutex of the parent directory. * - * @todo Implement actually writing/syncing dirty inodes, for now inodes should use the notify functions but they will + * @todo Implement actually writing/syncing dirty vnodes, for now vnodes should use the notify functions but they will * never actually be "cleaned." * * @{ */ /** - * @brief Inode structure. - * @struct inode_t + * @brief Vnode structure. + * @struct vnode_t * - * Inodes are owned by the filesystem, not the VFS. + * Vnodes are owned by the filesystem, not the VFS. */ -typedef struct inode +typedef struct vnode { ref_t ref; ino_t number; - itype_t type; - _Atomic(uint64_t) dentryCount; ///< The number of dentries pointing to this inode. + vtype_t type; + _Atomic(uint64_t) dentryCount; ///< The number of dentries pointing to this vnode. size_t size; size_t blocks; - time_t accessTime; ///< Unix time stamp for the last inode access. + time_t accessTime; ///< Unix time stamp for the last vnode access. time_t modifyTime; ///< Unix time stamp for last file content alteration. time_t changeTime; ///< Unix time stamp for the last file metadata alteration. - time_t createTime; ///< Unix time stamp for the inode creation. + time_t createTime; ///< Unix time stamp for the vnode creation. void* data; superblock_t* superblock; - const inode_ops_t* ops; + const vnode_ops_t* ops; const file_ops_t* fileOps; const verb_table_t* verbs; rcu_entry_t rcu; mutex_t mutex; -} inode_t; +} vnode_t; /** - * @brief Inode operations structure. - * @struct inode_ops_t + * @brief Vnode operations structure. + * @struct vnode_ops_t * - * Note that the inodes mutex will be acquired by the vfs. + * Note that the vnodes mutex will be acquired by the vfs. */ -typedef struct inode_ops +typedef struct vnode_ops { /** - * @brief Look up a dentry in a directory inode. + * @brief Look up a dentry in a directory vnode. * - * Should set the target dentry to be positive (give it an inode), if the entry does not exist the operation + * Should set the target dentry to be positive (give it an vnode), if the entry does not exist the operation * should still return success but leave the dentry negative. * - * @param dir The directory inode to look in. + * @param dir The directory vnode to look in. * @param target The dentry to look up. * @return On success, `0`. On failure, returns `ERR` and `errno` is set. */ - uint64_t (*lookup)(inode_t* dir, dentry_t* target); + uint64_t (*lookup)(vnode_t* dir, dentry_t* target); /** * @brief Handles both directories and files depending on mode. * - * Takes in a negative dentry and creates the corresponding inode to make the dentry positive. + * Takes in a negative dentry and creates the corresponding vnode to make the dentry positive. * - * @param dir The directory inode to create the entry in. + * @param dir The directory vnode to create the entry in. * @param target The negative dentry to create. * @param mode The mode to create the entry with. * @return On success, `0`. On failure, returns `ERR` and `errno` is set. */ - uint64_t (*create)(inode_t* dir, dentry_t* target, mode_t mode); + uint64_t (*create)(vnode_t* dir, dentry_t* target, mode_t mode); /** - * @brief Set the inode size to zero. + * @brief Set the vnode size to zero. * - * @param target The inode to truncate. + * @param target The vnode to truncate. */ - void (*truncate)(inode_t* target); + void (*truncate)(vnode_t* target); /** - * @brief Make the same file inode appear twice in the filesystem. + * @brief Make the same file vnode appear twice in the filesystem. * - * @param dir The directory inode to create the link in. - * @param old The existing dentry containing the inode to link to. - * @param new The negative dentry to store the same inode as old. + * @param dir The directory vnode to create the link in. + * @param old The existing dentry containing the vnode to link to. + * @param new The negative dentry to store the same vnode as old. * @return On success, `0`. On failure, returns `ERR` and `errno` is set. */ - uint64_t (*link)(inode_t* dir, dentry_t* old, dentry_t* new); + uint64_t (*link)(vnode_t* dir, dentry_t* old, dentry_t* new); /** * @brief Retrieve the path of the symbolic link. * - * @param inode The symbolic link inode. + * @param vnode The symbolic link vnode. * @param buffer The buffer to store the path in. * @param size The size of the buffer. * @return On success, the number of bytes read. On failure, returns `ERR` and `errno` is set. */ - uint64_t (*readlink)(inode_t* inode, char* buffer, uint64_t size); + uint64_t (*readlink)(vnode_t* vnode, char* buffer, uint64_t size); /** * @brief Create a symbolic link. * - * @param dir The directory inode to create the symbolic link in. + * @param dir The directory vnode to create the symbolic link in. * @param target The negative dentry to create. * @param dest The path to which the symbolic link will point. * @return On success, `0`. On failure, returns `ERR` and `errno` is set. */ - uint64_t (*symlink)(inode_t* dir, dentry_t* target, const char* dest); + uint64_t (*symlink)(vnode_t* dir, dentry_t* target, const char* dest); /** * @brief Remove a file or directory. * - * @param dir The directory inode containing the target. + * @param dir The directory vnode containing the target. * @param target The dentry to remove. * @return On success, `0`. On failure, returns `ERR` and `errno` is set. */ - uint64_t (*remove)(inode_t* dir, dentry_t* target); + uint64_t (*remove)(vnode_t* dir, dentry_t* target); /** - * @brief Cleanup function called when the inode is being freed. + * @brief Cleanup function called when the vnode is being freed. * - * @param inode The inode being freed. + * @param vnode The vnode being freed. */ - void (*cleanup)(inode_t* inode); -} inode_ops_t; + void (*cleanup)(vnode_t* vnode); +} vnode_ops_t; /** - * @brief Create a new inode. + * @brief Create a new vnode. * - * This DOES add the inode to the inode cache. It also does not associate the inode with a dentry, that is done when a + * This DOES add the vnode to the vnode cache. It also does not associate the vnode with a dentry, that is done when a * dentry is made positive with `dentry_make_positive()`. * - * There is no `inode_free()` instead use `UNREF()`. + * There is no `vnode_free()` instead use `UNREF()`. * - * @param superblock The superblock the inode belongs to. - * @param number The inode number, for a generic filesystem `vfs_id_get()` can be used. - * @param type The inode type. - * @param ops The inode operations. - * @param fileOps The file operations for files opened on this inode. - * @return On success, the new inode. On failure, returns `NULL` and `errno` is set. + * @param superblock The superblock the vnode belongs to. + * @param number The vnode number, for a generic filesystem `vfs_id_get()` can be used. + * @param type The vnode type. + * @param ops The vnode operations. + * @param fileOps The file operations for files opened on this vnode. + * @return On success, the new vnode. On failure, returns `NULL` and `errno` is set. */ -inode_t* inode_new(superblock_t* superblock, ino_t number, itype_t type, const inode_ops_t* ops, +vnode_t* vnode_new(superblock_t* superblock, ino_t number, vtype_t type, const vnode_ops_t* ops, const file_ops_t* fileOps); /** - * @brief Notify the inode that it has been accessed. + * @brief Notify the vnode that it has been accessed. * * This updates the access time. * - * @param inode The inode to notify. + * @param vnode The vnode to notify. */ -void inode_notify_access(inode_t* inode); +void vnode_notify_access(vnode_t* vnode); /** - * @brief Notify the inode that its content has been modified. + * @brief Notify the vnode that its content has been modified. * * This updates the modify time and change time. * - * @param inode The inode to notify. + * @param vnode The vnode to notify. */ -void inode_notify_modify(inode_t* inode); +void vnode_notify_modify(vnode_t* vnode); /** - * @brief Notify the inode that its metadata has changed. + * @brief Notify the vnode that its metadata has changed. * * This updates the change time. * - * @param inode The inode to notify. + * @param vnode The vnode to notify. */ -void inode_notify_change(inode_t* inode); +void vnode_notify_change(vnode_t* vnode); /** - * @brief Truncate the inode. + * @brief Truncate the vnode. * - * The filesystem should implement the actual truncation in the inode ops truncate function, this is just a helper to + * The filesystem should implement the actual truncation in the vnode ops truncate function, this is just a helper to * call it. * - * @param inode The inode to truncate. + * @param vnode The vnode to truncate. */ -void inode_truncate(inode_t* inode); +void vnode_truncate(vnode_t* vnode); /** - * @brief Helper to generate a consistent inode number for an entry in a directory. + * @brief Helper to generate a consistent vnode number for an entry in a directory. * - * This is useful for in-memory filesystems or filesystem that dont provide native inode numbers. + * This is useful for in-memory filesystems or filesystem that dont provide native vnode numbers. * - * @param parentNumber The inode number of the parent directory. + * @param parentNumber The vnode number of the parent directory. * @param name The name of the entry. - * @return The generated inode number. + * @return The generated vnode number. */ ino_t ino_gen(ino_t parentNumber, const char* name); diff --git a/include/kernel/fs/superblock.h b/include/kernel/fs/superblock.h index ba4a6bc6e..e4e2a150b 100644 --- a/include/kernel/fs/superblock.h +++ b/include/kernel/fs/superblock.h @@ -11,7 +11,7 @@ typedef struct filesystem filesystem_t; typedef struct superblock superblock_t; typedef struct superblock_ops superblock_ops_t; typedef struct dentry_ops dentry_ops_t; -typedef struct inode inode_t; +typedef struct vnode vnode_t; typedef struct dentry dentry_t; /** diff --git a/include/kernel/fs/sysfs.h b/include/kernel/fs/sysfs.h index 164a298c1..781e5fc77 100644 --- a/include/kernel/fs/sysfs.h +++ b/include/kernel/fs/sysfs.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include typedef struct file file_t; @@ -40,23 +40,23 @@ void sysfs_init(void); * * @param parent The parent directory, if `NULL` then the root is used. * @param name The name of the new directory. - * @param inodeOps The inode operations for the new directory, can be `NULL`. - * @param private Private data to store in the inode of the new directory, can be `NULL`. + * @param vnodeOps The vnode operations for the new directory, can be `NULL`. + * @param private Private data to store in the vnode of the new directory, can be `NULL`. * @return On success, the new sysfs directory. On failure, `NULL` and `errno` is set. */ -dentry_t* sysfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); +dentry_t* sysfs_dir_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, void* data); /** * @brief Create a new file inside a mounted sysfs instance. * * @param parent The parent directory, if `NULL` then the root is used. * @param name The name of the new file. - * @param inodeOps The inode operations for the new file, can be `NULL`. + * @param vnodeOps The vnode operations for the new file, can be `NULL`. * @param fileOps The file operations for the new file, can be `NULL`. - * @param private Private data to store in the inode of the new file, can be `NULL`. + * @param private Private data to store in the vnode of the new file, can be `NULL`. * @return On success, the new sysfs file. On failure, `NULL` and `errno` is set. */ -dentry_t* sysfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, const file_ops_t* fileOps, +dentry_t* sysfs_file_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, const file_ops_t* fileOps, void* data); /** @@ -64,11 +64,11 @@ dentry_t* sysfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* * * @param parent The parent directory, if `NULL` then the root is used. * @param name The name of the new symbolic link. - * @param inodeOps The inode operations for the new symbolic link. - * @param private Private data to store in the inode of the new symbolic link, can be `NULL`. + * @param vnodeOps The vnode operations for the new symbolic link. + * @param private Private data to store in the vnode of the new symbolic link, can be `NULL`. * @return On success, the new sysfs symbolic link. On failure, `NULL` and `errno` is set. */ -dentry_t* sysfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data); +dentry_t* sysfs_symlink_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, void* data); /** * @brief Descriptor for batch file creation. @@ -77,9 +77,9 @@ dentry_t* sysfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_ typedef struct sysfs_file_desc { const char* name; ///< Name of the file, `NULL` marks end of array. - const inode_ops_t* inodeOps; ///< Inode operations, can be `NULL`. + const vnode_ops_t* vnodeOps; ///< Vnode operations, can be `NULL`. const file_ops_t* fileOps; ///< File operations, can be `NULL`. - void* data; ///< Private data to store in the inode of the file. + void* data; ///< Private data to store in the vnode of the file. } sysfs_file_desc_t; /** diff --git a/include/kernel/fs/tmpfs.h b/include/kernel/fs/tmpfs.h index a137c528f..1e6869bcf 100644 --- a/include/kernel/fs/tmpfs.h +++ b/include/kernel/fs/tmpfs.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include diff --git a/include/kernel/fs/vfs.h b/include/kernel/fs/vfs.h index d3070e05b..eea960094 100644 --- a/include/kernel/fs/vfs.h +++ b/include/kernel/fs/vfs.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -162,13 +162,13 @@ uint64_t vfs_link(const pathname_t* oldPathname, const pathname_t* newPathname, /** * @brief Read the path in a symbolic link. * - * @param symlink The symbolic link inode. + * @param symlink The symbolic link vnode. * @param buffer The buffer to store the path in. * @param size The size of the buffer. * @param process The process performing the readlink. * @return On success, the number of bytes read. On failure, `ERR` and `errno` is set. */ -size_t vfs_readlink(inode_t* symlink, char* buffer, size_t size); +size_t vfs_readlink(vnode_t* symlink, char* buffer, size_t size); /** * @brief Create a symbolic link. diff --git a/include/kernel/fs/vnode.h b/include/kernel/fs/vnode.h new file mode 100644 index 000000000..d894db7c6 --- /dev/null +++ b/include/kernel/fs/vnode.h @@ -0,0 +1,170 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +typedef struct vnode vnode_t; +typedef struct vnode_ops vnode_ops_t; +typedef struct superblock superblock_t; +typedef struct file_ops file_ops_t; +typedef struct dentry dentry_t; + +/** + * @brief Virtual node. + * @defgroup kernel_fs_vnode Vnode + * @ingroup kernel_fs + * + * A vnode represents the actual data and metadata of a file. It is referenced by dentries, which represent the name or + * "location" of the file but a vnode can appear in multiple dentries due to hardlinks or mounts. + * + * @note Despite the name vnodes are in no way "nodes" in any kind of tree structure, that would be the dentries. + * + * ## Synchronization + * + * vnodes have an additional purpose within the Virtual File System (VFS) as they act as the primary means of + * synchronization. All dentries synchronize upon their vnodes mutex, open files synchronize upon the mutex of the + * underlying vnode and operations like create, remove, etc synchronize upon the vnode mutex of the parent directory. + * + * @todo Implement actually writing/syncing dirty vnodes, for now vnodes should use the notify functions but they will + * never actually be "cleaned." + * + * @{ + */ + +/** + * @brief vnode structure. + * @struct vnode_t + * + * vnodes are owned by the filesystem, not the VFS. + */ +typedef struct vnode +{ + ref_t ref; + vtype_t type; + _Atomic(uint64_t) dentryCount; ///< The number of dentries pointing to this vnode. + void* data; ///< Filesystem defined data. + uint64_t size; ///< Used for convenience by certain filesystems, does not represent the file size. + superblock_t* superblock; + const vnode_ops_t* ops; + const file_ops_t* fileOps; + const verb_table_t* verbs; + rcu_entry_t rcu; + mutex_t mutex; +} vnode_t; + +/** + * @brief vnode operations structure. + * @struct vnode_ops_t + * + * Note that the vnodes mutex will be acquired by the vfs. + */ +typedef struct vnode_ops +{ + /** + * @brief Look up a dentry in a directory vnode. + * + * Should set the target dentry to be positive (give it an vnode), if the entry does not exist the operation + * should still return success but leave the dentry negative. + * + * @param dir The directory vnode to look in. + * @param target The dentry to look up. + * @return On success, `0`. On failure, returns `ERR` and `errno` is set. + */ + uint64_t (*lookup)(vnode_t* dir, dentry_t* target); + /** + * @brief Handles both directories and files depending on mode. + * + * Takes in a negative dentry and creates the corresponding vnode to make the dentry positive. + * + * @param dir The directory vnode to create the entry in. + * @param target The negative dentry to create. + * @param mode The mode to create the entry with. + * @return On success, `0`. On failure, returns `ERR` and `errno` is set. + */ + uint64_t (*create)(vnode_t* dir, dentry_t* target, mode_t mode); + /** + * @brief Set the vnode size to zero. + * + * @param target The vnode to truncate. + */ + void (*truncate)(vnode_t* target); + /** + * @brief Make the same file vnode appear twice in the filesystem. + * + * @param dir The directory vnode to create the link in. + * @param old The existing dentry containing the vnode to link to. + * @param new The negative dentry to store the same vnode as old. + * @return On success, `0`. On failure, returns `ERR` and `errno` is set. + */ + uint64_t (*link)(vnode_t* dir, dentry_t* old, dentry_t* new); + /** + * @brief Retrieve the path of the symbolic link. + * + * @param vnode The symbolic link vnode. + * @param buffer The buffer to store the path in. + * @param size The size of the buffer. + * @return On success, the number of bytes read. On failure, returns `ERR` and `errno` is set. + */ + uint64_t (*readlink)(vnode_t* vnode, char* buffer, uint64_t size); + /** + * @brief Create a symbolic link. + * + * @param dir The directory vnode to create the symbolic link in. + * @param target The negative dentry to create. + * @param dest The path to which the symbolic link will point. + * @return On success, `0`. On failure, returns `ERR` and `errno` is set. + */ + uint64_t (*symlink)(vnode_t* dir, dentry_t* target, const char* dest); + /** + * @brief Remove a file or directory. + * + * @param dir The directory vnode containing the target. + * @param target The dentry to remove. + * @return On success, `0`. On failure, returns `ERR` and `errno` is set. + */ + uint64_t (*remove)(vnode_t* dir, dentry_t* target); + /** + * @brief Cleanup function called when the vnode is being freed. + * + * @param vnode The vnode being freed. + */ + void (*cleanup)(vnode_t* vnode); +} vnode_ops_t; + +/** + * @brief Create a new vnode. + * + * Does not associate the vnode with a dentry, that is done when a dentry is made positive with `dentry_make_positive()`. + * + * There is no `vnode_free()` instead use `UNREF()`. + * + * @param superblock The superblock the vnode belongs to. + * @param type The vnode type. + * @param ops The vnode operations. + * @param fileOps The file operations for files opened on this vnode. + * @return On success, the new vnode. On failure, returns `NULL` and `errno` is set. + */ +vnode_t* vnode_new(superblock_t* superblock, vtype_t type, const vnode_ops_t* ops, + const file_ops_t* fileOps); + +/** + * @brief Truncate the vnode. + * + * The filesystem should implement the actual truncation in the vnode ops truncate function, this is just a helper to + * call it. + * + * @param vnode The vnode to truncate. + */ +void vnode_truncate(vnode_t* vnode); + +/** @} */ diff --git a/include/kernel/io/io.h b/include/kernel/io/io.h index 4c1c3bd2c..82a61ea4b 100644 --- a/include/kernel/io/io.h +++ b/include/kernel/io/io.h @@ -133,6 +133,28 @@ * @param arg4 Unused * @result The number of bytes read. * + * ### `VERB_WRITE` + * + * Writes data to a file descriptor. + * + * @param fd The file descriptor to write to. + * @param buffer The buffer to write the data from. + * @param count The number of bytes to write. + * @param offset The offset to write to, or `IO_CUR` to use the current position. + * @param arg4 Unused + * @result The number of bytes written. + * + * ### `VERB_POLL` + * + * Polls a file descriptor for events. + * + * @param fd The file descriptor to poll. + * @param events The events to wait for. + * @param arg2 Unused + * @param arg3 Unused + * @param arg4 Unused + * @result The events that occurred. + * * @{ */ diff --git a/include/kernel/io/irp.h b/include/kernel/io/irp.h index af09cb7e8..8f6b00653 100644 --- a/include/kernel/io/irp.h +++ b/include/kernel/io/irp.h @@ -323,7 +323,8 @@ typedef struct ALIGNED(64) irp union { uint64_t arg1; - mdl_t* buffer; + mdl_t* buffer; + events_t events; }; union { diff --git a/include/libstd/sys/fs.h b/include/libstd/sys/fs.h index f61499b74..1dcd04f5e 100644 --- a/include/libstd/sys/fs.h +++ b/include/libstd/sys/fs.h @@ -335,29 +335,41 @@ uint64_t poll(pollfd_t* fds, uint64_t amount, clock_t timeout); poll_events_t poll1(fd_t fd, poll_events_t events, clock_t timeout); /** - * @brief Inode type enum. - * @enum itype_t + * @brief Vnode type enum. + * @enum vtype_t */ typedef enum { - INODE_REGULAR, ///< Is a regular file. - INODE_DIR, ///< Is a directory. - INODE_SYMLINK, ///< Is a symbolic link. -} itype_t; + VREG, ///< Is a regular file. + VDIR, ///< Is a directory. + VSYMLINK, ///< Is a symbolic link. +} vtype_t; /** - * @brief A inode number that uniquely identifies the inode within its filesystem. + * @brief A suberblock identifier that uniquely identifies a superblock within the system. * - * When combined with a superblock ID, this can uniquely identify an inode within the entire system. + * When combined with a vnode number, this can uniquely identify an vnode within the entire system. */ -typedef uint64_t ino_t; +typedef uint64_t sbid_t; /** - * @brief A suberblock identifier that uniquely identifies a superblock within the system. - * - * When combined with a inode number, this can uniquely identify an inode within the entire system. + * @brief Vnode attributes structure. + * @struct vattr_t */ -typedef uint64_t sbid_t; +typedef struct vattr +{ + vtype_t type; + uint64_t nlink; + uint64_t id; + uint64_t size; + uint64_t blocks; + uint64_t blockSize; + uint64_t rdev; + time_t atime; + time_t mtime; + time_t ctime; + uint8_t padding[64]; ///< Padding to leave space for future expansion. +} vattr_t; /** * @brief Stat type. @@ -366,17 +378,17 @@ typedef uint64_t sbid_t; typedef struct { sbid_t sbid; ///< The superblock ID of the filesystem containing the entry. - ino_t number; ///< The number of the entries inode. - itype_t type; ///< The type of the entries inode. + uint64_t number; ///< The number of the entries vnode. + vtype_t type; ///< The type of the entries vnode. uint64_t size; ///< The size of the file that is visible outside the filesystem. uint64_t blocks; ///< The amount of blocks used on disk to store the file. uint64_t blockSize; ///< The preferred block size of the filesystem. uint64_t maxFileSize; ///< The maximum size of a file on this filesystem. - uint64_t linkAmount; ///< The amount of times the inode appears in dentries. - time_t accessTime; ///< Unix time stamp for the last inode access. + uint64_t linkAmount; ///< The amount of times the vnode appears in dentries. + time_t accessTime; ///< Unix time stamp for the last vnode access. time_t modifyTime; ///< Unix time stamp for last file content alteration. time_t changeTime; ///< Unix time stamp for the last file metadata alteration. - time_t createTime; ///< Unix time stamp for the creation of the inode. + time_t createTime; ///< Unix time stamp for the creation of the vnode. char name[MAX_PATH]; ///< The name of the entry, not the full filepath. Includes the flags of the paths mount. uint8_t padding[64]; ///< Padding to leave space for future expansion. } stat_t; @@ -442,8 +454,7 @@ typedef enum */ typedef struct { - ino_t number; - itype_t type; + vtype_t type; dirent_flags_t flags; char path[MAX_PATH]; ///< The relative path of the entry. char mode[MAX_PATH]; ///< The flags of the paths mount. diff --git a/include/libstd/sys/ioring.h b/include/libstd/sys/ioring.h index 96a55aa6b..866fb69a3 100644 --- a/include/libstd/sys/ioring.h +++ b/include/libstd/sys/ioring.h @@ -39,9 +39,9 @@ typedef uint64_t events_t; ///< Poll events type. typedef uint32_t verb_t; ///< Verb type. #define VERB_NOP 0 ///< No-op verb. #define VERB_READ 1 ///< Read verb. -// #define VERB_WRITE 2 ///< Write verb. -// #define VERB_POLL 3 ///< Poll verb. -#define VERB_MAX 2 ///< The maximum number of verbs. +#define VERB_WRITE 2 ///< Write verb. +#define VERB_POLL 3 ///< Poll verb. +#define VERB_MAX 4 ///< The maximum number of verbs. typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. @@ -106,6 +106,7 @@ typedef struct sqe { uint64_t arg1; void* buffer; + events_t events; }; union { diff --git a/lib/OVMFbin/OVMF_VARS-pure-efi.fd b/lib/OVMFbin/OVMF_VARS-pure-efi.fd index b728cd32916c645e3def714dd4f885908f57f930..52f4bb61960c47f6f7a086bdfd3174c231faba2a 100644 GIT binary patch delta 49 zcmZo@;Am*z*bwBtd6R_`3(Gxi28PK2-YzWnL5vAz8k-et4fwWy(Pca_wdupM4~zl; Dr@#^D delta 40 wcmZo@;Am*z*bwBt*(&TG^YjJUj4mwqv>6ySJ4WrFx;Y?fC12BrWgi#?06yRlApigX diff --git a/src/kernel/drivers/abstract/fb.c b/src/kernel/drivers/abstract/fb.c index 8da016eef..af176bbce 100644 --- a/src/kernel/drivers/abstract/fb.c +++ b/src/kernel/drivers/abstract/fb.c @@ -17,7 +17,7 @@ static dentry_t* dir = NULL; static size_t fb_name_read(file_t* file, void* buffer, size_t count, size_t* offset) { - fb_t* fb = file->inode->data; + fb_t* fb = file->vnode->data; assert(fb != NULL); uint64_t length = strlen(fb->name); @@ -30,7 +30,7 @@ static file_ops_t nameOps = { static size_t fb_data_read(file_t* file, void* buffer, size_t count, size_t* offset) { - fb_t* fb = file->inode->data; + fb_t* fb = file->vnode->data; assert(fb != NULL); if (fb->ops->read == NULL) @@ -44,7 +44,7 @@ static size_t fb_data_read(file_t* file, void* buffer, size_t count, size_t* off static size_t fb_data_write(file_t* file, const void* buffer, size_t count, size_t* offset) { - fb_t* fb = file->inode->data; + fb_t* fb = file->vnode->data; assert(fb != NULL); if (fb->ops->write == NULL) @@ -58,7 +58,7 @@ static size_t fb_data_write(file_t* file, const void* buffer, size_t count, size static void* fb_data_mmap(file_t* file, void* addr, size_t length, size_t* offset, pml_flags_t flags) { - fb_t* fb = file->inode->data; + fb_t* fb = file->vnode->data; assert(fb != NULL); if (fb->ops->mmap == NULL) @@ -78,7 +78,7 @@ static file_ops_t dataOps = { static size_t fb_info_read(file_t* file, void* buffer, size_t count, size_t* offset) { - fb_t* fb = file->inode->data; + fb_t* fb = file->vnode->data; assert(fb != NULL); if (fb->ops->info == NULL) @@ -115,9 +115,9 @@ static file_ops_t infoOps = { .read = fb_info_read, }; -static void fb_dir_cleanup(inode_t* inode) +static void fb_dir_cleanup(vnode_t* vnode) { - fb_t* fb = inode->data; + fb_t* fb = vnode->data; if (fb->ops->cleanup != NULL) { @@ -127,7 +127,7 @@ static void fb_dir_cleanup(inode_t* inode) free(fb); } -static inode_ops_t dirInodeOps = { +static vnode_ops_t dirVnodeOps = { .cleanup = fb_dir_cleanup, }; @@ -169,7 +169,7 @@ fb_t* fb_new(const char* name, const fb_ops_t* ops, void* data) return NULL; } - fb->dir = devfs_dir_new(dir, id, &dirInodeOps, fb); + fb->dir = devfs_dir_new(dir, id, &dirVnodeOps, fb); if (fb->dir == NULL) { free(fb); @@ -179,19 +179,19 @@ fb_t* fb_new(const char* name, const fb_ops_t* ops, void* data) devfs_file_desc_t files[] = { { .name = "name", - .inodeOps = NULL, + .vnodeOps = NULL, .fileOps = &nameOps, .data = fb, }, { .name = "info", - .inodeOps = NULL, + .vnodeOps = NULL, .fileOps = &infoOps, .data = fb, }, { .name = "data", - .inodeOps = NULL, + .vnodeOps = NULL, .fileOps = &dataOps, .data = fb, }, diff --git a/src/kernel/drivers/abstract/kbd.c b/src/kernel/drivers/abstract/kbd.c index a608cf64c..99eb276c9 100644 --- a/src/kernel/drivers/abstract/kbd.c +++ b/src/kernel/drivers/abstract/kbd.c @@ -24,7 +24,7 @@ static atomic_uint64_t newId = ATOMIC_VAR_INIT(0); static size_t kbd_name_read(file_t* file, void* buffer, size_t count, size_t* offset) { - kbd_t* kbd = file->inode->data; + kbd_t* kbd = file->vnode->data; assert(kbd != NULL); size_t length = strlen(kbd->name); @@ -37,7 +37,7 @@ static file_ops_t nameOps = { static uint64_t kbd_events_open(file_t* file) { - kbd_t* kbd = file->inode->data; + kbd_t* kbd = file->vnode->data; assert(kbd != NULL); kbd_client_t* client = calloc(1, sizeof(kbd_client_t)); @@ -59,7 +59,7 @@ static uint64_t kbd_events_open(file_t* file) static void kbd_events_close(file_t* file) { - kbd_t* kbd = file->inode->data; + kbd_t* kbd = file->vnode->data; assert(kbd != NULL); kbd_client_t* client = file->data; @@ -84,7 +84,7 @@ static size_t kbd_events_read(file_t* file, void* buffer, size_t count, size_t* return 0; } - kbd_t* kbd = file->inode->data; + kbd_t* kbd = file->vnode->data; assert(kbd != NULL); kbd_client_t* client = file->data; assert(client != NULL); @@ -110,7 +110,7 @@ static size_t kbd_events_read(file_t* file, void* buffer, size_t count, size_t* static wait_queue_t* kbd_events_poll(file_t* file, poll_events_t* revents) { - kbd_t* kbd = file->inode->data; + kbd_t* kbd = file->vnode->data; assert(kbd != NULL); kbd_client_t* client = file->data; assert(client != NULL); @@ -131,9 +131,9 @@ static file_ops_t eventsOps = { .poll = kbd_events_poll, }; -static void kbd_dir_cleanup(inode_t* inode) +static void kbd_dir_cleanup(vnode_t* vnode) { - kbd_t* kbd = inode->data; + kbd_t* kbd = vnode->data; if (kbd == NULL) { return; @@ -143,7 +143,7 @@ static void kbd_dir_cleanup(inode_t* inode) free(kbd); } -static inode_ops_t dirInodeOps = { +static vnode_ops_t dirVnodeOps = { .cleanup = kbd_dir_cleanup, }; @@ -187,7 +187,7 @@ kbd_t* kbd_new(const char* name) return NULL; } - kbd->dir = devfs_dir_new(dir, id, &dirInodeOps, kbd); + kbd->dir = devfs_dir_new(dir, id, &dirVnodeOps, kbd); if (kbd->dir == NULL) { wait_queue_deinit(&kbd->waitQueue); diff --git a/src/kernel/drivers/abstract/mouse.c b/src/kernel/drivers/abstract/mouse.c index 42edf6285..86bf00ed3 100644 --- a/src/kernel/drivers/abstract/mouse.c +++ b/src/kernel/drivers/abstract/mouse.c @@ -18,7 +18,7 @@ static atomic_uint64_t newId = ATOMIC_VAR_INIT(0); static size_t mouse_name_read(file_t* file, void* buffer, size_t count, size_t* offset) { - mouse_t* mouse = file->inode->data; + mouse_t* mouse = file->vnode->data; assert(mouse != NULL); size_t length = strlen(mouse->name); @@ -31,7 +31,7 @@ static file_ops_t nameOps = { static uint64_t mouse_events_open(file_t* file) { - mouse_t* mouse = file->inode->data; + mouse_t* mouse = file->vnode->data; assert(mouse != NULL); mouse_client_t* client = calloc(1, sizeof(mouse_client_t)); @@ -53,7 +53,7 @@ static uint64_t mouse_events_open(file_t* file) static void mouse_events_close(file_t* file) { - mouse_t* mouse = file->inode->data; + mouse_t* mouse = file->vnode->data; assert(mouse != NULL); mouse_client_t* client = file->data; @@ -78,7 +78,7 @@ static size_t mouse_events_read(file_t* file, void* buffer, size_t count, size_t return 0; } - mouse_t* mouse = file->inode->data; + mouse_t* mouse = file->vnode->data; assert(mouse != NULL); mouse_client_t* client = file->data; assert(client != NULL); @@ -104,7 +104,7 @@ static size_t mouse_events_read(file_t* file, void* buffer, size_t count, size_t static wait_queue_t* mouse_events_poll(file_t* file, poll_events_t* revents) { - mouse_t* mouse = file->inode->data; + mouse_t* mouse = file->vnode->data; assert(mouse != NULL); mouse_client_t* client = file->data; assert(client != NULL); @@ -125,9 +125,9 @@ static file_ops_t eventsOps = { .poll = mouse_events_poll, }; -static void mouse_dir_cleanup(inode_t* inode) +static void mouse_dir_cleanup(vnode_t* vnode) { - mouse_t* mouse = inode->data; + mouse_t* mouse = vnode->data; if (mouse == NULL) { return; @@ -137,7 +137,7 @@ static void mouse_dir_cleanup(inode_t* inode) free(mouse); } -static inode_ops_t dirInodeOps = { +static vnode_ops_t dirVnodeOps = { .cleanup = mouse_dir_cleanup, }; @@ -181,7 +181,7 @@ mouse_t* mouse_new(const char* name) return NULL; } - mouse->dir = devfs_dir_new(dir, id, &dirInodeOps, mouse); + mouse->dir = devfs_dir_new(dir, id, &dirVnodeOps, mouse); if (mouse->dir == NULL) { wait_queue_deinit(&mouse->waitQueue); diff --git a/src/kernel/fs/dentry.c b/src/kernel/fs/dentry.c index 3712de65f..ede8de3b5 100644 --- a/src/kernel/fs/dentry.c +++ b/src/kernel/fs/dentry.c @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include #include @@ -80,11 +80,11 @@ static void dentry_free(dentry_t* dentry) if (!DENTRY_IS_ROOT(dentry)) { assert(dentry->parent != NULL); - assert(dentry->parent->inode != NULL); + assert(dentry->parent->vnode != NULL); - mutex_acquire(&dentry->parent->inode->mutex); + mutex_acquire(&dentry->parent->vnode->mutex); list_remove(&dentry->siblingEntry); - mutex_release(&dentry->parent->inode->mutex); + mutex_release(&dentry->parent->vnode->mutex); UNREF(dentry->parent); dentry->parent = NULL; @@ -96,11 +96,11 @@ static void dentry_free(dentry_t* dentry) } dentry->data = NULL; - if (dentry->inode != NULL) + if (dentry->vnode != NULL) { - atomic_fetch_sub_explicit(&dentry->inode->dentryCount, 1, memory_order_relaxed); - UNREF(dentry->inode); - dentry->inode = NULL; + atomic_fetch_sub_explicit(&dentry->vnode->dentryCount, 1, memory_order_relaxed); + UNREF(dentry->vnode); + dentry->vnode = NULL; } UNREF(dentry->superblock); @@ -116,7 +116,7 @@ static void dentry_ctor(void* ptr) dentry->ref = (ref_t){0}; dentry->id = vfs_id_get(); dentry->name[0] = '\0'; - dentry->inode = NULL; + dentry->vnode = NULL; dentry->parent = NULL; list_entry_init(&dentry->siblingEntry); list_init(&dentry->children); @@ -262,7 +262,7 @@ dentry_t* dentry_lookup(dentry_t* parent, const char* name, size_t length) assert(rflags_read() & RFLAGS_INTERRUPT_ENABLE); - inode_t* dir = parent->inode; + vnode_t* dir = parent->vnode; if (dir->ops == NULL || dir->ops->lookup == NULL) { return dentry; // Leave it as negative. @@ -286,15 +286,15 @@ dentry_t* dentry_lookup(dentry_t* parent, const char* name, size_t length) return dentry; } -void dentry_make_positive(dentry_t* dentry, inode_t* inode) +void dentry_make_positive(dentry_t* dentry, vnode_t* vnode) { - if (dentry == NULL || inode == NULL) + if (dentry == NULL || vnode == NULL) { return; } - atomic_fetch_add_explicit(&inode->dentryCount, 1, memory_order_relaxed); - dentry->inode = REF(inode); + atomic_fetch_add_explicit(&vnode->dentryCount, 1, memory_order_relaxed); + dentry->vnode = REF(vnode); if (!DENTRY_IS_ROOT(dentry)) { list_push_back(&dentry->parent->children, &dentry->siblingEntry); @@ -305,7 +305,7 @@ bool dentry_iterate_dots(dentry_t* dentry, dir_ctx_t* ctx) { if (ctx->index++ >= ctx->pos) { - if (!ctx->emit(ctx, ".", dentry->inode->number, dentry->inode->type)) + if (!ctx->emit(ctx, ".", dentry->vnode->type)) { return false; } @@ -313,7 +313,7 @@ bool dentry_iterate_dots(dentry_t* dentry, dir_ctx_t* ctx) if (ctx->index++ >= ctx->pos) { - if (!ctx->emit(ctx, "..", dentry->parent->inode->number, dentry->parent->inode->type)) + if (!ctx->emit(ctx, "..", dentry->parent->vnode->type)) { return false; } @@ -335,7 +335,7 @@ uint64_t dentry_generic_iterate(dentry_t* dentry, dir_ctx_t* ctx) if (ctx->index++ >= ctx->pos) { assert(DENTRY_IS_POSITIVE(child)); - if (!ctx->emit(ctx, child->name, child->inode->number, child->inode->type)) + if (!ctx->emit(ctx, child->name, child->vnode->type)) { return 0; } diff --git a/src/kernel/fs/devfs.c b/src/kernel/fs/devfs.c index 4b24b9697..573cce03b 100644 --- a/src/kernel/fs/devfs.c +++ b/src/kernel/fs/devfs.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -24,10 +24,6 @@ static dentry_t* root = NULL; -static file_ops_t dirOps = { - .seek = file_generic_seek, -}; - static dentry_ops_t dentryOps = { .iterate = dentry_generic_iterate, }; @@ -65,12 +61,12 @@ void devfs_init(void) } UNREF_DEFER(superblock); - inode_t* inode = inode_new(superblock, vfs_id_get(), INODE_DIR, NULL, &dirOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(superblock, VDIR, NULL, NULL); + if (vnode == NULL) { - panic(NULL, "Failed to create devfs root inode"); + panic(NULL, "Failed to create devfs root vnode"); } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); dentry_t* dentry = dentry_new(superblock, NULL, NULL); if (dentry == NULL) @@ -78,12 +74,12 @@ void devfs_init(void) panic(NULL, "Failed to create devfs root dentry"); } - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); superblock->root = dentry; root = dentry; } -dentry_t* devfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) +dentry_t* devfs_dir_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, void* data) { if (name == NULL) { @@ -109,20 +105,20 @@ dentry_t* devfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* i } UNREF_DEFER(dir); - inode_t* inode = inode_new(parent->superblock, vfs_id_get(), INODE_DIR, inodeOps, &dirOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(parent->superblock, VDIR, vnodeOps, NULL); + if (vnode == NULL) { return NULL; } - UNREF_DEFER(inode); - inode->data = data; + UNREF_DEFER(vnode); + vnode->data = data; - dentry_make_positive(dir, inode); + dentry_make_positive(dir, vnode); return REF(dir); } -dentry_t* devfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, const file_ops_t* fileOps, +dentry_t* devfs_file_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, const file_ops_t* fileOps, void* data) { if (name == NULL) @@ -149,22 +145,22 @@ dentry_t* devfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* } UNREF_DEFER(dentry); - inode_t* inode = inode_new(parent->superblock, vfs_id_get(), INODE_REGULAR, inodeOps, fileOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(parent->superblock, VREG, vnodeOps, fileOps); + if (vnode == NULL) { return NULL; } - UNREF_DEFER(inode); - inode->data = data; + UNREF_DEFER(vnode); + vnode->data = data; - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); return REF(dentry); } -dentry_t* devfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) +dentry_t* devfs_symlink_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, void* data) { - if (parent == NULL || name == NULL || inodeOps == NULL) + if (parent == NULL || name == NULL || vnodeOps == NULL) { errno = EINVAL; return NULL; @@ -183,15 +179,15 @@ dentry_t* devfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_ } UNREF_DEFER(dentry); - inode_t* inode = inode_new(parent->superblock, vfs_id_get(), INODE_SYMLINK, inodeOps, NULL); - if (inode == NULL) + vnode_t* vnode = vnode_new(parent->superblock, VSYMLINK, vnodeOps, NULL); + if (vnode == NULL) { return NULL; } - UNREF_DEFER(inode); - inode->data = data; + UNREF_DEFER(vnode); + vnode->data = data; - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); return REF(dentry); } @@ -220,7 +216,7 @@ uint64_t devfs_files_new(list_t* out, dentry_t* parent, const devfs_file_desc_t* uint64_t count = 0; for (const devfs_file_desc_t* desc = descs; desc->name != NULL; desc++) { - dentry_t* file = devfs_file_new(parent, desc->name, desc->inodeOps, desc->fileOps, desc->data); + dentry_t* file = devfs_file_new(parent, desc->name, desc->vnodeOps, desc->fileOps, desc->data); if (file == NULL) { while (!list_is_empty(&createdList)) diff --git a/src/kernel/fs/file.c b/src/kernel/fs/file.c index b99946690..995977d25 100644 --- a/src/kernel/fs/file.c +++ b/src/kernel/fs/file.c @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include @@ -28,8 +28,8 @@ static void file_free(file_t* file) file->ops->close(file); } - UNREF(file->inode); - file->inode = NULL; + UNREF(file->vnode); + file->vnode = NULL; path_put(&file->path); cache_free(file); @@ -66,17 +66,17 @@ file_t* file_new(const path_t* path, mode_t mode) ref_init(&file->ref, file_free); file->pos = 0; file->mode = mode; - file->inode = REF(path->dentry->inode); + file->vnode = REF(path->dentry->vnode); file->path = PATH_CREATE(path->mount, path->dentry); - file->ops = path->dentry->inode->fileOps; - file->verbs = file->inode->verbs; + file->ops = path->dentry->vnode->fileOps; + file->verbs = file->vnode->verbs; file->data = NULL; return file; } size_t file_generic_seek(file_t* file, ssize_t offset, seek_origin_t origin) { - MUTEX_SCOPE(&file->inode->mutex); + MUTEX_SCOPE(&file->vnode->mutex); size_t newPos; switch (origin) @@ -88,7 +88,7 @@ size_t file_generic_seek(file_t* file, ssize_t offset, seek_origin_t origin) newPos = file->pos + offset; break; case SEEK_END: - newPos = file->inode->size + offset; + newPos = file->vnode->size + offset; break; default: errno = EINVAL; diff --git a/src/kernel/fs/filesystem.c b/src/kernel/fs/filesystem.c index 19ace62d0..d9bc1189a 100644 --- a/src/kernel/fs/filesystem.c +++ b/src/kernel/fs/filesystem.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -44,7 +44,7 @@ static map_key_t filesystem_key(const char* name) static size_t superblock_read(file_t* file, void* buffer, size_t count, size_t* offset) { - superblock_t* sb = file->inode->data; + superblock_t* sb = file->vnode->data; assert(sb != NULL); char info[MAX_PATH]; @@ -58,27 +58,27 @@ static size_t superblock_read(file_t* file, void* buffer, size_t count, size_t* return BUFFER_READ(buffer, count, offset, info, (size_t)length); } -static void superblock_cleanup(inode_t* inode) +static void superblock_cleanup(vnode_t* vnode) { - superblock_t* sb = inode->data; + superblock_t* sb = vnode->data; if (sb == NULL) { return; } UNREF(sb); - inode->data = NULL; + vnode->data = NULL; } static file_ops_t sbFileOps = { .read = superblock_read, }; -static inode_ops_t sbInodeOps = { +static vnode_ops_t sbVnodeOps = { .cleanup = superblock_cleanup, }; -static uint64_t filesystem_lookup(inode_t* dir, dentry_t* dentry) +static uint64_t filesystem_lookup(vnode_t* dir, dentry_t* dentry) { filesystem_t* fs = dir->data; assert(fs != NULL); @@ -99,14 +99,14 @@ static uint64_t filesystem_lookup(inode_t* dir, dentry_t* dentry) continue; } - inode_t* inode = - inode_new(dentry->superblock, ino_gen(dir->number, dentry->name), INODE_REGULAR, NULL, &sbFileOps); - if (inode == NULL) + vnode_t* vnode = + vnode_new(dentry->superblock, VREG, NULL, &sbFileOps); + if (vnode == NULL) { return ERR; } - inode->data = REF(sb); - dentry_make_positive(dentry, inode); + vnode->data = REF(sb); + dentry_make_positive(dentry, vnode); return 0; } @@ -120,7 +120,7 @@ static uint64_t filesystem_iterate(dentry_t* dentry, dir_ctx_t* ctx) return 0; } - filesystem_t* fs = dentry->inode->data; + filesystem_t* fs = dentry->vnode->data; assert(fs != NULL); RWLOCK_READ_SCOPE(&fs->lock); @@ -136,7 +136,7 @@ static uint64_t filesystem_iterate(dentry_t* dentry, dir_ctx_t* ctx) char name[MAX_NAME]; snprintf(name, MAX_NAME, "%llu", sb->id); - if (!ctx->emit(ctx, name, ino_gen(dentry->inode->number, name), INODE_REGULAR)) + if (!ctx->emit(ctx, name, VREG)) { return 0; } @@ -145,7 +145,7 @@ static uint64_t filesystem_iterate(dentry_t* dentry, dir_ctx_t* ctx) return 0; } -static inode_ops_t fsInodeOps = { +static vnode_ops_t fsVnodeOps = { .lookup = filesystem_lookup, }; @@ -153,8 +153,10 @@ static dentry_ops_t fsDentryOps = { .iterate = filesystem_iterate, }; -static uint64_t filesystem_dir_lookup(inode_t* dir, dentry_t* dentry) +static uint64_t filesystem_dir_lookup(vnode_t* dir, dentry_t* dentry) { + UNUSED(dir); + RWLOCK_READ_SCOPE(&lock); map_key_t key = filesystem_key(dentry->name); @@ -164,16 +166,16 @@ static uint64_t filesystem_dir_lookup(inode_t* dir, dentry_t* dentry) return 0; } - inode_t* inode = inode_new(dentry->superblock, ino_gen(dir->number, fs->name), INODE_DIR, &fsInodeOps, NULL); - if (inode == NULL) + vnode_t* vnode = vnode_new(dentry->superblock, VDIR, &fsVnodeOps, NULL); + if (vnode == NULL) { return ERR; } - UNREF_DEFER(inode); - inode->data = fs; + UNREF_DEFER(vnode); + vnode->data = fs; dentry->ops = &fsDentryOps; - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); return 0; } @@ -194,7 +196,7 @@ static uint64_t filesystem_dir_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, fs->name, ino_gen(dentry->inode->number, fs->name), INODE_DIR)) + if (!ctx->emit(ctx, fs->name, VDIR)) { return 0; } @@ -203,7 +205,7 @@ static uint64_t filesystem_dir_iterate(dentry_t* dentry, dir_ctx_t* ctx) return 0; } -static inode_ops_t dirInodeOps = { +static vnode_ops_t dirVnodeOps = { .lookup = filesystem_dir_lookup, }; @@ -221,7 +223,7 @@ void filesystem_expose(void) return; } - dir = sysfs_dir_new(NULL, "fs", &dirInodeOps, NULL); + dir = sysfs_dir_new(NULL, "fs", &dirVnodeOps, NULL); if (dir == NULL) { panic(NULL, "failed to expose filesystem sysfs directory"); @@ -320,7 +322,7 @@ filesystem_t* filesystem_get_by_path(const char* path, process_t* process) return NULL; } - return target.dentry->inode->data; + return target.dentry->vnode->data; } bool options_next(const char** iter, char* buffer, size_t size, char** key, char** value) diff --git a/src/kernel/fs/inode.c b/src/kernel/fs/inode.c deleted file mode 100644 index eed894572..000000000 --- a/src/kernel/fs/inode.c +++ /dev/null @@ -1,148 +0,0 @@ -#include - -#include -#include -#include -#include -#include - -#include - -static void inode_free(inode_t* inode) -{ - if (inode == NULL) - { - return; - } - - if (inode->ops != NULL && inode->ops->cleanup != NULL) - { - inode->ops->cleanup(inode); - } - inode->data = NULL; - - if (inode->superblock != NULL) - { - UNREF(inode->superblock); - inode->superblock = NULL; - } - - rcu_call(&inode->rcu, rcu_call_cache_free, inode); -} - -static void inode_ctor(void* ptr) -{ - inode_t* inode = (inode_t*)ptr; - - inode->ref = (ref_t){0}; - inode->number = 0; - inode->type = 0; - atomic_init(&inode->dentryCount, 0); - inode->size = 0; - inode->blocks = 0; - inode->accessTime = 0; - inode->modifyTime = 0; - inode->changeTime = 0; - inode->createTime = 0; - inode->data = NULL; - inode->superblock = NULL; - inode->ops = NULL; - inode->fileOps = NULL; - inode->rcu = (rcu_entry_t){0}; - mutex_init(&inode->mutex); -} - -static cache_t cache = CACHE_CREATE(cache, "inode", sizeof(inode_t), CACHE_LINE, inode_ctor, NULL); - -inode_t* inode_new(superblock_t* superblock, ino_t number, itype_t type, const inode_ops_t* ops, - const file_ops_t* fileOps) -{ - if (superblock == NULL) - { - errno = EINVAL; - return NULL; - } - - inode_t* inode = cache_alloc(&cache); - if (inode == NULL) - { - errno = ENOMEM; - return NULL; - } - - ref_init(&inode->ref, inode_free); - inode->number = number; - inode->type = type; - inode->superblock = REF(superblock); - inode->ops = ops; - inode->fileOps = fileOps; - inode->verbs = superblock->verbs; - return inode; -} - -void inode_notify_access(inode_t* inode) -{ - if (inode == NULL) - { - return; - } - - MUTEX_SCOPE(&inode->mutex); - - inode->accessTime = clock_epoch(); -} - -void inode_notify_modify(inode_t* inode) -{ - if (inode == NULL) - { - return; - } - - MUTEX_SCOPE(&inode->mutex); - inode->modifyTime = clock_epoch(); - inode->changeTime = inode->modifyTime; -} - -void inode_notify_change(inode_t* inode) -{ - if (inode == NULL) - { - return; - } - - MUTEX_SCOPE(&inode->mutex); - inode->changeTime = clock_epoch(); -} - -void inode_truncate(inode_t* inode) -{ - if (inode == NULL) - { - return; - } - - if (inode->ops != NULL && inode->ops->truncate != NULL) - { - MUTEX_SCOPE(&inode->mutex); - assert(rflags_read() & RFLAGS_INTERRUPT_ENABLE); - inode->ops->truncate(inode); - } -} - -ino_t ino_gen(ino_t parentNumber, const char* name) -{ - uint64_t hash = 0xcbf29ce484222325; - const uint64_t prime = 0x100000001b3; - - hash ^= parentNumber; - hash *= prime; - - while (*name != '\0') - { - hash ^= (uint8_t)*name++; - hash *= prime; - } - - return hash; -} \ No newline at end of file diff --git a/src/kernel/fs/netfs.c b/src/kernel/fs/netfs.c index a86dd3241..0c1219c03 100644 --- a/src/kernel/fs/netfs.c +++ b/src/kernel/fs/netfs.c @@ -72,7 +72,7 @@ typedef struct socket_file static uint64_t netfs_data_open(file_t* file) { - socket_t* sock = file->inode->data; + socket_t* sock = file->vnode->data; assert(sock != NULL); file->data = REF(sock); @@ -160,7 +160,7 @@ static file_ops_t dataOps = { static uint64_t netfs_accept_open(file_t* file) { - socket_t* sock = file->inode->data; + socket_t* sock = file->vnode->data; assert(sock != NULL); if (sock->family->accept == NULL) @@ -203,7 +203,7 @@ static uint64_t netfs_ctl_bind(file_t* file, uint64_t argc, const char** argv) { UNUSED(argc); - socket_t* sock = file->inode->data; + socket_t* sock = file->vnode->data; assert(sock != NULL); if (sock->family->bind == NULL) @@ -252,7 +252,7 @@ static uint64_t netfs_ctl_listen(file_t* file, uint64_t argc, const char** argv) } } - socket_t* sock = file->inode->data; + socket_t* sock = file->vnode->data; assert(sock != NULL); if (sock->family->listen == NULL) @@ -282,7 +282,7 @@ static uint64_t netfs_ctl_connect(file_t* file, uint64_t argc, const char** argv { UNUSED(argc); - socket_t* sock = file->inode->data; + socket_t* sock = file->vnode->data; assert(sock != NULL); if (sock->family->connect == NULL) @@ -325,7 +325,7 @@ static socket_file_t socketFiles[] = { {.name = "ctl", .fileOps = &ctlOps}, }; -static uint64_t netfs_socket_lookup(inode_t* dir, dentry_t* dentry) +static uint64_t netfs_socket_lookup(vnode_t* dir, dentry_t* dentry) { for (size_t i = 0; i < ARRAY_SIZE(socketFiles); i++) { @@ -334,25 +334,25 @@ static uint64_t netfs_socket_lookup(inode_t* dir, dentry_t* dentry) continue; } - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, socketFiles[i].name), INODE_REGULAR, NULL, + vnode_t* vnode = vnode_new(dir->superblock, VREG, NULL, socketFiles[i].fileOps); - if (inode == NULL) + if (vnode == NULL) { return ERR; } - UNREF_DEFER(inode); - inode->data = dir->data; // No reference + UNREF_DEFER(vnode); + vnode->data = dir->data; // No reference - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); return 0; } return 0; } -static void netfs_socket_cleanup(inode_t* inode) +static void netfs_socket_cleanup(vnode_t* vnode) { - socket_t* socket = (socket_t*)inode->data; + socket_t* socket = (socket_t*)vnode->data; if (socket == NULL) { return; @@ -375,7 +375,7 @@ static uint64_t netfs_socket_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, socketFiles[i].name, ino_gen(dentry->inode->number, socketFiles[i].name), INODE_REGULAR)) + if (!ctx->emit(ctx, socketFiles[i].name, VREG)) { return 0; } @@ -384,7 +384,7 @@ static uint64_t netfs_socket_iterate(dentry_t* dentry, dir_ctx_t* ctx) return 0; } -static inode_ops_t socketInodeOps = { +static vnode_ops_t socketVnodeOps = { .lookup = netfs_socket_lookup, .cleanup = netfs_socket_cleanup, }; @@ -414,9 +414,9 @@ static void socket_weak_ptr_callback(void* arg) static uint64_t netfs_factory_open(file_t* file) { - netfs_family_file_ctx_t* ctx = file->inode->data; + netfs_family_file_ctx_t* ctx = file->vnode->data; assert(ctx != NULL); - dentry_t* root = file->inode->superblock->root; + dentry_t* root = file->vnode->superblock->root; assert(root != NULL); socket_t* socket = socket_new(ctx->family, ctx->fileInfo->type); @@ -473,7 +473,7 @@ static file_ops_t factoryFileOps = { static size_t netfs_addrs_read(file_t* file, void* buffer, size_t count, size_t* offset) { - netfs_family_file_ctx_t* ctx = file->inode->data; + netfs_family_file_ctx_t* ctx = file->vnode->data; assert(ctx != NULL); RWMUTEX_READ_SCOPE(&ctx->family->mutex); @@ -525,23 +525,23 @@ static netfs_family_file_t familyFiles[] = { {.name = "addrs", .type = 0, .fileOps = &addrsFileOps}, }; -static void netfs_file_cleanup(inode_t* inode) +static void netfs_file_cleanup(vnode_t* vnode) { - netfs_family_file_ctx_t* ctx = inode->data; + netfs_family_file_ctx_t* ctx = vnode->data; if (ctx == NULL) { return; } free(ctx); - inode->data = NULL; + vnode->data = NULL; } -static inode_ops_t familyFileInodeOps = { +static vnode_ops_t familyFileVnodeOps = { .cleanup = netfs_file_cleanup, }; -static uint64_t netfs_family_lookup(inode_t* dir, dentry_t* dentry) +static uint64_t netfs_family_lookup(vnode_t* dir, dentry_t* dentry) { netfs_family_t* family = dir->data; assert(family != NULL); @@ -553,13 +553,13 @@ static uint64_t netfs_family_lookup(inode_t* dir, dentry_t* dentry) continue; } - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, familyFiles[i].name), INODE_REGULAR, - &familyFileInodeOps, familyFiles[i].fileOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(dir->superblock, VREG, + &familyFileVnodeOps, familyFiles[i].fileOps); + if (vnode == NULL) { return ERR; } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); netfs_family_file_ctx_t* ctx = malloc(sizeof(netfs_family_file_ctx_t)); if (ctx == NULL) @@ -568,9 +568,9 @@ static uint64_t netfs_family_lookup(inode_t* dir, dentry_t* dentry) } ctx->family = family; ctx->fileInfo = &familyFiles[i]; - inode->data = ctx; + vnode->data = ctx; - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); return 0; } @@ -608,17 +608,17 @@ static uint64_t netfs_family_lookup(inode_t* dir, dentry_t* dentry) continue; } - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, socket->id), INODE_DIR, &socketInodeOps, NULL); - if (inode == NULL) + vnode_t* vnode = vnode_new(dir->superblock, VDIR, &socketVnodeOps, NULL); + if (vnode == NULL) { return ERR; } - UNREF_DEFER(inode); - inode->data = REF(socket); + UNREF_DEFER(vnode); + vnode->data = REF(socket); dentry->ops = &socketDentryOps; - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); return 0; } @@ -627,7 +627,7 @@ static uint64_t netfs_family_lookup(inode_t* dir, dentry_t* dentry) static uint64_t netfs_family_iterate(dentry_t* dentry, dir_ctx_t* ctx) { - netfs_family_t* family = dentry->inode->data; + netfs_family_t* family = dentry->vnode->data; assert(family != NULL); if (!dentry_iterate_dots(dentry, ctx)) @@ -642,7 +642,7 @@ static uint64_t netfs_family_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, familyFiles[i].name, ino_gen(dentry->inode->number, familyFiles[i].name), INODE_REGULAR)) + if (!ctx->emit(ctx, familyFiles[i].name, VREG)) { return 0; } @@ -682,7 +682,7 @@ static uint64_t netfs_family_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, socket->id, ino_gen(dentry->inode->number, socket->id), INODE_DIR)) + if (!ctx->emit(ctx, socket->id, VDIR)) { return 0; } @@ -691,7 +691,7 @@ static uint64_t netfs_family_iterate(dentry_t* dentry, dir_ctx_t* ctx) return 0; } -static inode_ops_t familyInodeOps = { +static vnode_ops_t familyVnodeOps = { .lookup = netfs_family_lookup, }; @@ -699,7 +699,7 @@ static dentry_ops_t familyDentryOps = { .iterate = netfs_family_iterate, }; -static uint64_t netfs_lookup(inode_t* dir, dentry_t* dentry) +static uint64_t netfs_lookup(vnode_t* dir, dentry_t* dentry) { RWMUTEX_READ_SCOPE(&familiesMutex); @@ -711,18 +711,18 @@ static uint64_t netfs_lookup(inode_t* dir, dentry_t* dentry) continue; } - inode_t* inode = - inode_new(dir->superblock, ino_gen(dir->number, family->name), INODE_DIR, &familyInodeOps, NULL); - if (inode == NULL) + vnode_t* vnode = + vnode_new(dir->superblock, VDIR, &familyVnodeOps, NULL); + if (vnode == NULL) { return ERR; } - UNREF_DEFER(inode); - inode->data = family; + UNREF_DEFER(vnode); + vnode->data = family; dentry->ops = &familyDentryOps; - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); return 0; } @@ -746,7 +746,7 @@ static uint64_t netfs_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, family->name, ino_gen(dentry->inode->number, family->name), INODE_DIR)) + if (!ctx->emit(ctx, family->name, VDIR)) { return 0; } @@ -755,7 +755,7 @@ static uint64_t netfs_iterate(dentry_t* dentry, dir_ctx_t* ctx) return 0; } -static inode_ops_t netInodeOps = { +static vnode_ops_t netVnodeOps = { .lookup = netfs_lookup, }; @@ -780,12 +780,12 @@ static dentry_t* netfs_mount(filesystem_t* fs, const char* options, void* data) } UNREF_DEFER(superblock); - inode_t* inode = inode_new(superblock, 0, INODE_DIR, &netInodeOps, NULL); - if (inode == NULL) + vnode_t* vnode = vnode_new(superblock, VDIR, &netVnodeOps, NULL); + if (vnode == NULL) { return NULL; } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); dentry_t* dentry = dentry_new(superblock, NULL, NULL); if (dentry == NULL) @@ -794,7 +794,7 @@ static dentry_t* netfs_mount(filesystem_t* fs, const char* options, void* data) } dentry->ops = &netDentryOps; - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); superblock->root = dentry; return superblock->root; diff --git a/src/kernel/fs/path.c b/src/kernel/fs/path.c index 04e13d3cd..1c0dca2d2 100644 --- a/src/kernel/fs/path.c +++ b/src/kernel/fs/path.c @@ -432,7 +432,7 @@ static uint64_t path_rcu_symlink(path_walk_ctx_t* ctx, dentry_t* symlink) } char symlinkPath[MAX_PATH]; - size_t readCount = vfs_readlink(symlink->inode, symlinkPath, MAX_PATH - 1); + size_t readCount = vfs_readlink(symlink->vnode, symlinkPath, MAX_PATH - 1); path_walk_release(ctx); diff --git a/src/kernel/fs/procfs.c b/src/kernel/fs/procfs.c index a59a5f736..bb4782326 100644 --- a/src/kernel/fs/procfs.c +++ b/src/kernel/fs/procfs.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -31,7 +31,7 @@ static uint64_t procfs_revalidate_hide(dentry_t* dentry) { process_t* current = process_current(); assert(current != NULL); - process_t* process = dentry->inode->data; + process_t* process = dentry->vnode->data; assert(process != NULL); namespace_t* currentNs = process_get_ns(current); @@ -63,7 +63,7 @@ static dentry_ops_t hideDentryOps = { static size_t procfs_prio_read(file_t* file, void* buffer, size_t count, size_t* offset) { - process_t* process = file->inode->data; + process_t* process = file->vnode->data; priority_t priority = atomic_load(&process->priority); @@ -76,7 +76,7 @@ static size_t procfs_prio_write(file_t* file, const void* buffer, size_t count, { UNUSED(offset); - process_t* process = file->inode->data; + process_t* process = file->vnode->data; char prioStr[MAX_NAME]; if (count >= MAX_NAME) @@ -111,7 +111,7 @@ static file_ops_t prioOps = { static size_t procfs_cwd_read(file_t* file, void* buffer, size_t count, size_t* offset) { - process_t* process = file->inode->data; + process_t* process = file->vnode->data; namespace_t* ns = process_get_ns(process); if (ns == NULL) @@ -137,7 +137,7 @@ static size_t procfs_cwd_write(file_t* file, const void* buffer, size_t count, s { UNUSED(offset); - process_t* process = file->inode->data; + process_t* process = file->vnode->data; char cwdStr[MAX_PATH]; if (count >= MAX_PATH) @@ -193,7 +193,7 @@ static file_ops_t cwdOps = { static size_t procfs_cmdline_read(file_t* file, void* buffer, size_t count, size_t* offset) { - process_t* process = file->inode->data; + process_t* process = file->vnode->data; if (process->argv == NULL || process->argc == 0) { @@ -250,7 +250,7 @@ static size_t procfs_note_write(file_t* file, const void* buffer, size_t count, return ERR; } - process_t* process = file->inode->data; + process_t* process = file->vnode->data; RCU_READ_SCOPE(); @@ -289,7 +289,7 @@ static size_t procfs_notegroup_write(file_t* file, const void* buffer, size_t co errno = EINVAL; return ERR; } - process_t* process = file->inode->data; + process_t* process = file->vnode->data; char string[NOTE_MAX] = {0}; memcpy_s(string, NOTE_MAX, buffer, count); @@ -307,7 +307,7 @@ static file_ops_t notegroupOps = { static uint64_t procfs_group_open(file_t* file) { - process_t* process = file->inode->data; + process_t* process = file->vnode->data; group_t* group = group_get(&process->group); if (group == NULL) @@ -338,7 +338,7 @@ static file_ops_t groupOps = { static size_t procfs_pid_read(file_t* file, void* buffer, size_t count, size_t* offset) { - process_t* process = file->inode->data; + process_t* process = file->vnode->data; char pidStr[MAX_NAME]; uint32_t length = snprintf(pidStr, MAX_NAME, "%llu", process->id); @@ -351,7 +351,7 @@ static file_ops_t pidOps = { static size_t procfs_wait_read(file_t* file, void* buffer, size_t count, size_t* offset) { - process_t* process = file->inode->data; + process_t* process = file->vnode->data; if (WAIT_BLOCK(&process->dyingQueue, atomic_load(&process->flags) & PROCESS_DYING) == ERR) { @@ -366,7 +366,7 @@ static size_t procfs_wait_read(file_t* file, void* buffer, size_t count, size_t* static wait_queue_t* procfs_wait_poll(file_t* file, poll_events_t* revents) { - process_t* process = file->inode->data; + process_t* process = file->vnode->data; if (atomic_load(&process->flags) & PROCESS_DYING) { *revents |= POLLIN; @@ -383,7 +383,7 @@ static file_ops_t waitOps = { static size_t procfs_perf_read(file_t* file, void* buffer, size_t count, size_t* offset) { - process_t* process = file->inode->data; + process_t* process = file->vnode->data; size_t userPages = space_user_page_count(&process->space); RCU_READ_SCOPE(); @@ -413,7 +413,7 @@ static file_ops_t perfOps = { static uint64_t procfs_ns_open(file_t* file) { - process_t* process = file->inode->data; + process_t* process = file->vnode->data; namespace_t* ns = process_get_ns(process); if (ns == NULL) @@ -449,7 +449,7 @@ static uint64_t procfs_ctl_close(file_t* file, uint64_t argc, const char** argv) return ERR; } - process_t* process = file->inode->data; + process_t* process = file->vnode->data; if (argc == 2) { @@ -498,7 +498,7 @@ static uint64_t procfs_ctl_dup2(file_t* file, uint64_t argc, const char** argv) return ERR; } - process_t* process = file->inode->data; + process_t* process = file->vnode->data; fd_t oldFd; if (sscanf(argv[1], "%lld", &oldFd) != 1) @@ -530,7 +530,7 @@ static uint64_t procfs_ctl_bind(file_t* file, uint64_t argc, const char** argv) return ERR; } - process_t* process = file->inode->data; + process_t* process = file->vnode->data; process_t* writing = process_current(); pathname_t targetName; @@ -593,7 +593,7 @@ static uint64_t procfs_ctl_mount(file_t* file, uint64_t argc, const char** argv) } process_t* writing = process_current(); - process_t* process = file->inode->data; + process_t* process = file->vnode->data; pathname_t mountname; if (pathname_init(&mountname, argv[1]) == ERR) @@ -640,7 +640,7 @@ static uint64_t procfs_ctl_touch(file_t* file, uint64_t argc, const char** argv) return ERR; } - process_t* process = file->inode->data; + process_t* process = file->vnode->data; pathname_t pathname; if (pathname_init(&pathname, argv[1]) == ERR) @@ -667,7 +667,7 @@ static uint64_t procfs_ctl_start(file_t* file, uint64_t argc, const char** argv) return ERR; } - process_t* process = file->inode->data; + process_t* process = file->vnode->data; atomic_fetch_and(&process->flags, ~PROCESS_SUSPENDED); wait_unblock(&process->suspendQueue, WAIT_ALL, EOK); @@ -679,7 +679,7 @@ static uint64_t procfs_ctl_kill(file_t* file, uint64_t argc, const char** argv) { UNUSED(argv); - process_t* process = file->inode->data; + process_t* process = file->vnode->data; if (argc == 2) { @@ -700,7 +700,7 @@ static uint64_t procfs_ctl_setns(file_t* file, uint64_t argc, const char** argv) return ERR; } - process_t* process = file->inode->data; + process_t* process = file->vnode->data; fd_t fd; if (sscanf(argv[1], "%llu", &fd) != 1) @@ -741,7 +741,7 @@ static uint64_t procfs_ctl_setgroup(file_t* file, uint64_t argc, const char** ar return ERR; } - process_t* process = file->inode->data; + process_t* process = file->vnode->data; fd_t fd; if (sscanf(argv[1], "%llu", &fd) != 1) @@ -790,7 +790,7 @@ CTL_STANDARD_OPS_DEFINE(ctlOps, static size_t procfs_env_read(file_t* file, void* buffer, size_t count, size_t* offset) { - process_t* process = file->inode->data; + process_t* process = file->vnode->data; const char* value = env_get(&process->env, file->path.dentry->name); if (value == NULL) @@ -806,7 +806,7 @@ static size_t procfs_env_write(file_t* file, const void* buffer, size_t count, s { UNUSED(offset); - process_t* process = file->inode->data; + process_t* process = file->vnode->data; char value[MAX_NAME]; if (count >= MAX_NAME) @@ -831,7 +831,7 @@ static file_ops_t envVarOps = { .write = procfs_env_write, }; -static uint64_t procfs_env_lookup(inode_t* dir, dentry_t* target) +static uint64_t procfs_env_lookup(vnode_t* dir, dentry_t* target) { UNUSED(target); @@ -843,20 +843,20 @@ static uint64_t procfs_env_lookup(inode_t* dir, dentry_t* target) return 0; } - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, target->name), INODE_REGULAR, NULL, &envVarOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(dir->superblock, VREG, NULL, &envVarOps); + if (vnode == NULL) { return ERR; } - UNREF_DEFER(inode); - inode->data = process; // No reference + UNREF_DEFER(vnode); + vnode->data = process; // No reference - dentry_make_positive(target, inode); + dentry_make_positive(target, vnode); return 0; } -static uint64_t procfs_env_create(inode_t* dir, dentry_t* target, mode_t mode) +static uint64_t procfs_env_create(vnode_t* dir, dentry_t* target, mode_t mode) { if (mode & MODE_DIRECTORY) { @@ -872,19 +872,19 @@ static uint64_t procfs_env_create(inode_t* dir, dentry_t* target, mode_t mode) return ERR; } - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, target->name), INODE_REGULAR, NULL, &envVarOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(dir->superblock, VREG, NULL, &envVarOps); + if (vnode == NULL) { return ERR; } - UNREF_DEFER(inode); - inode->data = process; // No reference + UNREF_DEFER(vnode); + vnode->data = process; // No reference - dentry_make_positive(target, inode); + dentry_make_positive(target, vnode); return 0; } -static uint64_t procfs_env_remove(inode_t* dir, dentry_t* target) +static uint64_t procfs_env_remove(vnode_t* dir, dentry_t* target) { process_t* process = dir->data; assert(process != NULL); @@ -897,7 +897,7 @@ static uint64_t procfs_env_remove(inode_t* dir, dentry_t* target) return 0; } -static inode_ops_t envInodeOps = { +static vnode_ops_t envVnodeOps = { .lookup = procfs_env_lookup, .create = procfs_env_create, .remove = procfs_env_remove, @@ -910,7 +910,7 @@ static uint64_t procfs_env_iterate(dentry_t* dentry, dir_ctx_t* ctx) return 0; } - process_t* process = dentry->inode->data; + process_t* process = dentry->vnode->data; assert(process != NULL); MUTEX_SCOPE(&process->env.mutex); @@ -922,8 +922,8 @@ static uint64_t procfs_env_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, process->env.vars[i].key, ino_gen(dentry->inode->number, process->env.vars[i].key), - INODE_REGULAR)) + if (!ctx->emit(ctx, process->env.vars[i].key, + VREG)) { return 0; } @@ -937,9 +937,9 @@ static dentry_ops_t envDentryOps = { .revalidate = procfs_revalidate_hide, }; -static uint64_t procfs_self_readlink(inode_t* inode, char* buffer, uint64_t count) +static uint64_t procfs_self_readlink(vnode_t* vnode, char* buffer, uint64_t count) { - UNUSED(inode); + UNUSED(vnode); process_t* process = process_current(); int ret = snprintf(buffer, count, "%llu", process->id); @@ -951,15 +951,15 @@ static uint64_t procfs_self_readlink(inode_t* inode, char* buffer, uint64_t coun return ret; } -static inode_ops_t selfOps = { +static vnode_ops_t selfOps = { .readlink = procfs_self_readlink, }; typedef struct { const char* name; - itype_t type; - const inode_ops_t* inodeOps; + vtype_t type; + const vnode_ops_t* vnodeOps; const file_ops_t* fileOps; const dentry_ops_t* dentryOps; } procfs_entry_t; @@ -967,70 +967,70 @@ typedef struct static const procfs_entry_t pidEntries[] = { { .name = "prio", - .type = INODE_REGULAR, + .type = VREG, .fileOps = &prioOps, .dentryOps = &hideDentryOps, }, { .name = "cwd", - .type = INODE_REGULAR, + .type = VREG, .fileOps = &cwdOps, .dentryOps = &hideDentryOps, }, { .name = "cmdline", - .type = INODE_REGULAR, + .type = VREG, .fileOps = &cmdlineOps, }, { .name = "note", - .type = INODE_REGULAR, + .type = VREG, .fileOps = ¬eOps, .dentryOps = &hideDentryOps, }, { .name = "notegroup", - .type = INODE_REGULAR, + .type = VREG, .fileOps = ¬egroupOps, .dentryOps = &hideDentryOps, }, { .name = "group", - .type = INODE_REGULAR, + .type = VREG, .fileOps = &groupOps, .dentryOps = &hideDentryOps, }, { .name = "pid", - .type = INODE_REGULAR, + .type = VREG, .fileOps = &pidOps, }, { .name = "wait", - .type = INODE_REGULAR, + .type = VREG, .fileOps = &waitOps, }, { .name = "perf", - .type = INODE_REGULAR, + .type = VREG, .fileOps = &perfOps, }, { .name = "ns", - .type = INODE_REGULAR, + .type = VREG, .fileOps = &nsOps, .dentryOps = &hideDentryOps, }, { .name = "ctl", - .type = INODE_REGULAR, + .type = VREG, .fileOps = &ctlOps, .dentryOps = &hideDentryOps, }, { .name = "env", - .type = INODE_DIR, - .inodeOps = &envInodeOps, + .type = VDIR, + .vnodeOps = &envVnodeOps, .dentryOps = &envDentryOps, }, }; @@ -1038,13 +1038,13 @@ static const procfs_entry_t pidEntries[] = { static procfs_entry_t procEntries[] = { { .name = "self", - .type = INODE_SYMLINK, - .inodeOps = &selfOps, + .type = VSYMLINK, + .vnodeOps = &selfOps, .fileOps = NULL, }, }; -static uint64_t procfs_pid_lookup(inode_t* dir, dentry_t* target) +static uint64_t procfs_pid_lookup(vnode_t* dir, dentry_t* target) { process_t* process = dir->data; assert(process != NULL); @@ -1056,37 +1056,37 @@ static uint64_t procfs_pid_lookup(inode_t* dir, dentry_t* target) continue; } - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, pidEntries[i].name), pidEntries[i].type, - pidEntries[i].inodeOps, pidEntries[i].fileOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(dir->superblock, pidEntries[i].type, + pidEntries[i].vnodeOps, pidEntries[i].fileOps); + if (vnode == NULL) { return 0; } - UNREF_DEFER(inode); - inode->data = process; // No reference + UNREF_DEFER(vnode); + vnode->data = process; // No reference if (pidEntries[i].dentryOps != NULL) { target->ops = pidEntries[i].dentryOps; } - dentry_make_positive(target, inode); + dentry_make_positive(target, vnode); return 0; } return 0; } -static void procfs_pid_cleanup(inode_t* inode) +static void procfs_pid_cleanup(vnode_t* vnode) { - process_t* process = inode->data; + process_t* process = vnode->data; if (process == NULL) { return; } UNREF(process); - inode->data = NULL; + vnode->data = NULL; } static uint64_t procfs_pid_iterate(dentry_t* dentry, dir_ctx_t* ctx) @@ -1097,7 +1097,7 @@ static uint64_t procfs_pid_iterate(dentry_t* dentry, dir_ctx_t* ctx) } process_t* current = process_current(); - process_t* process = dentry->inode->data; + process_t* process = dentry->vnode->data; assert(process != NULL); for (size_t i = 0; i < ARRAY_SIZE(pidEntries); i++) @@ -1121,7 +1121,7 @@ static uint64_t procfs_pid_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, pidEntries[i].name, ino_gen(dentry->inode->number, pidEntries[i].name), pidEntries[i].type)) + if (!ctx->emit(ctx, pidEntries[i].name, pidEntries[i].type)) { return 0; } @@ -1130,7 +1130,7 @@ static uint64_t procfs_pid_iterate(dentry_t* dentry, dir_ctx_t* ctx) return 0; } -static inode_ops_t pidInodeOps = { +static vnode_ops_t pidVnodeOps = { .lookup = procfs_pid_lookup, .cleanup = procfs_pid_cleanup, }; @@ -1139,7 +1139,7 @@ static dentry_ops_t pidDentryOps = { .iterate = procfs_pid_iterate, }; -static uint64_t procfs_lookup(inode_t* dir, dentry_t* target) +static uint64_t procfs_lookup(vnode_t* dir, dentry_t* target) { for (size_t i = 0; i < ARRAY_SIZE(procEntries); i++) { @@ -1148,15 +1148,15 @@ static uint64_t procfs_lookup(inode_t* dir, dentry_t* target) continue; } - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, target->name), procEntries[i].type, - procEntries[i].inodeOps, procEntries[i].fileOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(dir->superblock, procEntries[i].type, + procEntries[i].vnodeOps, procEntries[i].fileOps); + if (vnode == NULL) { return ERR; } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); - dentry_make_positive(target, inode); + dentry_make_positive(target, vnode); return 0; } @@ -1173,17 +1173,17 @@ static uint64_t procfs_lookup(inode_t* dir, dentry_t* target) } UNREF_DEFER(process); - inode_t* inode = inode_new(dir->superblock, ino_gen(dir->number, target->name), INODE_DIR, &pidInodeOps, NULL); - if (inode == NULL) + vnode_t* vnode = vnode_new(dir->superblock, VDIR, &pidVnodeOps, NULL); + if (vnode == NULL) { return ERR; } - UNREF_DEFER(inode); - inode->data = REF(process); + UNREF_DEFER(vnode); + vnode->data = REF(process); target->ops = &pidDentryOps; - dentry_make_positive(target, inode); + dentry_make_positive(target, vnode); return 0; } @@ -1201,7 +1201,7 @@ static uint64_t procfs_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, procEntries[i].name, ino_gen(dentry->inode->number, procEntries[i].name), + if (!ctx->emit(ctx, procEntries[i].name, procEntries[i].type)) { return 0; @@ -1220,7 +1220,7 @@ static uint64_t procfs_iterate(dentry_t* dentry, dir_ctx_t* ctx) char name[MAX_NAME]; snprintf(name, sizeof(name), "%llu", process->id); - if (!ctx->emit(ctx, name, ino_gen(dentry->inode->number, name), INODE_DIR)) + if (!ctx->emit(ctx, name, VDIR)) { return 0; } @@ -1229,7 +1229,7 @@ static uint64_t procfs_iterate(dentry_t* dentry, dir_ctx_t* ctx) return 0; } -static inode_ops_t procInodeOps = { +static vnode_ops_t procVnodeOps = { .lookup = procfs_lookup, }; @@ -1254,12 +1254,12 @@ static dentry_t* procfs_mount(filesystem_t* fs, const char* options, void* data) } UNREF_DEFER(superblock); - inode_t* inode = inode_new(superblock, 0, INODE_DIR, &procInodeOps, NULL); - if (inode == NULL) + vnode_t* vnode = vnode_new(superblock, VDIR, &procVnodeOps, NULL); + if (vnode == NULL) { return NULL; } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); dentry_t* dentry = dentry_new(superblock, NULL, NULL); if (dentry == NULL) @@ -1268,7 +1268,7 @@ static dentry_t* procfs_mount(filesystem_t* fs, const char* options, void* data) } dentry->ops = &procDentryOps; - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); superblock->root = dentry; return superblock->root; diff --git a/src/kernel/fs/sysfs.c b/src/kernel/fs/sysfs.c index b06a881b2..b7d1a72fc 100644 --- a/src/kernel/fs/sysfs.c +++ b/src/kernel/fs/sysfs.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -24,10 +24,6 @@ static dentry_t* root = NULL; -static file_ops_t dirOps = { - .seek = file_generic_seek, -}; - static dentry_ops_t dentryOps = { .iterate = dentry_generic_iterate, }; @@ -65,12 +61,12 @@ void sysfs_init(void) } UNREF_DEFER(superblock); - inode_t* inode = inode_new(superblock, vfs_id_get(), INODE_DIR, NULL, &dirOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(superblock, VDIR, NULL, NULL); + if (vnode == NULL) { - panic(NULL, "Failed to create sysfs root inode"); + panic(NULL, "Failed to create sysfs root vnode"); } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); dentry_t* dentry = dentry_new(superblock, NULL, NULL); if (dentry == NULL) @@ -78,7 +74,7 @@ void sysfs_init(void) panic(NULL, "Failed to create sysfs root dentry"); } - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); superblock->root = dentry; root = dentry; @@ -115,7 +111,7 @@ void sysfs_init(void) LOG_INFO("sysfs mounted to '/sys'\n"); } -dentry_t* sysfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) +dentry_t* sysfs_dir_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, void* data) { if (name == NULL) { @@ -141,20 +137,20 @@ dentry_t* sysfs_dir_new(dentry_t* parent, const char* name, const inode_ops_t* i } UNREF_DEFER(dir); - inode_t* inode = inode_new(parent->superblock, vfs_id_get(), INODE_DIR, inodeOps, &dirOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(parent->superblock, VDIR, vnodeOps, NULL); + if (vnode == NULL) { return NULL; } - UNREF_DEFER(inode); - inode->data = data; + UNREF_DEFER(vnode); + vnode->data = data; - dentry_make_positive(dir, inode); + dentry_make_positive(dir, vnode); return REF(dir); } -dentry_t* sysfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, const file_ops_t* fileOps, +dentry_t* sysfs_file_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, const file_ops_t* fileOps, void* data) { if (name == NULL) @@ -181,22 +177,22 @@ dentry_t* sysfs_file_new(dentry_t* parent, const char* name, const inode_ops_t* } UNREF_DEFER(dentry); - inode_t* inode = inode_new(parent->superblock, vfs_id_get(), INODE_REGULAR, inodeOps, fileOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(parent->superblock, VREG, vnodeOps, fileOps); + if (vnode == NULL) { return NULL; } - UNREF_DEFER(inode); - inode->data = data; + UNREF_DEFER(vnode); + vnode->data = data; - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); return REF(dentry); } -dentry_t* sysfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_t* inodeOps, void* data) +dentry_t* sysfs_symlink_new(dentry_t* parent, const char* name, const vnode_ops_t* vnodeOps, void* data) { - if (parent == NULL || name == NULL || inodeOps == NULL) + if (parent == NULL || name == NULL || vnodeOps == NULL) { errno = EINVAL; return NULL; @@ -215,15 +211,15 @@ dentry_t* sysfs_symlink_new(dentry_t* parent, const char* name, const inode_ops_ } UNREF_DEFER(dentry); - inode_t* inode = inode_new(parent->superblock, vfs_id_get(), INODE_SYMLINK, inodeOps, NULL); - if (inode == NULL) + vnode_t* vnode = vnode_new(parent->superblock, VSYMLINK, vnodeOps, NULL); + if (vnode == NULL) { return NULL; } - UNREF_DEFER(inode); - inode->data = data; + UNREF_DEFER(vnode); + vnode->data = data; - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); return REF(dentry); } @@ -252,7 +248,7 @@ uint64_t sysfs_files_new(list_t* out, dentry_t* parent, const sysfs_file_desc_t* uint64_t count = 0; for (const sysfs_file_desc_t* desc = descs; desc->name != NULL; desc++) { - dentry_t* file = sysfs_file_new(parent, desc->name, desc->inodeOps, desc->fileOps, desc->data); + dentry_t* file = sysfs_file_new(parent, desc->name, desc->vnodeOps, desc->fileOps, desc->data); if (file == NULL) { while (!list_is_empty(&createdList)) diff --git a/src/kernel/fs/tmpfs.c b/src/kernel/fs/tmpfs.c index bbfc7e06f..e0a13e4f6 100644 --- a/src/kernel/fs/tmpfs.c +++ b/src/kernel/fs/tmpfs.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -27,7 +27,7 @@ static bool initialized = false; -static inode_t* tmpfs_inode_new(superblock_t* superblock, itype_t type, void* buffer, uint64_t size); +static vnode_t* tmpfs_vnode_new(superblock_t* superblock, vtype_t type, void* buffer, uint64_t size); static void tmpfs_dentry_add(dentry_t* dentry) { @@ -53,34 +53,34 @@ static void tmpfs_dentry_remove(dentry_t* dentry) static size_t tmpfs_read(file_t* file, void* buffer, size_t count, size_t* offset) { - MUTEX_SCOPE(&file->inode->mutex); + MUTEX_SCOPE(&file->vnode->mutex); - if (file->inode->data == NULL) + if (file->vnode->data == NULL) { return 0; } - return BUFFER_READ(buffer, count, offset, file->inode->data, file->inode->size); + return BUFFER_READ(buffer, count, offset, file->vnode->data, file->vnode->size); } static size_t tmpfs_write(file_t* file, const void* buffer, size_t count, size_t* offset) { - MUTEX_SCOPE(&file->inode->mutex); + MUTEX_SCOPE(&file->vnode->mutex); size_t requiredSize = *offset + count; - if (requiredSize > file->inode->size) + if (requiredSize > file->vnode->size) { - void* newData = realloc(file->inode->data, requiredSize); + void* newData = realloc(file->vnode->data, requiredSize); if (newData == NULL) { return ERR; } - memset(newData + file->inode->size, 0, requiredSize - file->inode->size); - file->inode->data = newData; - file->inode->size = requiredSize; + memset(newData + file->vnode->size, 0, requiredSize - file->vnode->size); + file->vnode->data = newData; + file->vnode->size = requiredSize; } - return BUFFER_WRITE(buffer, count, offset, file->inode->data, file->inode->size); + return BUFFER_WRITE(buffer, count, offset, file->vnode->data, file->vnode->size); } static file_ops_t fileOps = { @@ -89,86 +89,86 @@ static file_ops_t fileOps = { .seek = file_generic_seek, }; -static uint64_t tmpfs_create(inode_t* dir, dentry_t* target, mode_t mode) +static uint64_t tmpfs_create(vnode_t* dir, dentry_t* target, mode_t mode) { MUTEX_SCOPE(&dir->mutex); - inode_t* inode = tmpfs_inode_new(dir->superblock, mode & MODE_DIRECTORY ? INODE_DIR : INODE_REGULAR, NULL, 0); - if (inode == NULL) + vnode_t* vnode = tmpfs_vnode_new(dir->superblock, mode & MODE_DIRECTORY ? VDIR : VREG, NULL, 0); + if (vnode == NULL) { return ERR; } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); - dentry_make_positive(target, inode); + dentry_make_positive(target, vnode); tmpfs_dentry_add(target); return 0; } -static void tmpfs_truncate(inode_t* inode) +static void tmpfs_truncate(vnode_t* vnode) { - MUTEX_SCOPE(&inode->mutex); + MUTEX_SCOPE(&vnode->mutex); - if (inode->data != NULL) + if (vnode->data != NULL) { - free(inode->data); - inode->data = NULL; + free(vnode->data); + vnode->data = NULL; } - inode->size = 0; + vnode->size = 0; } -static uint64_t tmpfs_link(inode_t* dir, dentry_t* old, dentry_t* target) +static uint64_t tmpfs_link(vnode_t* dir, dentry_t* old, dentry_t* target) { MUTEX_SCOPE(&dir->mutex); - dentry_make_positive(target, old->inode); + dentry_make_positive(target, old->vnode); tmpfs_dentry_add(target); return 0; } -static uint64_t tmpfs_readlink(inode_t* inode, char* buffer, uint64_t count) +static uint64_t tmpfs_readlink(vnode_t* vnode, char* buffer, uint64_t count) { - MUTEX_SCOPE(&inode->mutex); + MUTEX_SCOPE(&vnode->mutex); - if (inode->data == NULL) + if (vnode->data == NULL) { errno = EINVAL; return ERR; } - uint64_t copySize = MIN(count, inode->size); - memcpy(buffer, inode->data, copySize); + uint64_t copySize = MIN(count, vnode->size); + memcpy(buffer, vnode->data, copySize); return copySize; } -static uint64_t tmpfs_symlink(inode_t* dir, dentry_t* target, const char* dest) +static uint64_t tmpfs_symlink(vnode_t* dir, dentry_t* target, const char* dest) { MUTEX_SCOPE(&dir->mutex); - inode_t* inode = tmpfs_inode_new(dir->superblock, INODE_SYMLINK, (void*)dest, strlen(dest)); - if (inode == NULL) + vnode_t* vnode = tmpfs_vnode_new(dir->superblock, VSYMLINK, (void*)dest, strlen(dest)); + if (vnode == NULL) { return ERR; } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); - dentry_make_positive(target, inode); + dentry_make_positive(target, vnode); tmpfs_dentry_add(target); return 0; } -static uint64_t tmpfs_remove(inode_t* dir, dentry_t* target) +static uint64_t tmpfs_remove(vnode_t* dir, dentry_t* target) { MUTEX_SCOPE(&dir->mutex); - if (target->inode->type == INODE_REGULAR || target->inode->type == INODE_SYMLINK) + if (target->vnode->type == VREG || target->vnode->type == VSYMLINK) { tmpfs_dentry_remove(target); } - else if (target->inode->type == INODE_DIR) + else if (target->vnode->type == VDIR) { if (!list_is_empty(&target->children)) { @@ -182,24 +182,24 @@ static uint64_t tmpfs_remove(inode_t* dir, dentry_t* target) return 0; } -static void tmpfs_inode_cleanup(inode_t* inode) +static void tmpfs_vnode_cleanup(vnode_t* vnode) { - if (inode->data != NULL) + if (vnode->data != NULL) { - free(inode->data); - inode->data = NULL; - inode->size = 0; + free(vnode->data); + vnode->data = NULL; + vnode->size = 0; } } -static inode_ops_t inodeOps = { +static vnode_ops_t vnodeOps = { .create = tmpfs_create, .truncate = tmpfs_truncate, .link = tmpfs_link, .readlink = tmpfs_readlink, .symlink = tmpfs_symlink, .remove = tmpfs_remove, - .cleanup = tmpfs_inode_cleanup, + .cleanup = tmpfs_vnode_cleanup, }; static dentry_ops_t dentryOps = { @@ -228,14 +228,14 @@ static dentry_t* tmpfs_load_file(superblock_t* superblock, dentry_t* parent, con tmpfs_dentry_add(dentry); - inode_t* inode = tmpfs_inode_new(superblock, INODE_REGULAR, in->data, in->size); - if (inode == NULL) + vnode_t* vnode = tmpfs_vnode_new(superblock, VREG, in->data, in->size); + if (vnode == NULL) { - panic(NULL, "Failed to create tmpfs file inode"); + panic(NULL, "Failed to create tmpfs file vnode"); } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); return REF(dentry); } @@ -251,15 +251,15 @@ static dentry_t* tmpfs_load_dir(superblock_t* superblock, dentry_t* parent, cons } UNREF_DEFER(dentry); - inode_t* inode = tmpfs_inode_new(superblock, INODE_DIR, NULL, 0); - if (inode == NULL) + vnode_t* vnode = tmpfs_vnode_new(superblock, VDIR, NULL, 0); + if (vnode == NULL) { - panic(NULL, "Failed to create tmpfs inode"); + panic(NULL, "Failed to create tmpfs vnode"); } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); tmpfs_dentry_add(dentry); - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); boot_file_t* file; LIST_FOR_EACH(file, &in->files, entry) @@ -327,48 +327,46 @@ static dentry_t* tmpfs_mount(filesystem_t* fs, const char* options, void* data) } UNREF_DEFER(dentry); - inode_t* inode = tmpfs_inode_new(superblock, INODE_DIR, NULL, 0); - if (inode == NULL) + vnode_t* vnode = tmpfs_vnode_new(superblock, VDIR, NULL, 0); + if (vnode == NULL) { return NULL; } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); tmpfs_dentry_add(dentry); - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); superblock->root = dentry; return REF(superblock->root); } -static inode_t* tmpfs_inode_new(superblock_t* superblock, itype_t type, void* buffer, uint64_t size) +static vnode_t* tmpfs_vnode_new(superblock_t* superblock, vtype_t type, void* buffer, uint64_t size) { - inode_t* inode = inode_new(superblock, vfs_id_get(), type, &inodeOps, &fileOps); - if (inode == NULL) + vnode_t* vnode = vnode_new(superblock, type, &vnodeOps, &fileOps); + if (vnode == NULL) { return NULL; } - UNREF_DEFER(inode); - - inode->blocks = 0; + UNREF_DEFER(vnode); if (buffer != NULL) { - inode->data = malloc(size); - if (inode->data == NULL) + vnode->data = malloc(size); + if (vnode->data == NULL) { return NULL; } - memcpy(inode->data, buffer, size); - inode->size = size; + memcpy(vnode->data, buffer, size); + vnode->size = size; } else { - inode->data = NULL; - inode->size = 0; + vnode->data = NULL; + vnode->size = 0; } - return REF(inode); + return REF(vnode); } static filesystem_t tmpfs = { diff --git a/src/kernel/fs/vfs.c b/src/kernel/fs/vfs.c index 8529618a2..38c47bc65 100644 --- a/src/kernel/fs/vfs.c +++ b/src/kernel/fs/vfs.c @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include @@ -101,7 +101,7 @@ static uint64_t vfs_create(path_t* path, const pathname_t* pathname, namespace_t return ERR; } - inode_t* dir = parent.dentry->inode; + vnode_t* dir = parent.dentry->vnode; if (dir->ops == NULL || dir->ops->create == NULL) { errno = EPERM; @@ -200,9 +200,9 @@ uint64_t vfs_open2(const pathname_t* pathname, file_t* files[2], process_t* proc return ERR; } - if (pathname->mode & MODE_TRUNCATE && files[0]->inode->type == INODE_REGULAR) + if (pathname->mode & MODE_TRUNCATE && files[0]->vnode->type == VREG) { - inode_truncate(files[0]->inode); + vnode_truncate(files[0]->vnode); } if (files[0]->ops != NULL && files[0]->ops->open2 != NULL) @@ -217,7 +217,6 @@ uint64_t vfs_open2(const pathname_t* pathname, file_t* files[2], process_t* proc } } - inode_notify_access(files[0]->inode); return 0; } @@ -258,9 +257,9 @@ file_t* vfs_openat(const path_t* from, const pathname_t* pathname, process_t* pr return NULL; } - if (pathname->mode & MODE_TRUNCATE && file->inode->type == INODE_REGULAR) + if (pathname->mode & MODE_TRUNCATE && file->vnode->type == VREG) { - inode_truncate(file->inode); + vnode_truncate(file->vnode); } if (file->ops != NULL && file->ops->open != NULL) @@ -274,7 +273,6 @@ file_t* vfs_openat(const path_t* from, const pathname_t* pathname, process_t* pr } } - inode_notify_access(file->inode); return file; } @@ -286,7 +284,7 @@ size_t vfs_read(file_t* file, void* buffer, size_t count) return ERR; } - if (file->inode->type == INODE_DIR) + if (file->vnode->type == VDIR) { errno = EISDIR; return ERR; @@ -309,11 +307,6 @@ size_t vfs_read(file_t* file, void* buffer, size_t count) size_t result = file->ops->read(file, buffer, count, &offset); file->pos = offset; - if (result != ERR) - { - inode_notify_access(file->inode); - } - return result; } @@ -325,7 +318,7 @@ size_t vfs_write(file_t* file, const void* buffer, size_t count) return ERR; } - if (file->inode->type == INODE_DIR) + if (file->vnode->type == VDIR) { errno = EISDIR; return ERR; @@ -356,11 +349,6 @@ size_t vfs_write(file_t* file, const void* buffer, size_t count) size_t result = file->ops->write(file, buffer, count, &offset); file->pos = offset; - if (result != ERR) - { - inode_notify_modify(file->inode); - } - return result; } @@ -390,7 +378,7 @@ uint64_t vfs_ioctl(file_t* file, uint64_t request, void* argp, size_t size) return ERR; } - if (file->inode->type == INODE_DIR) + if (file->vnode->type == VDIR) { errno = EISDIR; return ERR; @@ -403,12 +391,7 @@ uint64_t vfs_ioctl(file_t* file, uint64_t request, void* argp, size_t size) } assert(rflags_read() & RFLAGS_INTERRUPT_ENABLE); - uint64_t result = file->ops->ioctl(file, request, argp, size); - if (result != ERR) - { - inode_notify_access(file->inode); - } - return result; + return file->ops->ioctl(file, request, argp, size); } void* vfs_mmap(file_t* file, void* address, size_t length, pml_flags_t flags) @@ -419,7 +402,7 @@ void* vfs_mmap(file_t* file, void* address, size_t length, pml_flags_t flags) return NULL; } - if (file->inode->type == INODE_DIR) + if (file->vnode->type == VDIR) { errno = EISDIR; return NULL; @@ -436,7 +419,6 @@ void* vfs_mmap(file_t* file, void* address, size_t length, pml_flags_t flags) void* result = file->ops->mmap(file, address, length, &offset, flags); if (result != NULL) { - inode_notify_access(file->inode); file->pos = offset; } return result; @@ -534,7 +516,7 @@ uint64_t vfs_poll(poll_file_t* files, uint64_t amount, clock_t timeout) return ERR; } - if (files[i].file->inode->type == INODE_DIR) + if (files[i].file->vnode->type == VDIR) { errno = EISDIR; return ERR; @@ -612,7 +594,7 @@ typedef struct namespace_t* ns; } vfs_dir_ctx_t; -static bool vfs_dir_emit(dir_ctx_t* ctx, const char* name, ino_t number, itype_t type) +static bool vfs_dir_emit(dir_ctx_t* ctx, const char* name, vtype_t type) { vfs_dir_ctx_t* vctx = (vfs_dir_ctx_t*)ctx; if (vctx->written + sizeof(dirent_t) > vctx->count) @@ -631,8 +613,7 @@ static bool vfs_dir_emit(dir_ctx_t* ctx, const char* name, ino_t number, itype_t dentry_t* dentry = child; if (namespace_rcu_traverse(vctx->ns, &mount, &dentry)) { - number = dentry->inode->number; - type = dentry->inode->type; + type = dentry->vnode->type; mode = mount->mode; flags |= DIRENT_MOUNTED; } @@ -641,7 +622,6 @@ static bool vfs_dir_emit(dir_ctx_t* ctx, const char* name, ino_t number, itype_t rcu_read_unlock(); dirent_t* d = (dirent_t*)((uint8_t*)vctx->buffer + vctx->written); - d->number = number; d->type = type; d->flags = flags; strncpy(d->path, name, MAX_PATH - 1); @@ -718,7 +698,7 @@ static uint64_t vfs_getdents_recursive_step(path_t* path, mode_t mode, getdents_ } ctx->currentOffset += sizeof(dirent_t); - if ((d->type == INODE_DIR || d->type == INODE_SYMLINK) && strcmp(d->path, ".") != 0 && + if ((d->type == VDIR || d->type == VSYMLINK) && strcmp(d->path, ".") != 0 && strcmp(d->path, "..") != 0) { path_t childPath = PATH_CREATE(path->mount, path->dentry); @@ -769,12 +749,11 @@ static uint64_t vfs_remove_recursive(path_t* path, process_t* process) if (!DENTRY_IS_DIR(path->dentry)) { - inode_t* dir = path->dentry->parent->inode; + vnode_t* dir = path->dentry->parent->vnode; if (dir->ops->remove(dir, path->dentry) == ERR) { return ERR; } - inode_notify_modify(dir); return 0; } @@ -847,7 +826,7 @@ static uint64_t vfs_remove_recursive(path_t* path, process_t* process) free(buf); - inode_t* dir = path->dentry->parent->inode; + vnode_t* dir = path->dentry->parent->vnode; if (dir->ops == NULL || dir->ops->remove == NULL) { errno = EPERM; @@ -861,7 +840,6 @@ static uint64_t vfs_remove_recursive(path_t* path, process_t* process) return ERR; } - inode_notify_modify(dir); return 0; } @@ -873,7 +851,7 @@ size_t vfs_getdents(file_t* file, dirent_t* buffer, size_t count) return ERR; } - if (file->inode == NULL || file->inode->type != INODE_DIR) + if (file->vnode == NULL || file->vnode->type != VDIR) { errno = ENOTDIR; return ERR; @@ -907,7 +885,7 @@ size_t vfs_getdents(file_t* file, dirent_t* buffer, size_t count) } UNREF_DEFER(ns); - MUTEX_SCOPE(&file->inode->mutex); + MUTEX_SCOPE(&file->vnode->mutex); if (file->mode & MODE_RECURSIVE) { @@ -940,7 +918,6 @@ size_t vfs_getdents(file_t* file, dirent_t* buffer, size_t count) if (result != ERR) { - inode_notify_access(file->inode); return ctx.written; } return result; @@ -983,17 +960,18 @@ uint64_t vfs_stat(const pathname_t* pathname, stat_t* buffer, process_t* process return ERR; } - inode_t* inode = path.dentry->inode; - mutex_acquire(&inode->mutex); - buffer->number = inode->number; - buffer->type = inode->type; - buffer->size = inode->size; - buffer->blocks = inode->blocks; - buffer->linkAmount = atomic_load(&inode->dentryCount); - buffer->accessTime = inode->accessTime; - buffer->modifyTime = inode->modifyTime; - buffer->changeTime = inode->changeTime; - buffer->createTime = inode->createTime; + /// @todo Reimplement this after the async system. + vnode_t* vnode = path.dentry->vnode; + mutex_acquire(&vnode->mutex); + buffer->number = 0; + buffer->type = vnode->type; + buffer->size = vnode->size; + buffer->blocks = 0; + buffer->linkAmount = atomic_load(&vnode->dentryCount); + buffer->accessTime = 0; + buffer->modifyTime = 0; + buffer->changeTime = 0; + buffer->createTime = 0; char mode[MAX_PATH]; if (mode_to_string(path.mount->mode, mode, MAX_PATH) == ERR) @@ -1007,7 +985,7 @@ uint64_t vfs_stat(const pathname_t* pathname, stat_t* buffer, process_t* process return ERR; } - mutex_release(&inode->mutex); + mutex_release(&vnode->mutex); return 0; } @@ -1073,7 +1051,7 @@ uint64_t vfs_link(const pathname_t* oldPathname, const pathname_t* newPathname, return ERR; } - if (newParent.dentry->inode->ops == NULL || newParent.dentry->inode->ops->link == NULL) + if (newParent.dentry->vnode->ops == NULL || newParent.dentry->vnode->ops->link == NULL) { errno = EPERM; return ERR; @@ -1092,18 +1070,15 @@ uint64_t vfs_link(const pathname_t* oldPathname, const pathname_t* newPathname, } assert(rflags_read() & RFLAGS_INTERRUPT_ENABLE); - if (newParent.dentry->inode->ops->link(newParent.dentry->inode, old.dentry, new.dentry) == ERR) + if (newParent.dentry->vnode->ops->link(newParent.dentry->vnode, old.dentry, new.dentry) == ERR) { return ERR; } - inode_notify_modify(newParent.dentry->inode); - inode_notify_change(old.dentry->inode); - return 0; } -size_t vfs_readlink(inode_t* symlink, char* buffer, size_t count) +size_t vfs_readlink(vnode_t* symlink, char* buffer, size_t count) { if (symlink == NULL || buffer == NULL || count == 0) { @@ -1111,7 +1086,7 @@ size_t vfs_readlink(inode_t* symlink, char* buffer, size_t count) return ERR; } - if (symlink->type != INODE_SYMLINK) + if (symlink->type != VSYMLINK) { errno = EINVAL; return ERR; @@ -1124,12 +1099,7 @@ size_t vfs_readlink(inode_t* symlink, char* buffer, size_t count) } assert(rflags_read() & RFLAGS_INTERRUPT_ENABLE); - size_t result = symlink->ops->readlink(symlink, buffer, count); - if (result != ERR) - { - inode_notify_access(symlink); - } - return result; + return symlink->ops->readlink(symlink, buffer, count); } uint64_t vfs_symlink(const pathname_t* oldPathname, const pathname_t* newPathname, process_t* process) @@ -1172,7 +1142,7 @@ uint64_t vfs_symlink(const pathname_t* oldPathname, const pathname_t* newPathnam return ERR; } - if (newParent.dentry->inode->ops == NULL || newParent.dentry->inode->ops->symlink == NULL) + if (newParent.dentry->vnode->ops == NULL || newParent.dentry->vnode->ops->symlink == NULL) { errno = EPERM; return ERR; @@ -1185,13 +1155,7 @@ uint64_t vfs_symlink(const pathname_t* oldPathname, const pathname_t* newPathnam } assert(rflags_read() & RFLAGS_INTERRUPT_ENABLE); - if (newParent.dentry->inode->ops->symlink(newParent.dentry->inode, new.dentry, oldPathname->string) == ERR) - { - return ERR; - } - - inode_notify_modify(newParent.dentry->inode); - return 0; + return newParent.dentry->vnode->ops->symlink(newParent.dentry->vnode, new.dentry, oldPathname->string); } uint64_t vfs_remove(const pathname_t* pathname, process_t* process) @@ -1258,7 +1222,7 @@ uint64_t vfs_remove(const pathname_t* pathname, process_t* process) return vfs_remove_recursive(&target, process); } - inode_t* dir = parent.dentry->inode; + vnode_t* dir = parent.dentry->vnode; if (dir->ops == NULL || dir->ops->remove == NULL) { errno = EPERM; @@ -1267,13 +1231,7 @@ uint64_t vfs_remove(const pathname_t* pathname, process_t* process) assert(rflags_read() & RFLAGS_INTERRUPT_ENABLE); - if (dir->ops->remove(dir, target.dentry) == ERR) - { - return ERR; - } - - inode_notify_modify(dir); - return 0; + return dir->ops->remove(dir, target.dentry); } uint64_t vfs_id_get(void) @@ -1651,7 +1609,7 @@ SYSCALL_DEFINE(SYS_READLINK, uint64_t, const char* pathString, char* buffer, uin { return ERR; } - uint64_t result = vfs_readlink(path.dentry->inode, buffer, count); + uint64_t result = vfs_readlink(path.dentry->vnode, buffer, count); space_unpin(&process->space, buffer, count); return result; } diff --git a/src/kernel/fs/vnode.c b/src/kernel/fs/vnode.c new file mode 100644 index 000000000..e702fcd68 --- /dev/null +++ b/src/kernel/fs/vnode.c @@ -0,0 +1,89 @@ +#include + +#include +#include +#include +#include +#include + +#include + +static void vnode_free(vnode_t* vnode) +{ + if (vnode == NULL) + { + return; + } + + if (vnode->ops != NULL && vnode->ops->cleanup != NULL) + { + vnode->ops->cleanup(vnode); + } + vnode->data = NULL; + + if (vnode->superblock != NULL) + { + UNREF(vnode->superblock); + vnode->superblock = NULL; + } + + rcu_call(&vnode->rcu, rcu_call_cache_free, vnode); +} + +static void vnode_ctor(void* ptr) +{ + vnode_t* vnode = (vnode_t*)ptr; + + vnode->ref = (ref_t){0}; + vnode->type = 0; + atomic_init(&vnode->dentryCount, 0); + vnode->data = NULL; + vnode->size = 0; + vnode->superblock = NULL; + vnode->ops = NULL; + vnode->fileOps = NULL; + vnode->rcu = (rcu_entry_t){0}; + mutex_init(&vnode->mutex); +} + +static cache_t cache = CACHE_CREATE(cache, "vnode", sizeof(vnode_t), CACHE_LINE, vnode_ctor, NULL); + +vnode_t* vnode_new(superblock_t* superblock, vtype_t type, const vnode_ops_t* ops, + const file_ops_t* fileOps) +{ + if (superblock == NULL) + { + errno = EINVAL; + return NULL; + } + + vnode_t* vnode = cache_alloc(&cache); + if (vnode == NULL) + { + errno = ENOMEM; + return NULL; + } + + ref_init(&vnode->ref, vnode_free); + vnode->type = type; + vnode->superblock = REF(superblock); + vnode->ops = ops; + vnode->fileOps = fileOps; + vnode->verbs = superblock->verbs; + return vnode; +} + +void vnode_truncate(vnode_t* vnode) +{ + if (vnode == NULL) + { + return; + } + + if (vnode->ops != NULL && vnode->ops->truncate != NULL) + { + MUTEX_SCOPE(&vnode->mutex); + assert(rflags_read() & RFLAGS_INTERRUPT_ENABLE); + vnode->ops->truncate(vnode); + } +} \ No newline at end of file diff --git a/src/kernel/io/verb.c b/src/kernel/io/verb.c index baf14ab74..d2530d95a 100644 --- a/src/kernel/io/verb.c +++ b/src/kernel/io/verb.c @@ -66,17 +66,30 @@ static void verb_dispatch_file(irp_t* irp) file_t* file = irp->file; assert(file != NULL); + switch (irp->verb) + { + case VERB_READ: + if (!(file->mode & MODE_READ)) + { + irp_error(irp, EBADF); + return; + } + break; + default: + break; + } + if (verb_invoke(irp, file->verbs)) { return; } - if (verb_invoke(irp, file->inode->verbs)) + if (verb_invoke(irp, file->vnode->verbs)) { return; } - if (verb_invoke(irp, file->inode->superblock->verbs)) + if (verb_invoke(irp, file->vnode->superblock->verbs)) { return; } diff --git a/src/kernel/module/module.c b/src/kernel/module/module.c index 60e5fc870..af0795b72 100644 --- a/src/kernel/module/module.c +++ b/src/kernel/module/module.c @@ -624,7 +624,7 @@ static uint64_t module_cache_build(void) for (uint64_t i = 0; i < readCount / sizeof(dirent_t); i++) { - if (buffer[i].path[0] == '.' || buffer[i].type != INODE_REGULAR) + if (buffer[i].path[0] == '.' || buffer[i].type != VREG) { continue; } diff --git a/src/kernel/proc/group.c b/src/kernel/proc/group.c index d9e791640..4e1812efe 100644 --- a/src/kernel/proc/group.c +++ b/src/kernel/proc/group.c @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include diff --git a/src/modules/acpi/tables.c b/src/modules/acpi/tables.c index a0c1327d4..4bccebc1b 100644 --- a/src/modules/acpi/tables.c +++ b/src/modules/acpi/tables.c @@ -29,7 +29,7 @@ static uint64_t acpi_table_read(file_t* file, void* buffer, size_t count, size_t return ERR; } - sdt_header_t* table = file->inode->data; + sdt_header_t* table = file->vnode->data; if (table == NULL) { errno = EINVAL; diff --git a/src/modules/fs/9p/9p.c b/src/modules/fs/9p/9p.c index f9b26ff5c..b02116d3c 100644 --- a/src/modules/fs/9p/9p.c +++ b/src/modules/fs/9p/9p.c @@ -130,12 +130,12 @@ static dentry_t* ninep_mount(filesystem_t* fs, const char* options, void* data) superblock->data = ninep; - inode_t* inode = inode_new(superblock, 0, INODE_DIR, NULL, NULL); - if (inode == NULL) + vnode_t* vnode = vnode_new(superblock, VDIR, NULL, NULL); + if (vnode == NULL) { return NULL; } - UNREF_DEFER(inode); + UNREF_DEFER(vnode); dentry_t* dentry = dentry_new(superblock, NULL, NULL); if (dentry == NULL) @@ -143,7 +143,7 @@ static dentry_t* ninep_mount(filesystem_t* fs, const char* options, void* data) return NULL; } - dentry_make_positive(dentry, inode); + dentry_make_positive(dentry, vnode); superblock->root = dentry; return superblock->root; diff --git a/src/programs/core/init/main.c b/src/programs/core/init/main.c index 6034b006d..0f1ac8bff 100644 --- a/src/programs/core/init/main.c +++ b/src/programs/core/init/main.c @@ -149,7 +149,7 @@ static void init_create_pkg_links(void) for (uint64_t i = 0; i < amount; i++) { - if (dirents[i].type != INODE_DIR || dirents[i].path[0] == '.') + if (dirents[i].type != VDIR || dirents[i].path[0] == '.') { continue; } diff --git a/src/programs/core/shell/pipeline.c b/src/programs/core/shell/pipeline.c index 190ef8539..5328e256d 100644 --- a/src/programs/core/shell/pipeline.c +++ b/src/programs/core/shell/pipeline.c @@ -341,7 +341,7 @@ static pid_t pipeline_execute_cmd(cmd_t* cmd) else if (strchr(argv[0], '/') != NULL) { stat_t info; - if (stat(argv[0], &info) != ERR && info.type != INODE_DIR) + if (stat(argv[0], &info) != ERR && info.type != VDIR) { result = spawn(argv, SPAWN_STDIO_FDS); } @@ -369,7 +369,7 @@ static pid_t pipeline_execute_cmd(cmd_t* cmd) if (snprintf(path, MAX_PATH, "%s/%s", token, argv[0]) < MAX_PATH) { stat_t info; - if (stat(path, &info) != ERR && info.type != INODE_DIR) + if (stat(path, &info) != ERR && info.type != VDIR) { const char* newArgv[argc + 1]; newArgv[0] = path; diff --git a/src/programs/utils/ls/main.c b/src/programs/utils/ls/main.c index 0f0571023..2b4792a66 100644 --- a/src/programs/utils/ls/main.c +++ b/src/programs/utils/ls/main.c @@ -131,7 +131,7 @@ static uint64_t print_dir(const char* path) for (uint64_t i = 0; i < count; i++) { uint64_t len = strlen(entries[i].path); - if (entries[i].type == INODE_DIR || entries[i].type == INODE_SYMLINK) + if (entries[i].type == VDIR || entries[i].type == VSYMLINK) { len++; } @@ -175,12 +175,12 @@ static uint64_t print_dir(const char* path) int len = strlen(name); const char* modifier = (ent->flags & DIRENT_MOUNTED) ? "\033[4m" : ""; - if (ent->type == INODE_DIR) + if (ent->type == VDIR) { printf("%s\033[34m%s%s\033[0m/", modifier, name, showFlags ? ent->mode : ""); len++; } - else if (ent->type == INODE_SYMLINK) + else if (ent->type == VSYMLINK) { printf("%s\033[36m%s%s\033[0m@", modifier, name, showFlags ? ent->mode : ""); len++; diff --git a/src/programs/utils/stat/main.c b/src/programs/utils/stat/main.c index cce905c82..f320e302d 100644 --- a/src/programs/utils/stat/main.c +++ b/src/programs/utils/stat/main.c @@ -6,15 +6,15 @@ #include #include -static const char* type_to_string(itype_t type) +static const char* type_to_string(vtype_t type) { switch (type) { - case INODE_REGULAR: + case VREG: return "file"; - case INODE_DIR: + case VDIR: return "directory"; - case INODE_SYMLINK: + case VSYMLINK: return "symlink"; default: return "unknown"; @@ -33,7 +33,7 @@ static void print_stat(const char* path) printf(" File: %s\n", path); printf(" Size: %llu\t\tBlocks: %llu\t IO Block: %llu %s\n", buffer.size, buffer.blocks, buffer.blockSize, type_to_string(buffer.type)); - printf("Superblock: %llu\tInode: %llu\tLinks: %llu\n", buffer.sbid, buffer.number, buffer.linkAmount); + printf("Superblock: %llu\tVnode: %llu\tLinks: %llu\n", buffer.sbid, buffer.number, buffer.linkAmount); printf(" Max: %llu\n", buffer.maxFileSize); printf(" Name: %s\n", buffer.name); printf("Access: %s", ctime(&buffer.accessTime)); From 1e5d9e7784b58306f86a2dc3a03b1025546e8847 Mon Sep 17 00:00:00 2001 From: KN Date: Thu, 22 Jan 2026 23:05:19 +0100 Subject: [PATCH 21/23] refactor: standardize sys/ioring.h names and move IRP arguments to irp_frame_t --- include/kernel/fs/dentry.h | 2 +- include/kernel/fs/filesystem.h | 2 +- include/kernel/fs/inode.h | 216 ----------- include/kernel/fs/tmpfs.h | 2 +- include/kernel/fs/vfs.h | 2 +- include/kernel/fs/vnode.h | 13 +- include/kernel/io/io.h | 25 +- include/kernel/io/irp.h | 348 +++++++----------- include/kernel/io/verb.h | 15 +- include/kernel/mem/mdl.h | 4 +- include/libstd/sys/fs.h | 7 +- include/libstd/sys/ioring.h | 148 ++++---- meta/doxy/Doxyfile | 2 +- src/kernel/fs/dentry.c | 2 +- src/kernel/fs/devfs.c | 2 +- src/kernel/fs/file.c | 2 +- src/kernel/fs/filesystem.c | 5 +- src/kernel/fs/netfs.c | 9 +- src/kernel/fs/procfs.c | 15 +- src/kernel/fs/sysfs.c | 2 +- src/kernel/fs/tmpfs.c | 2 +- src/kernel/fs/vfs.c | 7 +- src/kernel/fs/vnode.c | 3 +- src/kernel/io/io.c | 26 +- src/kernel/io/irp.c | 39 +- src/kernel/io/verb.c | 29 +- src/kernel/mem/mdl.c | 2 +- src/kernel/proc/group.c | 2 +- src/libstd/user/common/syscalls.h | 8 +- .../ioring/{enter.c => ioring_enter.c} | 2 +- .../user/functions/ioring/ioring_setup.c | 13 + .../ioring/{teardown.c => ioring_teardown.c} | 2 +- src/libstd/user/functions/ioring/setup.c | 13 - src/programs/utils/ringtest/main.c | 12 +- 34 files changed, 333 insertions(+), 650 deletions(-) delete mode 100644 include/kernel/fs/inode.h rename src/libstd/user/functions/ioring/{enter.c => ioring_enter.c} (75%) create mode 100644 src/libstd/user/functions/ioring/ioring_setup.c rename src/libstd/user/functions/ioring/{teardown.c => ioring_teardown.c} (82%) delete mode 100644 src/libstd/user/functions/ioring/setup.c diff --git a/include/kernel/fs/dentry.h b/include/kernel/fs/dentry.h index c753e3513..d868a3de6 100644 --- a/include/kernel/fs/dentry.h +++ b/include/kernel/fs/dentry.h @@ -1,7 +1,7 @@ #pragma once -#include #include +#include #include #include #include diff --git a/include/kernel/fs/filesystem.h b/include/kernel/fs/filesystem.h index 6380872d1..440bbc492 100644 --- a/include/kernel/fs/filesystem.h +++ b/include/kernel/fs/filesystem.h @@ -3,10 +3,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include diff --git a/include/kernel/fs/inode.h b/include/kernel/fs/inode.h deleted file mode 100644 index 5f23f4579..000000000 --- a/include/kernel/fs/inode.h +++ /dev/null @@ -1,216 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -typedef struct vnode vnode_t; -typedef struct vnode_ops vnode_ops_t; -typedef struct superblock superblock_t; -typedef struct file_ops file_ops_t; -typedef struct dentry dentry_t; - -/** - * @brief Index node. - * @defgroup kernel_fs_vnode Vnode - * @ingroup kernel_fs - * - * A vnode represents the actual data and metadata of a file. It is referenced by dentries, which represent the name or - * "location" of the file but a vnode can appear in multiple dentries due to hardlinks or mounts. - * - * @note Despite the name vnodes are in no way "nodes" in any kind of tree structure, that would be the dentries. - * - * ## Synchronization - * - * Vnodes have an additional purpose within the Virtual File System (VFS) as they act as the primary means of - * synchronization. All dentries synchronize upon their vnodes mutex, open files synchronize upon the mutex of the - * underlying vnode and operations like create, remove, etc synchronize upon the vnode mutex of the parent directory. - * - * @todo Implement actually writing/syncing dirty vnodes, for now vnodes should use the notify functions but they will - * never actually be "cleaned." - * - * @{ - */ - -/** - * @brief Vnode structure. - * @struct vnode_t - * - * Vnodes are owned by the filesystem, not the VFS. - */ -typedef struct vnode -{ - ref_t ref; - ino_t number; - vtype_t type; - _Atomic(uint64_t) dentryCount; ///< The number of dentries pointing to this vnode. - size_t size; - size_t blocks; - time_t accessTime; ///< Unix time stamp for the last vnode access. - time_t modifyTime; ///< Unix time stamp for last file content alteration. - time_t changeTime; ///< Unix time stamp for the last file metadata alteration. - time_t createTime; ///< Unix time stamp for the vnode creation. - void* data; - superblock_t* superblock; - const vnode_ops_t* ops; - const file_ops_t* fileOps; - const verb_table_t* verbs; - rcu_entry_t rcu; - mutex_t mutex; -} vnode_t; - -/** - * @brief Vnode operations structure. - * @struct vnode_ops_t - * - * Note that the vnodes mutex will be acquired by the vfs. - */ -typedef struct vnode_ops -{ - /** - * @brief Look up a dentry in a directory vnode. - * - * Should set the target dentry to be positive (give it an vnode), if the entry does not exist the operation - * should still return success but leave the dentry negative. - * - * @param dir The directory vnode to look in. - * @param target The dentry to look up. - * @return On success, `0`. On failure, returns `ERR` and `errno` is set. - */ - uint64_t (*lookup)(vnode_t* dir, dentry_t* target); - /** - * @brief Handles both directories and files depending on mode. - * - * Takes in a negative dentry and creates the corresponding vnode to make the dentry positive. - * - * @param dir The directory vnode to create the entry in. - * @param target The negative dentry to create. - * @param mode The mode to create the entry with. - * @return On success, `0`. On failure, returns `ERR` and `errno` is set. - */ - uint64_t (*create)(vnode_t* dir, dentry_t* target, mode_t mode); - /** - * @brief Set the vnode size to zero. - * - * @param target The vnode to truncate. - */ - void (*truncate)(vnode_t* target); - /** - * @brief Make the same file vnode appear twice in the filesystem. - * - * @param dir The directory vnode to create the link in. - * @param old The existing dentry containing the vnode to link to. - * @param new The negative dentry to store the same vnode as old. - * @return On success, `0`. On failure, returns `ERR` and `errno` is set. - */ - uint64_t (*link)(vnode_t* dir, dentry_t* old, dentry_t* new); - /** - * @brief Retrieve the path of the symbolic link. - * - * @param vnode The symbolic link vnode. - * @param buffer The buffer to store the path in. - * @param size The size of the buffer. - * @return On success, the number of bytes read. On failure, returns `ERR` and `errno` is set. - */ - uint64_t (*readlink)(vnode_t* vnode, char* buffer, uint64_t size); - /** - * @brief Create a symbolic link. - * - * @param dir The directory vnode to create the symbolic link in. - * @param target The negative dentry to create. - * @param dest The path to which the symbolic link will point. - * @return On success, `0`. On failure, returns `ERR` and `errno` is set. - */ - uint64_t (*symlink)(vnode_t* dir, dentry_t* target, const char* dest); - /** - * @brief Remove a file or directory. - * - * @param dir The directory vnode containing the target. - * @param target The dentry to remove. - * @return On success, `0`. On failure, returns `ERR` and `errno` is set. - */ - uint64_t (*remove)(vnode_t* dir, dentry_t* target); - /** - * @brief Cleanup function called when the vnode is being freed. - * - * @param vnode The vnode being freed. - */ - void (*cleanup)(vnode_t* vnode); -} vnode_ops_t; - -/** - * @brief Create a new vnode. - * - * This DOES add the vnode to the vnode cache. It also does not associate the vnode with a dentry, that is done when a - * dentry is made positive with `dentry_make_positive()`. - * - * There is no `vnode_free()` instead use `UNREF()`. - * - * @param superblock The superblock the vnode belongs to. - * @param number The vnode number, for a generic filesystem `vfs_id_get()` can be used. - * @param type The vnode type. - * @param ops The vnode operations. - * @param fileOps The file operations for files opened on this vnode. - * @return On success, the new vnode. On failure, returns `NULL` and `errno` is set. - */ -vnode_t* vnode_new(superblock_t* superblock, ino_t number, vtype_t type, const vnode_ops_t* ops, - const file_ops_t* fileOps); - -/** - * @brief Notify the vnode that it has been accessed. - * - * This updates the access time. - * - * @param vnode The vnode to notify. - */ -void vnode_notify_access(vnode_t* vnode); - -/** - * @brief Notify the vnode that its content has been modified. - * - * This updates the modify time and change time. - * - * @param vnode The vnode to notify. - */ -void vnode_notify_modify(vnode_t* vnode); - -/** - * @brief Notify the vnode that its metadata has changed. - * - * This updates the change time. - * - * @param vnode The vnode to notify. - */ -void vnode_notify_change(vnode_t* vnode); - -/** - * @brief Truncate the vnode. - * - * The filesystem should implement the actual truncation in the vnode ops truncate function, this is just a helper to - * call it. - * - * @param vnode The vnode to truncate. - */ -void vnode_truncate(vnode_t* vnode); - -/** - * @brief Helper to generate a consistent vnode number for an entry in a directory. - * - * This is useful for in-memory filesystems or filesystem that dont provide native vnode numbers. - * - * @param parentNumber The vnode number of the parent directory. - * @param name The name of the entry. - * @return The generated vnode number. - */ -ino_t ino_gen(ino_t parentNumber, const char* name); - -/** @} */ diff --git a/include/kernel/fs/tmpfs.h b/include/kernel/fs/tmpfs.h index 1e6869bcf..7c2732a1b 100644 --- a/include/kernel/fs/tmpfs.h +++ b/include/kernel/fs/tmpfs.h @@ -1,8 +1,8 @@ #pragma once #include -#include #include +#include #include diff --git a/include/kernel/fs/vfs.h b/include/kernel/fs/vfs.h index eea960094..5932b6435 100644 --- a/include/kernel/fs/vfs.h +++ b/include/kernel/fs/vfs.h @@ -3,10 +3,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include diff --git a/include/kernel/fs/vnode.h b/include/kernel/fs/vnode.h index d894db7c6..9f8e4190a 100644 --- a/include/kernel/fs/vnode.h +++ b/include/kernel/fs/vnode.h @@ -35,9 +35,6 @@ typedef struct dentry dentry_t; * synchronization. All dentries synchronize upon their vnodes mutex, open files synchronize upon the mutex of the * underlying vnode and operations like create, remove, etc synchronize upon the vnode mutex of the parent directory. * - * @todo Implement actually writing/syncing dirty vnodes, for now vnodes should use the notify functions but they will - * never actually be "cleaned." - * * @{ */ @@ -52,8 +49,8 @@ typedef struct vnode ref_t ref; vtype_t type; _Atomic(uint64_t) dentryCount; ///< The number of dentries pointing to this vnode. - void* data; ///< Filesystem defined data. - uint64_t size; ///< Used for convenience by certain filesystems, does not represent the file size. + void* data; ///< Filesystem defined data. + uint64_t size; ///< Used for convenience by certain filesystems, does not represent the file size. superblock_t* superblock; const vnode_ops_t* ops; const file_ops_t* fileOps; @@ -144,7 +141,8 @@ typedef struct vnode_ops /** * @brief Create a new vnode. * - * Does not associate the vnode with a dentry, that is done when a dentry is made positive with `dentry_make_positive()`. + * Does not associate the vnode with a dentry, that is done when a dentry is made positive with + * `dentry_make_positive()`. * * There is no `vnode_free()` instead use `UNREF()`. * @@ -154,8 +152,7 @@ typedef struct vnode_ops * @param fileOps The file operations for files opened on this vnode. * @return On success, the new vnode. On failure, returns `NULL` and `errno` is set. */ -vnode_t* vnode_new(superblock_t* superblock, vtype_t type, const vnode_ops_t* ops, - const file_ops_t* fileOps); +vnode_t* vnode_new(superblock_t* superblock, vtype_t type, const vnode_ops_t* ops, const file_ops_t* fileOps); /** * @brief Truncate the vnode. diff --git a/include/kernel/io/io.h b/include/kernel/io/io.h index 82a61ea4b..2c3dccfd8 100644 --- a/include/kernel/io/io.h +++ b/include/kernel/io/io.h @@ -68,17 +68,20 @@ * * ## Arguments * - * Arguments within a SQE are stored in five 64-bit values, `arg1` through `arg5`. For convenience, each argument value is stored as a union with various types. - * - * To avoid nameing conflicts and to avoid having to define new arguments for each verb, we define a convention to be used for the arguments. - * + * Arguments within a SQE are stored in five 64-bit values, `arg1` through `arg5`. For convenience, each argument value + * is stored as a union with various types. + * + * To avoid nameing conflicts and to avoid having to define new arguments for each verb, we define a convention to be + * used for the arguments. + * * - `arg0`: The noun or subject of the verb, for example, a `fd_t` for file operations. * - `arg1`: The source or payload of the verb, for example, a buffer or path. * - `arg2`: The magnitude of the operation, for example, a size or encoding. * - `arg3`: The location or a modifier to the operation, for example, an offset or flags. * - `arg4`: An auxiliary argument, for example, additional flags or options. - * - * It may not always be possible for a verb to follow these conventions, but they should be followed whenever reasonable. + * + * It may not always be possible for a verb to follow these conventions, but they should be followed whenever + * reasonable. * * @note The kernels internal I/O Request Packet structure uses a similar system but with the kernel equivalents * of the arguments, for example, a `file_t*` instead of a `fd_t`. @@ -108,7 +111,7 @@ * ## Verbs * * Included below is a list of all currently implemented verbs. - * + * * The arguments of each verb is specified in order as `arg0`, `arg1`, `arg2`, `arg3`, `arg4`. * * ### `VERB_NOP` @@ -121,7 +124,7 @@ * @param arg3 Unused * @param arg4 Unused * @result None - * + * * ### `VERB_READ` * * Reads data from a file descriptor. @@ -132,11 +135,11 @@ * @param offset The offset to read from, or `IO_CUR` to use the current position. * @param arg4 Unused * @result The number of bytes read. - * + * * ### `VERB_WRITE` * * Writes data to a file descriptor. - * + * * @param fd The file descriptor to write to. * @param buffer The buffer to write the data from. * @param count The number of bytes to write. @@ -154,7 +157,7 @@ * @param arg3 Unused * @param arg4 Unused * @result The events that occurred. - * + * * @{ */ diff --git a/include/kernel/io/irp.h b/include/kernel/io/irp.h index 8f6b00653..df7153d61 100644 --- a/include/kernel/io/irp.h +++ b/include/kernel/io/irp.h @@ -15,6 +15,9 @@ #include #include +typedef struct file file_t; +typedef struct process process_t; + typedef struct irp irp_t; /** @@ -33,22 +36,15 @@ typedef struct irp irp_t; * * ## Completion * - * The IRP system is designed around the concept of layered completions as it may take more than one subsystem within - * the kernel to complete a IRP. - * the kernel to complete an IRP. + * The IRP system is designed to allow multiple subsystems or functions to asynchronously "call" each other. * * Consider a traditional synchronous set of functions: * * ``` - * int fun_c(void) - * { - * wait_until_data_ready(); - * return data; - * } - * * int fun_b(int val) * { - * return fun_c(val) + 1; + * wait_until_data_ready(); + * return val * get_data(); * } * * int fun_a(int val) @@ -56,140 +52,103 @@ typedef struct irp irp_t; * return fun_b(val) * 2; * } * - * int result = fun_a(); + * int result = fun_a(5); * // Do stuff with the result * ``` * - * When the code is executed, `fun_a()` would be called, which calls `fun_b()`, which in turn calls `fun_c()`. At this - * point `fun_c()` will block, causing the scheduler to switch to another thread until the data is ready. Once the data - * is ready, `fun_c()` will "complete" and return, followed by `fun_b()` and finally `fun_a()`, with the final result - * being stored in `result`. - * - * The above may seem obvious, but in a asynchronous kernel we are not allowed to block but must still be able to - * achieve the same result. As such, we need a way of representing the layered calls and their completions. + * The above may seem obvious, but in a asynchronous kernel we are not allowed to block, as such `fun_b()` should not be + * implemented this way. We must however still be able to achieve the same result. * * @note In practice its possible that more than just one layer needs to block, as such the IRP system needs to handle * such cases as well. * + * The idea behind IRPs is to effectively create a call stack which is detached from the actual CPU stack. Each frame in + * the IRP stack represents a function call with associated arguments and a "return address" in the form of a function + * pointer and "local variables" in the form of a context pointer. + * * Using the IRP system, the above code would be written as: * * ``` - * void fun_c_complete(irp_t* irp, void* ctx) + * void fun_b_interrupt(void) * { - * irp->result = get_data(); + * irp_t* irp = pop_irp_from_list(); + * irp_frame_t* frame = irp_current(irp); + * irp->res.u64 = frame->args[0] * get_data(); * irp_complete(irp); * } * - * void fun_b_complete(irp_t* irp, void* ctx) - * { - * irp->result += 1; - * irp_complete(irp); - * } - * - * void fun_a_complete(irp_t* irp, void* ctx) - * { - * irp->result *= 2; - * irp_complete(irp); - * } - * - * void fun_c(irp_t* irp) + * void fun_b(irp_t* irp) * { + * irp_frame_t* frame = irp_current(irp); * if (can_complete_now()) * { - * irp->result = get_data(); + * irp->res.u64 = frame->args[0] * get_data(); * irp_complete(irp); + * return; * } - * else - * { - * irp_push(irp, fun_c_complete, NULL); - * } + * + * add_irp_to_list(irp); * } * - * void fun_b(irp_t* irp) + * void fun_a_return_address(irp_t* irp, void* ctx) * { - * irp_push(irp, fun_b_complete, NULL); - * fun_c(irp); + * irp->res.u64 *= 2; + * irp_complete(irp); + * return; * } * * void fun_a(irp_t* irp) * { - * irp_push(irp, fun_a_complete, NULL); - * fun_b(irp); - * } + * irp_frame_t* current = irp_current(irp); + * irp_frame_t* next = irp_next(irp); * - * void my_completion(irp_t* irp) - * { - * // Do stuff with the result in irp->result. - * irp_free(irp); - * } + * next->args[0] = current->args[0]; + * next->ret = fun_a_return_address; * - * irp_t* irp = irp_new(pool, NULL); - * // We can set arguments here if we want. - * irp_push(irp, my_completion, NULL); // Our completion to handle cleanup. - * fun_a_do(irp); - * // Continue executing even if fun_c() cannot complete immediately. - * ``` - * - * When `fun_a()` is called, it pushes its completion onto the IRP stack, followed by `fun_b()` pushing its completion, - * and finally `fun_c()` which may either complete immediately or push its completion if it cannot complete right away. - * - * Each time a completion is called via `irp_complete()`, the next completion on the stack is called until the stack is - * empty, at which point the IRP is considered fully completed. - * - * A real world example of this would be the ring system allocating an IRP, pushing a completion which will add a - * `cqe_t` to its rings, before passing the IRP to the VFS which may pass it to a filesystem. Each layer pushing its own - * completion to handle its part of the operation. - * - * Finally, it is also possible to use the `irp_dispatch()` function. This function allows us to dispatch the IRP to a - * appropriate handler depending on the IRPs specified verb. For example: + * irp_call(irp, fun_b); + * } * - * ``` - * void my_completion(irp_t* irp) + * void my_return_address(irp_t* irp, void* ctx) * { * // Do stuff with the result in irp->result. * irp_free(irp); * } * - * irp_t* irp = irp_new(pool, NULL); + * irp_t* irp = irp_new(pool); * - * // Set our desired verb and arguments. - * irp->verb = VERB_READ; - * irp->rw.file = file; - * irp->rw.buffer = buffer; - * irp->rw.len = len; - * irp->rw.off = off; + * // Setup the "stack frame" for our function call. + * irp_frame_t* next = irp_next(irp); + * next->args[0] = 5; + * next->ret = my_return_address; * - * // Our completion to receive the result. - * irp_push(irp, my_completion, NULL); + * // Call `fun_a()` and advance the stack frame. + * irp_call(irp, fun_a); * - * // Finally, dispatch the IRP to the appropriate handler. - * verb_dispatch(irp); - * // Continue executing even if the operation cannot complete immediately. + * // Continue executing even if fun_b() cannot complete immediately. * ``` * * ## Cancellation * * The current owner of a IRP is responsible for handling cancellation. The current owner being the last subsystem to - * push a completion onto the IRP stack. + * advance the IRP stack frame. * * @note Intuitively, we can think of "cancelling" a IRP to be equivalent to causing the last completion to fail, thus - * resulting in all the other completions to fail as well. In the examples from the Completion section, it would be as - * though the synchronous `fun_c()` returned an error code instead of the data. + * resulting in all the other return addresses to fail as well. In the examples from the Completion section, it would be + * as though the synchronous `fun_c()` returned an error code instead of the data. * - * The owner implements cancellation by calling `irp_set_cancel()` to set a cancellation callback when it pushes its - * completion. When an IRP is to be cancelled or timed out the cancellation callback will be invoked and atomically - * exchanged with a `IRP_CANCELLED` sentinel value. At which point the owner should cleanup the IRP and call - * `irp_complete()`. + * The owner implements cancellation by calling `irp_set_cancel()` to set a cancellation callback. When an IRP is to be + * cancelled or timed out the cancellation callback will be invoked and atomically exchanged with a `IRP_CANCELLED` + * sentinel value. At which point the owner should cleanup the IRP and call `irp_complete(irp)`. * - * It is not possible for the IRP system to perform this atomic exchange for completions. As such, to avoid race + * It is not possible for the IRP system to perform this atomic exchange for return addresses. As such, to avoid race * conditions while completing an IRP, it is vital that the owner of the IRP atomically exchanges the cancellation * callback with the `IRP_CANCELLED` sentinel value. For the sake of convenience, the `irp_claim()` function is provided * to perform this operation. * - * Below is an example of how to safely implement a completion with an associated cancellation callback: + * Below is an example of how to safely implement a return address with an associated cancellation callback: * * ``` - * void my_completion(irp_t* irp, void* ctx) + * void my_return_address(irp_t* irp, void* ctx) * { * if (!irp_claim(irp)) * { @@ -248,15 +207,13 @@ typedef struct irp irp_t; * @{ */ -#define IRP_LOC_MAX 5 ///< The maximum number of locations in a IRP. - /** - * @brief IRP completion callback type. + * @brief IRP return address type. * * @param irp Pointer to the IRP. * @param ctx Context pointer. */ -typedef void (*irp_complete_t)(irp_t* irp, void* ctx); +typedef void (*irp_ret_t)(irp_t* irp, void* ctx); /** * @brief IRP cancellation callback type. @@ -271,15 +228,30 @@ typedef uint64_t (*irp_cancel_t)(irp_t* irp); */ #define IRP_CANCELLED ((irp_cancel_t)1) +#define IRP_ARGS_MAX 5 ///< The maximum number of 64-bit arguments in a IRP frame. + /** - * @brief IRP location structure. - * @struct irp_loc + * @brief IRP stack frame structure. + * @struct irp_frame_t */ -typedef struct irp_loc +typedef struct irp_frame { - void* ctx; - irp_complete_t complete; -} irp_loc_t; + irp_ret_t ret; ///< Return Address. + void* local; ///< Local context. + union { + struct + { + file_t* file; + mdl_t* buffer; + size_t len; + ssize_t off; + } read; + uint64_t args[IRP_ARGS_MAX]; + sqe_args_t sqe; + }; +} irp_frame_t; + +#define IRP_FRAME_MAX 5 ///< The maximum number of frames in a IRP stack. /** * @brief I/O Request Packet structure. @@ -289,14 +261,7 @@ typedef struct irp_loc * no need for any allocation beyond the allocation of the IRP itself. This does require careful consideration of * padding, alignment and field sizes to keep it within a reasonable size. * - * @warning The `sqe` field is only valid if the IRP is a user IRP and only until the IRP is entered into the kernel via - * `irp_dispatch()`. - * - * @note We need the ability to store both the original arguments from a SQE and the parsed arguments. For example, - * opening a `fd_t` into a `file_t*`. As such, to avoid using another cache line, the SQE is stored in a union with the - * parsed arguments. - * - * @todo Consider raising `IRP_LOC_MAX` to 9 if needed, it will add another cache line tho. + * @todo Consider raising `IRP_FRAME_MAX` if needed, it will add more cache lines tho. * * @see kernel_io for more information for each possible verb. */ @@ -306,70 +271,25 @@ typedef struct ALIGNED(64) irp list_entry_t timeoutEntry; ///< Used to store the IRP in the timeout queue. _Atomic(irp_cancel_t) cancel; ///< Cancellation callback, must be atomic to ensure an IRP is only cancelled once. union { - struct - { - verb_t verb; ///< Verb specifying the action to perform. - sqe_flags_t flags; ///< Submission flags. - union { - clock_t timeout; ///< The timeout starting from when the IRP is added to a timeout queue. - clock_t deadline; ///< The time at which the IRP will be removed from a timeout queue. - }; - void* data; ///< Private data for the operation, will be returned in the completion entry. - union - { - uint64_t arg0; - file_t* file; - }; - union - { - uint64_t arg1; - mdl_t* buffer; - events_t events; - }; - union - { - uint64_t arg2; - size_t count; - }; - union - { - uint64_t arg3; - ssize_t offset; - }; - union - { - uint64_t arg4; - }; - }; - sqe_t sqe; ///< The original SQE for this IRP. + clock_t timeout; ///< The timeout starting from when the IRP is added to a timeout queue. + clock_t deadline; ///< The time at which the IRP will be removed from a timeout queue. }; + void* data; ///< Private data for the operation, will be returned in the completion entry. union { - file_t* file; - size_t count; - void* ptr; - events_t events; - uint64_t _raw; + uint64_t u64; + int64_t s64; + size_t read; } res; - mdl_t mdl; ///< A preallocated memory descriptor list for use by the IRP. - pool_idx_t index; ///< Index of the IRP in its pool. - pool_idx_t next; ///< Index of the next IRP in a chain or in the free list. - cpu_id_t cpu; ///< The CPU whose timeout queue the IRP is in. - uint8_t err; ///< The error code of the operation, also used to specify its current state. - uint8_t location; ///< The index of the current location in the stack. - irp_loc_t stack[IRP_LOC_MAX]; ///< The location stack, grows downwards. + mdl_t mdl; ///< A preallocated memory descriptor list for use by the IRP. + pool_idx_t index; ///< Index of the IRP in its pool. + pool_idx_t next; ///< Index of the next IRP in a chain or in the free list. + cpu_id_t cpu; ///< The CPU whose timeout queue the IRP is in. + uint8_t err; ///< The error code of the operation, also used to specify its current state. + uint8_t frame; ///< The index of the current frame in the stack. + irp_frame_t stack[IRP_FRAME_MAX]; ///< The frame stack, grows downwards. } irp_t; -static_assert(offsetof(irp_t, verb) == offsetof(irp_t, sqe.verb), "verb offset mismatch"); -static_assert(offsetof(irp_t, flags) == offsetof(irp_t, sqe.flags), "flags offset mismatch"); -static_assert(offsetof(irp_t, timeout) == offsetof(irp_t, sqe.timeout), "timeout offset mismatch"); -static_assert(offsetof(irp_t, data) == offsetof(irp_t, sqe.data), "data offset mismatch"); -static_assert(offsetof(irp_t, arg0) == offsetof(irp_t, sqe.arg0), "arg0 offset mismatch"); -static_assert(offsetof(irp_t, arg1) == offsetof(irp_t, sqe.arg1), "arg1 offset mismatch"); -static_assert(offsetof(irp_t, arg2) == offsetof(irp_t, sqe.arg2), "arg2 offset mismatch"); -static_assert(offsetof(irp_t, arg3) == offsetof(irp_t, sqe.arg3), "arg3 offset mismatch"); -static_assert(offsetof(irp_t, arg4) == offsetof(irp_t, sqe.arg4), "arg4 offset mismatch"); - -static_assert(sizeof(irp_t) == 256, "irp_t is not 256 bytes"); +static_assert(sizeof(irp_t) == 448, "irp_t is not 448 bytes"); /** * @brief Request pool structure. @@ -426,16 +346,13 @@ void irp_timeouts_check(void); * function. * * @param pool Pointer to the IRP pool. - * @param sqe The Submission Queue Entry associated with the IRP, if `NULL` the IRP will be a kernel IRP. * @return On success, a pointer to the allocated IRP. On failure, `NULL` and `errno` is set. */ -irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe); +irp_t* irp_new(irp_pool_t* pool); /** * @brief Free a IRP back to its pool. * - * If the IRP is a user IRP, the `irp_handler_t::leave` callback will be invoked before freeing the IRP. - * * @param irp Pointer to the IRP to free. */ void irp_free(irp_t* irp); @@ -451,6 +368,17 @@ static inline irp_pool_t* irp_get_pool(irp_t* irp) return CONTAINER_OF(irp, irp_pool_t, irps[irp->index]); } +/** + * @brief Retrieve the context of the IRP pool that an IRP was allocated from. + * + * @param irp Pointer to the IRP. + * @return Pointer to the context. + */ +static inline void* irp_get_ctx(irp_t* irp) +{ + return irp_get_pool(irp)->ctx; +} + /** * @brief Retrieve the process that owns an IRP. * @@ -486,7 +414,7 @@ static inline irp_cancel_t irp_set_cancel(irp_t* irp, irp_cancel_t cancel) * @brief Attempt to claim an IRP for completion. * * @param irp Pointer to the IRP. - * @return `true` if the IRP was successfully claimed, `false` if it was already cancelled or claimed. + * @return `true` if the IRP was successfully claimed, `false` if it was already cancelled. */ static inline bool irp_claim(irp_t* irp) { @@ -494,23 +422,12 @@ static inline bool irp_claim(irp_t* irp) } /** - * @brief Retrieve the context of the IRP pool that an IRP was allocated from. - * - * @param irp Pointer to the IRP. - * @return Pointer to the context. - */ -static inline void* irp_get_ctx(irp_t* irp) -{ - return irp_get_pool(irp)->ctx; -} - -/** - * @brief Retrieve the next IRP and clear the next field. + * @brief Retrieve the next IRP in a chain and clear the next field. * * @param irp Pointer to the current IRP. * @return Pointer to the next IRP, or `NULL` if there is no next IRP. */ -static inline irp_t* irp_next(irp_t* irp) +static inline irp_t* irp_chain_next(irp_t* irp) { irp_pool_t* pool = irp_get_pool(irp); if (irp->next == POOL_IDX_MAX) @@ -524,70 +441,65 @@ static inline irp_t* irp_next(irp_t* irp) } /** - * @brief Retrieve the current location in the IRP stack. + * @brief Retrieve the current frame in the IRP stack. * - * @param irp Pointer to the IRP to retrieve the location from. - * @return Pointer to the current location. + * @param irp Pointer to the IRP to retrieve the frame from. + * @return Pointer to the current frame. */ -static inline irp_loc_t* irp_current(irp_t* irp) +static inline irp_frame_t* irp_current(irp_t* irp) { - assert(irp->location <= IRP_LOC_MAX); - return &irp->stack[irp->location]; + assert(irp->frame < IRP_FRAME_MAX); + return &irp->stack[irp->frame]; } /** - * @brief Retrieve the next location in the IRP stack. + * @brief Retrieve the next frame in the IRP stack. * - * @param irp Pointer to the IRP to retrieve the location from. - * @return Pointer to the next location, or `NULL` if we are at the bottom of the stack. + * @param irp Pointer to the IRP to retrieve the frame from. + * @return Pointer to the next frame, or `NULL` if we are at the bottom of the stack. */ -static inline irp_loc_t* irp_next_loc(irp_t* irp) +static inline irp_frame_t* irp_next(irp_t* irp) { - if (irp->location == 0) + if (irp->frame == 0) { return NULL; } - return &irp->stack[irp->location - 1]; + return &irp->stack[irp->frame - 1]; } - /** - * @brief Push a new location onto the IRP stack. - * - * @param irp Pointer to the IRP to push to. - * @param complete The completion callback. - * @param ctx The context pointer. + * @brief Call a function with an IRP, advancing the frame in the IRP stack. + * + * @param irp Pointer to the IRP. + * @param func The function to call. */ -static inline void irp_push(irp_t* irp, irp_complete_t complete, void* ctx) +static inline void irp_call(irp_t* irp, void (*func)(irp_t* irp)) { - assert(irp->location > 0); - assert(complete != NULL); - irp_loc_t* loc = &irp->stack[irp->location - 1]; - loc->complete = complete; - loc->ctx = ctx; - irp->location--; + assert(irp->frame > 0); + irp->frame--; + func(irp); } /** - * @brief Complete the current location in the IRP stack. + * @brief Complete the current frame in the IRP stack. * * @param irp Pointer to the IRP to complete. */ static inline void irp_complete(irp_t* irp) { - if (irp->location == IRP_LOC_MAX) + if (irp->frame == IRP_FRAME_MAX) { return; } - irp_loc_t* loc = irp_current(irp); - irp->location++; + irp_frame_t* loc = irp_current(irp); + irp->frame++; - if (irp->location == IRP_LOC_MAX) + if (irp->frame == IRP_FRAME_MAX) { irp_timeout_remove(irp); } - loc->complete(irp, loc->ctx); + loc->ret(irp, loc->local); } /** diff --git a/include/kernel/io/verb.h b/include/kernel/io/verb.h index c3724d163..b5ed3fe41 100644 --- a/include/kernel/io/verb.h +++ b/include/kernel/io/verb.h @@ -14,7 +14,7 @@ /** * @brief Verb function type. - * + * * @param irp Pointer to the IRP. */ typedef void (*verb_func_t)(irp_t* irp); @@ -28,20 +28,11 @@ typedef struct verb_table verb_func_t handlers[VERB_MAX]; } verb_table_t; -/** - * @brief Cleanup the arguments used by a verb. - * - * Handles both kernel IRPs and parsed user IRPs. - * - * @param irp Pointer to the IRP. - */ -void verb_args_cleanup(irp_t* irp); - /** * @brief Dispatch an IRP to the appropriate verb handler. - * + * * If the IRP is a user IRP, the arguments will be parsed before invoking the handler. - * + * * @param irp Pointer to the IRP. */ void verb_dispatch(irp_t* irp); diff --git a/include/kernel/mem/mdl.h b/include/kernel/mem/mdl.h index 1d66e9c63..9822662a2 100644 --- a/include/kernel/mem/mdl.h +++ b/include/kernel/mem/mdl.h @@ -89,11 +89,11 @@ void mdl_deinit(mdl_t* mdl); * @brief Free a Memory Descriptor List chain. * * Will traverse the entire chain to deinitialize and free each MDL structure using the provided `free` function. - * + * * @param mdl Pointer to the first MDL in the chain. * @param free Function to free the MDL structure itself, or `NULL` to only deinitialize. */ -void mdl_free_chain(mdl_t* mdl, void(*free)(void*)); +void mdl_free_chain(mdl_t* mdl, void (*free)(void*)); /** * @brief Initialize a Memory Descriptor List from a memory region. diff --git a/include/libstd/sys/fs.h b/include/libstd/sys/fs.h index 1dcd04f5e..57785925d 100644 --- a/include/libstd/sys/fs.h +++ b/include/libstd/sys/fs.h @@ -340,7 +340,7 @@ poll_events_t poll1(fd_t fd, poll_events_t events, clock_t timeout); */ typedef enum { - VREG, ///< Is a regular file. + VREG, ///< Is a regular file. VDIR, ///< Is a directory. VSYMLINK, ///< Is a symbolic link. } vtype_t; @@ -364,11 +364,10 @@ typedef struct vattr uint64_t size; uint64_t blocks; uint64_t blockSize; - uint64_t rdev; time_t atime; time_t mtime; time_t ctime; - uint8_t padding[64]; ///< Padding to leave space for future expansion. + uint8_t padding[64]; ///< Padding to leave space for future expansion. } vattr_t; /** @@ -378,7 +377,7 @@ typedef struct vattr typedef struct { sbid_t sbid; ///< The superblock ID of the filesystem containing the entry. - uint64_t number; ///< The number of the entries vnode. + uint64_t number; ///< The number of the entries vnode. vtype_t type; ///< The type of the entries vnode. uint64_t size; ///< The size of the file that is visible outside the filesystem. uint64_t blocks; ///< The amount of blocks used on disk to store the file. diff --git a/include/libstd/sys/ioring.h b/include/libstd/sys/ioring.h index 866fb69a3..2f3e0b1cc 100644 --- a/include/libstd/sys/ioring.h +++ b/include/libstd/sys/ioring.h @@ -24,38 +24,39 @@ extern "C" * @{ */ -typedef uint64_t whence_t; ///< Seek origin type. -#define IO_SET ((ssize_t) - 3) ///< Use the start of the file. -#define IO_END ((ssize_t) - 2) ///< Use the end of the file. -#define IO_CUR ((ssize_t) - 1) ///< Use the current file offset. - -typedef uint64_t events_t; ///< Poll events type. -#define IO_READABLE (1 << 0) ///< File descriptor is ready to read. -#define IO_WRITABLE (1 << 1) ///< File descriptor is ready to write -#define IO_ERROR (1 << 2) ///< File descriptor caused an error. -#define IO_CLOSED (1 << 3) ///< File descriptor is closed. -#define IO_INVALID (1 << 4) ///< Invalid file descriptor. - -typedef uint32_t verb_t; ///< Verb type. -#define VERB_NOP 0 ///< No-op verb. -#define VERB_READ 1 ///< Read verb. -#define VERB_WRITE 2 ///< Write verb. -#define VERB_POLL 3 ///< Poll verb. -#define VERB_MAX 4 ///< The maximum number of verbs. +#define IO_OFF_CUR ((ssize_t) - 1) ///< Use the current file offset. + +typedef uint64_t io_whence_t; ///< Seek origin type. +#define IO_SEEK_SET (1) ///< Use the start of the file. +#define IO_SEEK_END (2) ///< Use the end of the file. +#define IO_SEEK_CUR (3) ///< Use the current file offset. + +typedef uint64_t io_events_t; ///< Poll events type. +#define IO_POLL_READ (1 << 0) ///< File descriptor is ready to read. +#define IO_POLL_WRITE (1 << 1) ///< File descriptor is ready to write +#define IO_POLL_ERROR (1 << 2) ///< File descriptor caused an error. +#define IO_POLL_HUP (1 << 3) ///< File descriptor is closed. +#define IO_POLL_NVAL (1 << 4) ///< Invalid file descriptor. + +typedef uint32_t ioring_op_t; ///< I/O operation code type. +#define IORING_NOP 0 ///< No-op operation. +#define IORING_READ 1 ///< Read operation. +#define IORING_WRITE 2 ///< Write operation. +#define IORING_POLL 3 ///< Poll operation. +#define IORING_MAX 4 ///< The maximum number of operation. typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. - -#define SQE_REG0 (0) ///< The first register. -#define SQE_REG1 (1) ///< The second register. -#define SQE_REG2 (2) ///< The third register. -#define SQE_REG3 (3) ///< The fourth register. -#define SQE_REG4 (4) ///< The fifth register. -#define SQE_REG5 (5) ///< The sixth register. -#define SQE_REG6 (6) ///< The seventh register. -#define SQE_REG_NONE (7) ///< No register. -#define SQE_REGS_MAX (7) ///< The maximum number of registers. -#define SQE_REG_SHIFT (3) ///< The bitshift for each register specifier in a `sqe_flags_t`. -#define SQE_REG_MASK (0b111) ///< The bitmask for a register specifier in a `sqe_flags_t`. +#define SQE_REG0 (0) ///< The first register. +#define SQE_REG1 (1) ///< The second register. +#define SQE_REG2 (2) ///< The third register. +#define SQE_REG3 (3) ///< The fourth register. +#define SQE_REG4 (4) ///< The fifth register. +#define SQE_REG5 (5) ///< The sixth register. +#define SQE_REG6 (6) ///< The seventh register. +#define SQE_REG_NONE (7) ///< No register. +#define SQE_REGS_MAX (7) ///< The maximum number of registers. +#define SQE_REG_SHIFT (3) ///< The bitshift for each register specifier in a `sqe_flags_t`. +#define SQE_REG_MASK (0b111) ///< The bitmask for a register specifier in a `sqe_flags_t`. #define SQE_LOAD0 (0) ///< The offset to specify the register to load into the first argument. #define SQE_LOAD1 (SQE_LOAD0 + SQE_REG_SHIFT) ///< The offset to specify the register to load into the second argument. @@ -66,62 +67,61 @@ typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. #define _SQE_FLAGS (SQE_SAVE + SQE_REG_SHIFT) ///< The bitshift for where bit flags start in a `sqe_flags_t`. -#ifdef _KERNEL_ -/** - * The operation was created by the kernel, used internally by the kernel. - */ -#define SQE_KERNEL (1 << (_SQE_FLAGS)) -#endif - /** * Only process the next SQE when this one completes successfully) only applies within one `enter()` call. */ -#define SQE_LINK (1 << (_SQE_FLAGS + 2)) +#define SQE_LINK (1 << (_SQE_FLAGS)) /** * Like `SQE_LINK` but will process the next SQE even if this one fails. */ -#define SQE_HARDLINK (1 << (_SQE_FLAGS + 3)) +#define SQE_HARDLINK (1 << (_SQE_FLAGS + 1)) /** - * @brief Asynchronous submission queue entry (SQE). - * @struct sqe_t - * - * @warning It is the responsibility of userspace to ensure that any pointers - * passed to the kernel remain valid until the operation is complete. + * @brief Asynchronous submission queue entry (SQE) arguments. + * @struct sqe_args_t * - * @see kernel_io for more information for each possible verb. + * @see kernel_io for more information for each possible operation. */ -typedef struct sqe +typedef struct sqe_args { - verb_t verb; ///< Verb specifying the action to perform. - sqe_flags_t flags; ///< Submission flags. - clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. - void* data; ///< Private data for the operation, will be returned in the completion entry. - union - { - uint64_t arg0; + union { + uint64_t arg0; fd_t fd; }; - union - { + union { uint64_t arg1; void* buffer; - events_t events; + io_events_t events; }; - union - { + union { uint64_t arg2; size_t count; }; - union - { + union { uint64_t arg3; ssize_t offset; }; - union - { + union { uint64_t arg4; }; +} sqe_args_t; + +/** + * @brief Asynchronous submission queue entry (SQE). + * @struct sqe_t + * + * @warning It is the responsibility of userspace to ensure that any pointers + * passed to the kernel remain valid until the operation is complete. + * + * @see kernel_io for more information for each possible operation. + */ +typedef struct sqe +{ + ioring_op_t op; ///< The operation to perform. + sqe_flags_t flags; ///< Submission flags. + clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. + void* data; ///< Private data for the operation, will be returned in the completion entry. + sqe_args_t args; } sqe_t; #ifdef static_assert @@ -131,14 +131,14 @@ static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); /** * @brief Macro to create an asynchronous submission queue entry (SQE). * - * @param _verb Operation verb. + * @param _op The operation to perform. * @param _flags Submission flags. * @param _timeout Timeout for the operation, `CLOCKS_NEVER` for no timeout. * @param _data Private data for the operation. */ -#define SQE_CREATE(_verb, _flags, _timeout, _data) \ +#define SQE_CREATE(_op, _flags, _timeout, _data) \ { \ - .verb = (_verb), \ + .op = (_op), \ .flags = (_flags), \ .timeout = (_timeout), \ .data = (void*)(_data), \ @@ -152,14 +152,14 @@ static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); */ typedef struct cqe { - verb_t verb; ///< Verb specifying the action that was performed. - errno_t error; ///< Error code, if not equal to `EOK` an error occurred. - void* data; ///< Private data from the submission entry. + ioring_op_t op; ///< The operation that was performed. + errno_t error; ///< Error code, if not equal to `EOK` an error occurred. + void* data; ///< Private data from the submission entry. union { fd_t fd; size_t count; void* ptr; - events_t events; + io_events_t events; uint64_t _result; }; uint64_t _padding[1]; @@ -203,7 +203,7 @@ typedef struct ALIGNED(64) ioring_ctrl * @{ */ -typedef uint64_t io_id_t; ///< I/O ring ID type. +typedef uint64_t ioring_id_t; ///< I/O ring ID type. /** * @brief User I/O ring structure. @@ -214,7 +214,7 @@ typedef uint64_t io_id_t; ///< I/O ring ID type. typedef struct ioring { ioring_ctrl_t* ctrl; ///< Pointer to the shared control structure. - io_id_t id; ///< The ID of the ring. + ioring_id_t id; ///< The ID of the ring. sqe_t* squeue; ///< Pointer to the submission queue. size_t sentries; ///< Number of entries in the submission queue. size_t smask; ///< Bitmask for submission queue (sentries - 1). @@ -235,7 +235,7 @@ typedef struct ioring * @param centries Number of entries to allocate for the completion queue, must be a power of two. * @return On success, the ID of the new I/O ring. On failure, `ERR` and `errno` is set. */ -io_id_t setup(ioring_t* ring, void* address, size_t sentries, size_t centries); +ioring_id_t ioring_setup(ioring_t* ring, void* address, size_t sentries, size_t centries); /** * @brief System call to deinitialize the I/O ring. @@ -243,7 +243,7 @@ io_id_t setup(ioring_t* ring, void* address, size_t sentries, size_t centries); * @param id The ID of the I/O ring to teardown. * @return On success, `0`. On failure, `ERR` and `errno` is set. */ -uint64_t teardown(io_id_t id); +uint64_t ioring_teardown(ioring_id_t id); /** * @brief System call to notify the kernel of new submission queue entries (SQEs). @@ -253,7 +253,7 @@ uint64_t teardown(io_id_t id); * @param wait The minimum number of completion queue entries (CQEs) to wait for. * @return On success, the number of SQEs successfully processed. On failure, `ERR` and `errno` is set. */ -uint64_t enter(io_id_t id, size_t amount, size_t wait); +uint64_t ioring_enter(ioring_id_t id, size_t amount, size_t wait); /** * @brief Pushes a submission queue entry (SQE) to the submission queue. diff --git a/meta/doxy/Doxyfile b/meta/doxy/Doxyfile index 556ff35d1..4dd4f9b0e 100644 --- a/meta/doxy/Doxyfile +++ b/meta/doxy/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = "PatchworkOS" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "4fd4030b-dirty" +PROJECT_NUMBER = "94ae9881" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewers a diff --git a/src/kernel/fs/dentry.c b/src/kernel/fs/dentry.c index ede8de3b5..358fe0c8c 100644 --- a/src/kernel/fs/dentry.c +++ b/src/kernel/fs/dentry.c @@ -4,8 +4,8 @@ #include #include -#include #include +#include #include #include #include diff --git a/src/kernel/fs/devfs.c b/src/kernel/fs/devfs.c index 573cce03b..336fcdc4f 100644 --- a/src/kernel/fs/devfs.c +++ b/src/kernel/fs/devfs.c @@ -3,12 +3,12 @@ #include #include #include -#include #include #include #include #include #include +#include #include #include #include diff --git a/src/kernel/fs/file.c b/src/kernel/fs/file.c index 995977d25..36b5aa144 100644 --- a/src/kernel/fs/file.c +++ b/src/kernel/fs/file.c @@ -2,10 +2,10 @@ #include #include -#include #include #include #include +#include #include #include #include diff --git a/src/kernel/fs/filesystem.c b/src/kernel/fs/filesystem.c index d9bc1189a..7c9b12f65 100644 --- a/src/kernel/fs/filesystem.c +++ b/src/kernel/fs/filesystem.c @@ -4,12 +4,12 @@ #include #include #include -#include #include #include #include #include #include +#include #include #include #include @@ -99,8 +99,7 @@ static uint64_t filesystem_lookup(vnode_t* dir, dentry_t* dentry) continue; } - vnode_t* vnode = - vnode_new(dentry->superblock, VREG, NULL, &sbFileOps); + vnode_t* vnode = vnode_new(dentry->superblock, VREG, NULL, &sbFileOps); if (vnode == NULL) { return ERR; diff --git a/src/kernel/fs/netfs.c b/src/kernel/fs/netfs.c index 0c1219c03..70cf82aae 100644 --- a/src/kernel/fs/netfs.c +++ b/src/kernel/fs/netfs.c @@ -334,8 +334,7 @@ static uint64_t netfs_socket_lookup(vnode_t* dir, dentry_t* dentry) continue; } - vnode_t* vnode = vnode_new(dir->superblock, VREG, NULL, - socketFiles[i].fileOps); + vnode_t* vnode = vnode_new(dir->superblock, VREG, NULL, socketFiles[i].fileOps); if (vnode == NULL) { return ERR; @@ -553,8 +552,7 @@ static uint64_t netfs_family_lookup(vnode_t* dir, dentry_t* dentry) continue; } - vnode_t* vnode = vnode_new(dir->superblock, VREG, - &familyFileVnodeOps, familyFiles[i].fileOps); + vnode_t* vnode = vnode_new(dir->superblock, VREG, &familyFileVnodeOps, familyFiles[i].fileOps); if (vnode == NULL) { return ERR; @@ -711,8 +709,7 @@ static uint64_t netfs_lookup(vnode_t* dir, dentry_t* dentry) continue; } - vnode_t* vnode = - vnode_new(dir->superblock, VDIR, &familyVnodeOps, NULL); + vnode_t* vnode = vnode_new(dir->superblock, VDIR, &familyVnodeOps, NULL); if (vnode == NULL) { return ERR; diff --git a/src/kernel/fs/procfs.c b/src/kernel/fs/procfs.c index bb4782326..67e828aff 100644 --- a/src/kernel/fs/procfs.c +++ b/src/kernel/fs/procfs.c @@ -5,12 +5,12 @@ #include #include #include -#include #include #include #include #include #include +#include #include #include #include @@ -922,8 +922,7 @@ static uint64_t procfs_env_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, process->env.vars[i].key, - VREG)) + if (!ctx->emit(ctx, process->env.vars[i].key, VREG)) { return 0; } @@ -1056,8 +1055,7 @@ static uint64_t procfs_pid_lookup(vnode_t* dir, dentry_t* target) continue; } - vnode_t* vnode = vnode_new(dir->superblock, pidEntries[i].type, - pidEntries[i].vnodeOps, pidEntries[i].fileOps); + vnode_t* vnode = vnode_new(dir->superblock, pidEntries[i].type, pidEntries[i].vnodeOps, pidEntries[i].fileOps); if (vnode == NULL) { return 0; @@ -1148,8 +1146,8 @@ static uint64_t procfs_lookup(vnode_t* dir, dentry_t* target) continue; } - vnode_t* vnode = vnode_new(dir->superblock, procEntries[i].type, - procEntries[i].vnodeOps, procEntries[i].fileOps); + vnode_t* vnode = + vnode_new(dir->superblock, procEntries[i].type, procEntries[i].vnodeOps, procEntries[i].fileOps); if (vnode == NULL) { return ERR; @@ -1201,8 +1199,7 @@ static uint64_t procfs_iterate(dentry_t* dentry, dir_ctx_t* ctx) continue; } - if (!ctx->emit(ctx, procEntries[i].name, - procEntries[i].type)) + if (!ctx->emit(ctx, procEntries[i].name, procEntries[i].type)) { return 0; } diff --git a/src/kernel/fs/sysfs.c b/src/kernel/fs/sysfs.c index b7d1a72fc..7369abcd7 100644 --- a/src/kernel/fs/sysfs.c +++ b/src/kernel/fs/sysfs.c @@ -3,12 +3,12 @@ #include #include #include -#include #include #include #include #include #include +#include #include #include #include diff --git a/src/kernel/fs/tmpfs.c b/src/kernel/fs/tmpfs.c index e0a13e4f6..e4f06e77a 100644 --- a/src/kernel/fs/tmpfs.c +++ b/src/kernel/fs/tmpfs.c @@ -4,11 +4,11 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include diff --git a/src/kernel/fs/vfs.c b/src/kernel/fs/vfs.c index 38c47bc65..d95513751 100644 --- a/src/kernel/fs/vfs.c +++ b/src/kernel/fs/vfs.c @@ -6,10 +6,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include @@ -698,8 +698,7 @@ static uint64_t vfs_getdents_recursive_step(path_t* path, mode_t mode, getdents_ } ctx->currentOffset += sizeof(dirent_t); - if ((d->type == VDIR || d->type == VSYMLINK) && strcmp(d->path, ".") != 0 && - strcmp(d->path, "..") != 0) + if ((d->type == VDIR || d->type == VSYMLINK) && strcmp(d->path, ".") != 0 && strcmp(d->path, "..") != 0) { path_t childPath = PATH_CREATE(path->mount, path->dentry); PATH_DEFER(&childPath); @@ -960,7 +959,7 @@ uint64_t vfs_stat(const pathname_t* pathname, stat_t* buffer, process_t* process return ERR; } - /// @todo Reimplement this after the async system. + /// @todo Reimplement this after the async system. vnode_t* vnode = path.dentry->vnode; mutex_acquire(&vnode->mutex); buffer->number = 0; diff --git a/src/kernel/fs/vnode.c b/src/kernel/fs/vnode.c index e702fcd68..c26175473 100644 --- a/src/kernel/fs/vnode.c +++ b/src/kernel/fs/vnode.c @@ -48,8 +48,7 @@ static void vnode_ctor(void* ptr) static cache_t cache = CACHE_CREATE(cache, "vnode", sizeof(vnode_t), CACHE_LINE, vnode_ctor, NULL); -vnode_t* vnode_new(superblock_t* superblock, vtype_t type, const vnode_ops_t* ops, - const file_ops_t* fileOps) +vnode_t* vnode_new(superblock_t* superblock, vtype_t type, const vnode_ops_t* ops, const file_ops_t* fileOps) { if (superblock == NULL) { diff --git a/src/kernel/io/io.c b/src/kernel/io/io.c index b7bd1ac1e..6765c60b0 100644 --- a/src/kernel/io/io.c +++ b/src/kernel/io/io.c @@ -32,7 +32,7 @@ static inline void io_ctx_release(io_ctx_t* ctx) atomic_fetch_and(&ctx->flags, ~IO_CTX_BUSY); } -static inline uint64_t io_ctx_map(io_ctx_t* ctx, process_t* process, io_id_t id, ioring_t* userRing, void* address, +static inline uint64_t io_ctx_map(io_ctx_t* ctx, process_t* process, ioring_id_t id, ioring_t* userRing, void* address, size_t sentries, size_t centries) { ioring_t* kernelRing = &ctx->ring; @@ -220,7 +220,7 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) { while (true) { - irp_t* next = irp_next(irp); + irp_t* next = irp_chain_next(irp); if (next == NULL) { break; @@ -231,7 +231,7 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) } else { - irp_t* next = irp_next(irp); + irp_t* next = irp_chain_next(irp); if (next != NULL) { io_ctx_dispatch(next); @@ -251,31 +251,31 @@ static void io_ctx_dispatch(irp_t* irp) sqe_flags_t reg = (irp->flags >> SQE_LOAD0) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - irp->sqe.arg0 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + irp->arg0 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } reg = (irp->flags >> SQE_LOAD1) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - irp->sqe.arg1 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + irp->arg1 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } reg = (irp->flags >> SQE_LOAD2) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - irp->sqe.arg2 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + irp->arg2 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } reg = (irp->flags >> SQE_LOAD3) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - irp->sqe.arg3 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + irp->arg3 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } reg = (irp->flags >> SQE_LOAD4) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - irp->sqe.arg4 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + irp->arg4 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } irp_push(irp, io_ctx_complete, NULL); @@ -319,7 +319,7 @@ static uint64_t io_ctx_sqe_pop(io_ctx_t* ctx, io_ctx_notify_ctx_t* notify) list_push_back(¬ify->irps, &irp->entry); } - if (irp->sqe.flags & SQE_LINK || irp->sqe.flags & SQE_HARDLINK) + if (irp->flags & SQE_LINK || irp->flags & SQE_HARDLINK) { notify->link = irp; } @@ -385,7 +385,7 @@ uint64_t io_ctx_notify(io_ctx_t* ctx, size_t amount, size_t wait) return processed; } -SYSCALL_DEFINE(SYS_SETUP, io_id_t, ioring_t* userRing, void* address, size_t sentries, size_t centries) +SYSCALL_DEFINE(SYS_SETUP, ioring_id_t, ioring_t* userRing, void* address, size_t sentries, size_t centries) { if (userRing == NULL || sentries == 0 || centries == 0 || !IS_POW2(sentries) || !IS_POW2(centries)) { @@ -396,7 +396,7 @@ SYSCALL_DEFINE(SYS_SETUP, io_id_t, ioring_t* userRing, void* address, size_t sen process_t* process = process_current(); io_ctx_t* ctx = NULL; - io_id_t id = 0; + ioring_id_t id = 0; for (id = 0; id < ARRAY_SIZE(process->rings); id++) { io_ctx_flags_t expected = IO_CTX_NONE; @@ -423,7 +423,7 @@ SYSCALL_DEFINE(SYS_SETUP, io_id_t, ioring_t* userRing, void* address, size_t sen return id; } -SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, io_id_t id) +SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, ioring_id_t id) { process_t* process = process_current(); if (id >= ARRAY_SIZE(process->rings)) @@ -463,7 +463,7 @@ SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, io_id_t id) return 0; } -SYSCALL_DEFINE(SYS_ENTER, uint64_t, io_id_t id, size_t amount, size_t wait) +SYSCALL_DEFINE(SYS_ENTER, uint64_t, ioring_id_t id, size_t amount, size_t wait) { process_t* process = process_current(); if (id >= ARRAY_SIZE(process->rings)) diff --git a/src/kernel/io/irp.c b/src/kernel/io/irp.c index 4abe30ea2..639e8d244 100644 --- a/src/kernel/io/irp.c +++ b/src/kernel/io/irp.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -9,7 +10,7 @@ #include #include #include -#include +#include #include @@ -61,12 +62,11 @@ irp_pool_t* irp_pool_new(size_t size, process_t* process, void* ctx) irp->arg4 = 0; irp->res._raw = 0; mdl_init(&irp->mdl, NULL); - irp->sqe = (sqe_t){0}; irp->index = i; irp->err = EOK; irp->cpu = CPU_ID_INVALID; - irp->location = IRP_LOC_MAX; - for (size_t j = 0; j < IRP_LOC_MAX; j++) + irp->location = IRP_FRAME_MAX; + for (size_t j = 0; j < IRP_FRAME_MAX; j++) { irp->stack[j].ctx = NULL; irp->stack[j].complete = NULL; @@ -93,7 +93,7 @@ irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) } irp_t* irp = &pool->irps[idx]; - irp->location = IRP_LOC_MAX; + irp->location = IRP_FRAME_MAX; irp->next = POOL_IDX_MAX; irp->err = EINPROGRESS; irp->res._raw = 0; @@ -101,18 +101,34 @@ irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) if (sqe == NULL) { - irp->sqe = (sqe_t){0}; + irp->verb = 0; + irp->flags = 0; + irp->timeout = 0; + irp->data = NULL; + irp->arg0 = 0; + irp->arg1 = 0; + irp->arg2 = 0; + irp->arg3 = 0; + irp->arg4 = 0; irp->flags |= SQE_KERNEL; } else { - irp->sqe = *sqe; - irp->sqe.flags &= ~SQE_KERNEL; + irp->verb = sqe->verb; + irp->flags = sqe->flags; + irp->timeout = sqe->timeout; + irp->data = sqe->data; + irp->arg0 = sqe->arg0; + irp->arg1 = sqe->arg1; + irp->arg2 = sqe->arg2; + irp->arg3 = sqe->arg3; + irp->arg4 = sqe->arg4; + irp->flags &= ~SQE_KERNEL; } irp->next = POOL_IDX_MAX; irp->cpu = CPU_ID_INVALID; - for (size_t j = 0; j < IRP_LOC_MAX; j++) + for (size_t j = 0; j < IRP_FRAME_MAX; j++) { irp->stack[j].ctx = NULL; irp->stack[j].complete = NULL; @@ -126,7 +142,6 @@ irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) return irp; } - void irp_free(irp_t* irp) { if (irp == NULL) @@ -136,12 +151,10 @@ void irp_free(irp_t* irp) irp_timeout_remove(irp); - assert(irp->location == IRP_LOC_MAX); + assert(irp->location == IRP_FRAME_MAX); assert(irp->next == POOL_IDX_MAX); assert(irp->cpu == CPU_ID_INVALID); - verb_args_cleanup(irp); - mdl_t* next = irp->mdl.next; mdl_deinit(&irp->mdl); mdl_free_chain(next, free); diff --git a/src/kernel/io/verb.c b/src/kernel/io/verb.c index d2530d95a..49c6f5827 100644 --- a/src/kernel/io/verb.c +++ b/src/kernel/io/verb.c @@ -1,23 +1,16 @@ -#include -#include -#include #include +#include +#include #include #include +#include -void verb_args_cleanup(irp_t* irp) +static void verb_args_read_complete(irp_t* irp, void* ctx) { - switch (irp->verb) - { - case VERB_READ: - { - UNREF(irp->file); - irp->file = NULL; - } - break; - default: - break; - } + UNUSED(ctx); + + UNREF(irp->file); + irp->file = NULL; } static void verb_args_user(irp_t* irp) @@ -46,8 +39,8 @@ static void verb_args_user(irp_t* irp) irp->file = file; irp->buffer = &irp->mdl; - irp->count = irp->sqe.count; - irp->offset = irp->sqe.offset; + + irp_push(irp, verb_args_read_complete, NULL); } break; default: @@ -74,7 +67,7 @@ static void verb_dispatch_file(irp_t* irp) irp_error(irp, EBADF); return; } - break; + break; default: break; } diff --git a/src/kernel/mem/mdl.c b/src/kernel/mem/mdl.c index e033027fb..2566b3860 100644 --- a/src/kernel/mem/mdl.c +++ b/src/kernel/mem/mdl.c @@ -31,7 +31,7 @@ void mdl_deinit(mdl_t* mdl) mdl->capacity = 0; } -void mdl_free_chain(mdl_t* mdl, void(*free)(void*)) +void mdl_free_chain(mdl_t* mdl, void (*free)(void*)) { while (mdl != NULL) { diff --git a/src/kernel/proc/group.c b/src/kernel/proc/group.c index 4e1812efe..a05370787 100644 --- a/src/kernel/proc/group.c +++ b/src/kernel/proc/group.c @@ -1,7 +1,7 @@ #include #include -#include #include +#include #include #include #include diff --git a/src/libstd/user/common/syscalls.h b/src/libstd/user/common/syscalls.h index 89a8b4f7d..2c398b43b 100644 --- a/src/libstd/user/common/syscalls.h +++ b/src/libstd/user/common/syscalls.h @@ -291,12 +291,12 @@ static inline uint64_t _syscall_setup(ioring_t* ring, void* address, size_t sent return _SYSCALL4(uint64_t, SYS_SETUP, ioring_t*, ring, void*, address, size_t, sentries, size_t, centries); } -static inline uint64_t _syscall_teardown(io_id_t id) +static inline uint64_t _syscall_teardown(ioring_id_t id) { - return _SYSCALL1(uint64_t, SYS_TEARDOWN, io_id_t, id); + return _SYSCALL1(uint64_t, SYS_TEARDOWN, ioring_id_t, id); } -static inline uint64_t _syscall_enter(io_id_t id, size_t amount, size_t wait) +static inline uint64_t _syscall_enter(ioring_id_t id, size_t amount, size_t wait) { - return _SYSCALL3(uint64_t, SYS_ENTER, io_id_t, id, size_t, amount, size_t, wait); + return _SYSCALL3(uint64_t, SYS_ENTER, ioring_id_t, id, size_t, amount, size_t, wait); } \ No newline at end of file diff --git a/src/libstd/user/functions/ioring/enter.c b/src/libstd/user/functions/ioring/ioring_enter.c similarity index 75% rename from src/libstd/user/functions/ioring/enter.c rename to src/libstd/user/functions/ioring/ioring_enter.c index b4e2838ed..a6dcfda7c 100644 --- a/src/libstd/user/functions/ioring/enter.c +++ b/src/libstd/user/functions/ioring/ioring_enter.c @@ -2,7 +2,7 @@ #include "user/common/syscalls.h" -uint64_t enter(io_id_t id, size_t amount, size_t wait) +uint64_t ioring_enter(ioring_id_t id, size_t amount, size_t wait) { uint64_t result = _syscall_enter(id, amount, wait); if (result == ERR) diff --git a/src/libstd/user/functions/ioring/ioring_setup.c b/src/libstd/user/functions/ioring/ioring_setup.c new file mode 100644 index 000000000..07cb85dcd --- /dev/null +++ b/src/libstd/user/functions/ioring/ioring_setup.c @@ -0,0 +1,13 @@ +#include + +#include "user/common/syscalls.h" + +ioring_id_t ioring_setup(ioring_t* ring, void* address, size_t sentries, size_t centries) +{ + ioring_id_t result = _syscall_setup(ring, address, sentries, centries); + if (result == ERR) + { + errno = _syscall_errno(); + } + return result; +} \ No newline at end of file diff --git a/src/libstd/user/functions/ioring/teardown.c b/src/libstd/user/functions/ioring/ioring_teardown.c similarity index 82% rename from src/libstd/user/functions/ioring/teardown.c rename to src/libstd/user/functions/ioring/ioring_teardown.c index bbba4d2e6..d56c0ddba 100644 --- a/src/libstd/user/functions/ioring/teardown.c +++ b/src/libstd/user/functions/ioring/ioring_teardown.c @@ -2,7 +2,7 @@ #include "user/common/syscalls.h" -uint64_t teardown(io_id_t id) +uint64_t ioring_teardown(ioring_id_t id) { uint64_t result = _syscall_teardown(id); if (result == ERR) diff --git a/src/libstd/user/functions/ioring/setup.c b/src/libstd/user/functions/ioring/setup.c deleted file mode 100644 index 3cc9a0bd4..000000000 --- a/src/libstd/user/functions/ioring/setup.c +++ /dev/null @@ -1,13 +0,0 @@ -#include - -#include "user/common/syscalls.h" - -io_id_t setup(ioring_t* ring, void* address, size_t sentries, size_t centries) -{ - io_id_t result = _syscall_setup(ring, address, sentries, centries); - if (result == ERR) - { - errno = _syscall_errno(); - } - return result; -} \ No newline at end of file diff --git a/src/programs/utils/ringtest/main.c b/src/programs/utils/ringtest/main.c index 4aff330b6..942079c6a 100644 --- a/src/programs/utils/ringtest/main.c +++ b/src/programs/utils/ringtest/main.c @@ -10,7 +10,7 @@ int main() { printf("setting up ring test...\n"); ioring_t ring; - io_id_t id = setup(&ring, NULL, SENTRIES, CENTRIES); + ioring_id_t id = ioring_setup(&ring, NULL, SENTRIES, CENTRIES); if (id == ERR) { printf("failed to set up ring\n"); @@ -20,15 +20,15 @@ int main() memset(&ring.ctrl->regs, -1, sizeof(ring.ctrl->regs)); printf("pushing nop sqe to ring %llu...\n", ring.id); - sqe_t sqe = SQE_CREATE(VERB_NOP, SQE_HARDLINK | (SQE_REG0 << SQE_SAVE), CLOCKS_PER_SEC, 0x1234); + sqe_t sqe = SQE_CREATE(IORING_NOP, SQE_HARDLINK | (SQE_REG0 << SQE_SAVE), CLOCKS_PER_SEC, 0x1234); sqe_push(&ring, &sqe); printf("pushing nop sqe to ring %llu...\n", ring.id); - sqe = (sqe_t)SQE_CREATE(VERB_NOP, SQE_LINK, CLOCKS_PER_SEC, 0x5678); + sqe = (sqe_t)SQE_CREATE(IORING_NOP, SQE_LINK, CLOCKS_PER_SEC, 0x5678); sqe_push(&ring, &sqe); printf("entering ring...\n"); - if (enter(id, 2, 2) == ERR) + if (ioring_enter(id, 2, 2) == ERR) { printf("failed to enter ring\n"); return errno; @@ -40,7 +40,7 @@ int main() printf("cqe:\n"); printf("cqe data: %p\n", cqe.data); - printf("cqe verb: %d\n", cqe.verb); + printf("cqe op: %d\n", cqe.op); printf("cqe error: %s\n", strerror(cqe.error)); printf("cqe result: %llu\n", cqe._result); } @@ -52,6 +52,6 @@ int main() } printf("tearing down ring...\n"); - teardown(id); + ioring_teardown(id); return 0; } From 72e05163ce9283c5800ca7b7cc280a48e8161829 Mon Sep 17 00:00:00 2001 From: KN Date: Fri, 23 Jan 2026 22:17:17 +0100 Subject: [PATCH 22/23] feat(kernel:irp): implement IRP function numbers and vtable --- include/kernel/fs/file.h | 2 - include/kernel/fs/superblock.h | 3 +- include/kernel/fs/vnode.h | 6 +- include/kernel/io/io.h | 2 + include/kernel/io/irp.h | 490 ++++++++++++++++++--------------- include/kernel/io/verb.h | 82 ------ include/kernel/mem/mdl.h | 14 +- include/libstd/sys/ioring.h | 31 +-- src/kernel/fs/file.c | 1 - src/kernel/fs/superblock.c | 2 +- src/kernel/fs/vnode.c | 2 +- src/kernel/io/io.c | 67 +++-- src/kernel/io/irp.c | 343 ++++++++++++++--------- src/kernel/io/verb.c | 143 ---------- 14 files changed, 546 insertions(+), 642 deletions(-) delete mode 100644 include/kernel/io/verb.h delete mode 100644 src/kernel/io/verb.c diff --git a/include/kernel/fs/file.h b/include/kernel/fs/file.h index a778bea37..0d8cb8d17 100644 --- a/include/kernel/fs/file.h +++ b/include/kernel/fs/file.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include @@ -44,7 +43,6 @@ typedef struct file vnode_t* vnode; path_t path; const file_ops_t* ops; - const verb_table_t* verbs; void* data; } file_t; diff --git a/include/kernel/fs/superblock.h b/include/kernel/fs/superblock.h index e4e2a150b..611f828c1 100644 --- a/include/kernel/fs/superblock.h +++ b/include/kernel/fs/superblock.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -39,7 +40,7 @@ typedef struct superblock dentry_t* root; ///< Root dentry of the filesystem, should not take a reference. const superblock_ops_t* ops; const dentry_ops_t* dentryOps; - const verb_table_t* verbs; + const irp_vtable_t* vtable; filesystem_t* fs; /** * The number of mounts of this superblock. diff --git a/include/kernel/fs/vnode.h b/include/kernel/fs/vnode.h index 9f8e4190a..1aa6b048f 100644 --- a/include/kernel/fs/vnode.h +++ b/include/kernel/fs/vnode.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include @@ -31,7 +31,7 @@ typedef struct dentry dentry_t; * * ## Synchronization * - * vnodes have an additional purpose within the Virtual File System (VFS) as they act as the primary means of + * Vnodes have an additional purpose within the Virtual File System (VFS) as they act as the primary means of * synchronization. All dentries synchronize upon their vnodes mutex, open files synchronize upon the mutex of the * underlying vnode and operations like create, remove, etc synchronize upon the vnode mutex of the parent directory. * @@ -54,7 +54,7 @@ typedef struct vnode superblock_t* superblock; const vnode_ops_t* ops; const file_ops_t* fileOps; - const verb_table_t* verbs; + const irp_vtable_t* vtable; rcu_entry_t rcu; mutex_t mutex; } vnode_t; diff --git a/include/kernel/io/io.h b/include/kernel/io/io.h index 2c3dccfd8..7df9d9e97 100644 --- a/include/kernel/io/io.h +++ b/include/kernel/io/io.h @@ -19,6 +19,8 @@ * @todo The I/O ring system is primarily a design document for now as it remains very work in progress and subject to * change, currently being mostly unimplemented. * + * @todo Rewrite the Kernel-side I/O Ring Interface documentation to match the new system. + * * The I/O ring provides the core of all interfaces in PatchworkOS, where user-space submits Submission Queue Entries * (SQEs) and receives Completion Queue Entries (CQEs) from it, all within shared memory. Allowing for highly efficient * and asynchronous I/O operations, especially since PatchworkOS is designed to be natively asynchronous. diff --git a/include/kernel/io/irp.h b/include/kernel/io/irp.h index df7153d61..183c09cda 100644 --- a/include/kernel/io/irp.h +++ b/include/kernel/io/irp.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -17,6 +18,7 @@ typedef struct file file_t; typedef struct process process_t; +typedef struct vnode vnode_t; typedef struct irp irp_t; @@ -25,138 +27,37 @@ typedef struct irp irp_t; * @defgroup kernel_io_irp I/O Request Packet * @ingroup kernel_io * - * The I/O Request Packet is a lock-less, self-contained, layered, completion-based request that act as the primary - * structure used internally by the kernel for asynchronous operations. + * The I/O Request Packet (IRP) is a lock-less, self-contained, layered, continuation-passing request that acts as the + * primary structure used internally by the kernel for asynchronous operations. * - * The IRP system is designed to be generic enough to be used by any system in the kernel, however it is primarily used - * by the ring system. + * The IRP structure is designed to be generic enough to be used by any system in the kernel, however it is primarily + * used by the I/O ring system. * * @warning While the cancellation or completion of an IRP is thread safe, the setup of an IRP is not (as in pushing - * layers to it). As such, its up to the caller to ensure that only one thread is manipulating it during setup. + * layers to it). It is assumed that only one thread is manipulating an IRP during its setup. * * ## Completion * - * The IRP system is designed to allow multiple subsystems or functions to asynchronously "call" each other. - * - * Consider a traditional synchronous set of functions: - * - * ``` - * int fun_b(int val) - * { - * wait_until_data_ready(); - * return val * get_data(); - * } - * - * int fun_a(int val) - * { - * return fun_b(val) * 2; - * } - * - * int result = fun_a(5); - * // Do stuff with the result - * ``` - * - * The above may seem obvious, but in a asynchronous kernel we are not allowed to block, as such `fun_b()` should not be - * implemented this way. We must however still be able to achieve the same result. - * - * @note In practice its possible that more than just one layer needs to block, as such the IRP system needs to handle - * such cases as well. - * - * The idea behind IRPs is to effectively create a call stack which is detached from the actual CPU stack. Each frame in - * the IRP stack represents a function call with associated arguments and a "return address" in the form of a function - * pointer and "local variables" in the form of a context pointer. - * - * Using the IRP system, the above code would be written as: - * - * ``` - * void fun_b_interrupt(void) - * { - * irp_t* irp = pop_irp_from_list(); - * irp_frame_t* frame = irp_current(irp); - * irp->res.u64 = frame->args[0] * get_data(); - * irp_complete(irp); - * } - * - * void fun_b(irp_t* irp) - * { - * irp_frame_t* frame = irp_current(irp); - * if (can_complete_now()) - * { - * irp->res.u64 = frame->args[0] * get_data(); - * irp_complete(irp); - * return; - * } - * - * add_irp_to_list(irp); - * } - * - * void fun_a_return_address(irp_t* irp, void* ctx) - * { - * irp->res.u64 *= 2; - * irp_complete(irp); - * return; - * } - * - * void fun_a(irp_t* irp) - * { - * irp_frame_t* current = irp_current(irp); - * irp_frame_t* next = irp_next(irp); - * - * next->args[0] = current->args[0]; - * next->ret = fun_a_return_address; - * - * irp_call(irp, fun_b); - * } - * - * void my_return_address(irp_t* irp, void* ctx) - * { - * // Do stuff with the result in irp->result. - * irp_free(irp); - * } - * - * irp_t* irp = irp_new(pool); - * - * // Setup the "stack frame" for our function call. - * irp_frame_t* next = irp_next(irp); - * next->args[0] = 5; - * next->ret = my_return_address; - * - * // Call `fun_a()` and advance the stack frame. - * irp_call(irp, fun_a); - * - * // Continue executing even if fun_b() cannot complete immediately. - * ``` + * @todo Write the IRP documentation. * * ## Cancellation * - * The current owner of a IRP is responsible for handling cancellation. The current owner being the last subsystem to - * advance the IRP stack frame. + * Cancelling an IRP can intuitively be considered equivalent to forcing the last completion to fail, thus resulting in + * all the other completions to fail as well. * - * @note Intuitively, we can think of "cancelling" a IRP to be equivalent to causing the last completion to fail, thus - * resulting in all the other return addresses to fail as well. In the examples from the Completion section, it would be - * as though the synchronous `fun_c()` returned an error code instead of the data. + * The current owner of a IRP is responsible for handling cancellation by specifying a cancellation callback via + * `irp_set_cancel()`. The current owner being the last target of a `irp_call()` or `irp_call_direct()`. * - * The owner implements cancellation by calling `irp_set_cancel()` to set a cancellation callback. When an IRP is to be - * cancelled or timed out the cancellation callback will be invoked and atomically exchanged with a `IRP_CANCELLED` - * sentinel value. At which point the owner should cleanup the IRP and call `irp_complete(irp)`. + * When an IRP is cancelled or timed out the cancellation callback will be invoked and atomically exchanged with a + * `IRP_CANCELLED` sentinel value. At which point the owner should perform whatever logic is needed to cancel the IRP, + * if it is not possible immediately cancel the IRP it should return `ERR`. * - * It is not possible for the IRP system to perform this atomic exchange for return addresses. As such, to avoid race - * conditions while completing an IRP, it is vital that the owner of the IRP atomically exchanges the cancellation - * callback with the `IRP_CANCELLED` sentinel value. For the sake of convenience, the `irp_claim()` function is provided - * to perform this operation. - * - * Below is an example of how to safely implement a return address with an associated cancellation callback: + * Below is an example of how to implement a completion with an associated cancellation callback: * * ``` - * void my_return_address(irp_t* irp, void* ctx) + * void my_completion(irp_t* irp, void* ctx) * { - * if (!irp_claim(irp)) - * { - * // The IRP has already been cancelled, nothing to do here. - * return; - * } - * - * // Complete the IRP... + * // Do stuff... * * irp_complete(irp); * } @@ -174,7 +75,7 @@ typedef struct irp irp_t; * // We were explicitly cancelled. * } * - * // Perform cancellation... + * // Do stuff... * * uint64_t result = ...; * if (result == ERR) // If an error occurs we can reassign the cancellation callback. @@ -183,7 +84,6 @@ typedef struct irp irp_t; * return ERR; * } * - * irp_complete(irp); * return 0; * } * ``` @@ -193,12 +93,11 @@ typedef struct irp irp_t; * The IRP system uses the `err` field to indicate both the current state of the IRP as well as any error that may have * occurred during its processing. * - * Included below are a list of "special" values which the IRP system will recognize: + * Included below are a list of "special" values which the IRP system will set: * * - `EOK`: Operation completed successfully. * - `ECANCELED`: Operation was cancelled. * - `ETIMEDOUT`: Operation timed out. - * - `EINPROGRESS`: Operation is in a timeout queue. * * @see kernel_io for the ring system. * @see [Wikipedia](https://en.wikipedia.org/wiki/I/O_request_packet) for more information about IRPs. @@ -208,17 +107,17 @@ typedef struct irp irp_t; */ /** - * @brief IRP return address type. + * @brief IRP complete callback type. * - * @param irp Pointer to the IRP. - * @param ctx Context pointer. + * @param irp The IRP. + * @param ctx The contxt pointer from the `irp_frame_t` structure. */ -typedef void (*irp_ret_t)(irp_t* irp, void* ctx); +typedef void (*irp_complete_t)(irp_t* irp, void* ctx); /** * @brief IRP cancellation callback type. * - * @param irp Pointer to the IRP. + * @param irp The IRP. * @return On success, `0`. On failure, `ERR`. */ typedef uint64_t (*irp_cancel_t)(irp_t* irp); @@ -228,7 +127,16 @@ typedef uint64_t (*irp_cancel_t)(irp_t* irp); */ #define IRP_CANCELLED ((irp_cancel_t)1) -#define IRP_ARGS_MAX 5 ///< The maximum number of 64-bit arguments in a IRP frame. +typedef uint16_t irp_major_t; +#define IRP_MJ_READ 0 +#define IRP_MJ_WRITE 1 +#define IRP_MJ_POLL 2 +#define IRP_MJ_MAX 3 + +typedef uint16_t irp_minor_t; +#define IRP_MN_NORMAL 0 + +#define IRP_ARGS_MAX 4 ///< The maximum number of 64-bit arguments in an `irp_frame_t`. /** * @brief IRP stack frame structure. @@ -236,21 +144,35 @@ typedef uint64_t (*irp_cancel_t)(irp_t* irp); */ typedef struct irp_frame { - irp_ret_t ret; ///< Return Address. - void* local; ///< Local context. + irp_major_t major; ///< Major function number. + irp_minor_t minor; ///< Minor function number. + uint8_t _reserved[4]; + irp_complete_t complete; ///< Completion callback. + void* ctx; ///< Local context. + vnode_t* vnode; ///< Vnode associated with the operation. union { struct { - file_t* file; mdl_t* buffer; - size_t len; - ssize_t off; + uint64_t off; + void* data; + uint32_t len; + mode_t mode; } read; - uint64_t args[IRP_ARGS_MAX]; - sqe_args_t sqe; + struct + { + mdl_t* buffer; + uint64_t off; + void* data; + uint32_t len; + mode_t mode; + } write; + uint64_t args[IRP_ARGS_MAX]; ///< Generic arguments. }; } irp_frame_t; +static_assert(sizeof(irp_frame_t) == 64, "irp_frame_t is not 64 bytes"); + #define IRP_FRAME_MAX 5 ///< The maximum number of frames in a IRP stack. /** @@ -261,8 +183,6 @@ typedef struct irp_frame * no need for any allocation beyond the allocation of the IRP itself. This does require careful consideration of * padding, alignment and field sizes to keep it within a reasonable size. * - * @todo Consider raising `IRP_FRAME_MAX` if needed, it will add more cache lines tho. - * * @see kernel_io for more information for each possible verb. */ typedef struct ALIGNED(64) irp @@ -270,15 +190,11 @@ typedef struct ALIGNED(64) irp list_entry_t entry; ///< Used to store the IRP in various lists. list_entry_t timeoutEntry; ///< Used to store the IRP in the timeout queue. _Atomic(irp_cancel_t) cancel; ///< Cancellation callback, must be atomic to ensure an IRP is only cancelled once. + clock_t deadline; ///< The time at which the IRP will be removed from a timeout queue. union { - clock_t timeout; ///< The timeout starting from when the IRP is added to a timeout queue. - clock_t deadline; ///< The time at which the IRP will be removed from a timeout queue. - }; - void* data; ///< Private data for the operation, will be returned in the completion entry. - union { - uint64_t u64; - int64_t s64; size_t read; + size_t write; + uint64_t _raw; } res; mdl_t mdl; ///< A preallocated memory descriptor list for use by the IRP. pool_idx_t index; ///< Index of the IRP in its pool. @@ -287,9 +203,11 @@ typedef struct ALIGNED(64) irp uint8_t err; ///< The error code of the operation, also used to specify its current state. uint8_t frame; ///< The index of the current frame in the stack. irp_frame_t stack[IRP_FRAME_MAX]; ///< The frame stack, grows downwards. + sqe_t sqe; // A copy of the submission queue entry associated with this IRP. + uint8_t reserved[8]; } irp_t; -static_assert(sizeof(irp_t) == 448, "irp_t is not 448 bytes"); +static_assert(sizeof(irp_t) == 512, "irp_t is not 512 bytes"); /** * @brief Request pool structure. @@ -298,11 +216,26 @@ static_assert(sizeof(irp_t) == 448, "irp_t is not 448 bytes"); typedef struct irp_pool { void* ctx; - process_t* process; ///< Will only hold a reference if there is at least one allocated IRP. + process_t* process; ///< Will only hold a reference if there is at least one active IRP. + atomic_size_t active; pool_t pool; irp_t irps[]; } irp_pool_t; +/** + * @brief IRP function type. + */ +typedef void (*irp_func_t)(irp_t* irp); + +/** + * @brief IRP vtable structure. + * @struct irp_vtable_t + */ +typedef struct irp_vtable +{ + irp_func_t funcs[IRP_MJ_MAX]; +} irp_vtable_t; + /** * @brief Allocate a new IRP pool. * @@ -316,21 +249,22 @@ irp_pool_t* irp_pool_new(size_t size, process_t* process, void* ctx); /** * @brief Free a IRP pool. * - * @param pool Pointer to the IRP pool to free. + * @param pool The IRP pool to free. */ void irp_pool_free(irp_pool_t* pool); /** - * @brief Add an IRP to a per-CPU timeout queue with the timeout specified in the IRP. + * @brief Add an IRP to a per-CPU timeout queue. * - * @param irp Pointer to the IRP to add. + * @param irp The IRP to add. + * @param timeout The timeout of the IRP. */ -void irp_timeout_add(irp_t* irp); +void irp_timeout_add(irp_t* irp, clock_t timeout); /** * @brief Remove an IRP from its per-CPU timeout queue. * - * @param irp Pointer to the IRP to remove. + * @param irp The IRP to remove. */ void irp_timeout_remove(irp_t* irp); @@ -345,23 +279,28 @@ void irp_timeouts_check(void); * The pool that the IRP was allocated from, and its context, can be retrieved using the `irp_get_pool()` * function. * - * @param pool Pointer to the IRP pool. + * @param pool The IRP pool. * @return On success, a pointer to the allocated IRP. On failure, `NULL` and `errno` is set. */ irp_t* irp_new(irp_pool_t* pool); /** - * @brief Free a IRP back to its pool. + * @brief Retrieve a memory descriptor list and associate it with an IRP. + * + * All MDLs associated with a IRP will be cleaned up when finished. * - * @param irp Pointer to the IRP to free. + * @param irp The IRP to associate the MDL with. + * @param addr The virtual address of the memory region to add to the MDL, or `NULL` for a blank MDL. + * @param size The size of the memory region. + * @return On success, a pointer to the MDL. On failure, `NULL` and `errno` is set. */ -void irp_free(irp_t* irp); +mdl_t* irp_get_mdl(irp_t* irp, const void* addr, size_t size); /** * @brief Retrieve the IRP pool that an IRP was allocated from. * - * @param irp Pointer to the IRP. - * @return Pointer to the IRP pool. + * @param irp The IRP. + * @return The IRP pool. */ static inline irp_pool_t* irp_get_pool(irp_t* irp) { @@ -371,8 +310,8 @@ static inline irp_pool_t* irp_get_pool(irp_t* irp) /** * @brief Retrieve the context of the IRP pool that an IRP was allocated from. * - * @param irp Pointer to the IRP. - * @return Pointer to the context. + * @param irp The IRP. + * @return The context. */ static inline void* irp_get_ctx(irp_t* irp) { @@ -382,8 +321,8 @@ static inline void* irp_get_ctx(irp_t* irp) /** * @brief Retrieve the process that owns an IRP. * - * @param irp Pointer to the IRP. - * @return Pointer to the process. + * @param irp The IRP. + * @return The process. */ static inline process_t* irp_get_process(irp_t* irp) { @@ -391,60 +330,58 @@ static inline process_t* irp_get_process(irp_t* irp) } /** - * @brief Set the cancellation callback for an IRP. + * @brief Retrieve the next IRP in a chain and advance the chain. * - * @param irp Pointer to the IRP. - * @param cancel The cancellation callback. - * @return The previous cancellation callback. + * @param irp The current IRP. + * @return The next IRP, or `NULL` if there is no next IRP. */ -static inline irp_cancel_t irp_set_cancel(irp_t* irp, irp_cancel_t cancel) +static inline irp_t* irp_chain_next(irp_t* irp) { - irp_cancel_t expected = atomic_load(&irp->cancel); - while (expected != IRP_CANCELLED) + irp_pool_t* pool = irp_get_pool(irp); + if (irp->next == POOL_IDX_MAX) { - if (atomic_compare_exchange_weak(&irp->cancel, &expected, cancel)) - { - return expected; - } + return NULL; } - return IRP_CANCELLED; + + irp_t* next = &pool->irps[irp->next]; + irp->next = next->next; + next->next = POOL_IDX_MAX; + return next; } /** - * @brief Attempt to claim an IRP for completion. + * @brief Attempt to cancel an IRP. * - * @param irp Pointer to the IRP. - * @return `true` if the IRP was successfully claimed, `false` if it was already cancelled. + * @param irp The IRP to cancel. + * @return On success, `0`. On failure, `ERR` and `errno` is set. */ -static inline bool irp_claim(irp_t* irp) -{ - return irp_set_cancel(irp, NULL) != IRP_CANCELLED; -} +uint64_t irp_cancel(irp_t* irp); /** - * @brief Retrieve the next IRP in a chain and clear the next field. + * @brief Set the cancellation callback for an IRP. * - * @param irp Pointer to the current IRP. - * @return Pointer to the next IRP, or `NULL` if there is no next IRP. + * @param irp The IRP. + * @param cancel The cancellation callback. + * @return The previous cancellation callback. */ -static inline irp_t* irp_chain_next(irp_t* irp) +static inline irp_cancel_t irp_set_cancel(irp_t* irp, irp_cancel_t cancel) { - irp_pool_t* pool = irp_get_pool(irp); - if (irp->next == POOL_IDX_MAX) + irp_cancel_t expected = atomic_load(&irp->cancel); + while (expected != IRP_CANCELLED) { - return NULL; + if (atomic_compare_exchange_weak(&irp->cancel, &expected, cancel)) + { + return expected; + } } - - irp_t* next = &pool->irps[irp->next]; - irp->next = POOL_IDX_MAX; - return next; + return IRP_CANCELLED; } /** * @brief Retrieve the current frame in the IRP stack. * - * @param irp Pointer to the IRP to retrieve the frame from. - * @return Pointer to the current frame. + * @param irp The IRP to retrieve the frame from. + * @return The current frame. */ static inline irp_frame_t* irp_current(irp_t* irp) { @@ -455,8 +392,8 @@ static inline irp_frame_t* irp_current(irp_t* irp) /** * @brief Retrieve the next frame in the IRP stack. * - * @param irp Pointer to the IRP to retrieve the frame from. - * @return Pointer to the next frame, or `NULL` if we are at the bottom of the stack. + * @param irp The IRP to retrieve the frame from. + * @return The next frame, or `NULL` if we are at the bottom of the stack. */ static inline irp_frame_t* irp_next(irp_t* irp) { @@ -466,46 +403,96 @@ static inline irp_frame_t* irp_next(irp_t* irp) } return &irp->stack[irp->frame - 1]; } + /** - * @brief Call a function with an IRP, advancing the frame in the IRP stack. - * - * @param irp Pointer to the IRP. - * @param func The function to call. + * @brief Copy the current frame in the IRP stack to the next. + * + * @param irp The IRP. */ -static inline void irp_call(irp_t* irp, void (*func)(irp_t* irp)) +static inline void irp_copy_to_next(irp_t* irp) { - assert(irp->frame > 0); - irp->frame--; - func(irp); + irp_frame_t* current = irp_current(irp); + irp_frame_t* next = irp_next(irp); + + if (next->vnode != NULL) + { + UNREF(next->vnode); + next->vnode = NULL; + } + + *next = *current; + next->vnode = NULL; + next->complete = NULL; + next->ctx = NULL; } /** - * @brief Complete the current frame in the IRP stack. + * @brief Skip the current stack frame, meaning the next call will run in the same stack frame. * - * @param irp Pointer to the IRP to complete. + * @param irp The IRP. */ -static inline void irp_complete(irp_t* irp) +static inline void irp_skip(irp_t* irp) { - if (irp->frame == IRP_FRAME_MAX) + irp_frame_t* frame = irp_current(irp); + if (frame->vnode != NULL) { - return; + UNREF(frame->vnode); + frame->vnode = NULL; } - - irp_frame_t* loc = irp_current(irp); + assert(irp->frame < IRP_FRAME_MAX); irp->frame++; +} - if (irp->frame == IRP_FRAME_MAX) - { - irp_timeout_remove(irp); - } +/** + * @brief Send an IRP to a specified vnode. + * + * Will advance the IRP stack. + * + * @param irp The IRP to send. + * @param vnode The vnode to associated with the next IRP stack frame. + */ +void irp_call(irp_t* irp, vnode_t* vnode); + +/** + * @brief Send an IRP to a specified function directly. + * + * Will advance the IRP stack. + * + * @param irp The IRP to send. + * @param func The function to call. + */ +void irp_call_direct(irp_t* irp, irp_func_t func); + +/** + * @brief Complete the current frame in the IRP stack. + * + * If the current frame does not have a completion, it will automatically complete the next frame in the stack. + * + * If the last frame is reached, the IRP is considered finished. Which will causing its resources to be freed and for + * the IRP to be returned to its pool. + * + * @param irp The IRP to complete. + */ +void irp_complete(irp_t* irp); - loc->ret(irp, loc->local); +/** + * @brief Set the completion callback and context for the next frame in the IRP stack. + * + * @param irp The IRP to set. + * @param complete The completion callback. + * @param ctx The context pointer to pass to the completion callback. + */ +static inline void irp_set_complete(irp_t* irp, irp_complete_t complete, void* ctx) +{ + irp_frame_t* next = irp_next(irp); + next->complete = complete; + next->ctx = ctx; } /** * @brief Helper to set an error code and complete the IRP. - * - * @param irp Pointer to the IRP. + * + * @param irp The IRP to error. * @param err The error code to set. */ static inline void irp_error(irp_t* irp, uint8_t err) @@ -515,11 +502,72 @@ static inline void irp_error(irp_t* irp, uint8_t err) } /** - * @brief Attempt to cancel an IRP. - * - * @param irp Pointer to the IRP to cancel. - * @return On success, `0`. On failure, `ERR` and `errno` is set. + * @brief Prepares the next IRP stack frame for a generic operation. + * + * @param irp The IRP. + * @param major The major function number. + * @param arg0 Generic argument 0. + * @param arg1 Generic argument 1. + * @param arg2 Generic argument 2. + * @param arg3 Generic argument 3. */ -uint64_t irp_cancel(irp_t* irp); +static inline void irp_prepare_generic(irp_t* irp, irp_major_t major, uint64_t arg0, uint64_t arg1, uint64_t arg2, + uint64_t arg3) +{ + irp_frame_t* next = irp_next(irp); + assert(next != NULL); + + next->major = major; + next->args[0] = arg0; + next->args[1] = arg1; + next->args[2] = arg2; + next->args[3] = arg3; +} + +/** + * @brief Prepares the next IRP stack frame for a read operation. + * + * @param irp The IRP. + * @param buffer The memory descriptor list to read into. + * @param data The buffer to read into. + * @param off The offset in the file to read from. + * @param len The number of bytes to read. + * @param mode The mode. + */ +static inline void irp_prepare_read(irp_t* irp, mdl_t* buffer, void* data, uint64_t off, uint32_t len, mode_t mode) +{ + irp_frame_t* next = irp_next(irp); + assert(next != NULL); + + next->major = IRP_MJ_READ; + next->read.buffer = buffer; + next->read.data = data; + next->read.off = off; + next->read.len = len; + next->read.mode = mode; +} + +/** + * @brief Prepares the next IRP stack frame for a write operation. + * + * @param irp The IRP. + * @param buffer The memory descriptor list to write from. + * @param data The buffer to write from. + * @param off The offset in the file to write to. + * @param len The number of bytes to write. + * @param mode The mode. + */ +static inline void irp_prepare_write(irp_t* irp, mdl_t* buffer, void* data, uint64_t off, uint32_t len, mode_t mode) +{ + irp_frame_t* next = irp_next(irp); + assert(next != NULL); + + next->major = IRP_MJ_WRITE; + next->write.buffer = buffer; + next->write.data = data; + next->write.off = off; + next->write.len = len; + next->write.mode = mode; +} /** @} */ \ No newline at end of file diff --git a/include/kernel/io/verb.h b/include/kernel/io/verb.h deleted file mode 100644 index b5ed3fe41..000000000 --- a/include/kernel/io/verb.h +++ /dev/null @@ -1,82 +0,0 @@ -#pragma once - -#include - -#include - -/** - * @brief I/O Request Packet verbs. - * @defgroup kernel_io_verb I/O Request Packet Verbs - * @ingroup kernel_io - * - * @{ - */ - -/** - * @brief Verb function type. - * - * @param irp Pointer to the IRP. - */ -typedef void (*verb_func_t)(irp_t* irp); - -/** - * @brief Verb table structure. - * @struct verb_table_t - */ -typedef struct verb_table -{ - verb_func_t handlers[VERB_MAX]; -} verb_table_t; - -/** - * @brief Dispatch an IRP to the appropriate verb handler. - * - * If the IRP is a user IRP, the arguments will be parsed before invoking the handler. - * - * @param irp Pointer to the IRP. - */ -void verb_dispatch(irp_t* irp); - -/** - * @brief Invoke the appropriate verb handler from a verb table. - * - * @param irp Pointer to the IRP. - * @param table Pointer to the verb table. - * @return `true` if the IRP was completed, `false` otherwise. - */ -static inline bool verb_invoke(irp_t* irp, const verb_table_t* table) -{ - if (UNLIKELY(irp->verb >= VERB_MAX)) - { - irp_error(irp, EINVAL); - return true; - } - - if (table == NULL) - { - return false; - } - - verb_func_t handler = table->handlers[irp->verb]; - if (handler == NULL) - { - return false; - } - - handler(irp); - return true; -} - -/** - * @brief Execute an IRP synchronously. - * - * This function will dispatch the IRP and blocks the current thread until the operation is complete. - * - * @warning This function should only be used when the alternative of using asynchronous operations is simply not worth - * the complexity. For example, while loading modules. - * - * @param irp Pointer to the IRP to execute. - */ -void verb_run(irp_t* irp); - -/** @} */ \ No newline at end of file diff --git a/include/kernel/mem/mdl.h b/include/kernel/mem/mdl.h index 9822662a2..cc9a05fc7 100644 --- a/include/kernel/mem/mdl.h +++ b/include/kernel/mem/mdl.h @@ -63,19 +63,19 @@ typedef struct mdl /** * @brief Initialize a Memory Descriptor List. * - * @param mdl Pointer to the MDL. + * @param next Pointer to the MDL. * @param prev Pointer to the previous MDL in the chain, or `NULL` if none. */ -static inline void mdl_init(mdl_t* mdl, mdl_t* prev) +static inline void mdl_init(mdl_t* next, mdl_t* prev) { if (prev != NULL) { - prev->next = mdl; + prev->next = next; } - mdl->next = NULL; - mdl->segments = mdl->small; - mdl->amount = 0; - mdl->capacity = MDL_SEGS_SMALL_MAX; + next->next = NULL; + next->segments = next->small; + next->amount = 0; + next->capacity = MDL_SEGS_SMALL_MAX; } /** diff --git a/include/libstd/sys/ioring.h b/include/libstd/sys/ioring.h index 2f3e0b1cc..58738ebef 100644 --- a/include/libstd/sys/ioring.h +++ b/include/libstd/sys/ioring.h @@ -77,13 +77,20 @@ typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. #define SQE_HARDLINK (1 << (_SQE_FLAGS + 1)) /** - * @brief Asynchronous submission queue entry (SQE) arguments. - * @struct sqe_args_t + * @brief Asynchronous submission queue entry (SQE). + * @struct sqe_t + * + * @warning It is the responsibility of userspace to ensure that any pointers + * passed to the kernel remain valid until the operation is complete. * * @see kernel_io for more information for each possible operation. */ -typedef struct sqe_args +typedef struct sqe { + clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. + void* data; ///< Private data for the operation, will be returned in the completion entry. + ioring_op_t op; ///< The operation to perform. + sqe_flags_t flags; ///< Submission flags. union { uint64_t arg0; fd_t fd; @@ -104,24 +111,6 @@ typedef struct sqe_args union { uint64_t arg4; }; -} sqe_args_t; - -/** - * @brief Asynchronous submission queue entry (SQE). - * @struct sqe_t - * - * @warning It is the responsibility of userspace to ensure that any pointers - * passed to the kernel remain valid until the operation is complete. - * - * @see kernel_io for more information for each possible operation. - */ -typedef struct sqe -{ - ioring_op_t op; ///< The operation to perform. - sqe_flags_t flags; ///< Submission flags. - clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. - void* data; ///< Private data for the operation, will be returned in the completion entry. - sqe_args_t args; } sqe_t; #ifdef static_assert diff --git a/src/kernel/fs/file.c b/src/kernel/fs/file.c index 36b5aa144..c02cdc882 100644 --- a/src/kernel/fs/file.c +++ b/src/kernel/fs/file.c @@ -69,7 +69,6 @@ file_t* file_new(const path_t* path, mode_t mode) file->vnode = REF(path->dentry->vnode); file->path = PATH_CREATE(path->mount, path->dentry); file->ops = path->dentry->vnode->fileOps; - file->verbs = file->vnode->verbs; file->data = NULL; return file; } diff --git a/src/kernel/fs/superblock.c b/src/kernel/fs/superblock.c index 63bd7f4da..a69dd9848 100644 --- a/src/kernel/fs/superblock.c +++ b/src/kernel/fs/superblock.c @@ -54,7 +54,7 @@ superblock_t* superblock_new(filesystem_t* fs, const superblock_ops_t* ops, cons superblock->root = NULL; superblock->ops = ops; superblock->dentryOps = dentryOps; - superblock->verbs = NULL; + superblock->vtable = NULL; superblock->fs = fs; atomic_init(&superblock->mountCount, 0); diff --git a/src/kernel/fs/vnode.c b/src/kernel/fs/vnode.c index c26175473..c49c067e6 100644 --- a/src/kernel/fs/vnode.c +++ b/src/kernel/fs/vnode.c @@ -68,7 +68,7 @@ vnode_t* vnode_new(superblock_t* superblock, vtype_t type, const vnode_ops_t* op vnode->superblock = REF(superblock); vnode->ops = ops; vnode->fileOps = fileOps; - vnode->verbs = superblock->verbs; + vnode->vtable = NULL; return vnode; } diff --git a/src/kernel/io/io.c b/src/kernel/io/io.c index 6765c60b0..d38ffb932 100644 --- a/src/kernel/io/io.c +++ b/src/kernel/io/io.c @@ -192,7 +192,7 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) io_ctx_t* ctx = irp_get_ctx(irp); ioring_t* ring = &ctx->ring; - sqe_flags_t reg = (irp->flags >> SQE_SAVE) & SQE_REG_MASK; + sqe_flags_t reg = (irp->sqe.flags >> SQE_SAVE) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { atomic_store_explicit(&ring->ctrl->regs[reg], irp->res._raw, memory_order_release); @@ -208,15 +208,15 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) } cqe_t* cqe = &ring->cqueue[tail & ring->cmask]; - cqe->verb = irp->verb; + cqe->op = irp->sqe.op; cqe->error = irp->err; - cqe->data = irp->data; + cqe->data = irp->sqe.data; cqe->_result = irp->res._raw; atomic_store_explicit(&ring->ctrl->ctail, tail + 1, memory_order_release); wait_unblock(&ctx->waitQueue, WAIT_ALL, EOK); - if (irp->err != EOK && !(irp->flags & SQE_HARDLINK)) + if (irp->err != EOK && !(irp->sqe.flags & SQE_HARDLINK)) { while (true) { @@ -226,7 +226,7 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) break; } - irp_free(next); + irp_complete(next); } } else @@ -234,11 +234,18 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) irp_t* next = irp_chain_next(irp); if (next != NULL) { - io_ctx_dispatch(next); + irp_set_complete(next, io_ctx_complete, NULL); + irp_call_direct(next, io_ctx_dispatch); } } - irp_free(irp); + irp_complete(irp); +} + +static uint64_t nop_cancel(irp_t* irp) +{ + irp_complete(irp); + return 0; } static void io_ctx_dispatch(irp_t* irp) @@ -248,38 +255,46 @@ static void io_ctx_dispatch(irp_t* irp) // Ugly but the alternative is a super messy SQE structure. - sqe_flags_t reg = (irp->flags >> SQE_LOAD0) & SQE_REG_MASK; + sqe_flags_t reg = (irp->sqe.flags >> SQE_LOAD0) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - irp->arg0 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + irp->sqe.arg0 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } - reg = (irp->flags >> SQE_LOAD1) & SQE_REG_MASK; + reg = (irp->sqe.flags >> SQE_LOAD1) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - irp->arg1 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + irp->sqe.arg1 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } - reg = (irp->flags >> SQE_LOAD2) & SQE_REG_MASK; + reg = (irp->sqe.flags >> SQE_LOAD2) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - irp->arg2 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + irp->sqe.arg2 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } - reg = (irp->flags >> SQE_LOAD3) & SQE_REG_MASK; + reg = (irp->sqe.flags >> SQE_LOAD3) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - irp->arg3 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + irp->sqe.arg3 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } - reg = (irp->flags >> SQE_LOAD4) & SQE_REG_MASK; + reg = (irp->sqe.flags >> SQE_LOAD4) & SQE_REG_MASK; if (reg != SQE_REG_NONE) { - irp->arg4 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); + irp->sqe.arg4 = atomic_load_explicit(&ring->ctrl->regs[reg], memory_order_acquire); } - irp_push(irp, io_ctx_complete, NULL); - verb_dispatch(irp); + switch (irp->sqe.op) + { + case IORING_NOP: + irp_set_cancel(irp, nop_cancel); + irp_timeout_add(irp, irp->sqe.timeout); + break; + default: + irp_error(irp, EINVAL); + break; + } } typedef struct @@ -291,6 +306,7 @@ typedef struct static uint64_t io_ctx_sqe_pop(io_ctx_t* ctx, io_ctx_notify_ctx_t* notify) { ioring_t* ring = &ctx->ring; + uint32_t stail = atomic_load_explicit(&ring->ctrl->stail, memory_order_acquire); uint32_t shead = atomic_load_explicit(&ring->ctrl->shead, memory_order_relaxed); @@ -300,12 +316,12 @@ static uint64_t io_ctx_sqe_pop(io_ctx_t* ctx, io_ctx_notify_ctx_t* notify) return ERR; } - sqe_t* sqe = &ring->squeue[shead & ring->smask]; - irp_t* irp = irp_new(ctx->irps, sqe); + irp_t* irp = irp_new(ctx->irps); if (irp == NULL) { return ERR; } + irp->sqe = ring->squeue[shead & ring->smask]; atomic_store_explicit(&ring->ctrl->shead, shead + 1, memory_order_release); @@ -319,7 +335,7 @@ static uint64_t io_ctx_sqe_pop(io_ctx_t* ctx, io_ctx_notify_ctx_t* notify) list_push_back(¬ify->irps, &irp->entry); } - if (irp->flags & SQE_LINK || irp->flags & SQE_HARDLINK) + if (irp->sqe.flags & SQE_LINK || irp->sqe.flags & SQE_HARDLINK) { notify->link = irp; } @@ -347,13 +363,12 @@ uint64_t io_ctx_notify(io_ctx_t* ctx, size_t amount, size_t wait) return ERR; } - size_t processed = 0; - io_ctx_notify_ctx_t notify = { .irps = LIST_CREATE(notify.irps), .link = NULL, }; + size_t processed = 0; while (processed < amount) { if (io_ctx_sqe_pop(ctx, ¬ify) == ERR) @@ -366,7 +381,9 @@ uint64_t io_ctx_notify(io_ctx_t* ctx, size_t amount, size_t wait) while (!list_is_empty(¬ify.irps)) { irp_t* irp = CONTAINER_OF(list_pop_front(¬ify.irps), irp_t, entry); - io_ctx_dispatch(irp); + + irp_set_complete(irp, io_ctx_complete, NULL); + irp_call_direct(irp, io_ctx_dispatch); } if (wait == 0) diff --git a/src/kernel/io/irp.c b/src/kernel/io/irp.c index 639e8d244..185819377 100644 --- a/src/kernel/io/irp.c +++ b/src/kernel/io/irp.c @@ -1,15 +1,16 @@ #include #include #include -#include #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -45,32 +46,12 @@ irp_pool_t* irp_pool_new(size_t size, process_t* process, void* ctx) pool->ctx = ctx; pool->process = process; - for (pool_idx_t i = 0; i < (pool_idx_t)size; i++) + atomic_init(&pool->active, 0); + memset(&pool->irps, 0, sizeof(irp_t) * size); + for (size_t i = 0; i < size; i++) { irp_t* irp = &pool->irps[i]; - list_entry_init(&irp->entry); - list_entry_init(&irp->timeoutEntry); - atomic_init(&irp->cancel, NULL); - irp->verb = VERB_MAX; - irp->flags = 0; - irp->timeout = CLOCKS_NEVER; - irp->data = NULL; - irp->arg0 = 0; - irp->arg1 = 0; - irp->arg2 = 0; - irp->arg3 = 0; - irp->arg4 = 0; - irp->res._raw = 0; - mdl_init(&irp->mdl, NULL); irp->index = i; - irp->err = EOK; - irp->cpu = CPU_ID_INVALID; - irp->location = IRP_FRAME_MAX; - for (size_t j = 0; j < IRP_FRAME_MAX; j++) - { - irp->stack[j].ctx = NULL; - irp->stack[j].complete = NULL; - } } pool_init(&pool->pool, pool->irps, size, sizeof(irp_t), offsetof(irp_t, next)); @@ -83,115 +64,9 @@ void irp_pool_free(irp_pool_t* pool) free(pool); } -irp_t* irp_new(irp_pool_t* pool, sqe_t* sqe) -{ - pool_idx_t idx = pool_alloc(&pool->pool); - if (idx == POOL_IDX_MAX) - { - errno = ENOSPC; - return NULL; - } - - irp_t* irp = &pool->irps[idx]; - irp->location = IRP_FRAME_MAX; - irp->next = POOL_IDX_MAX; - irp->err = EINPROGRESS; - irp->res._raw = 0; - atomic_store_explicit(&irp->cancel, NULL, memory_order_relaxed); - - if (sqe == NULL) - { - irp->verb = 0; - irp->flags = 0; - irp->timeout = 0; - irp->data = NULL; - irp->arg0 = 0; - irp->arg1 = 0; - irp->arg2 = 0; - irp->arg3 = 0; - irp->arg4 = 0; - irp->flags |= SQE_KERNEL; - } - else - { - irp->verb = sqe->verb; - irp->flags = sqe->flags; - irp->timeout = sqe->timeout; - irp->data = sqe->data; - irp->arg0 = sqe->arg0; - irp->arg1 = sqe->arg1; - irp->arg2 = sqe->arg2; - irp->arg3 = sqe->arg3; - irp->arg4 = sqe->arg4; - irp->flags &= ~SQE_KERNEL; - } - - irp->next = POOL_IDX_MAX; - irp->cpu = CPU_ID_INVALID; - for (size_t j = 0; j < IRP_FRAME_MAX; j++) - { - irp->stack[j].ctx = NULL; - irp->stack[j].complete = NULL; - } - - if (atomic_load(&pool->pool.used) == 1) - { - REF(pool->process); - } - - return irp; -} - -void irp_free(irp_t* irp) +void irp_timeout_add(irp_t* irp, clock_t timeout) { - if (irp == NULL) - { - return; - } - - irp_timeout_remove(irp); - - assert(irp->location == IRP_FRAME_MAX); - assert(irp->next == POOL_IDX_MAX); - assert(irp->cpu == CPU_ID_INVALID); - - mdl_t* next = irp->mdl.next; - mdl_deinit(&irp->mdl); - mdl_free_chain(next, free); - - irp_pool_t* pool = irp_get_pool(irp); - pool_free(&pool->pool, irp->index); - - if (atomic_load(&pool->pool.used) == 0) - { - UNREF(pool->process); - } -} - -uint64_t irp_cancel(irp_t* irp) -{ - irp_cancel_t handler = atomic_exchange(&irp->cancel, IRP_CANCELLED); - if (handler == IRP_CANCELLED) - { - errno = EBUSY; - return ERR; - } - - if (handler == NULL) - { - errno = EBUSY; - return ERR; - } - - irp_timeout_remove(irp); - - irp->err = ECANCELED; - return handler(irp); -} - -void irp_timeout_add(irp_t* irp) -{ - if (irp->timeout == CLOCKS_NEVER) + if (timeout == CLOCKS_NEVER) { return; } @@ -202,7 +77,7 @@ void irp_timeout_add(irp_t* irp) irp->cpu = SELF->id; clock_t now = clock_uptime(); - irp->deadline = CLOCKS_DEADLINE(irp->timeout, now); + irp->deadline = CLOCKS_DEADLINE(timeout, now); irp_t* entry; LIST_FOR_EACH(entry, &ctx->timeouts, timeoutEntry) @@ -231,7 +106,7 @@ void irp_timeout_remove(irp_t* irp) assert(ctx != NULL); LOCK_SCOPE(&ctx->lock); - if (irp->cpu != cpu) + if (irp->cpu != cpu) // Check for race condition { return; } @@ -240,6 +115,48 @@ void irp_timeout_remove(irp_t* irp) irp->cpu = CPU_ID_INVALID; } +static void irp_perform_completion(irp_t* irp) +{ + while (irp->frame < IRP_FRAME_MAX) + { + irp_frame_t* frame = irp_current(irp); + irp->frame++; + + if (irp->frame == IRP_FRAME_MAX) + { + irp_timeout_remove(irp); + } + + if (frame->vnode != NULL) + { + UNREF(frame->vnode); + frame->vnode = NULL; + } + + if (frame->complete != NULL) + { + frame->complete(irp, frame->ctx); + return; + } + } + + assert(irp->frame == IRP_FRAME_MAX); + assert(irp->next == POOL_IDX_MAX); + assert(irp->cpu == CPU_ID_INVALID); + + mdl_t* next = irp->mdl.next; + mdl_deinit(&irp->mdl); + mdl_free_chain(next, free); + + irp_pool_t* pool = irp_get_pool(irp); + pool_free(&pool->pool, irp->index); + + if (atomic_fetch_sub(&pool->active, 1) == 1) + { + UNREF(pool->process); + } +} + void irp_timeouts_check(void) { irp_ctx_t* ctx = SELF_PTR(pcpu_irps); @@ -278,10 +195,168 @@ void irp_timeouts_check(void) { irp->err = ETIMEDOUT; handler(irp); + irp_perform_completion(irp); + } + else + { + atomic_store(&irp->cancel, NULL); } lock_acquire(&ctx->lock); } lock_release(&ctx->lock); +} + +irp_t* irp_new(irp_pool_t* pool) +{ + pool_idx_t idx = pool_alloc(&pool->pool); + if (idx == POOL_IDX_MAX) + { + errno = ENOSPC; + return NULL; + } + + if (atomic_fetch_add(&pool->active, 1) == 0) + { + REF(pool->process); + } + + irp_t* irp = &pool->irps[idx]; + assert(irp->index == idx); + + list_entry_init(&irp->entry); + list_entry_init(&irp->timeoutEntry); + atomic_init(&irp->cancel, NULL); + irp->deadline = CLOCKS_NEVER; + irp->res._raw = 0; + mdl_init(&irp->mdl, NULL); + irp->next = POOL_IDX_MAX; + irp->cpu = CPU_ID_INVALID; + irp->err = EOK; + irp->frame = IRP_FRAME_MAX; + return irp; +} + +mdl_t* irp_get_mdl(irp_t* irp, const void* addr, size_t size) +{ + if (irp == NULL) + { + errno = EINVAL; + return NULL; + } + + process_t* process = irp_get_process(irp); + if (process == NULL) + { + errno = EINVAL; + return NULL; + } + + mdl_t* mdl = &irp->mdl; + while (mdl->amount > 0) + { + if (mdl->next != NULL) + { + mdl = mdl->next; + continue; + } + + mdl_t* next = malloc(sizeof(mdl_t)); + if (next == NULL) + { + errno = ENOMEM; + return NULL; + } + mdl_init(next, mdl); + mdl = next; + } + + if (mdl_add(mdl, &process->space, addr, size) == ERR) + { + return NULL; + } + + return mdl; +} + +void irp_call(irp_t* irp, vnode_t* vnode) +{ + assert(irp->frame > 0); + irp->frame--; + + irp_frame_t* frame = irp_current(irp); + if (UNLIKELY(frame->major >= IRP_MJ_MAX)) + { + irp_error(irp, EINVAL); + return; + } + + if (vnode == NULL || vnode->vtable == NULL) + { + irp_error(irp, EINVAL); + return; + } + + irp_func_t func = vnode->vtable->funcs[frame->major]; + if (func == NULL) + { + irp_error(irp, ENOSYS); + return; + } + + atomic_store_explicit(&irp->cancel, NULL, memory_order_relaxed); + frame->vnode = REF(vnode); + + func(irp); +} + +void irp_call_direct(irp_t* irp, irp_func_t func) +{ + assert(irp->frame > 0); + irp->frame--; + + irp_frame_t* frame = irp_current(irp); + + if (frame->vnode != NULL) + { + UNREF(frame->vnode); + frame->vnode = NULL; + } + + atomic_store_explicit(&irp->cancel, NULL, memory_order_relaxed); + func(irp); +} + +uint64_t irp_cancel(irp_t* irp) +{ + irp_cancel_t handler = atomic_exchange(&irp->cancel, IRP_CANCELLED); + if (handler == IRP_CANCELLED) + { + errno = EBUSY; + return ERR; + } + + if (handler == NULL) + { + atomic_store(&irp->cancel, NULL); + errno = EBUSY; + return ERR; + } + + irp_timeout_remove(irp); + + irp->err = ECANCELED; + uint64_t result = handler(irp); + irp_perform_completion(irp); + return result; +} + +void irp_complete(irp_t* irp) +{ + if (irp_set_cancel(irp, NULL) == IRP_CANCELLED) + { + return; + } + irp_perform_completion(irp); } \ No newline at end of file diff --git a/src/kernel/io/verb.c b/src/kernel/io/verb.c deleted file mode 100644 index 49c6f5827..000000000 --- a/src/kernel/io/verb.c +++ /dev/null @@ -1,143 +0,0 @@ -#include -#include -#include -#include -#include -#include - -static void verb_args_read_complete(irp_t* irp, void* ctx) -{ - UNUSED(ctx); - - UNREF(irp->file); - irp->file = NULL; -} - -static void verb_args_user(irp_t* irp) -{ - assert(!(irp->flags & SQE_KERNEL)); - - process_t* process = irp_get_process(irp); - - switch (irp->verb) - { - case VERB_READ: - { - file_t* file = file_table_get(&process->fileTable, irp->sqe.fd); - if (file == NULL) - { - irp->err = EBADF; - return; - } - - if (mdl_from_region(&irp->mdl, NULL, &process->space, irp->sqe.buffer, irp->sqe.count) == ERR) - { - UNREF(file); - irp->err = EFAULT; - return; - } - - irp->file = file; - irp->buffer = &irp->mdl; - - irp_push(irp, verb_args_read_complete, NULL); - } - break; - default: - break; - } -} - -static uint64_t nop_cancel(irp_t* irp) -{ - irp_complete(irp); - return 0; -} - -static void verb_dispatch_file(irp_t* irp) -{ - file_t* file = irp->file; - assert(file != NULL); - - switch (irp->verb) - { - case VERB_READ: - if (!(file->mode & MODE_READ)) - { - irp_error(irp, EBADF); - return; - } - break; - default: - break; - } - - if (verb_invoke(irp, file->verbs)) - { - return; - } - - if (verb_invoke(irp, file->vnode->verbs)) - { - return; - } - - if (verb_invoke(irp, file->vnode->superblock->verbs)) - { - return; - } - - irp_error(irp, ENOSYS); -} - -void verb_dispatch(irp_t* irp) -{ - if (!(irp->flags & SQE_KERNEL)) - { - verb_args_user(irp); - } - - if (irp->err != EINPROGRESS) - { - irp_complete(irp); - return; - } - - if (irp->timeout != CLOCKS_NEVER) - { - irp_timeout_add(irp); - } - - switch (irp->verb) - { - case VERB_NOP: - irp_set_cancel(irp, nop_cancel); - break; - case VERB_READ: - verb_dispatch_file(irp); - break; - default: - break; - }; -} - -static void verb_run_completion(irp_t* irp, void* ctx) -{ - UNUSED(irp); - - wait_queue_t* wait = (wait_queue_t*)ctx; - wait_unblock(wait, WAIT_ALL, EOK); -} - -void verb_run(irp_t* irp) -{ - wait_queue_t wait; - wait_queue_init(&wait); - - irp_push(irp, verb_run_completion, &wait); - verb_dispatch(irp); - - WAIT_BLOCK(&wait, irp->err != EINPROGRESS); - - wait_queue_deinit(&wait); -} From caff5df45d7c40db5ef74c8ca2d0e800f745d74e Mon Sep 17 00:00:00 2001 From: KN Date: Sat, 24 Jan 2026 10:30:46 +0100 Subject: [PATCH 23/23] refactor: flatten libstd include dir and merge kernel+module include dirs --- Make.defaults | 5 +- .../_internal => _libstd}/CONTAINER_OF.h | 0 include/{libstd/_internal => _libstd}/ERR.h | 0 .../{libstd/_internal => _libstd}/MAX_NAME.h | 0 .../{libstd/_internal => _libstd}/MAX_PATH.h | 0 include/{libstd/_internal => _libstd}/NULL.h | 0 .../{libstd/_internal => _libstd}/PAGE_SIZE.h | 0 include/{libstd/_internal => _libstd}/SEEK.h | 0 include/{libstd/_internal => _libstd}/ascii.h | 0 .../{libstd/_internal => _libstd}/clock_t.h | 0 .../{libstd/_internal => _libstd}/config.h | 0 .../{libstd/_internal => _libstd}/errno_t.h | 0 include/{libstd/_internal => _libstd}/fd_t.h | 0 include/{libstd/_internal => _libstd}/init.h | 0 include/{libstd/_internal => _libstd}/off_t.h | 0 include/{libstd/_internal => _libstd}/pid_t.h | 0 .../{libstd/_internal => _libstd}/ptrdiff_t.h | 0 .../{libstd/_internal => _libstd}/rsize_t.h | 0 .../{libstd/_internal => _libstd}/size_t.h | 0 .../{libstd/_internal => _libstd}/ssize_t.h | 0 include/{libstd/_internal => _libstd}/tid_t.h | 0 .../{libstd/_internal => _libstd}/time_t.h | 0 .../{libstd/_internal => _libstd}/timespec.h | 0 .../{libstd/_internal => _libstd}/wchar_t.h | 0 include/{libstd => }/alloca.h | 2 +- include/{libstd => }/assert.h | 2 +- include/boot/boot_info.h | 2 +- include/{libstd => }/ctype.h | 4 +- include/{libstd => }/errno.h | 6 +- include/{libstd => }/float.h | 2 +- include/{libstd => }/inttypes.h | 2 +- include/{modules => kernel}/acpi/acpi.h | 4 +- include/{modules => kernel}/acpi/aml/aml.h | 12 +- include/{modules => kernel}/acpi/aml/debug.h | 8 +- .../acpi/aml/encoding/arg.h | 4 +- .../acpi/aml/encoding/data.h | 6 +- .../acpi/aml/encoding/data_integers.h | 2 +- .../acpi/aml/encoding/debug.h | 4 +- .../acpi/aml/encoding/expression.h | 10 +- .../acpi/aml/encoding/local.h | 4 +- .../acpi/aml/encoding/name.h | 4 +- .../acpi/aml/encoding/named.h | 8 +- .../acpi/aml/encoding/namespace_modifier.h | 4 +- .../acpi/aml/encoding/package_length.h | 4 +- .../acpi/aml/encoding/statement.h | 6 +- .../acpi/aml/encoding/term.h | 6 +- .../{modules => kernel}/acpi/aml/integer.h | 4 +- .../{modules => kernel}/acpi/aml/namespace.h | 6 +- include/{modules => kernel}/acpi/aml/object.h | 14 +- .../{modules => kernel}/acpi/aml/patch_up.h | 4 +- .../{modules => kernel}/acpi/aml/predefined.h | 6 +- .../acpi/aml/runtime/access_type.h | 6 +- .../acpi/aml/runtime/buffer_field.h | 6 +- .../acpi/aml/runtime/compare.h | 6 +- .../acpi/aml/runtime/concat.h | 6 +- .../acpi/aml/runtime/convert.h | 6 +- .../acpi/aml/runtime/copy.h | 6 +- .../acpi/aml/runtime/eisa_id.h | 4 +- .../acpi/aml/runtime/evaluate.h | 6 +- .../acpi/aml/runtime/field_unit.h | 6 +- .../acpi/aml/runtime/method.h | 6 +- .../acpi/aml/runtime/mid.h | 6 +- .../acpi/aml/runtime/mutex.h | 6 +- .../acpi/aml/runtime/store.h | 6 +- include/{modules => kernel}/acpi/aml/state.h | 12 +- .../{modules => kernel}/acpi/aml/to_string.h | 8 +- include/{modules => kernel}/acpi/aml/token.h | 8 +- include/{modules => kernel}/acpi/devices.h | 8 +- include/{modules => kernel}/acpi/resources.h | 6 +- include/{modules => kernel}/acpi/tables.h | 6 +- include/kernel/config.h | 1 + include/kernel/cpu/cpu.h | 2 +- include/kernel/drivers/abstract/fb.h | 2 +- .../drivers/apic/apic_timer.h | 4 +- .../{modules => kernel}/drivers/apic/ioapic.h | 4 +- .../{modules => kernel}/drivers/apic/lapic.h | 4 +- .../{modules => kernel}/drivers/pci/config.h | 6 +- include/{modules => kernel}/drivers/pci/pci.h | 4 +- include/kernel/io/io.h | 216 +----------------- include/kernel/io/irp.h | 5 +- include/kernel/io/ring.h | 216 ++++++++++++++++++ include/kernel/kernel.h | 4 +- include/kernel/log/log.h | 4 +- include/kernel/mem/mem.h | 2 +- include/kernel/mem/paging.h | 2 +- include/kernel/mem/paging_types.h | 2 +- .../{modules/linker.lds => kernel/module.lds} | 0 include/kernel/module/module.h | 4 +- include/kernel/proc/process.h | 6 +- include/kernel/sync/sync.h | 2 +- include/{libstd => }/libstd.h | 0 include/{libstd => }/limits.h | 2 +- include/{libstd => }/locale.h | 2 +- include/{libstd => }/math.h | 2 +- include/{libpatchwork => patchwork}/cmd.h | 0 include/{libpatchwork => patchwork}/config.h | 0 include/{libpatchwork => patchwork}/display.h | 0 .../{libpatchwork => patchwork}/drawable.h | 0 include/{libpatchwork => patchwork}/element.h | 0 .../{libpatchwork => patchwork}/element_id.h | 0 include/{libpatchwork => patchwork}/event.h | 0 include/{libpatchwork => patchwork}/font.h | 0 include/{libpatchwork => patchwork}/image.h | 0 .../{libpatchwork => patchwork}/patchwork.h | 0 include/{libpatchwork => patchwork}/pixel.h | 0 include/{libpatchwork => patchwork}/point.h | 0 include/{libpatchwork => patchwork}/polygon.h | 0 include/{libpatchwork => patchwork}/popup.h | 0 .../{libpatchwork => patchwork}/procedure.h | 0 include/{libpatchwork => patchwork}/rect.h | 0 include/{libpatchwork => patchwork}/surface.h | 0 include/{libpatchwork => patchwork}/theme.h | 0 include/{libpatchwork => patchwork}/widgets.h | 0 include/{libpatchwork => patchwork}/window.h | 0 include/{libstd => }/setjmp.h | 2 +- include/{libstd => }/signal.h | 2 +- include/{libstd => }/stdarg.h | 0 include/{libstd => }/stdatomic.h | 0 include/{libstd => }/stdbool.h | 0 include/{libstd => }/stddef.h | 14 +- include/{libstd => }/stdint.h | 0 include/{libstd => }/stdio.h | 16 +- include/{libstd => }/stdlib.h | 10 +- include/{libstd => }/string.h | 10 +- include/{libstd => }/strings.h | 6 +- include/{libstd => }/sys/9p.h | 0 include/{libstd => }/sys/argsplit.h | 2 +- include/{libstd => }/sys/bitmap.h | 0 include/{libstd => }/sys/cpuid.h | 0 include/{libstd => }/sys/defs.h | 0 include/{libstd => }/sys/elf.h | 0 include/{libstd => }/sys/fs.h | 20 +- include/{libstd => }/sys/ioring.h | 28 +-- include/{libstd => }/sys/kbd.h | 2 +- include/{libstd => }/sys/list.h | 4 +- include/{libstd => }/sys/math.h | 0 include/{libstd => }/sys/proc.h | 18 +- include/{libstd => }/threads.h | 8 +- include/{libstd => }/time.h | 16 +- lib/OVMFbin/OVMF_VARS-pure-efi.fd | Bin 131072 -> 131072 bytes meta/doxy/Doxyfile | 2 +- meta/doxy/DoxygenLayout.xml | 2 +- src/boxes/apps/calculator/main.c | 2 +- src/boxes/apps/clock/main.c | 2 +- src/boxes/apps/terminal/ansi.h | 2 +- src/boxes/apps/terminal/terminal.c | 2 +- src/boxes/apps/terminal/terminal.h | 2 +- src/boxes/apps/tetris/main.c | 2 +- src/boxes/core/cursor/main.c | 2 +- src/boxes/core/dwm/client.h | 4 +- src/boxes/core/dwm/dwm.c | 2 +- src/boxes/core/dwm/dwm.h | 2 +- src/boxes/core/dwm/kbd.h | 2 +- src/boxes/core/dwm/region.h | 2 +- src/boxes/core/dwm/surface.h | 10 +- src/boxes/core/taskbar/main.c | 2 +- src/boxes/core/taskbar/start_menu.c | 2 +- src/boxes/core/taskbar/start_menu.h | 2 +- src/boxes/core/taskbar/taskbar.c | 10 +- src/boxes/core/taskbar/taskbar.h | 2 +- src/boxes/core/wall/main.c | 2 +- src/kernel/fs/path.c | 2 +- src/kernel/fs/procfs.c | 2 +- src/kernel/fs/vfs.c | 2 +- src/kernel/init/init.c | 2 +- src/kernel/io/{io.c => ring.c} | 112 ++++----- src/kernel/proc/process.c | 4 +- src/libpatchwork/config.c | 2 +- src/libpatchwork/internal.h | 2 +- src/libpatchwork/polygon.c | 2 +- src/libpatchwork/theme.c | 4 +- src/libpatchwork/window.c | 4 +- src/libstd/common/ascii_table.c | 2 +- src/libstd/common/digits.h | 2 +- src/libstd/common/init.c | 2 +- src/modules/acpi/acpi.c | 8 +- src/modules/acpi/aml/aml.c | 16 +- src/modules/acpi/aml/debug.c | 4 +- src/modules/acpi/aml/encoding/arg.c | 10 +- src/modules/acpi/aml/encoding/data.c | 18 +- src/modules/acpi/aml/encoding/debug.c | 6 +- src/modules/acpi/aml/encoding/expression.c | 34 +-- src/modules/acpi/aml/encoding/local.c | 10 +- src/modules/acpi/aml/encoding/name.c | 14 +- src/modules/acpi/aml/encoding/named.c | 22 +- .../acpi/aml/encoding/namespace_modifier.c | 18 +- .../acpi/aml/encoding/package_length.c | 8 +- src/modules/acpi/aml/encoding/statement.c | 12 +- src/modules/acpi/aml/encoding/term.c | 20 +- src/modules/acpi/aml/integer.c | 4 +- src/modules/acpi/aml/namespace.c | 10 +- src/modules/acpi/aml/object.c | 10 +- src/modules/acpi/aml/patch_up.c | 8 +- src/modules/acpi/aml/predefined.c | 8 +- src/modules/acpi/aml/runtime/access_type.c | 2 +- src/modules/acpi/aml/runtime/buffer_field.c | 2 +- src/modules/acpi/aml/runtime/compare.c | 4 +- src/modules/acpi/aml/runtime/concat.c | 4 +- src/modules/acpi/aml/runtime/convert.c | 14 +- src/modules/acpi/aml/runtime/copy.c | 10 +- src/modules/acpi/aml/runtime/eisa_id.c | 2 +- src/modules/acpi/aml/runtime/evaluate.c | 10 +- src/modules/acpi/aml/runtime/field_unit.c | 16 +- src/modules/acpi/aml/runtime/method.c | 10 +- src/modules/acpi/aml/runtime/mid.c | 2 +- src/modules/acpi/aml/runtime/mutex.c | 2 +- src/modules/acpi/aml/runtime/store.c | 8 +- src/modules/acpi/aml/state.c | 6 +- src/modules/acpi/aml/tests.c | 14 +- src/modules/acpi/aml/to_string.c | 2 +- src/modules/acpi/aml/token.c | 2 +- src/modules/acpi/devices.c | 24 +- src/modules/acpi/resources.c | 24 +- src/modules/acpi/tables.c | 4 +- src/modules/drivers/apic/apic.c | 10 +- src/modules/drivers/apic/apic_timer.c | 4 +- src/modules/drivers/apic/ioapic.c | 6 +- src/modules/drivers/apic/lapic.c | 4 +- src/modules/drivers/const/const.c | 4 +- src/modules/drivers/drivers.h | 4 +- src/modules/drivers/gop/gop.c | 4 +- src/modules/drivers/hpet/hpet.c | 6 +- src/modules/drivers/pci/config.c | 4 +- src/modules/drivers/ps2/ps2.c | 8 +- src/modules/drivers/ps2/ps2.h | 4 +- src/modules/drivers/ps2/ps2_kbd.h | 2 +- src/modules/drivers/ps2/ps2_mouse.h | 2 +- src/modules/drivers/ps2/ps2_scanmap.h | 2 +- src/modules/drivers/rtc/rtc.c | 4 +- src/modules/fs/9p/9p.c | 4 +- src/modules/fs/fs.h | 4 +- src/modules/ipc/ipc.h | 4 +- src/modules/ipc/pipe/pipe.c | 4 +- src/modules/ipc/shmem/shmem.c | 4 +- src/modules/modules.h | 2 +- src/modules/net/local/local_listen.h | 2 +- src/modules/net/net.h | 2 +- src/modules/smp/smp.c | 6 +- src/modules/smp/trampoline.h | 6 +- src/programs/core/init/main.c | 2 +- src/programs/core/shell/pipeline.c | 2 +- {include => src}/programs/programs.h | 0 src/programs/utils/ringtest/main.c | 4 +- src/programs/utils/tail/main.c | 2 +- 244 files changed, 827 insertions(+), 818 deletions(-) rename include/{libstd/_internal => _libstd}/CONTAINER_OF.h (100%) rename include/{libstd/_internal => _libstd}/ERR.h (100%) rename include/{libstd/_internal => _libstd}/MAX_NAME.h (100%) rename include/{libstd/_internal => _libstd}/MAX_PATH.h (100%) rename include/{libstd/_internal => _libstd}/NULL.h (100%) rename include/{libstd/_internal => _libstd}/PAGE_SIZE.h (100%) rename include/{libstd/_internal => _libstd}/SEEK.h (100%) rename include/{libstd/_internal => _libstd}/ascii.h (100%) rename include/{libstd/_internal => _libstd}/clock_t.h (100%) rename include/{libstd/_internal => _libstd}/config.h (100%) rename include/{libstd/_internal => _libstd}/errno_t.h (100%) rename include/{libstd/_internal => _libstd}/fd_t.h (100%) rename include/{libstd/_internal => _libstd}/init.h (100%) rename include/{libstd/_internal => _libstd}/off_t.h (100%) rename include/{libstd/_internal => _libstd}/pid_t.h (100%) rename include/{libstd/_internal => _libstd}/ptrdiff_t.h (100%) rename include/{libstd/_internal => _libstd}/rsize_t.h (100%) rename include/{libstd/_internal => _libstd}/size_t.h (100%) rename include/{libstd/_internal => _libstd}/ssize_t.h (100%) rename include/{libstd/_internal => _libstd}/tid_t.h (100%) rename include/{libstd/_internal => _libstd}/time_t.h (100%) rename include/{libstd/_internal => _libstd}/timespec.h (100%) rename include/{libstd/_internal => _libstd}/wchar_t.h (100%) rename include/{libstd => }/alloca.h (84%) rename include/{libstd => }/assert.h (96%) rename include/{libstd => }/ctype.h (95%) rename include/{libstd => }/errno.h (98%) rename include/{libstd => }/float.h (98%) rename include/{libstd => }/inttypes.h (99%) rename include/{modules => kernel}/acpi/acpi.h (99%) rename include/{modules => kernel}/acpi/aml/aml.h (91%) rename include/{modules => kernel}/acpi/aml/debug.h (83%) rename include/{modules => kernel}/acpi/aml/encoding/arg.h (93%) rename include/{modules => kernel}/acpi/aml/encoding/data.h (98%) rename include/{modules => kernel}/acpi/aml/encoding/data_integers.h (77%) rename include/{modules => kernel}/acpi/aml/encoding/debug.h (89%) rename include/{modules => kernel}/acpi/aml/encoding/expression.h (99%) rename include/{modules => kernel}/acpi/aml/encoding/local.h (93%) rename include/{modules => kernel}/acpi/aml/encoding/name.h (99%) rename include/{modules => kernel}/acpi/aml/encoding/named.h (99%) rename include/{modules => kernel}/acpi/aml/encoding/namespace_modifier.h (94%) rename include/{modules => kernel}/acpi/aml/encoding/package_length.h (95%) rename include/{modules => kernel}/acpi/aml/encoding/statement.h (97%) rename include/{modules => kernel}/acpi/aml/encoding/term.h (97%) rename include/{modules => kernel}/acpi/aml/integer.h (93%) rename include/{modules => kernel}/acpi/aml/namespace.h (99%) rename include/{modules => kernel}/acpi/aml/object.h (99%) rename include/{modules => kernel}/acpi/aml/patch_up.h (97%) rename include/{modules => kernel}/acpi/aml/predefined.h (94%) rename include/{modules => kernel}/acpi/aml/runtime/access_type.h (90%) rename include/{modules => kernel}/acpi/aml/runtime/buffer_field.h (90%) rename include/{modules => kernel}/acpi/aml/runtime/compare.h (93%) rename include/{modules => kernel}/acpi/aml/runtime/concat.h (85%) rename include/{modules => kernel}/acpi/aml/runtime/convert.h (98%) rename include/{modules => kernel}/acpi/aml/runtime/copy.h (91%) rename include/{modules => kernel}/acpi/aml/runtime/eisa_id.h (92%) rename include/{modules => kernel}/acpi/aml/runtime/evaluate.h (84%) rename include/{modules => kernel}/acpi/aml/runtime/field_unit.h (94%) rename include/{modules => kernel}/acpi/aml/runtime/method.h (94%) rename include/{modules => kernel}/acpi/aml/runtime/mid.h (87%) rename include/{modules => kernel}/acpi/aml/runtime/mutex.h (94%) rename include/{modules => kernel}/acpi/aml/runtime/store.h (89%) rename include/{modules => kernel}/acpi/aml/state.h (91%) rename include/{modules => kernel}/acpi/aml/to_string.h (90%) rename include/{modules => kernel}/acpi/aml/token.h (98%) rename include/{modules => kernel}/acpi/devices.h (97%) rename include/{modules => kernel}/acpi/resources.h (99%) rename include/{modules => kernel}/acpi/tables.h (98%) rename include/{modules => kernel}/drivers/apic/apic_timer.h (92%) rename include/{modules => kernel}/drivers/apic/ioapic.h (98%) rename include/{modules => kernel}/drivers/apic/lapic.h (98%) rename include/{modules => kernel}/drivers/pci/config.h (96%) rename include/{modules => kernel}/drivers/pci/pci.h (58%) create mode 100644 include/kernel/io/ring.h rename include/{modules/linker.lds => kernel/module.lds} (100%) rename include/{libstd => }/libstd.h (100%) rename include/{libstd => }/limits.h (96%) rename include/{libstd => }/locale.h (98%) rename include/{libstd => }/math.h (99%) rename include/{libpatchwork => patchwork}/cmd.h (100%) rename include/{libpatchwork => patchwork}/config.h (100%) rename include/{libpatchwork => patchwork}/display.h (100%) rename include/{libpatchwork => patchwork}/drawable.h (100%) rename include/{libpatchwork => patchwork}/element.h (100%) rename include/{libpatchwork => patchwork}/element_id.h (100%) rename include/{libpatchwork => patchwork}/event.h (100%) rename include/{libpatchwork => patchwork}/font.h (100%) rename include/{libpatchwork => patchwork}/image.h (100%) rename include/{libpatchwork => patchwork}/patchwork.h (100%) rename include/{libpatchwork => patchwork}/pixel.h (100%) rename include/{libpatchwork => patchwork}/point.h (100%) rename include/{libpatchwork => patchwork}/polygon.h (100%) rename include/{libpatchwork => patchwork}/popup.h (100%) rename include/{libpatchwork => patchwork}/procedure.h (100%) rename include/{libpatchwork => patchwork}/rect.h (100%) rename include/{libpatchwork => patchwork}/surface.h (100%) rename include/{libpatchwork => patchwork}/theme.h (100%) rename include/{libpatchwork => patchwork}/widgets.h (100%) rename include/{libpatchwork => patchwork}/window.h (100%) rename include/{libstd => }/setjmp.h (94%) rename include/{libstd => }/signal.h (96%) rename include/{libstd => }/stdarg.h (100%) rename include/{libstd => }/stdatomic.h (100%) rename include/{libstd => }/stdbool.h (100%) rename include/{libstd => }/stddef.h (51%) rename include/{libstd => }/stdint.h (100%) rename include/{libstd => }/stdio.h (95%) rename include/{libstd => }/stdlib.h (95%) rename include/{libstd => }/string.h (94%) rename include/{libstd => }/strings.h (73%) rename include/{libstd => }/sys/9p.h (100%) rename include/{libstd => }/sys/argsplit.h (98%) rename include/{libstd => }/sys/bitmap.h (100%) rename include/{libstd => }/sys/cpuid.h (100%) rename include/{libstd => }/sys/defs.h (100%) rename include/{libstd => }/sys/elf.h (100%) rename include/{libstd => }/sys/fs.h (98%) rename include/{libstd => }/sys/ioring.h (93%) rename include/{libstd => }/sys/kbd.h (99%) rename include/{libstd => }/sys/list.h (99%) rename include/{libstd => }/sys/math.h (100%) rename include/{libstd => }/sys/proc.h (98%) rename include/{libstd => }/threads.h (94%) rename include/{libstd => }/time.h (84%) rename src/kernel/io/{io.c => ring.c} (77%) rename {include => src}/programs/programs.h (100%) diff --git a/Make.defaults b/Make.defaults index d778115b1..e71d32df3 100644 --- a/Make.defaults +++ b/Make.defaults @@ -28,8 +28,6 @@ CFLAGS := \ -Wno-unused-but-set-parameter \ -Wno-unused-command-line-argument \ -isystem include \ - -isystem include/libstd \ - -Iinclude \ -Ilib \ -I$(SRCDIR) \ -D_PATCHWORK_OS_ @@ -46,7 +44,6 @@ ASFLAGS := \ -ffreestanding \ -fno-builtin \ -Iinclude \ - -Iinclude/libstd \ -I$(SRCDIR) \ -D_PATCHWORK_OS_ @@ -94,7 +91,7 @@ ASFLAGS_MODULE := \ LDFLAGS_MODULE := \ -shared \ -fPIC \ - -Tinclude/modules/linker.lds + -Tinclude/kernel/module.lds define find_sources $(shell find $(1) -name "*.c" -o -name "*.S" 2>/dev/null) diff --git a/include/libstd/_internal/CONTAINER_OF.h b/include/_libstd/CONTAINER_OF.h similarity index 100% rename from include/libstd/_internal/CONTAINER_OF.h rename to include/_libstd/CONTAINER_OF.h diff --git a/include/libstd/_internal/ERR.h b/include/_libstd/ERR.h similarity index 100% rename from include/libstd/_internal/ERR.h rename to include/_libstd/ERR.h diff --git a/include/libstd/_internal/MAX_NAME.h b/include/_libstd/MAX_NAME.h similarity index 100% rename from include/libstd/_internal/MAX_NAME.h rename to include/_libstd/MAX_NAME.h diff --git a/include/libstd/_internal/MAX_PATH.h b/include/_libstd/MAX_PATH.h similarity index 100% rename from include/libstd/_internal/MAX_PATH.h rename to include/_libstd/MAX_PATH.h diff --git a/include/libstd/_internal/NULL.h b/include/_libstd/NULL.h similarity index 100% rename from include/libstd/_internal/NULL.h rename to include/_libstd/NULL.h diff --git a/include/libstd/_internal/PAGE_SIZE.h b/include/_libstd/PAGE_SIZE.h similarity index 100% rename from include/libstd/_internal/PAGE_SIZE.h rename to include/_libstd/PAGE_SIZE.h diff --git a/include/libstd/_internal/SEEK.h b/include/_libstd/SEEK.h similarity index 100% rename from include/libstd/_internal/SEEK.h rename to include/_libstd/SEEK.h diff --git a/include/libstd/_internal/ascii.h b/include/_libstd/ascii.h similarity index 100% rename from include/libstd/_internal/ascii.h rename to include/_libstd/ascii.h diff --git a/include/libstd/_internal/clock_t.h b/include/_libstd/clock_t.h similarity index 100% rename from include/libstd/_internal/clock_t.h rename to include/_libstd/clock_t.h diff --git a/include/libstd/_internal/config.h b/include/_libstd/config.h similarity index 100% rename from include/libstd/_internal/config.h rename to include/_libstd/config.h diff --git a/include/libstd/_internal/errno_t.h b/include/_libstd/errno_t.h similarity index 100% rename from include/libstd/_internal/errno_t.h rename to include/_libstd/errno_t.h diff --git a/include/libstd/_internal/fd_t.h b/include/_libstd/fd_t.h similarity index 100% rename from include/libstd/_internal/fd_t.h rename to include/_libstd/fd_t.h diff --git a/include/libstd/_internal/init.h b/include/_libstd/init.h similarity index 100% rename from include/libstd/_internal/init.h rename to include/_libstd/init.h diff --git a/include/libstd/_internal/off_t.h b/include/_libstd/off_t.h similarity index 100% rename from include/libstd/_internal/off_t.h rename to include/_libstd/off_t.h diff --git a/include/libstd/_internal/pid_t.h b/include/_libstd/pid_t.h similarity index 100% rename from include/libstd/_internal/pid_t.h rename to include/_libstd/pid_t.h diff --git a/include/libstd/_internal/ptrdiff_t.h b/include/_libstd/ptrdiff_t.h similarity index 100% rename from include/libstd/_internal/ptrdiff_t.h rename to include/_libstd/ptrdiff_t.h diff --git a/include/libstd/_internal/rsize_t.h b/include/_libstd/rsize_t.h similarity index 100% rename from include/libstd/_internal/rsize_t.h rename to include/_libstd/rsize_t.h diff --git a/include/libstd/_internal/size_t.h b/include/_libstd/size_t.h similarity index 100% rename from include/libstd/_internal/size_t.h rename to include/_libstd/size_t.h diff --git a/include/libstd/_internal/ssize_t.h b/include/_libstd/ssize_t.h similarity index 100% rename from include/libstd/_internal/ssize_t.h rename to include/_libstd/ssize_t.h diff --git a/include/libstd/_internal/tid_t.h b/include/_libstd/tid_t.h similarity index 100% rename from include/libstd/_internal/tid_t.h rename to include/_libstd/tid_t.h diff --git a/include/libstd/_internal/time_t.h b/include/_libstd/time_t.h similarity index 100% rename from include/libstd/_internal/time_t.h rename to include/_libstd/time_t.h diff --git a/include/libstd/_internal/timespec.h b/include/_libstd/timespec.h similarity index 100% rename from include/libstd/_internal/timespec.h rename to include/_libstd/timespec.h diff --git a/include/libstd/_internal/wchar_t.h b/include/_libstd/wchar_t.h similarity index 100% rename from include/libstd/_internal/wchar_t.h rename to include/_libstd/wchar_t.h diff --git a/include/libstd/alloca.h b/include/alloca.h similarity index 84% rename from include/libstd/alloca.h rename to include/alloca.h index 05db6e650..71910980a 100644 --- a/include/libstd/alloca.h +++ b/include/alloca.h @@ -6,7 +6,7 @@ extern "C" { #endif -#include "_internal/config.h" +#include "_libstd/config.h" #define alloca(size) __builtin_alloca(size) diff --git a/include/libstd/assert.h b/include/assert.h similarity index 96% rename from include/libstd/assert.h rename to include/assert.h index d6695d1d7..9a397880b 100644 --- a/include/libstd/assert.h +++ b/include/assert.h @@ -6,7 +6,7 @@ extern "C" { #endif -#include "_internal/config.h" +#include "_libstd/config.h" _PUBLIC void _assert_99(const char* const, const char* const, const char* const); _PUBLIC void _assert_89(const char* const); diff --git a/include/boot/boot_info.h b/include/boot/boot_info.h index 5948ad91e..622b8d56d 100644 --- a/include/boot/boot_info.h +++ b/include/boot/boot_info.h @@ -4,7 +4,7 @@ #include -#include <_internal/MAX_NAME.h> +#include <_libstd/MAX_NAME.h> #include #include #include diff --git a/include/libstd/ctype.h b/include/ctype.h similarity index 95% rename from include/libstd/ctype.h rename to include/ctype.h index 92435e0cb..a3d5be99d 100644 --- a/include/libstd/ctype.h +++ b/include/ctype.h @@ -6,8 +6,8 @@ extern "C" { #endif -#include "_internal/ascii.h" -#include "_internal/config.h" +#include "_libstd/ascii.h" +#include "_libstd/config.h" #define isalnum(c) ((int)(_asciiTable[(int)(c)].flags & (_ASCII_ALPHA | _ASCII_DIGIT))) diff --git a/include/libstd/errno.h b/include/errno.h similarity index 98% rename from include/libstd/errno.h rename to include/errno.h index 1c1f8ee10..68219e685 100644 --- a/include/libstd/errno.h +++ b/include/errno.h @@ -6,8 +6,8 @@ extern "C" { #endif -#include "_internal/ERR.h" -#include "_internal/config.h" +#include "_libstd/ERR.h" +#include "_libstd/config.h" int* _errno_get(void); @@ -694,7 +694,7 @@ int* _errno_get(void); /** @} */ #if _USE_ANNEX_K == 1 -#include "_internal/errno_t.h" +#include "_libstd/errno_t.h" #endif #if defined(__cplusplus) diff --git a/include/libstd/float.h b/include/float.h similarity index 98% rename from include/libstd/float.h rename to include/float.h index f7629abcd..30a194b93 100644 --- a/include/libstd/float.h +++ b/include/float.h @@ -1,7 +1,7 @@ #ifndef _FLOAT_H #define _FLOAT_H 1 -#include "_internal/config.h" +#include "_libstd/config.h" #define FLT_ROUNDS -1 #define FLT_EVAL_METHOD __FLT_EVAL_METHOD__ diff --git a/include/libstd/inttypes.h b/include/inttypes.h similarity index 99% rename from include/libstd/inttypes.h rename to include/inttypes.h index fdb48e38f..8db694482 100644 --- a/include/libstd/inttypes.h +++ b/include/inttypes.h @@ -6,7 +6,7 @@ extern "C" { #endif -#include "_internal/config.h" +#include "_libstd/config.h" typedef __INTMAX_TYPE__ intmax_t; typedef __UINTMAX_TYPE__ uintmax_t; diff --git a/include/modules/acpi/acpi.h b/include/kernel/acpi/acpi.h similarity index 99% rename from include/modules/acpi/acpi.h rename to include/kernel/acpi/acpi.h index 54bca47c0..7109a4053 100644 --- a/include/modules/acpi/acpi.h +++ b/include/kernel/acpi/acpi.h @@ -9,8 +9,8 @@ /** * @brief Advanced Configuration and Power Interface - * @defgroup modules_acpi ACPI - * @ingroup modules + * @defgroup kernel_acpi ACPI + * @ingroup kernel * * We use version 6.6 of the ACPI specification, but it contains minor mistakes or deprecated features that we use other * versions to straighten out. If the "ACPI specification" is ever sourced, without mentioning its version, assume diff --git a/include/modules/acpi/aml/aml.h b/include/kernel/acpi/aml/aml.h similarity index 91% rename from include/modules/acpi/aml/aml.h rename to include/kernel/acpi/aml/aml.h index bf3d958ab..be950876b 100644 --- a/include/modules/acpi/aml/aml.h +++ b/include/kernel/acpi/aml/aml.h @@ -1,17 +1,17 @@ #pragma once #include -#include -#include -#include -#include +#include +#include +#include +#include #include /** * @brief ACPI AML - * @defgroup modules_acpi_aml AML - * @ingroup modules_acpi + * @defgroup kernel_acpi_aml AML + * @ingroup kernel_acpi * * ACPI AML is a procedural turing complete bytecode language used to describe the hardware configuration of a computer * system. A hardware manufacturer creates the bytecode to describe their hardware, and we, as the kernel, parse it. The diff --git a/include/modules/acpi/aml/debug.h b/include/kernel/acpi/aml/debug.h similarity index 83% rename from include/modules/acpi/aml/debug.h rename to include/kernel/acpi/aml/debug.h index 7b0611b83..7d1c6fac4 100644 --- a/include/modules/acpi/aml/debug.h +++ b/include/kernel/acpi/aml/debug.h @@ -1,12 +1,12 @@ #pragma once -#include -#include +#include +#include /** * @brief Debugging - * @defgroup modules_acpi_aml_debug Debugging - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_debug Debugging + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/encoding/arg.h b/include/kernel/acpi/aml/encoding/arg.h similarity index 93% rename from include/modules/acpi/aml/encoding/arg.h rename to include/kernel/acpi/aml/encoding/arg.h index 362a4b554..e9e92fe98 100644 --- a/include/modules/acpi/aml/encoding/arg.h +++ b/include/kernel/acpi/aml/encoding/arg.h @@ -5,8 +5,8 @@ typedef struct aml_term_list_ctx aml_term_list_ctx_t; /** * @brief Arg Objecs Encoding - * @defgroup modules_acpi_aml_encoding_args Args - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_encoding_args Args + * @ingroup kernel_acpi_aml * * @see Section 20.2.6.1 of the ACPI specification. * diff --git a/include/modules/acpi/aml/encoding/data.h b/include/kernel/acpi/aml/encoding/data.h similarity index 98% rename from include/modules/acpi/aml/encoding/data.h rename to include/kernel/acpi/aml/encoding/data.h index d7c2a7bca..97568e88a 100644 --- a/include/modules/acpi/aml/encoding/data.h +++ b/include/kernel/acpi/aml/encoding/data.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include @@ -11,8 +11,8 @@ typedef struct aml_term_list_ctx aml_term_list_ctx_t; /** * @brief Data Objects Encoding - * @defgroup modules_acpi_aml_encoding_data Data Objects - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_encoding_data Data Objects + * @ingroup kernel_acpi_aml * * @see Section 20.2.3 of the ACPI specification for more details. * diff --git a/include/modules/acpi/aml/encoding/data_integers.h b/include/kernel/acpi/aml/encoding/data_integers.h similarity index 77% rename from include/modules/acpi/aml/encoding/data_integers.h rename to include/kernel/acpi/aml/encoding/data_integers.h index e3744214f..d267cd77f 100644 --- a/include/modules/acpi/aml/encoding/data_integers.h +++ b/include/kernel/acpi/aml/encoding/data_integers.h @@ -3,7 +3,7 @@ #include /** - * @addtogroup modules_acpi_aml_encoding_data + * @addtogroup kernel_acpi_aml_encoding_data * * @{ */ diff --git a/include/modules/acpi/aml/encoding/debug.h b/include/kernel/acpi/aml/encoding/debug.h similarity index 89% rename from include/modules/acpi/aml/encoding/debug.h rename to include/kernel/acpi/aml/encoding/debug.h index ed9c1adc5..c3b7fb7ac 100644 --- a/include/modules/acpi/aml/encoding/debug.h +++ b/include/kernel/acpi/aml/encoding/debug.h @@ -5,8 +5,8 @@ typedef struct aml_term_list_ctx aml_term_list_ctx_t; /** * @brief Debug Objects Encoding - * @defgroup modules_acpi_aml_encoding_debug Debug Objects - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_encoding_debug Debug Objects + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/encoding/expression.h b/include/kernel/acpi/aml/encoding/expression.h similarity index 99% rename from include/modules/acpi/aml/encoding/expression.h rename to include/kernel/acpi/aml/encoding/expression.h index 124d9c528..1913299d5 100644 --- a/include/modules/acpi/aml/encoding/expression.h +++ b/include/kernel/acpi/aml/encoding/expression.h @@ -1,16 +1,16 @@ #pragma once -#include -#include -#include +#include +#include +#include typedef struct aml_object aml_object_t; typedef struct aml_term_list_ctx aml_term_list_ctx_t; /** * @brief Expression Opcodes Encoding - * @defgroup modules_acpi_aml_encoding_expression Expression Opcodes - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_encoding_expression Expression Opcodes + * @ingroup kernel_acpi_aml * * @see Section 20.2.5.4 of the ACPI specification for more details. * diff --git a/include/modules/acpi/aml/encoding/local.h b/include/kernel/acpi/aml/encoding/local.h similarity index 93% rename from include/modules/acpi/aml/encoding/local.h rename to include/kernel/acpi/aml/encoding/local.h index b204c7c1d..c0cf4b756 100644 --- a/include/modules/acpi/aml/encoding/local.h +++ b/include/kernel/acpi/aml/encoding/local.h @@ -7,8 +7,8 @@ typedef struct aml_term_list_ctx aml_term_list_ctx_t; /** * @brief Local Objecs Encoding - * @defgroup modules_acpi_aml_encoding_local Locals - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_encoding_local Locals + * @ingroup kernel_acpi_aml * * @see Section 20.2.6.2 of the ACPI specification. * diff --git a/include/modules/acpi/aml/encoding/name.h b/include/kernel/acpi/aml/encoding/name.h similarity index 99% rename from include/modules/acpi/aml/encoding/name.h rename to include/kernel/acpi/aml/encoding/name.h index f9dc99ff6..07385e910 100644 --- a/include/modules/acpi/aml/encoding/name.h +++ b/include/kernel/acpi/aml/encoding/name.h @@ -9,8 +9,8 @@ typedef struct aml_term_list_ctx aml_term_list_ctx_t; /** * @brief Name Objects Encoding - * @defgroup modules_acpi_aml_encoding_name Name Objects - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_encoding_name Name Objects + * @ingroup kernel_acpi_aml * * Not to be confused with "ACPI AML Named Objects Encoding". * diff --git a/include/modules/acpi/aml/encoding/named.h b/include/kernel/acpi/aml/encoding/named.h similarity index 99% rename from include/modules/acpi/aml/encoding/named.h rename to include/kernel/acpi/aml/encoding/named.h index f4d490376..779fa5487 100644 --- a/include/modules/acpi/aml/encoding/named.h +++ b/include/kernel/acpi/aml/encoding/named.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include @@ -12,8 +12,8 @@ typedef struct aml_term_list_ctx aml_term_list_ctx_t; /** * @brief Named Objects Encoding - * @defgroup modules_acpi_aml_encoding_name Named Objects - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_encoding_name Named Objects + * @ingroup kernel_acpi_aml * * Not to be confused with "ACPI AML Name Objects Encoding". * diff --git a/include/modules/acpi/aml/encoding/namespace_modifier.h b/include/kernel/acpi/aml/encoding/namespace_modifier.h similarity index 94% rename from include/modules/acpi/aml/encoding/namespace_modifier.h rename to include/kernel/acpi/aml/encoding/namespace_modifier.h index dbf95b8f8..b1f1566d4 100644 --- a/include/modules/acpi/aml/encoding/namespace_modifier.h +++ b/include/kernel/acpi/aml/encoding/namespace_modifier.h @@ -9,8 +9,8 @@ typedef struct aml_term_list_ctx aml_term_list_ctx_t; /** * @brief Namespace Modifier Objects Encoding - * @defgroup modules_acpi_aml_encoding_namespace_modifier Namespace Modifier Objects - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_encoding_namespace_modifier Namespace Modifier Objects + * @ingroup kernel_acpi_aml * * @see Section 20.2.5.1 of the ACPI specification for more details. * diff --git a/include/modules/acpi/aml/encoding/package_length.h b/include/kernel/acpi/aml/encoding/package_length.h similarity index 95% rename from include/modules/acpi/aml/encoding/package_length.h rename to include/kernel/acpi/aml/encoding/package_length.h index f45d00172..207d75cf2 100644 --- a/include/modules/acpi/aml/encoding/package_length.h +++ b/include/kernel/acpi/aml/encoding/package_length.h @@ -7,8 +7,8 @@ typedef struct aml_term_list_ctx aml_term_list_ctx_t; /** * @brief Package Length Encoding - * @defgroup modules_acpi_aml_encoding_package_length Package Length - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_encoding_package_length Package Length + * @ingroup kernel_acpi_aml * * @see Section 20.2.4 of the ACPI specification. * diff --git a/include/modules/acpi/aml/encoding/statement.h b/include/kernel/acpi/aml/encoding/statement.h similarity index 97% rename from include/modules/acpi/aml/encoding/statement.h rename to include/kernel/acpi/aml/encoding/statement.h index 98d99fc8e..1e6bee2d8 100644 --- a/include/modules/acpi/aml/encoding/statement.h +++ b/include/kernel/acpi/aml/encoding/statement.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include @@ -10,8 +10,8 @@ typedef struct aml_term_list_ctx aml_term_list_ctx_t; /** * @brief Statement Opcodes Encoding - * @defgroup modules_acpi_aml_encoding_statement Statement Opcodes - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_encoding_statement Statement Opcodes + * @ingroup kernel_acpi_aml * * @see Section 20.2.5.3 of the ACPI specification for more details. * diff --git a/include/modules/acpi/aml/encoding/term.h b/include/kernel/acpi/aml/encoding/term.h similarity index 97% rename from include/modules/acpi/aml/encoding/term.h rename to include/kernel/acpi/aml/encoding/term.h index b15f1af06..234781392 100644 --- a/include/modules/acpi/aml/encoding/term.h +++ b/include/kernel/acpi/aml/encoding/term.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include @@ -8,8 +8,8 @@ typedef struct aml_state aml_state_t; /** * @brief Term Objects Encoding - * @defgroup modules_acpi_aml_encoding_term Term Objects - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_encoding_term Term Objects + * @ingroup kernel_acpi_aml * * @see Section 20.2.5 of the ACPI specification for more details. * diff --git a/include/modules/acpi/aml/integer.h b/include/kernel/acpi/aml/integer.h similarity index 93% rename from include/modules/acpi/aml/integer.h rename to include/kernel/acpi/aml/integer.h index 9edac26dd..5ff231fdf 100644 --- a/include/modules/acpi/aml/integer.h +++ b/include/kernel/acpi/aml/integer.h @@ -5,8 +5,8 @@ /** * @brief Integer revision handling - * @defgroup modules_acpi_aml_integer Integer - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_integer Integer + * @ingroup kernel_acpi_aml * * This module handles the varying size of integers in AML, which can be either 32 or 64 bits depending on the ACPI * revision. diff --git a/include/modules/acpi/aml/namespace.h b/include/kernel/acpi/aml/namespace.h similarity index 99% rename from include/modules/acpi/aml/namespace.h rename to include/kernel/acpi/aml/namespace.h index e6d82c3cf..557ba1aca 100644 --- a/include/modules/acpi/aml/namespace.h +++ b/include/kernel/acpi/aml/namespace.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include @@ -9,8 +9,8 @@ /** * @brief Namespace and Namespace Overlays - * @defgroup modules_acpi_aml_namespace Namespace - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_namespace Namespace + * @ingroup kernel_acpi_aml * * We need this slightly complex system as when a method runs it can create named objects that should not be visible * outside of the method, and when the method finishes these objects need to be removed. Additionally, if the method diff --git a/include/modules/acpi/aml/object.h b/include/kernel/acpi/aml/object.h similarity index 99% rename from include/modules/acpi/aml/object.h rename to include/kernel/acpi/aml/object.h index bfc7af5c9..7ee0b84cf 100644 --- a/include/modules/acpi/aml/object.h +++ b/include/kernel/acpi/aml/object.h @@ -2,11 +2,11 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include @@ -20,8 +20,8 @@ typedef struct aml_method aml_method_t; /** * @brief Object - * @defgroup modules_acpi_aml_object Object - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_object Object + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/patch_up.h b/include/kernel/acpi/aml/patch_up.h similarity index 97% rename from include/modules/acpi/aml/patch_up.h rename to include/kernel/acpi/aml/patch_up.h index 8e914e68d..622ab8e0c 100644 --- a/include/modules/acpi/aml/patch_up.h +++ b/include/kernel/acpi/aml/patch_up.h @@ -10,8 +10,8 @@ typedef struct aml_state aml_state_t; /** * @brief Patch-up system for forward references. - * @defgroup modules_acpi_aml_patch_up Patch-up - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_patch_up Patch-up + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/predefined.h b/include/kernel/acpi/aml/predefined.h similarity index 94% rename from include/modules/acpi/aml/predefined.h rename to include/kernel/acpi/aml/predefined.h index 8dc046be0..e2ee95f2d 100644 --- a/include/modules/acpi/aml/predefined.h +++ b/include/kernel/acpi/aml/predefined.h @@ -1,13 +1,13 @@ #pragma once -#include +#include #include /** * @brief Predefined AML names and objects - * @defgroup modules_acpi_aml_predefined Predefined - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_predefined Predefined + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/runtime/access_type.h b/include/kernel/acpi/aml/runtime/access_type.h similarity index 90% rename from include/modules/acpi/aml/runtime/access_type.h rename to include/kernel/acpi/aml/runtime/access_type.h index 22e834bdc..f6cb520ec 100644 --- a/include/modules/acpi/aml/runtime/access_type.h +++ b/include/kernel/acpi/aml/runtime/access_type.h @@ -1,14 +1,14 @@ #pragma once #pragma once -#include +#include #include /** * @brief Access Type Handling - * @defgroup modules_acpi_aml_access_type Access Type - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_access_type Access Type + * @ingroup kernel_acpi_aml * * This module provides functionality for handling access types, alignment with access types, etc. * diff --git a/include/modules/acpi/aml/runtime/buffer_field.h b/include/kernel/acpi/aml/runtime/buffer_field.h similarity index 90% rename from include/modules/acpi/aml/runtime/buffer_field.h rename to include/kernel/acpi/aml/runtime/buffer_field.h index b22d4c7e2..6f178bdf2 100644 --- a/include/modules/acpi/aml/runtime/buffer_field.h +++ b/include/kernel/acpi/aml/runtime/buffer_field.h @@ -1,13 +1,13 @@ #pragma once -#include +#include #include /** * @brief Buffer Field - * @defgroup modules_acpi_aml_buffer_field Buffer Field - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_buffer_field Buffer Field + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/runtime/compare.h b/include/kernel/acpi/aml/runtime/compare.h similarity index 93% rename from include/modules/acpi/aml/runtime/compare.h rename to include/kernel/acpi/aml/runtime/compare.h index c0f762556..a627ed901 100644 --- a/include/modules/acpi/aml/runtime/compare.h +++ b/include/kernel/acpi/aml/runtime/compare.h @@ -1,11 +1,11 @@ #pragma once -#include +#include /** * @brief Object Comparison - * @defgroup modules_acpi_aml_compare Compare - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_compare Compare + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/runtime/concat.h b/include/kernel/acpi/aml/runtime/concat.h similarity index 85% rename from include/modules/acpi/aml/runtime/concat.h rename to include/kernel/acpi/aml/runtime/concat.h index 486963566..4c313fc93 100644 --- a/include/modules/acpi/aml/runtime/concat.h +++ b/include/kernel/acpi/aml/runtime/concat.h @@ -1,11 +1,11 @@ #pragma once -#include +#include /** * @brief Object Concatenation - * @defgroup modules_acpi_aml_runtime_concat Concat - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_runtime_concat Concat + * @ingroup kernel_acpi_aml */ /** diff --git a/include/modules/acpi/aml/runtime/convert.h b/include/kernel/acpi/aml/runtime/convert.h similarity index 98% rename from include/modules/acpi/aml/runtime/convert.h rename to include/kernel/acpi/aml/runtime/convert.h index 0a24ec4fc..269e40b3d 100644 --- a/include/modules/acpi/aml/runtime/convert.h +++ b/include/kernel/acpi/aml/runtime/convert.h @@ -1,13 +1,13 @@ #pragma once -#include +#include #include /** * @brief Data Type Conversion - * @defgroup modules_acpi_aml_convert Convert - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_convert Convert + * @ingroup kernel_acpi_aml * * @see Section 19.3.5 of the ACPI specification for more details. * @see Section 19.3.5.7 table 19.6 for the conversion priority order. diff --git a/include/modules/acpi/aml/runtime/copy.h b/include/kernel/acpi/aml/runtime/copy.h similarity index 91% rename from include/modules/acpi/aml/runtime/copy.h rename to include/kernel/acpi/aml/runtime/copy.h index df37854e3..a716ea608 100644 --- a/include/modules/acpi/aml/runtime/copy.h +++ b/include/kernel/acpi/aml/runtime/copy.h @@ -1,13 +1,13 @@ #pragma once -#include +#include #include /** * @brief Copy - * @defgroup modules_acpi_aml_copy Copy - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_copy Copy + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/runtime/eisa_id.h b/include/kernel/acpi/aml/runtime/eisa_id.h similarity index 92% rename from include/modules/acpi/aml/runtime/eisa_id.h rename to include/kernel/acpi/aml/runtime/eisa_id.h index 461d8cc15..3263228b2 100644 --- a/include/modules/acpi/aml/runtime/eisa_id.h +++ b/include/kernel/acpi/aml/runtime/eisa_id.h @@ -5,8 +5,8 @@ /** * @brief EISA ID to string and vice versa conversion - * @defgroup modules_acpi_aml_runtime_eisa_id EISA ID - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_runtime_eisa_id EISA ID + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/runtime/evaluate.h b/include/kernel/acpi/aml/runtime/evaluate.h similarity index 84% rename from include/modules/acpi/aml/runtime/evaluate.h rename to include/kernel/acpi/aml/runtime/evaluate.h index 04a3498a1..9afc4625a 100644 --- a/include/modules/acpi/aml/runtime/evaluate.h +++ b/include/kernel/acpi/aml/runtime/evaluate.h @@ -1,11 +1,11 @@ #pragma once -#include +#include /** * @brief Object Runtime Evaluation - * @defgroup modules_acpi_aml_evaluate Runtime Evaluation - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_evaluate Runtime Evaluation + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/runtime/field_unit.h b/include/kernel/acpi/aml/runtime/field_unit.h similarity index 94% rename from include/modules/acpi/aml/runtime/field_unit.h rename to include/kernel/acpi/aml/runtime/field_unit.h index 637088e69..5aad08014 100644 --- a/include/modules/acpi/aml/runtime/field_unit.h +++ b/include/kernel/acpi/aml/runtime/field_unit.h @@ -1,13 +1,13 @@ #pragma once -#include +#include #include /** * @brief Opregion and Field Access - * @defgroup modules_acpi_aml_field Field Access - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_field Field Access + * @ingroup kernel_acpi_aml * * This module provides functionality for accessing Opregions and Fields. * diff --git a/include/modules/acpi/aml/runtime/method.h b/include/kernel/acpi/aml/runtime/method.h similarity index 94% rename from include/modules/acpi/aml/runtime/method.h rename to include/kernel/acpi/aml/runtime/method.h index 16ed15627..c8447ad9f 100644 --- a/include/modules/acpi/aml/runtime/method.h +++ b/include/kernel/acpi/aml/runtime/method.h @@ -1,13 +1,13 @@ #pragma once -#include +#include #include /** * @brief Method Evaluation - * @defgroup modules_acpi_aml_method Methods - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_method Methods + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/runtime/mid.h b/include/kernel/acpi/aml/runtime/mid.h similarity index 87% rename from include/modules/acpi/aml/runtime/mid.h rename to include/kernel/acpi/aml/runtime/mid.h index ee800f529..cc8edc1cb 100644 --- a/include/modules/acpi/aml/runtime/mid.h +++ b/include/kernel/acpi/aml/runtime/mid.h @@ -1,11 +1,11 @@ #pragma once -#include +#include /** * @brief Extract Portion of Buffer or String - * @defgroup modules_acpi_aml_mid Mid - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_mid Mid + * @ingroup kernel_acpi_aml * * @see Section 19.6.86 of the ACPI specification for more details. * diff --git a/include/modules/acpi/aml/runtime/mutex.h b/include/kernel/acpi/aml/runtime/mutex.h similarity index 94% rename from include/modules/acpi/aml/runtime/mutex.h rename to include/kernel/acpi/aml/runtime/mutex.h index 181f95007..e27ee8ca2 100644 --- a/include/modules/acpi/aml/runtime/mutex.h +++ b/include/kernel/acpi/aml/runtime/mutex.h @@ -1,14 +1,14 @@ #pragma once #include -#include +#include typedef struct aml_thread aml_thread_t; /** * @brief Mutex - * @defgroup modules_acpi_aml_mutex Mutex - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_mutex Mutex + * @ingroup kernel_acpi_aml * * This module provides functionality for acquiring and releasing AML mutexes. * diff --git a/include/modules/acpi/aml/runtime/store.h b/include/kernel/acpi/aml/runtime/store.h similarity index 89% rename from include/modules/acpi/aml/runtime/store.h rename to include/kernel/acpi/aml/runtime/store.h index c701b630e..a531b004c 100644 --- a/include/modules/acpi/aml/runtime/store.h +++ b/include/kernel/acpi/aml/runtime/store.h @@ -1,13 +1,13 @@ #pragma once -#include +#include #include /** * @brief Store - * @defgroup modules_acpi_aml_store Store - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_store Store + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/state.h b/include/kernel/acpi/aml/state.h similarity index 91% rename from include/modules/acpi/aml/state.h rename to include/kernel/acpi/aml/state.h index ab5d8e724..18db51819 100644 --- a/include/modules/acpi/aml/state.h +++ b/include/kernel/acpi/aml/state.h @@ -1,14 +1,14 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include +#include /** * @brief State - * @defgroup modules_acpi_aml_state State - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_state State + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/to_string.h b/include/kernel/acpi/aml/to_string.h similarity index 90% rename from include/modules/acpi/aml/to_string.h rename to include/kernel/acpi/aml/to_string.h index 836e5abbe..7c6017f32 100644 --- a/include/modules/acpi/aml/to_string.h +++ b/include/kernel/acpi/aml/to_string.h @@ -1,12 +1,12 @@ #pragma once -#include -#include +#include +#include /** * @brief String Conversion - * @defgroup modules_acpi_aml_string_conversion String Conversion - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_string_conversion String Conversion + * @ingroup kernel_acpi_aml * * @{ */ diff --git a/include/modules/acpi/aml/token.h b/include/kernel/acpi/aml/token.h similarity index 98% rename from include/modules/acpi/aml/token.h rename to include/kernel/acpi/aml/token.h index c46658a4e..9ce3d6a47 100644 --- a/include/modules/acpi/aml/token.h +++ b/include/kernel/acpi/aml/token.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include #include @@ -10,8 +10,8 @@ /** * @brief Tokens - * @defgroup modules_acpi_aml_token Tokens - * @ingroup modules_acpi_aml + * @defgroup kernel_acpi_aml_token Tokens + * @ingroup kernel_acpi_aml * * This module handles descriptions of all tokens that can be found in an AML byte stream, storing them and their * properties. diff --git a/include/modules/acpi/devices.h b/include/kernel/acpi/devices.h similarity index 97% rename from include/modules/acpi/devices.h rename to include/kernel/acpi/devices.h index 877993323..e31ee8096 100644 --- a/include/modules/acpi/devices.h +++ b/include/kernel/acpi/devices.h @@ -2,15 +2,15 @@ #include #include -#include -#include +#include +#include #include /** * @brief Device and Power Management - * @defgroup modules_acpi_devices Devices - * @ingroup modules_acpi + * @defgroup kernel_acpi_devices Devices + * @ingroup kernel_acpi * * Handles enumeration and configuration of ACPI devices, along with dynamic loading of device drivers. * diff --git a/include/modules/acpi/resources.h b/include/kernel/acpi/resources.h similarity index 99% rename from include/modules/acpi/resources.h rename to include/kernel/acpi/resources.h index 61dff7911..31a7ca61d 100644 --- a/include/modules/acpi/resources.h +++ b/include/kernel/acpi/resources.h @@ -1,13 +1,13 @@ #pragma once -#include +#include #include #include /** * @brief ACPI resource settings. - * @defgroup modules_acpi_resources Resources - * @ingroup modules_acpi + * @defgroup kernel_acpi_resources Resources + * @ingroup kernel_acpi * * In the AML namespace heirarchy each device uses a buffer object, usually returned by their `_CRS` method, to describe * the resources they require, for example IO ports, IRQs, DMA channels, etc. diff --git a/include/modules/acpi/tables.h b/include/kernel/acpi/tables.h similarity index 98% rename from include/modules/acpi/tables.h rename to include/kernel/acpi/tables.h index ab85a711f..f3870e132 100644 --- a/include/modules/acpi/tables.h +++ b/include/kernel/acpi/tables.h @@ -1,11 +1,11 @@ #pragma once -#include +#include /** * @brief System Description Tables - * @defgroup modules_acpi_tables Tables - * @ingroup modules_acpi + * @defgroup kernel_acpi_tables Tables + * @ingroup kernel_acpi * * This module defines the ACPI tables found in the ACPI specification, tables defined outside of the specification, for * example, MCFG is defined in their own files. diff --git a/include/kernel/config.h b/include/kernel/config.h index 69e96c824..87d8dfa6f 100644 --- a/include/kernel/config.h +++ b/include/kernel/config.h @@ -2,6 +2,7 @@ /** * @brief Kernel configuration. + * @ingroup kernel * @defgroup kernel_config Configuration * * @{ diff --git a/include/kernel/cpu/cpu.h b/include/kernel/cpu/cpu.h index 857904c6a..7b585c6a5 100644 --- a/include/kernel/cpu/cpu.h +++ b/include/kernel/cpu/cpu.h @@ -14,7 +14,7 @@ typedef struct cpu cpu_t; /** * @brief CPU - * @defgroup kernel_cpu CPU + * @defgroup kernel_cpu CPU Management * @ingroup kernel * * @{ diff --git a/include/kernel/drivers/abstract/fb.h b/include/kernel/drivers/abstract/fb.h index 645394d57..d76d5453f 100644 --- a/include/kernel/drivers/abstract/fb.h +++ b/include/kernel/drivers/abstract/fb.h @@ -1,6 +1,6 @@ #pragma once -#include <_internal/MAX_PATH.h> +#include <_libstd/MAX_PATH.h> #include #include diff --git a/include/modules/drivers/apic/apic_timer.h b/include/kernel/drivers/apic/apic_timer.h similarity index 92% rename from include/modules/drivers/apic/apic_timer.h rename to include/kernel/drivers/apic/apic_timer.h index 7531e3f2f..21cac0b42 100644 --- a/include/modules/drivers/apic/apic_timer.h +++ b/include/kernel/drivers/apic/apic_timer.h @@ -4,8 +4,8 @@ /** * @brief Advanced Programmable Interrupt Controller Timer. - * @defgroup modules_drivers_apic_timer APIC Timer - * @ingroup modules_drivers_apic + * @defgroup kernel_drivers_apic_timer APIC Timer + * @ingroup kernel_drivers_apic * * Each local APIC is associated with a timer which can be used to generate interrupts at specific intervals, or as we * use it, to generate a single interrupt after a specified time. diff --git a/include/modules/drivers/apic/ioapic.h b/include/kernel/drivers/apic/ioapic.h similarity index 98% rename from include/modules/drivers/apic/ioapic.h rename to include/kernel/drivers/apic/ioapic.h index c5c267bce..4ac4b6ce0 100644 --- a/include/modules/drivers/apic/ioapic.h +++ b/include/kernel/drivers/apic/ioapic.h @@ -5,8 +5,8 @@ /** * @brief Input / Output Advanced Programmable Interrupt Controller. - * @defgroup modules_drivers_apic_ioapic IO APIC - * @ingroup modules_drivers_apic + * @defgroup kernel_drivers_apic_ioapic IO APIC + * @ingroup kernel_drivers_apic * * The IO APICs are used to route external interrupts to a CPUs local APIC. Each IO APIC handles a range of Global * System Interrupts (GSIs) or in PatchworkOS terms, physical IRQs, which it receives from external devices such as a diff --git a/include/modules/drivers/apic/lapic.h b/include/kernel/drivers/apic/lapic.h similarity index 98% rename from include/modules/drivers/apic/lapic.h rename to include/kernel/drivers/apic/lapic.h index 4d1fd2b85..2780df1f7 100644 --- a/include/modules/drivers/apic/lapic.h +++ b/include/kernel/drivers/apic/lapic.h @@ -8,8 +8,8 @@ /** * @brief Local Advanced Programmable Interrupt Controller. - * @defgroup modules_drivers_apic_lapic Local APIC - * @ingroup modules_drivers_apic + * @defgroup kernel_drivers_apic_lapic Local APIC + * @ingroup kernel_drivers_apic * * ## Local APICs * diff --git a/include/modules/drivers/pci/config.h b/include/kernel/drivers/pci/config.h similarity index 96% rename from include/modules/drivers/pci/config.h rename to include/kernel/drivers/pci/config.h index 65b530081..1aac638e6 100644 --- a/include/modules/drivers/pci/config.h +++ b/include/kernel/drivers/pci/config.h @@ -1,13 +1,13 @@ #pragma once -#include +#include #include /** * @brief PCI configuration space - * @defgroup modules_drivers_pci_config PCI Configuration Space - * @ingroup modules_drivers_pci + * @defgroup kernel_drivers_pci_config PCI Configuration Space + * @ingroup kernel_drivers_pci * * Id like to use the PCI Firmware Specification as a reference for this, but unfortunately, its not freely available. * So we use the OSDev Wiki instead. diff --git a/include/modules/drivers/pci/pci.h b/include/kernel/drivers/pci/pci.h similarity index 58% rename from include/modules/drivers/pci/pci.h rename to include/kernel/drivers/pci/pci.h index 1bff6eaa1..db683b9ae 100644 --- a/include/modules/drivers/pci/pci.h +++ b/include/kernel/drivers/pci/pci.h @@ -2,8 +2,8 @@ /** * @brief Peripheral Component Interconnect (PCI) - * @defgroup modules_drivers_pci PCI - * @ingroup modules_drivers + * @defgroup kernel_drivers_pci PCI + * @ingroup kernel_drivers * * @{ */ diff --git a/include/kernel/io/io.h b/include/kernel/io/io.h index 7df9d9e97..a15cddbd0 100644 --- a/include/kernel/io/io.h +++ b/include/kernel/io/io.h @@ -1,216 +1,10 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include - -#include -#include - /** - * @brief Programmable submission/completion interface. - * @defgroup kernel_io Kernel-side I/O Ring Interface + * @brief I/O Subsystem. + * @defgroup kernel_io I/O Subsystem * @ingroup kernel - * - * @todo The I/O ring system is primarily a design document for now as it remains very work in progress and subject to - * change, currently being mostly unimplemented. - * - * @todo Rewrite the Kernel-side I/O Ring Interface documentation to match the new system. * - * The I/O ring provides the core of all interfaces in PatchworkOS, where user-space submits Submission Queue Entries - * (SQEs) and receives Completion Queue Entries (CQEs) from it, all within shared memory. Allowing for highly efficient - * and asynchronous I/O operations, especially since PatchworkOS is designed to be natively asynchronous. - * - * Each SQE specifies a verb (the operation to perform) and a set of up to `SQE_MAX_ARG` arguments, while each CQE - * returns the result of a previously submitted SQE. - * - * Synchronous operations are implemented on top of this API in userspace. - * - * @see libstd_sys_ioring for the userspace interface to the asynchronous ring. - * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`, the inspiration for this - * system. - * @see [Manpages](https://man7.org/linux/man-pages/man7/io_uring.7.html) for more information about `io_uring`. - * - * ## Syncronization - * - * The I/O ring structure is designed to be safe under the assumption that there is a single producer (one user-space - * thread) and a single consumer (the kernel). - * - * If an I/O ring needs multiple producers (needs to be accessed by multiple threads) it is the responsibility of - * the caller to ensure proper synchronization. - * - * @note The reason for this limitation is optimization for the common case, as the syncronization logic for multiple - * producers would add significant overhead. Additionally, it is rather straight forward for user-space to protect the - * ring with a mutex should it need to. - * - * Regarding the I/O ring structure itself, the structure can only be torndown as long as nothing is using it and there - * are no pending operations. - * - * ## Registers - * - * Operations performed on a I/O ring can load arguments from, and save their results to, seven 64-bit general purpose - * registers. All registers are stored in the shared control area of the I/O ring structure (`ioring_ctrl_t`), as such - * they can be inspected and modified by user space. - * - * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for each argument and - * one for the result. Each specifier is stored as three bits, with a `SQE_REG_NONE` value indicating no-op and any - * other value representing the n-th register. The offset of the specifier specifies its meaning, for example, bits - * `0-2` specify the register to load into the first argument, bits `3-5` specify the register to load into the second - * argument, and so on until bits `15-17` which specify the register to save the result into. - * - * This system, when combined with `SQE_LINK`, allows for multiple operations to be performed at once, for example, it - * would be possible to open a file, read from it, seek to a new position, write to it, and finally close the file, with - * a single `enter()` call. - * - * @see `sqe_flags_t` for more information about register specifiers and their formatting. - * - * ## Arguments - * - * Arguments within a SQE are stored in five 64-bit values, `arg1` through `arg5`. For convenience, each argument value - * is stored as a union with various types. - * - * To avoid nameing conflicts and to avoid having to define new arguments for each verb, we define a convention to be - * used for the arguments. - * - * - `arg0`: The noun or subject of the verb, for example, a `fd_t` for file operations. - * - `arg1`: The source or payload of the verb, for example, a buffer or path. - * - `arg2`: The magnitude of the operation, for example, a size or encoding. - * - `arg3`: The location or a modifier to the operation, for example, an offset or flags. - * - `arg4`: An auxiliary argument, for example, additional flags or options. - * - * It may not always be possible for a verb to follow these conventions, but they should be followed whenever - * reasonable. - * - * @note The kernels internal I/O Request Packet structure uses a similar system but with the kernel equivalents - * of the arguments, for example, a `file_t*` instead of a `fd_t`. - * - * ## Results - * - * The result of a SQE is stored in its corresponding CQE using a single 64-bit value. For convenience, the result is - * stored as a union of various types. Note that this does not actually change the stored value, just how it is - * interpreted. - * - * If a SQE fails, the error code will be stored separately from the result and the result it self may be undefined. - * Some verbs may allow partial failures in which case the result may still be valid even if an error code is present. - * - * @todo Decide if partial failures are a good idea or not. - * - * ## Errors - * - * The majority of errors are returned in the CQEs, certain errors (such as `ENOMEM`) may be - * reported directly from the `enter()` call. - * - * Error values that may be returned in a CQE include: - * - `EOK`: Success. - * - `ECANCELED`: The verb was cancelled. - * - `ETIMEDOUT`: The verb timed out. - * - Other values may be returned depending on the verb. - * - * ## Verbs - * - * Included below is a list of all currently implemented verbs. - * - * The arguments of each verb is specified in order as `arg0`, `arg1`, `arg2`, `arg3`, `arg4`. - * - * ### `VERB_NOP` - * - * A no-operation verb that does nothing but is useful for implementing sleeping. - * - * @param arg0 Unused - * @param arg1 Unused - * @param arg2 Unused - * @param arg3 Unused - * @param arg4 Unused - * @result None - * - * ### `VERB_READ` - * - * Reads data from a file descriptor. - * - * @param fd The file descriptor to read from. - * @param buffer The buffer to read the data into. - * @param count The number of bytes to read. - * @param offset The offset to read from, or `IO_CUR` to use the current position. - * @param arg4 Unused - * @result The number of bytes read. - * - * ### `VERB_WRITE` - * - * Writes data to a file descriptor. - * - * @param fd The file descriptor to write to. - * @param buffer The buffer to write the data from. - * @param count The number of bytes to write. - * @param offset The offset to write to, or `IO_CUR` to use the current position. - * @param arg4 Unused - * @result The number of bytes written. - * - * ### `VERB_POLL` - * - * Polls a file descriptor for events. - * - * @param fd The file descriptor to poll. - * @param events The events to wait for. - * @param arg2 Unused - * @param arg3 Unused - * @param arg4 Unused - * @result The events that occurred. - * - * @{ - */ - -/** - * @brief Ring context flags. - * @enum io_ctx_flags_t - */ -typedef enum -{ - IO_CTX_NONE = 0, ///< No flags set. - IO_CTX_BUSY = 1 << 0, ///< Context is currently being used, used for fast locking. - IO_CTX_MAPPED = 1 << 1, ///< Context is currently mapped into userspace. -} io_ctx_flags_t; - -/** - * @brief The kernel-side ring context structure. - * @struct io_ctx_t - */ -typedef struct io_ctx -{ - ioring_t ring; ///< The kernel-side ring structure. - irp_pool_t* irps; ///< Pool of preallocated IRPs. - void* userAddr; ///< Userspace address of the ring. - void* kernelAddr; ///< Kernel address of the ring. - size_t pageAmount; ///< Amount of pages mapped for the ring. - wait_queue_t waitQueue; ///< Wait queue for completions. - _Atomic(io_ctx_flags_t) flags; -} io_ctx_t; - -/** - * @brief Initialize a I/O context. - * - * @param ctx Pointer to the context to initialize. - */ -void io_ctx_init(io_ctx_t* ctx); - -/** - * @brief Deinitialize a I/O context. - * - * @param ctx Pointer to the context to deinitialize. - */ -void io_ctx_deinit(io_ctx_t* ctx); - -/** - * @brief Notify the context of new SQEs. - * - * @param ctx Pointer to the context. - * @param amount The number of SQEs to process. - * @param wait The minimum number of CQEs to wait for. - * @return On success, the number of SQEs processed. On failure, `ERR` and `errno` is set. - */ -uint64_t io_ctx_notify(io_ctx_t* ctx, size_t amount, size_t wait); - -/** @} */ \ No newline at end of file + * The I/O subsystem is responsible for asynchronous input and output operations within PatchworkOS. + * + */ \ No newline at end of file diff --git a/include/kernel/io/irp.h b/include/kernel/io/irp.h index 183c09cda..943604a68 100644 --- a/include/kernel/io/irp.h +++ b/include/kernel/io/irp.h @@ -28,10 +28,9 @@ typedef struct irp irp_t; * @ingroup kernel_io * * The I/O Request Packet (IRP) is a lock-less, self-contained, layered, continuation-passing request that acts as the - * primary structure used internally by the kernel for asynchronous operations. + * primary primitive used by the kernel for asynchronous operations. * - * The IRP structure is designed to be generic enough to be used by any system in the kernel, however it is primarily - * used by the I/O ring system. + * The IRP is designed to be generic enough to be used by any system in the kernel, however it is primarily used by the I/O ring system. * * @warning While the cancellation or completion of an IRP is thread safe, the setup of an IRP is not (as in pushing * layers to it). It is assumed that only one thread is manipulating an IRP during its setup. diff --git a/include/kernel/io/ring.h b/include/kernel/io/ring.h new file mode 100644 index 000000000..3f0bf361c --- /dev/null +++ b/include/kernel/io/ring.h @@ -0,0 +1,216 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/** + * @brief Programmable submission/completion interface. + * @defgroup kernel_io_ring Kernel-side I/O Ring Interface + * @ingroup kernel_io + * + * @todo The I/O ring system is primarily a design document for now as it remains very work in progress and subject to + * change, currently being mostly unimplemented. + * + * @todo Rewrite the Kernel-side I/O Ring Interface documentation to match the new system. + * + * The I/O ring provides the core of all interfaces in PatchworkOS, where user-space submits Submission Queue Entries + * (SQEs) and receives Completion Queue Entries (CQEs) from it, all within shared memory. Allowing for highly efficient + * and asynchronous I/O operations, especially since PatchworkOS is designed to be natively asynchronous. + * + * Each SQE specifies a verb (the operation to perform) and a set of up to `SQE_MAX_ARG` arguments, while each CQE + * returns the result of a previously submitted SQE. + * + * Synchronous operations are implemented on top of this API in userspace. + * + * @see libstd_sys_ioring for the userspace interface to the asynchronous ring. + * @see [Wikipedia](https://en.wikipedia.org/wiki/Io_uring) for information about `io_uring`, the inspiration for this + * system. + * @see [Manpages](https://man7.org/linux/man-pages/man7/io_uring.7.html) for more information about `io_uring`. + * + * ## Syncronization + * + * The I/O ring structure is designed to be safe under the assumption that there is a single producer (one user-space + * thread) and a single consumer (the kernel). + * + * If an I/O ring needs multiple producers (needs to be accessed by multiple threads) it is the responsibility of + * the caller to ensure proper synchronization. + * + * @note The reason for this limitation is optimization for the common case, as the syncronization logic for multiple + * producers would add significant overhead. Additionally, it is rather straight forward for user-space to protect the + * ring with a mutex should it need to. + * + * Regarding the I/O ring structure itself, the structure can only be torndown as long as nothing is using it and there + * are no pending operations. + * + * ## Registers + * + * Operations performed on a I/O ring can load arguments from, and save their results to, seven 64-bit general purpose + * registers. All registers are stored in the shared control area of the I/O ring structure (`ioring_ctrl_t`), as such + * they can be inspected and modified by user space. + * + * When a SQE is processed, the kernel will check six register specifiers in the SQE flags, one for each argument and + * one for the result. Each specifier is stored as three bits, with a `SQE_REG_NONE` value indicating no-op and any + * other value representing the n-th register. The offset of the specifier specifies its meaning, for example, bits + * `0-2` specify the register to load into the first argument, bits `3-5` specify the register to load into the second + * argument, and so on until bits `15-17` which specify the register to save the result into. + * + * This system, when combined with `SQE_LINK`, allows for multiple operations to be performed at once, for example, it + * would be possible to open a file, read from it, seek to a new position, write to it, and finally close the file, with + * a single `enter()` call. + * + * @see `sqe_flags_t` for more information about register specifiers and their formatting. + * + * ## Arguments + * + * Arguments within a SQE are stored in five 64-bit values, `arg1` through `arg5`. For convenience, each argument value + * is stored as a union with various types. + * + * To avoid nameing conflicts and to avoid having to define new arguments for each verb, we define a convention to be + * used for the arguments. + * + * - `arg0`: The noun or subject of the verb, for example, a `fd_t` for file operations. + * - `arg1`: The source or payload of the verb, for example, a buffer or path. + * - `arg2`: The magnitude of the operation, for example, a size or encoding. + * - `arg3`: The location or a modifier to the operation, for example, an offset or flags. + * - `arg4`: An auxiliary argument, for example, additional flags or options. + * + * It may not always be possible for a verb to follow these conventions, but they should be followed whenever + * reasonable. + * + * @note The kernels internal I/O Request Packet structure uses a similar system but with the kernel equivalents + * of the arguments, for example, a `file_t*` instead of a `fd_t`. + * + * ## Results + * + * The result of a SQE is stored in its corresponding CQE using a single 64-bit value. For convenience, the result is + * stored as a union of various types. Note that this does not actually change the stored value, just how it is + * interpreted. + * + * If a SQE fails, the error code will be stored separately from the result and the result it self may be undefined. + * Some verbs may allow partial failures in which case the result may still be valid even if an error code is present. + * + * @todo Decide if partial failures are a good idea or not. + * + * ## Errors + * + * The majority of errors are returned in the CQEs, certain errors (such as `ENOMEM`) may be + * reported directly from the `enter()` call. + * + * Error values that may be returned in a CQE include: + * - `EOK`: Success. + * - `ECANCELED`: The verb was cancelled. + * - `ETIMEDOUT`: The verb timed out. + * - Other values may be returned depending on the verb. + * + * ## Verbs + * + * Included below is a list of all currently implemented verbs. + * + * The arguments of each verb is specified in order as `arg0`, `arg1`, `arg2`, `arg3`, `arg4`. + * + * ### `VERB_NOP` + * + * A no-operation verb that does nothing but is useful for implementing sleeping. + * + * @param arg0 Unused + * @param arg1 Unused + * @param arg2 Unused + * @param arg3 Unused + * @param arg4 Unused + * @result None + * + * ### `VERB_READ` + * + * Reads data from a file descriptor. + * + * @param fd The file descriptor to read from. + * @param buffer The buffer to read the data into. + * @param count The number of bytes to read. + * @param offset The offset to read from, or `IO_CUR` to use the current position. + * @param arg4 Unused + * @result The number of bytes read. + * + * ### `VERB_WRITE` + * + * Writes data to a file descriptor. + * + * @param fd The file descriptor to write to. + * @param buffer The buffer to write the data from. + * @param count The number of bytes to write. + * @param offset The offset to write to, or `IO_CUR` to use the current position. + * @param arg4 Unused + * @result The number of bytes written. + * + * ### `VERB_POLL` + * + * Polls a file descriptor for events. + * + * @param fd The file descriptor to poll. + * @param events The events to wait for. + * @param arg2 Unused + * @param arg3 Unused + * @param arg4 Unused + * @result The events that occurred. + * + * @{ + */ + +/** + * @brief Ring context flags. + * @enum ioring_ctx_flags_t + */ +typedef enum +{ + IORING_CTX_NONE = 0, ///< No flags set. + IORING_CTX_BUSY = 1 << 0, ///< Context is currently being used, used for fast locking. + IORING_CTX_MAPPED = 1 << 1, ///< Context is currently mapped into userspace. +} ioring_ctx_flags_t; + +/** + * @brief The kernel-side ring context structure. + * @struct ioring_ctx_t + */ +typedef struct ioring_ctx +{ + ioring_t ring; ///< The kernel-side ring structure. + irp_pool_t* irps; ///< Pool of preallocated IRPs. + void* userAddr; ///< Userspace address of the ring. + void* kernelAddr; ///< Kernel address of the ring. + size_t pageAmount; ///< Amount of pages mapped for the ring. + wait_queue_t waitQueue; ///< Wait queue for completions. + _Atomic(ioring_ctx_flags_t) flags; +} ioring_ctx_t; + +/** + * @brief Initialize a I/O context. + * + * @param ctx Pointer to the context to initialize. + */ +void ioring_ctx_init(ioring_ctx_t* ctx); + +/** + * @brief Deinitialize a I/O context. + * + * @param ctx Pointer to the context to deinitialize. + */ +void ioring_ctx_deinit(ioring_ctx_t* ctx); + +/** + * @brief Notify the context of new SQEs. + * + * @param ctx Pointer to the context. + * @param amount The number of SQEs to process. + * @param wait The minimum number of CQEs to wait for. + * @return On success, the number of SQEs processed. On failure, `ERR` and `errno` is set. + */ +uint64_t ioring_ctx_notify(ioring_ctx_t* ctx, size_t amount, size_t wait); + +/** @} */ \ No newline at end of file diff --git a/include/kernel/kernel.h b/include/kernel/kernel.h index 7e54d726e..13d337141 100644 --- a/include/kernel/kernel.h +++ b/include/kernel/kernel.h @@ -1,5 +1,7 @@ /** - * @brief The kernel of PatchworkOS. + * @brief The modular of PatchworkOS. * @defgroup kernel Kernel + * + * The kernel and its associated modules is responsible for managing system resources and providing hardware abstractions. */ \ No newline at end of file diff --git a/include/kernel/log/log.h b/include/kernel/log/log.h index e880e16e7..315a6713b 100644 --- a/include/kernel/log/log.h +++ b/include/kernel/log/log.h @@ -12,8 +12,8 @@ #include /** - * @brief Logging - * @defgroup kernel_log Logging + * @brief Kernel logging and debugging. + * @defgroup kernel_log Logging Subsystem * @ingroup kernel * * @{ diff --git a/include/kernel/mem/mem.h b/include/kernel/mem/mem.h index 5b66f3175..1815e5bd3 100644 --- a/include/kernel/mem/mem.h +++ b/include/kernel/mem/mem.h @@ -1,6 +1,6 @@ /** * @brief Memory management. - * @defgroup kernel_mem Memory + * @defgroup kernel_mem Memory Subsystem * @ingroup kernel * */ \ No newline at end of file diff --git a/include/kernel/mem/paging.h b/include/kernel/mem/paging.h index 64d4b1a82..756febaca 100644 --- a/include/kernel/mem/paging.h +++ b/include/kernel/mem/paging.h @@ -3,7 +3,7 @@ #include #include -#include <_internal/PAGE_SIZE.h> +#include <_libstd/PAGE_SIZE.h> #include #include #include diff --git a/include/kernel/mem/paging_types.h b/include/kernel/mem/paging_types.h index 932ac5b90..10250d964 100644 --- a/include/kernel/mem/paging_types.h +++ b/include/kernel/mem/paging_types.h @@ -1,8 +1,8 @@ #pragma once -#include <_internal/size_t.h> #include #include +#include #include /** diff --git a/include/modules/linker.lds b/include/kernel/module.lds similarity index 100% rename from include/modules/linker.lds rename to include/kernel/module.lds diff --git a/include/kernel/module/module.h b/include/kernel/module/module.h index a61f0b68c..5e8eaabcd 100644 --- a/include/kernel/module/module.h +++ b/include/kernel/module/module.h @@ -1,13 +1,13 @@ #pragma once -#include <_internal/MAX_PATH.h> +#include <_libstd/MAX_PATH.h> #include #include #include #include #include #include -#include +#include #include #include diff --git a/include/kernel/proc/process.h b/include/kernel/proc/process.h index 84b7d8073..aaaf8fab0 100644 --- a/include/kernel/proc/process.h +++ b/include/kernel/proc/process.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -22,7 +22,7 @@ /** * @brief Process management. - * @defgroup kernel_proc Process + * @defgroup kernel_proc Process Subsystem * @ingroup kernel * * Processes store the shared resources for threads of execution, for example the address space and open files. @@ -88,7 +88,7 @@ typedef struct process file_table_t fileTable; futex_ctx_t futexCtx; perf_process_ctx_t perf; - io_ctx_t rings[CONFIG_MAX_RINGS]; + ioring_ctx_t rings[CONFIG_MAX_RINGS]; note_handler_t noteHandler; wait_queue_t suspendQueue; wait_queue_t dyingQueue; diff --git a/include/kernel/sync/sync.h b/include/kernel/sync/sync.h index a4be76aae..10647f6b6 100644 --- a/include/kernel/sync/sync.h +++ b/include/kernel/sync/sync.h @@ -1,6 +1,6 @@ /** * @brief Locks and concurrency primitives. - * @defgroup kernel_sync Synchronization + * @defgroup kernel_sync Syncronization Primitives * @ingroup kernel */ \ No newline at end of file diff --git a/include/libstd/libstd.h b/include/libstd.h similarity index 100% rename from include/libstd/libstd.h rename to include/libstd.h diff --git a/include/libstd/limits.h b/include/limits.h similarity index 96% rename from include/libstd/limits.h rename to include/limits.h index c2bf5be61..2a9bc4f35 100644 --- a/include/libstd/limits.h +++ b/include/limits.h @@ -6,7 +6,7 @@ extern "C" { #endif -#include "_internal/config.h" +#include "_libstd/config.h" /* TODO: Defined to 1 as multibyte characters are not supported yet. */ #define MB_LEN_MAX 1 diff --git a/include/libstd/locale.h b/include/locale.h similarity index 98% rename from include/libstd/locale.h rename to include/locale.h index 370284302..ed5708926 100644 --- a/include/libstd/locale.h +++ b/include/locale.h @@ -6,7 +6,7 @@ extern "C" { #endif -#include "_internal/config.h" +#include "_libstd/config.h" struct lconv { diff --git a/include/libstd/math.h b/include/math.h similarity index 99% rename from include/libstd/math.h rename to include/math.h index a4230edc1..caea9defd 100644 --- a/include/libstd/math.h +++ b/include/math.h @@ -6,7 +6,7 @@ extern "C" { #endif -#include "_internal/config.h" +#include "_libstd/config.h" #define M_PI 3.14159265358979323846 #define M_PI_2 1.57079632679489661923 diff --git a/include/libpatchwork/cmd.h b/include/patchwork/cmd.h similarity index 100% rename from include/libpatchwork/cmd.h rename to include/patchwork/cmd.h diff --git a/include/libpatchwork/config.h b/include/patchwork/config.h similarity index 100% rename from include/libpatchwork/config.h rename to include/patchwork/config.h diff --git a/include/libpatchwork/display.h b/include/patchwork/display.h similarity index 100% rename from include/libpatchwork/display.h rename to include/patchwork/display.h diff --git a/include/libpatchwork/drawable.h b/include/patchwork/drawable.h similarity index 100% rename from include/libpatchwork/drawable.h rename to include/patchwork/drawable.h diff --git a/include/libpatchwork/element.h b/include/patchwork/element.h similarity index 100% rename from include/libpatchwork/element.h rename to include/patchwork/element.h diff --git a/include/libpatchwork/element_id.h b/include/patchwork/element_id.h similarity index 100% rename from include/libpatchwork/element_id.h rename to include/patchwork/element_id.h diff --git a/include/libpatchwork/event.h b/include/patchwork/event.h similarity index 100% rename from include/libpatchwork/event.h rename to include/patchwork/event.h diff --git a/include/libpatchwork/font.h b/include/patchwork/font.h similarity index 100% rename from include/libpatchwork/font.h rename to include/patchwork/font.h diff --git a/include/libpatchwork/image.h b/include/patchwork/image.h similarity index 100% rename from include/libpatchwork/image.h rename to include/patchwork/image.h diff --git a/include/libpatchwork/patchwork.h b/include/patchwork/patchwork.h similarity index 100% rename from include/libpatchwork/patchwork.h rename to include/patchwork/patchwork.h diff --git a/include/libpatchwork/pixel.h b/include/patchwork/pixel.h similarity index 100% rename from include/libpatchwork/pixel.h rename to include/patchwork/pixel.h diff --git a/include/libpatchwork/point.h b/include/patchwork/point.h similarity index 100% rename from include/libpatchwork/point.h rename to include/patchwork/point.h diff --git a/include/libpatchwork/polygon.h b/include/patchwork/polygon.h similarity index 100% rename from include/libpatchwork/polygon.h rename to include/patchwork/polygon.h diff --git a/include/libpatchwork/popup.h b/include/patchwork/popup.h similarity index 100% rename from include/libpatchwork/popup.h rename to include/patchwork/popup.h diff --git a/include/libpatchwork/procedure.h b/include/patchwork/procedure.h similarity index 100% rename from include/libpatchwork/procedure.h rename to include/patchwork/procedure.h diff --git a/include/libpatchwork/rect.h b/include/patchwork/rect.h similarity index 100% rename from include/libpatchwork/rect.h rename to include/patchwork/rect.h diff --git a/include/libpatchwork/surface.h b/include/patchwork/surface.h similarity index 100% rename from include/libpatchwork/surface.h rename to include/patchwork/surface.h diff --git a/include/libpatchwork/theme.h b/include/patchwork/theme.h similarity index 100% rename from include/libpatchwork/theme.h rename to include/patchwork/theme.h diff --git a/include/libpatchwork/widgets.h b/include/patchwork/widgets.h similarity index 100% rename from include/libpatchwork/widgets.h rename to include/patchwork/widgets.h diff --git a/include/libpatchwork/window.h b/include/patchwork/window.h similarity index 100% rename from include/libpatchwork/window.h rename to include/patchwork/window.h diff --git a/include/libstd/setjmp.h b/include/setjmp.h similarity index 94% rename from include/libstd/setjmp.h rename to include/setjmp.h index eeb3a2380..101fb8457 100644 --- a/include/libstd/setjmp.h +++ b/include/setjmp.h @@ -6,7 +6,7 @@ extern "C" { #endif -#include "_internal/config.h" +#include "_libstd/config.h" /** * @brief Setjmp/Longjmp functions. diff --git a/include/libstd/signal.h b/include/signal.h similarity index 96% rename from include/libstd/signal.h rename to include/signal.h index 39ecd82c8..442f34107 100644 --- a/include/libstd/signal.h +++ b/include/signal.h @@ -8,7 +8,7 @@ extern "C" #include -#include "_internal/config.h" +#include "_libstd/config.h" /** * @brief Wrappers around "notes" for ANSI C signal handling. diff --git a/include/libstd/stdarg.h b/include/stdarg.h similarity index 100% rename from include/libstd/stdarg.h rename to include/stdarg.h diff --git a/include/libstd/stdatomic.h b/include/stdatomic.h similarity index 100% rename from include/libstd/stdatomic.h rename to include/stdatomic.h diff --git a/include/libstd/stdbool.h b/include/stdbool.h similarity index 100% rename from include/libstd/stdbool.h rename to include/stdbool.h diff --git a/include/libstd/stddef.h b/include/stddef.h similarity index 51% rename from include/libstd/stddef.h rename to include/stddef.h index 7701fe94b..9a3bbf63f 100644 --- a/include/libstd/stddef.h +++ b/include/stddef.h @@ -6,17 +6,17 @@ extern "C" { #endif -#include "_internal/ERR.h" -#include "_internal/NULL.h" -#include "_internal/config.h" -#include "_internal/ptrdiff_t.h" -#include "_internal/size_t.h" -#include "_internal/wchar_t.h" +#include "_libstd/ERR.h" +#include "_libstd/NULL.h" +#include "_libstd/config.h" +#include "_libstd/ptrdiff_t.h" +#include "_libstd/size_t.h" +#include "_libstd/wchar_t.h" #define offsetof(type, member) ((size_t)&(((type*)0)->member)) #if _USE_ANNEX_K == 1 -#include "_internal/rsize_t.h" +#include "_libstd/rsize_t.h" #endif #if defined(__cplusplus) diff --git a/include/libstd/stdint.h b/include/stdint.h similarity index 100% rename from include/libstd/stdint.h rename to include/stdint.h diff --git a/include/libstd/stdio.h b/include/stdio.h similarity index 95% rename from include/libstd/stdio.h rename to include/stdio.h index 3d7bb8093..684b5128c 100644 --- a/include/libstd/stdio.h +++ b/include/stdio.h @@ -8,12 +8,12 @@ extern "C" { #endif -#include "_internal/MAX_PATH.h" -#include "_internal/NULL.h" -#include "_internal/SEEK.h" -#include "_internal/config.h" -#include "_internal/fd_t.h" -#include "_internal/size_t.h" +#include "_libstd/MAX_PATH.h" +#include "_libstd/NULL.h" +#include "_libstd/SEEK.h" +#include "_libstd/config.h" +#include "_libstd/fd_t.h" +#include "_libstd/size_t.h" #define _IOFBF (1u << 0) #define _IOLBF (1u << 1) @@ -129,8 +129,8 @@ _PUBLIC void perror(const char* s); #define L_tmpnam_s L_tmpnam #define TMP_MAX_S TMP_MAX -#include "_internal/errno_t.h" -#include "_internal/rsize_t.h" +#include "_libstd/errno_t.h" +#include "_libstd/rsize_t.h" _PUBLIC errno_t tmpfile_s(FILE * _RESTRICT * _RESTRICT streamptr); diff --git a/include/libstd/stdlib.h b/include/stdlib.h similarity index 95% rename from include/libstd/stdlib.h rename to include/stdlib.h index 2e29679fc..bf4ec8f8a 100644 --- a/include/libstd/stdlib.h +++ b/include/stdlib.h @@ -6,9 +6,9 @@ extern "C" { #endif -#include "_internal/NULL.h" -#include "_internal/config.h" -#include "_internal/size_t.h" +#include "_libstd/NULL.h" +#include "_libstd/config.h" +#include "_libstd/size_t.h" _PUBLIC char* lltoa(long long number, char* str, int base); #define ltoa(number, str, base) lltoa(number, str, base) @@ -97,8 +97,8 @@ int system(const char* command); #if _USE_ANNEX_K == 1 -#include "_internal/errno_t.h" -#include "_internal/rsize_t.h" +#include "_libstd/errno_t.h" +#include "_libstd/rsize_t.h" typedef void (*constraint_handler_t)(const char* _RESTRICT msg, void* _RESTRICT ptr, errno_t err); diff --git a/include/libstd/string.h b/include/string.h similarity index 94% rename from include/libstd/string.h rename to include/string.h index cda7ff2b1..e12eba0b9 100644 --- a/include/libstd/string.h +++ b/include/string.h @@ -6,9 +6,9 @@ extern "C" { #endif -#include "_internal/NULL.h" -#include "_internal/config.h" -#include "_internal/size_t.h" +#include "_libstd/NULL.h" +#include "_libstd/config.h" +#include "_libstd/size_t.h" _PUBLIC void* memcpy(void* _RESTRICT s1, const void* _RESTRICT s2, size_t n); @@ -61,8 +61,8 @@ char* strdup(const char* src); #if (__STDC_WANT_LIB_EXT1__ + 0) != 0 -#include "_internal/errno_t.h" -#include "_internal/rsize_t.h" +#include "_libstd/errno_t.h" +#include "_libstd/rsize_t.h" _PUBLIC errno_t memcpy_s(void* _RESTRICT s1, rsize_t s1max, const void* _RESTRICT s2, rsize_t n); diff --git a/include/libstd/strings.h b/include/strings.h similarity index 73% rename from include/libstd/strings.h rename to include/strings.h index aaf8dceb0..370f119af 100644 --- a/include/libstd/strings.h +++ b/include/strings.h @@ -6,9 +6,9 @@ extern "C" { #endif -#include "_internal/NULL.h" -#include "_internal/config.h" -#include "_internal/size_t.h" +#include "_libstd/NULL.h" +#include "_libstd/config.h" +#include "_libstd/size_t.h" int strcasecmp(const char* s1, const char* s2); int strncasecmp(const char* s1, const char* s2, size_t n); diff --git a/include/libstd/sys/9p.h b/include/sys/9p.h similarity index 100% rename from include/libstd/sys/9p.h rename to include/sys/9p.h diff --git a/include/libstd/sys/argsplit.h b/include/sys/argsplit.h similarity index 98% rename from include/libstd/sys/argsplit.h rename to include/sys/argsplit.h index 0c9d4e0f2..ecbfdfa3b 100644 --- a/include/libstd/sys/argsplit.h +++ b/include/sys/argsplit.h @@ -8,7 +8,7 @@ extern "C" { #endif -#include "_internal/config.h" +#include "_libstd/config.h" /** * @brief Standardized argument parsing diff --git a/include/libstd/sys/bitmap.h b/include/sys/bitmap.h similarity index 100% rename from include/libstd/sys/bitmap.h rename to include/sys/bitmap.h diff --git a/include/libstd/sys/cpuid.h b/include/sys/cpuid.h similarity index 100% rename from include/libstd/sys/cpuid.h rename to include/sys/cpuid.h diff --git a/include/libstd/sys/defs.h b/include/sys/defs.h similarity index 100% rename from include/libstd/sys/defs.h rename to include/sys/defs.h diff --git a/include/libstd/sys/elf.h b/include/sys/elf.h similarity index 100% rename from include/libstd/sys/elf.h rename to include/sys/elf.h diff --git a/include/libstd/sys/fs.h b/include/sys/fs.h similarity index 98% rename from include/libstd/sys/fs.h rename to include/sys/fs.h index 57785925d..07a3b60b0 100644 --- a/include/libstd/sys/fs.h +++ b/include/sys/fs.h @@ -13,16 +13,16 @@ extern "C" { #endif -#include "_internal/ERR.h" -#include "_internal/MAX_NAME.h" -#include "_internal/MAX_PATH.h" -#include "_internal/NULL.h" -#include "_internal/SEEK.h" -#include "_internal/clock_t.h" -#include "_internal/config.h" -#include "_internal/fd_t.h" -#include "_internal/ssize_t.h" -#include "_internal/time_t.h" +#include "_libstd/ERR.h" +#include "_libstd/MAX_NAME.h" +#include "_libstd/MAX_PATH.h" +#include "_libstd/NULL.h" +#include "_libstd/SEEK.h" +#include "_libstd/clock_t.h" +#include "_libstd/config.h" +#include "_libstd/fd_t.h" +#include "_libstd/ssize_t.h" +#include "_libstd/time_t.h" /** * @brief Filesystem header. diff --git a/include/libstd/sys/ioring.h b/include/sys/ioring.h similarity index 93% rename from include/libstd/sys/ioring.h rename to include/sys/ioring.h index 58738ebef..36e359eae 100644 --- a/include/libstd/sys/ioring.h +++ b/include/sys/ioring.h @@ -12,12 +12,12 @@ extern "C" { #endif -#include "_internal/MAX_NAME.h" -#include "_internal/MAX_PATH.h" -#include "_internal/clock_t.h" -#include "_internal/errno_t.h" -#include "_internal/fd_t.h" -#include "_internal/ssize_t.h" +#include "_libstd/MAX_NAME.h" +#include "_libstd/MAX_PATH.h" +#include "_libstd/clock_t.h" +#include "_libstd/errno_t.h" +#include "_libstd/fd_t.h" +#include "_libstd/ssize_t.h" /** * @addtogroup kernel_io @@ -38,12 +38,12 @@ typedef uint64_t io_events_t; ///< Poll events type. #define IO_POLL_HUP (1 << 3) ///< File descriptor is closed. #define IO_POLL_NVAL (1 << 4) ///< Invalid file descriptor. -typedef uint32_t ioring_op_t; ///< I/O operation code type. -#define IORING_NOP 0 ///< No-op operation. -#define IORING_READ 1 ///< Read operation. -#define IORING_WRITE 2 ///< Write operation. -#define IORING_POLL 3 ///< Poll operation. -#define IORING_MAX 4 ///< The maximum number of operation. +typedef uint32_t io_op_t; ///< I/O operation code type. +#define IO_OP_NOP 0 ///< No-op operation. +#define IO_OP_READ 1 ///< Read operation. +#define IO_OP_WRITE 2 ///< Write operation. +#define IO_OP_POLL 3 ///< Poll operation. +#define IO_OP_MAX 4 ///< The maximum number of operation. typedef uint32_t sqe_flags_t; ///< Submission queue entry (SQE) flags. #define SQE_REG0 (0) ///< The first register. @@ -89,7 +89,7 @@ typedef struct sqe { clock_t timeout; ///< Timeout for the operation, `CLOCKS_NEVER` for no timeout. void* data; ///< Private data for the operation, will be returned in the completion entry. - ioring_op_t op; ///< The operation to perform. + io_op_t op; ///< The operation to perform. sqe_flags_t flags; ///< Submission flags. union { uint64_t arg0; @@ -141,7 +141,7 @@ static_assert(sizeof(sqe_t) == 64, "sqe_t is not 64 bytes"); */ typedef struct cqe { - ioring_op_t op; ///< The operation that was performed. + io_op_t op; ///< The operation that was performed. errno_t error; ///< Error code, if not equal to `EOK` an error occurred. void* data; ///< Private data from the submission entry. union { diff --git a/include/libstd/sys/kbd.h b/include/sys/kbd.h similarity index 99% rename from include/libstd/sys/kbd.h rename to include/sys/kbd.h index 9c894119f..a9c244a50 100644 --- a/include/libstd/sys/kbd.h +++ b/include/sys/kbd.h @@ -8,7 +8,7 @@ extern "C" { #endif -#include "_internal/clock_t.h" +#include "_libstd/clock_t.h" /** * @brief Keyboard keycodes. diff --git a/include/libstd/sys/list.h b/include/sys/list.h similarity index 99% rename from include/libstd/sys/list.h rename to include/sys/list.h index 87b81ca51..d8b440790 100644 --- a/include/libstd/sys/list.h +++ b/include/sys/list.h @@ -1,8 +1,8 @@ #ifndef _SYS_LIST_H #define _SYS_LIST_H 1 -#include "_internal/CONTAINER_OF.h" -#include "_internal/NULL.h" +#include "_libstd/CONTAINER_OF.h" +#include "_libstd/NULL.h" #include #include diff --git a/include/libstd/sys/math.h b/include/sys/math.h similarity index 100% rename from include/libstd/sys/math.h rename to include/sys/math.h diff --git a/include/libstd/sys/proc.h b/include/sys/proc.h similarity index 98% rename from include/libstd/sys/proc.h rename to include/sys/proc.h index 8cbf2b541..91ad38731 100644 --- a/include/libstd/sys/proc.h +++ b/include/sys/proc.h @@ -10,15 +10,15 @@ extern "C" { #endif -#include "_internal/ERR.h" -#include "_internal/NULL.h" -#include "_internal/PAGE_SIZE.h" -#include "_internal/clock_t.h" -#include "_internal/config.h" -#include "_internal/fd_t.h" - -#include "_internal/pid_t.h" -#include "_internal/tid_t.h" +#include "_libstd/ERR.h" +#include "_libstd/NULL.h" +#include "_libstd/PAGE_SIZE.h" +#include "_libstd/clock_t.h" +#include "_libstd/config.h" +#include "_libstd/fd_t.h" + +#include "_libstd/pid_t.h" +#include "_libstd/tid_t.h" /** * @brief Process management header. diff --git a/include/libstd/threads.h b/include/threads.h similarity index 94% rename from include/libstd/threads.h rename to include/threads.h index 589bff1fc..fdf248346 100644 --- a/include/libstd/threads.h +++ b/include/threads.h @@ -8,10 +8,10 @@ extern "C" { #endif -#include "_internal/config.h" -#include "_internal/pid_t.h" -#include "_internal/tid_t.h" -#include "_internal/timespec.h" +#include "_libstd/config.h" +#include "_libstd/pid_t.h" +#include "_libstd/tid_t.h" +#include "_libstd/timespec.h" #include diff --git a/include/libstd/time.h b/include/time.h similarity index 84% rename from include/libstd/time.h rename to include/time.h index 295aa6453..141be80f3 100644 --- a/include/libstd/time.h +++ b/include/time.h @@ -8,12 +8,12 @@ extern "C" { #endif -#include "_internal/NULL.h" -#include "_internal/clock_t.h" -#include "_internal/config.h" -#include "_internal/size_t.h" -#include "_internal/time_t.h" -#include "_internal/timespec.h" +#include "_libstd/NULL.h" +#include "_libstd/clock_t.h" +#include "_libstd/config.h" +#include "_libstd/size_t.h" +#include "_libstd/time_t.h" +#include "_libstd/timespec.h" #define TIME_UTC 1 @@ -55,8 +55,8 @@ _PUBLIC size_t strftime(char* _RESTRICT s, size_t maxsize, const char* _RESTRICT #if (__STDC_WANT_LIB_EXT1__ + 0) != 0 -#include "_internal/errno_t.h" -#include "_internal/rsize_t.h" +#include "_libstd/errno_t.h" +#include "_libstd/rsize_t.h" _PUBLIC errno_t asctime_s(char* s, rsize_t maxsize, const struct tm* timeptr); diff --git a/lib/OVMFbin/OVMF_VARS-pure-efi.fd b/lib/OVMFbin/OVMF_VARS-pure-efi.fd index 52f4bb61960c47f6f7a086bdfd3174c231faba2a..840e8790cbf0fd7586b9ca181c210403c6af0b02 100644 GIT binary patch delta 113 zcmZo@;Am*z*f7g`@*-`g&3C-xxF$=5sBKONTgSM$%_@(H@xkW1-m&O{53vb8!VuJ9 Wn!ZSz(TVXfP-byv(}!gr7zF?mOfAO% delta 35 tcmV+;0Nnq8fCzwq2(X$ - + diff --git a/src/boxes/apps/calculator/main.c b/src/boxes/apps/calculator/main.c index 39b7cb8b0..c9c054fb8 100644 --- a/src/boxes/apps/calculator/main.c +++ b/src/boxes/apps/calculator/main.c @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/boxes/apps/clock/main.c b/src/boxes/apps/clock/main.c index a62070828..514c05ea7 100644 --- a/src/boxes/apps/clock/main.c +++ b/src/boxes/apps/clock/main.c @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/boxes/apps/terminal/ansi.h b/src/boxes/apps/terminal/ansi.h index 54a9e2b5f..0e63d8112 100644 --- a/src/boxes/apps/terminal/ansi.h +++ b/src/boxes/apps/terminal/ansi.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/src/boxes/apps/terminal/terminal.c b/src/boxes/apps/terminal/terminal.c index 94c3a1a1a..f9d94ef1b 100644 --- a/src/boxes/apps/terminal/terminal.c +++ b/src/boxes/apps/terminal/terminal.c @@ -1,7 +1,7 @@ #include "terminal.h" #include "ansi.h" -#include +#include #include #include #include diff --git a/src/boxes/apps/terminal/terminal.h b/src/boxes/apps/terminal/terminal.h index 5fd3f4b0d..05f70012d 100644 --- a/src/boxes/apps/terminal/terminal.h +++ b/src/boxes/apps/terminal/terminal.h @@ -2,7 +2,7 @@ #include "ansi.h" -#include +#include #include #include #include diff --git a/src/boxes/apps/tetris/main.c b/src/boxes/apps/tetris/main.c index 8bfeee948..17da98f6a 100644 --- a/src/boxes/apps/tetris/main.c +++ b/src/boxes/apps/tetris/main.c @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/boxes/core/cursor/main.c b/src/boxes/core/cursor/main.c index 62c64bdac..668240e37 100644 --- a/src/boxes/core/cursor/main.c +++ b/src/boxes/core/cursor/main.c @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/boxes/core/dwm/client.h b/src/boxes/core/dwm/client.h index bf332f9b7..bef94d28f 100644 --- a/src/boxes/core/dwm/client.h +++ b/src/boxes/core/dwm/client.h @@ -2,8 +2,8 @@ #include "surface.h" -#include -#include +#include +#include #include #include diff --git a/src/boxes/core/dwm/dwm.c b/src/boxes/core/dwm/dwm.c index 2f7b09d2b..3a92a4cdc 100644 --- a/src/boxes/core/dwm/dwm.c +++ b/src/boxes/core/dwm/dwm.c @@ -7,7 +7,7 @@ #include "surface.h" #include -#include +#include #include #include #include diff --git a/src/boxes/core/dwm/dwm.h b/src/boxes/core/dwm/dwm.h index 045dce722..d2a631ce5 100644 --- a/src/boxes/core/dwm/dwm.h +++ b/src/boxes/core/dwm/dwm.h @@ -2,7 +2,7 @@ #include "surface.h" -#include +#include #include #include diff --git a/src/boxes/core/dwm/kbd.h b/src/boxes/core/dwm/kbd.h index c99663852..c5f738b92 100644 --- a/src/boxes/core/dwm/kbd.h +++ b/src/boxes/core/dwm/kbd.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include keycode_t kbd_translate(keycode_t code); diff --git a/src/boxes/core/dwm/region.h b/src/boxes/core/dwm/region.h index 076f48075..26121e070 100644 --- a/src/boxes/core/dwm/region.h +++ b/src/boxes/core/dwm/region.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/src/boxes/core/dwm/surface.h b/src/boxes/core/dwm/surface.h index 5c5299500..a3ae7929a 100644 --- a/src/boxes/core/dwm/surface.h +++ b/src/boxes/core/dwm/surface.h @@ -1,10 +1,10 @@ #pragma once -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include diff --git a/src/boxes/core/taskbar/main.c b/src/boxes/core/taskbar/main.c index 3f0f6bb38..7f880f926 100644 --- a/src/boxes/core/taskbar/main.c +++ b/src/boxes/core/taskbar/main.c @@ -1,6 +1,6 @@ #include "taskbar.h" -#include +#include #include #include diff --git a/src/boxes/core/taskbar/start_menu.c b/src/boxes/core/taskbar/start_menu.c index 56b48c9a6..fa35d6f32 100644 --- a/src/boxes/core/taskbar/start_menu.c +++ b/src/boxes/core/taskbar/start_menu.c @@ -2,7 +2,7 @@ #include "taskbar.h" -#include +#include #include #include #include diff --git a/src/boxes/core/taskbar/start_menu.h b/src/boxes/core/taskbar/start_menu.h index 8253ff158..c67e719f9 100644 --- a/src/boxes/core/taskbar/start_menu.h +++ b/src/boxes/core/taskbar/start_menu.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #define START_BUTTON_HEIGHT 32 diff --git a/src/boxes/core/taskbar/taskbar.c b/src/boxes/core/taskbar/taskbar.c index eb57507cf..a8d239af1 100644 --- a/src/boxes/core/taskbar/taskbar.c +++ b/src/boxes/core/taskbar/taskbar.c @@ -1,10 +1,10 @@ #include "taskbar.h" -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include diff --git a/src/boxes/core/taskbar/taskbar.h b/src/boxes/core/taskbar/taskbar.h index fe4217dbf..4f378cda8 100644 --- a/src/boxes/core/taskbar/taskbar.h +++ b/src/boxes/core/taskbar/taskbar.h @@ -2,7 +2,7 @@ #include "start_menu.h" -#include +#include #include #include diff --git a/src/boxes/core/wall/main.c b/src/boxes/core/wall/main.c index 6d0b4fbd0..3215af388 100644 --- a/src/boxes/core/wall/main.c +++ b/src/boxes/core/wall/main.c @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/kernel/fs/path.c b/src/kernel/fs/path.c index 1c0dca2d2..61e37d7fd 100644 --- a/src/kernel/fs/path.c +++ b/src/kernel/fs/path.c @@ -1,4 +1,4 @@ -#include <_internal/MAX_PATH.h> +#include <_libstd/MAX_PATH.h> #include #include diff --git a/src/kernel/fs/procfs.c b/src/kernel/fs/procfs.c index 67e828aff..4ea66d5d5 100644 --- a/src/kernel/fs/procfs.c +++ b/src/kernel/fs/procfs.c @@ -1,4 +1,4 @@ -#include <_internal/MAX_NAME.h> +#include <_libstd/MAX_NAME.h> #include #include diff --git a/src/kernel/fs/vfs.c b/src/kernel/fs/vfs.c index d95513751..72abc2769 100644 --- a/src/kernel/fs/vfs.c +++ b/src/kernel/fs/vfs.c @@ -1,4 +1,4 @@ -#include <_internal/MAX_PATH.h> +#include <_libstd/MAX_PATH.h> #include #include diff --git a/src/kernel/init/init.c b/src/kernel/init/init.c index 28eccd3da..ee02f1814 100644 --- a/src/kernel/init/init.c +++ b/src/kernel/init/init.c @@ -34,7 +34,7 @@ #include -#include +#include <_libstd/init.h> #ifdef _TESTING_ #include diff --git a/src/kernel/io/io.c b/src/kernel/io/ring.c similarity index 77% rename from src/kernel/io/io.c rename to src/kernel/io/ring.c index d38ffb932..0165fa95d 100644 --- a/src/kernel/io/io.c +++ b/src/kernel/io/ring.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include #include @@ -16,10 +16,10 @@ #include #include -static inline uint64_t io_ctx_acquire(io_ctx_t* ctx) +static inline uint64_t ioring_ctx_acquire(ioring_ctx_t* ctx) { - io_ctx_flags_t expected = atomic_load(&ctx->flags); - if (!(expected & IO_CTX_BUSY) && atomic_compare_exchange_strong(&ctx->flags, &expected, expected | IO_CTX_BUSY)) + ioring_ctx_flags_t expected = atomic_load(&ctx->flags); + if (!(expected & IORING_CTX_BUSY) && atomic_compare_exchange_strong(&ctx->flags, &expected, expected | IORING_CTX_BUSY)) { return 0; } @@ -27,12 +27,12 @@ static inline uint64_t io_ctx_acquire(io_ctx_t* ctx) return ERR; } -static inline void io_ctx_release(io_ctx_t* ctx) +static inline void ioring_ctx_release(ioring_ctx_t* ctx) { - atomic_fetch_and(&ctx->flags, ~IO_CTX_BUSY); + atomic_fetch_and(&ctx->flags, ~IORING_CTX_BUSY); } -static inline uint64_t io_ctx_map(io_ctx_t* ctx, process_t* process, ioring_id_t id, ioring_t* userRing, void* address, +static inline uint64_t ioring_ctx_map(ioring_ctx_t* ctx, process_t* process, ioring_id_t id, ioring_t* userRing, void* address, size_t sentries, size_t centries) { ioring_t* kernelRing = &ctx->ring; @@ -119,11 +119,11 @@ static inline uint64_t io_ctx_map(io_ctx_t* ctx, process_t* process, ioring_id_t ctx->kernelAddr = kernelAddr; ctx->pageAmount = pageAmount; - atomic_fetch_or(&ctx->flags, IO_CTX_MAPPED); + atomic_fetch_or(&ctx->flags, IORING_CTX_MAPPED); return 0; } -static inline uint64_t io_ctx_unmap(io_ctx_t* ctx) +static inline uint64_t ioring_ctx_unmap(ioring_ctx_t* ctx) { vmm_unmap(&ctx->irps->process->space, ctx->userAddr, ctx->pageAmount * PAGE_SIZE); vmm_unmap(NULL, ctx->kernelAddr, ctx->pageAmount * PAGE_SIZE); @@ -131,11 +131,11 @@ static inline uint64_t io_ctx_unmap(io_ctx_t* ctx) irp_pool_free(ctx->irps); ctx->irps = NULL; - atomic_fetch_and(&ctx->flags, ~IO_CTX_MAPPED); + atomic_fetch_and(&ctx->flags, ~IORING_CTX_MAPPED); return 0; } -static inline uint64_t io_ctx_avail_cqes(io_ctx_t* ctx) +static inline uint64_t ioring_ctx_avail_cqes(ioring_ctx_t* ctx) { ioring_t* ring = &ctx->ring; uint32_t ctail = atomic_load_explicit(&ring->ctrl->ctail, memory_order_relaxed); @@ -143,7 +143,7 @@ static inline uint64_t io_ctx_avail_cqes(io_ctx_t* ctx) return ctail - chead; } -void io_ctx_init(io_ctx_t* ctx) +void ioring_ctx_init(ioring_ctx_t* ctx) { if (ctx == NULL) { @@ -156,40 +156,40 @@ void io_ctx_init(io_ctx_t* ctx) ctx->kernelAddr = NULL; ctx->pageAmount = 0; wait_queue_init(&ctx->waitQueue); - atomic_init(&ctx->flags, IO_CTX_NONE); + atomic_init(&ctx->flags, IORING_CTX_NONE); } -void io_ctx_deinit(io_ctx_t* ctx) +void ioring_ctx_deinit(ioring_ctx_t* ctx) { if (ctx == NULL) { return; } - if (io_ctx_acquire(ctx) == ERR) + if (ioring_ctx_acquire(ctx) == ERR) { panic(NULL, "failed to acquire async context for deinitialization"); } - if (atomic_load(&ctx->flags) & IO_CTX_MAPPED) + if (atomic_load(&ctx->flags) & IORING_CTX_MAPPED) { - if (io_ctx_unmap(ctx) == ERR) + if (ioring_ctx_unmap(ctx) == ERR) { panic(NULL, "failed to deinitialize async context"); } } - io_ctx_release(ctx); + ioring_ctx_release(ctx); wait_queue_deinit(&ctx->waitQueue); } -static void io_ctx_dispatch(irp_t* irp); +static void ioring_ctx_dispatch(irp_t* irp); -static void io_ctx_complete(irp_t* irp, void* _ptr) +static void ioring_ctx_complete(irp_t* irp, void* _ptr) { UNUSED(_ptr); - io_ctx_t* ctx = irp_get_ctx(irp); + ioring_ctx_t* ctx = irp_get_ctx(irp); ioring_t* ring = &ctx->ring; sqe_flags_t reg = (irp->sqe.flags >> SQE_SAVE) & SQE_REG_MASK; @@ -234,8 +234,8 @@ static void io_ctx_complete(irp_t* irp, void* _ptr) irp_t* next = irp_chain_next(irp); if (next != NULL) { - irp_set_complete(next, io_ctx_complete, NULL); - irp_call_direct(next, io_ctx_dispatch); + irp_set_complete(next, ioring_ctx_complete, NULL); + irp_call_direct(next, ioring_ctx_dispatch); } } @@ -248,9 +248,9 @@ static uint64_t nop_cancel(irp_t* irp) return 0; } -static void io_ctx_dispatch(irp_t* irp) +static void ioring_ctx_dispatch(irp_t* irp) { - io_ctx_t* ctx = irp_get_ctx(irp); + ioring_ctx_t* ctx = irp_get_ctx(irp); ioring_t* ring = &ctx->ring; // Ugly but the alternative is a super messy SQE structure. @@ -287,7 +287,7 @@ static void io_ctx_dispatch(irp_t* irp) switch (irp->sqe.op) { - case IORING_NOP: + case IO_OP_NOP: irp_set_cancel(irp, nop_cancel); irp_timeout_add(irp, irp->sqe.timeout); break; @@ -301,9 +301,9 @@ typedef struct { list_t irps; irp_t* link; -} io_ctx_notify_ctx_t; +} ioring_ctx_notify_ctx_t; -static uint64_t io_ctx_sqe_pop(io_ctx_t* ctx, io_ctx_notify_ctx_t* notify) +static uint64_t ioring_ctx_sqe_pop(ioring_ctx_t* ctx, ioring_ctx_notify_ctx_t* notify) { ioring_t* ring = &ctx->ring; @@ -343,27 +343,27 @@ static uint64_t io_ctx_sqe_pop(io_ctx_t* ctx, io_ctx_notify_ctx_t* notify) return 0; } -uint64_t io_ctx_notify(io_ctx_t* ctx, size_t amount, size_t wait) +uint64_t ioring_ctx_notify(ioring_ctx_t* ctx, size_t amount, size_t wait) { if (amount == 0) { return 0; } - if (io_ctx_acquire(ctx) == ERR) + if (ioring_ctx_acquire(ctx) == ERR) { errno = EBUSY; return ERR; } - if (!(atomic_load(&ctx->flags) & IO_CTX_MAPPED)) + if (!(atomic_load(&ctx->flags) & IORING_CTX_MAPPED)) { - io_ctx_release(ctx); + ioring_ctx_release(ctx); errno = EINVAL; return ERR; } - io_ctx_notify_ctx_t notify = { + ioring_ctx_notify_ctx_t notify = { .irps = LIST_CREATE(notify.irps), .link = NULL, }; @@ -371,7 +371,7 @@ uint64_t io_ctx_notify(io_ctx_t* ctx, size_t amount, size_t wait) size_t processed = 0; while (processed < amount) { - if (io_ctx_sqe_pop(ctx, ¬ify) == ERR) + if (ioring_ctx_sqe_pop(ctx, ¬ify) == ERR) { break; } @@ -382,23 +382,23 @@ uint64_t io_ctx_notify(io_ctx_t* ctx, size_t amount, size_t wait) { irp_t* irp = CONTAINER_OF(list_pop_front(¬ify.irps), irp_t, entry); - irp_set_complete(irp, io_ctx_complete, NULL); - irp_call_direct(irp, io_ctx_dispatch); + irp_set_complete(irp, ioring_ctx_complete, NULL); + irp_call_direct(irp, ioring_ctx_dispatch); } if (wait == 0) { - io_ctx_release(ctx); + ioring_ctx_release(ctx); return processed; } - if (WAIT_BLOCK(&ctx->waitQueue, io_ctx_avail_cqes(ctx) >= wait) == ERR) + if (WAIT_BLOCK(&ctx->waitQueue, ioring_ctx_avail_cqes(ctx) >= wait) == ERR) { - io_ctx_release(ctx); + ioring_ctx_release(ctx); return processed > 0 ? processed : ERR; } - io_ctx_release(ctx); + ioring_ctx_release(ctx); return processed; } @@ -412,12 +412,12 @@ SYSCALL_DEFINE(SYS_SETUP, ioring_id_t, ioring_t* userRing, void* address, size_t process_t* process = process_current(); - io_ctx_t* ctx = NULL; + ioring_ctx_t* ctx = NULL; ioring_id_t id = 0; for (id = 0; id < ARRAY_SIZE(process->rings); id++) { - io_ctx_flags_t expected = IO_CTX_NONE; - if (atomic_compare_exchange_strong(&process->rings[id].flags, &expected, IO_CTX_BUSY)) + ioring_ctx_flags_t expected = IORING_CTX_NONE; + if (atomic_compare_exchange_strong(&process->rings[id].flags, &expected, IORING_CTX_BUSY)) { ctx = &process->rings[id]; break; @@ -430,13 +430,13 @@ SYSCALL_DEFINE(SYS_SETUP, ioring_id_t, ioring_t* userRing, void* address, size_t return ERR; } - if (io_ctx_map(ctx, process, id, userRing, address, sentries, centries) == ERR) + if (ioring_ctx_map(ctx, process, id, userRing, address, sentries, centries) == ERR) { - io_ctx_release(ctx); + ioring_ctx_release(ctx); return ERR; } - io_ctx_release(ctx); + ioring_ctx_release(ctx); return id; } @@ -449,34 +449,34 @@ SYSCALL_DEFINE(SYS_TEARDOWN, uint64_t, ioring_id_t id) return ERR; } - io_ctx_t* ctx = &process->rings[id]; - if (io_ctx_acquire(ctx) == ERR) + ioring_ctx_t* ctx = &process->rings[id]; + if (ioring_ctx_acquire(ctx) == ERR) { errno = EBUSY; return ERR; } - if (!(atomic_load(&ctx->flags) & IO_CTX_MAPPED)) + if (!(atomic_load(&ctx->flags) & IORING_CTX_MAPPED)) { - io_ctx_release(ctx); + ioring_ctx_release(ctx); errno = EINVAL; return ERR; } if (ctx->irps != NULL && atomic_load(&ctx->irps->pool.used) != 0) { - io_ctx_release(ctx); + ioring_ctx_release(ctx); errno = EBUSY; return ERR; } - if (io_ctx_unmap(ctx) == ERR) + if (ioring_ctx_unmap(ctx) == ERR) { - io_ctx_release(ctx); + ioring_ctx_release(ctx); return ERR; } - io_ctx_release(ctx); + ioring_ctx_release(ctx); return 0; } @@ -489,6 +489,6 @@ SYSCALL_DEFINE(SYS_ENTER, uint64_t, ioring_id_t id, size_t amount, size_t wait) return ERR; } - io_ctx_t* ctx = &process->rings[id]; - return io_ctx_notify(ctx, amount, wait); + ioring_ctx_t* ctx = &process->rings[id]; + return ioring_ctx_notify(ctx, amount, wait); } \ No newline at end of file diff --git a/src/kernel/proc/process.c b/src/kernel/proc/process.c index 6b5357819..90235bf09 100644 --- a/src/kernel/proc/process.c +++ b/src/kernel/proc/process.c @@ -113,7 +113,7 @@ static void process_free(process_t* process) futex_ctx_deinit(&process->futexCtx); for (uint64_t i = 0; i < ARRAY_SIZE(process->rings); i++) { - io_ctx_deinit(&process->rings[i]); + ioring_ctx_deinit(&process->rings[i]); } wait_queue_deinit(&process->dyingQueue); wait_queue_deinit(&process->suspendQueue); @@ -156,7 +156,7 @@ process_t* process_new(priority_t priority, group_member_t* group, namespace_t* perf_process_ctx_init(&process->perf); for (uint64_t i = 0; i < ARRAY_SIZE(process->rings); i++) { - io_ctx_init(&process->rings[i]); + ioring_ctx_init(&process->rings[i]); } note_handler_init(&process->noteHandler); wait_queue_init(&process->suspendQueue); diff --git a/src/libpatchwork/config.c b/src/libpatchwork/config.c index 929ce0f9d..17aa2ba06 100644 --- a/src/libpatchwork/config.c +++ b/src/libpatchwork/config.c @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/libpatchwork/internal.h b/src/libpatchwork/internal.h index 18ff13f1f..fdf791bbc 100644 --- a/src/libpatchwork/internal.h +++ b/src/libpatchwork/internal.h @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/libpatchwork/polygon.c b/src/libpatchwork/polygon.c index 08ed953e8..a2d0d6d01 100644 --- a/src/libpatchwork/polygon.c +++ b/src/libpatchwork/polygon.c @@ -1,4 +1,4 @@ -#include +#include #include void polygon_rotate(point_t* points, uint64_t pointCount, double angle, point_t center) diff --git a/src/libpatchwork/theme.c b/src/libpatchwork/theme.c index ce9bda19e..214cb50af 100644 --- a/src/libpatchwork/theme.c +++ b/src/libpatchwork/theme.c @@ -1,7 +1,7 @@ #include "internal.h" -#include -#include +#include +#include #include #include #include diff --git a/src/libpatchwork/window.c b/src/libpatchwork/window.c index e5324284d..a80467e1f 100644 --- a/src/libpatchwork/window.c +++ b/src/libpatchwork/window.c @@ -1,5 +1,5 @@ -#include -#include +#include +#include #define __STDC_WANT_LIB_EXT1__ 1 #include "internal.h" diff --git a/src/libstd/common/ascii_table.c b/src/libstd/common/ascii_table.c index 527f1f4d0..44d9a1566 100644 --- a/src/libstd/common/ascii_table.c +++ b/src/libstd/common/ascii_table.c @@ -1,4 +1,4 @@ -#include <_internal/ascii.h> +#include <_libstd/ascii.h> #include _ascii_entry_t _asciiTable[UINT8_MAX + 1] = { diff --git a/src/libstd/common/digits.h b/src/libstd/common/digits.h index b4e011976..2b7053616 100644 --- a/src/libstd/common/digits.h +++ b/src/libstd/common/digits.h @@ -1,6 +1,6 @@ #pragma once -#include <_internal/ascii.h> +#include <_libstd/ascii.h> #include extern const char _digits[]; diff --git a/src/libstd/common/init.c b/src/libstd/common/init.c index 5865c3efe..402b077e2 100644 --- a/src/libstd/common/init.c +++ b/src/libstd/common/init.c @@ -1,4 +1,4 @@ -#include "libstd/_internal/init.h" +#include "_libstd/init.h" #include "constraint_handler.h" #include "heap.h" diff --git a/src/modules/acpi/acpi.c b/src/modules/acpi/acpi.c index 27dc4f7bc..06a3c4573 100644 --- a/src/modules/acpi/acpi.c +++ b/src/modules/acpi/acpi.c @@ -1,9 +1,9 @@ #include -#include +#include -#include -#include -#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/aml/aml.c b/src/modules/acpi/aml/aml.c index de66a4e4b..0cc12ec5f 100644 --- a/src/modules/acpi/aml/aml.c +++ b/src/modules/acpi/aml/aml.c @@ -1,14 +1,14 @@ -#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include diff --git a/src/modules/acpi/aml/debug.c b/src/modules/acpi/aml/debug.c index d41a22583..eba1de7df 100644 --- a/src/modules/acpi/aml/debug.c +++ b/src/modules/acpi/aml/debug.c @@ -1,8 +1,8 @@ -#include +#include #include #include -#include +#include static void aml_debug_dump_print_line(const uint8_t* start, uint64_t lineStart, uint64_t lineEnd) { diff --git a/src/modules/acpi/aml/encoding/arg.c b/src/modules/acpi/aml/encoding/arg.c index cb50e0891..4c7e290ba 100644 --- a/src/modules/acpi/aml/encoding/arg.c +++ b/src/modules/acpi/aml/encoding/arg.c @@ -1,9 +1,9 @@ -#include +#include -#include -#include -#include -#include +#include +#include +#include +#include aml_object_t* aml_arg_obj_read(aml_term_list_ctx_t* ctx) { diff --git a/src/modules/acpi/aml/encoding/data.c b/src/modules/acpi/aml/encoding/data.c index 3a8c43c59..9a75a8d40 100644 --- a/src/modules/acpi/aml/encoding/data.c +++ b/src/modules/acpi/aml/encoding/data.c @@ -1,14 +1,14 @@ -#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/aml/encoding/debug.c b/src/modules/acpi/aml/encoding/debug.c index 0e4c9287b..414ee70aa 100644 --- a/src/modules/acpi/aml/encoding/debug.c +++ b/src/modules/acpi/aml/encoding/debug.c @@ -1,7 +1,7 @@ -#include +#include -#include -#include +#include +#include aml_object_t* aml_debug_obj_read(aml_term_list_ctx_t* ctx) { diff --git a/src/modules/acpi/aml/encoding/expression.c b/src/modules/acpi/aml/encoding/expression.c index ea88f3a2a..5a7f34de4 100644 --- a/src/modules/acpi/aml/encoding/expression.c +++ b/src/modules/acpi/aml/encoding/expression.c @@ -1,24 +1,24 @@ -#include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include diff --git a/src/modules/acpi/aml/encoding/local.c b/src/modules/acpi/aml/encoding/local.c index 6a549e9ab..acb017a1a 100644 --- a/src/modules/acpi/aml/encoding/local.c +++ b/src/modules/acpi/aml/encoding/local.c @@ -1,9 +1,9 @@ -#include +#include -#include -#include -#include -#include +#include +#include +#include +#include aml_object_t* aml_local_obj_read(aml_term_list_ctx_t* ctx) { diff --git a/src/modules/acpi/aml/encoding/name.c b/src/modules/acpi/aml/encoding/name.c index c3c34510c..4996c4af6 100644 --- a/src/modules/acpi/aml/encoding/name.c +++ b/src/modules/acpi/aml/encoding/name.c @@ -1,12 +1,12 @@ -#include +#include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/aml/encoding/named.c b/src/modules/acpi/aml/encoding/named.c index 2cbfa7934..2b53efeac 100644 --- a/src/modules/acpi/aml/encoding/named.c +++ b/src/modules/acpi/aml/encoding/named.c @@ -1,16 +1,16 @@ -#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include uint64_t aml_bank_value_read(aml_term_list_ctx_t* ctx, aml_uint_t* out) { diff --git a/src/modules/acpi/aml/encoding/namespace_modifier.c b/src/modules/acpi/aml/encoding/namespace_modifier.c index 4ad434484..2f0fe6a74 100644 --- a/src/modules/acpi/aml/encoding/namespace_modifier.c +++ b/src/modules/acpi/aml/encoding/namespace_modifier.c @@ -1,14 +1,14 @@ -#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include diff --git a/src/modules/acpi/aml/encoding/package_length.c b/src/modules/acpi/aml/encoding/package_length.c index 1a814879f..48d57874b 100644 --- a/src/modules/acpi/aml/encoding/package_length.c +++ b/src/modules/acpi/aml/encoding/package_length.c @@ -1,8 +1,8 @@ -#include +#include -#include -#include -#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/aml/encoding/statement.c b/src/modules/acpi/aml/encoding/statement.c index e852d6fbd..4ab52a985 100644 --- a/src/modules/acpi/aml/encoding/statement.c +++ b/src/modules/acpi/aml/encoding/statement.c @@ -1,11 +1,11 @@ -#include +#include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include diff --git a/src/modules/acpi/aml/encoding/term.c b/src/modules/acpi/aml/encoding/term.c index 7af7c3122..71b2c730d 100644 --- a/src/modules/acpi/aml/encoding/term.c +++ b/src/modules/acpi/aml/encoding/term.c @@ -1,16 +1,16 @@ -#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/aml/integer.c b/src/modules/acpi/aml/integer.c index f567e4f5f..e423b9da1 100644 --- a/src/modules/acpi/aml/integer.c +++ b/src/modules/acpi/aml/integer.c @@ -1,7 +1,7 @@ -#include +#include #include -#include +#include static uint8_t integerByteSize = 0; diff --git a/src/modules/acpi/aml/namespace.c b/src/modules/acpi/aml/namespace.c index 441f3283e..100a12ea2 100644 --- a/src/modules/acpi/aml/namespace.c +++ b/src/modules/acpi/aml/namespace.c @@ -1,13 +1,13 @@ -#include +#include #include #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include diff --git a/src/modules/acpi/aml/object.c b/src/modules/acpi/aml/object.c index d708389c9..b1d411a50 100644 --- a/src/modules/acpi/aml/object.c +++ b/src/modules/acpi/aml/object.c @@ -1,11 +1,11 @@ -#include +#include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/aml/patch_up.c b/src/modules/acpi/aml/patch_up.c index 4ccfd578b..3bf80dd32 100644 --- a/src/modules/acpi/aml/patch_up.c +++ b/src/modules/acpi/aml/patch_up.c @@ -1,10 +1,10 @@ -#include +#include #include #include -#include -#include -#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/aml/predefined.c b/src/modules/acpi/aml/predefined.c index 25f289219..1f1c71125 100644 --- a/src/modules/acpi/aml/predefined.c +++ b/src/modules/acpi/aml/predefined.c @@ -1,9 +1,9 @@ -#include +#include #include -#include -#include -#include +#include +#include +#include #include diff --git a/src/modules/acpi/aml/runtime/access_type.c b/src/modules/acpi/aml/runtime/access_type.c index 63b322df7..e5c33b68b 100644 --- a/src/modules/acpi/aml/runtime/access_type.c +++ b/src/modules/acpi/aml/runtime/access_type.c @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/src/modules/acpi/aml/runtime/buffer_field.c b/src/modules/acpi/aml/runtime/buffer_field.c index dff6e210a..401f5716a 100644 --- a/src/modules/acpi/aml/runtime/buffer_field.c +++ b/src/modules/acpi/aml/runtime/buffer_field.c @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/src/modules/acpi/aml/runtime/compare.c b/src/modules/acpi/aml/runtime/compare.c index aae432b94..1b33ef182 100644 --- a/src/modules/acpi/aml/runtime/compare.c +++ b/src/modules/acpi/aml/runtime/compare.c @@ -1,7 +1,7 @@ -#include +#include #include -#include +#include static inline aml_uint_t aml_compare_integers(aml_uint_t a, aml_uint_t b, aml_compare_operation_t operation) { diff --git a/src/modules/acpi/aml/runtime/concat.c b/src/modules/acpi/aml/runtime/concat.c index 2fef644e9..b15536c11 100644 --- a/src/modules/acpi/aml/runtime/concat.c +++ b/src/modules/acpi/aml/runtime/concat.c @@ -1,7 +1,7 @@ -#include +#include #include -#include +#include #include diff --git a/src/modules/acpi/aml/runtime/convert.c b/src/modules/acpi/aml/runtime/convert.c index e95dff7c0..806bdc910 100644 --- a/src/modules/acpi/aml/runtime/convert.c +++ b/src/modules/acpi/aml/runtime/convert.c @@ -1,12 +1,12 @@ -#include +#include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/aml/runtime/copy.c b/src/modules/acpi/aml/runtime/copy.c index 644d565ad..663c2e05f 100644 --- a/src/modules/acpi/aml/runtime/copy.c +++ b/src/modules/acpi/aml/runtime/copy.c @@ -1,10 +1,10 @@ -#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include diff --git a/src/modules/acpi/aml/runtime/eisa_id.c b/src/modules/acpi/aml/runtime/eisa_id.c index 411cff8df..6899ed1db 100644 --- a/src/modules/acpi/aml/runtime/eisa_id.c +++ b/src/modules/acpi/aml/runtime/eisa_id.c @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/modules/acpi/aml/runtime/evaluate.c b/src/modules/acpi/aml/runtime/evaluate.c index d247b773e..6044baed7 100644 --- a/src/modules/acpi/aml/runtime/evaluate.c +++ b/src/modules/acpi/aml/runtime/evaluate.c @@ -1,9 +1,9 @@ -#include -#include +#include +#include -#include -#include -#include +#include +#include +#include aml_object_t* aml_evaluate(aml_state_t* state, aml_object_t* object, aml_type_t targetTypes) { diff --git a/src/modules/acpi/aml/runtime/field_unit.c b/src/modules/acpi/aml/runtime/field_unit.c index 8a21c920a..0c254ba0f 100644 --- a/src/modules/acpi/aml/runtime/field_unit.c +++ b/src/modules/acpi/aml/runtime/field_unit.c @@ -2,14 +2,14 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/aml/runtime/method.c b/src/modules/acpi/aml/runtime/method.c index 039d649c1..461aee613 100644 --- a/src/modules/acpi/aml/runtime/method.c +++ b/src/modules/acpi/aml/runtime/method.c @@ -1,10 +1,10 @@ -#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include diff --git a/src/modules/acpi/aml/runtime/mid.c b/src/modules/acpi/aml/runtime/mid.c index ce31155a2..4dcd444c9 100644 --- a/src/modules/acpi/aml/runtime/mid.c +++ b/src/modules/acpi/aml/runtime/mid.c @@ -1,4 +1,4 @@ -#include +#include aml_object_t* aml_mid(aml_state_t* state, aml_object_t* bufferString, aml_uint_t index, aml_uint_t length) { diff --git a/src/modules/acpi/aml/runtime/mutex.c b/src/modules/acpi/aml/runtime/mutex.c index 6c06bd16a..b000e08df 100644 --- a/src/modules/acpi/aml/runtime/mutex.c +++ b/src/modules/acpi/aml/runtime/mutex.c @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/modules/acpi/aml/runtime/store.c b/src/modules/acpi/aml/runtime/store.c index ca3ebffd8..734b7f466 100644 --- a/src/modules/acpi/aml/runtime/store.c +++ b/src/modules/acpi/aml/runtime/store.c @@ -1,9 +1,9 @@ -#include +#include #include -#include -#include -#include +#include +#include +#include #include diff --git a/src/modules/acpi/aml/state.c b/src/modules/acpi/aml/state.c index a029182d1..959324abe 100644 --- a/src/modules/acpi/aml/state.c +++ b/src/modules/acpi/aml/state.c @@ -1,7 +1,7 @@ -#include +#include -#include -#include +#include +#include #include diff --git a/src/modules/acpi/aml/tests.c b/src/modules/acpi/aml/tests.c index fefeaa8d2..0b98171e4 100644 --- a/src/modules/acpi/aml/tests.c +++ b/src/modules/acpi/aml/tests.c @@ -6,13 +6,13 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/aml/to_string.c b/src/modules/acpi/aml/to_string.c index 46b38c4f4..c02edfeb3 100644 --- a/src/modules/acpi/aml/to_string.c +++ b/src/modules/acpi/aml/to_string.c @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/src/modules/acpi/aml/token.c b/src/modules/acpi/aml/token.c index 49d2e73c0..4ace9e66e 100644 --- a/src/modules/acpi/aml/token.c +++ b/src/modules/acpi/aml/token.c @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/src/modules/acpi/devices.c b/src/modules/acpi/devices.c index b0752bbc1..f19ebec83 100644 --- a/src/modules/acpi/devices.c +++ b/src/modules/acpi/devices.c @@ -1,6 +1,6 @@ -#include -#include -#include +#include +#include +#include #include #include @@ -8,15 +8,15 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/resources.c b/src/modules/acpi/resources.c index 9e116e7f0..240ac860c 100644 --- a/src/modules/acpi/resources.c +++ b/src/modules/acpi/resources.c @@ -1,21 +1,21 @@ -#include -#include +#include +#include -#include -#include +#include +#include #include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/modules/acpi/tables.c b/src/modules/acpi/tables.c index 4bccebc1b..cbfbd5c8a 100644 --- a/src/modules/acpi/tables.c +++ b/src/modules/acpi/tables.c @@ -1,12 +1,12 @@ #include -#include +#include #include #include #include #include #include -#include +#include #include diff --git a/src/modules/drivers/apic/apic.c b/src/modules/drivers/apic/apic.c index 30abc5d7e..3e6105f6c 100644 --- a/src/modules/drivers/apic/apic.c +++ b/src/modules/drivers/apic/apic.c @@ -1,6 +1,6 @@ -#include -#include -#include +#include +#include +#include #include #include @@ -11,8 +11,8 @@ /** * @brief Advanced Programmable Interrupt Controller. - * @defgroup modules_drivers_apic APIC - * @ingroup modules_drivers + * @defgroup kernel_drivers_apic APIC + * @ingroup kernel_drivers * * This module implements the Advanced Programmable Interrupt Controller (APIC) driver, which includes the per-CPU * local APICs, the IO APICs and the APIC timer. diff --git a/src/modules/drivers/apic/apic_timer.c b/src/modules/drivers/apic/apic_timer.c index 8bab1679e..fae6a35c1 100644 --- a/src/modules/drivers/apic/apic_timer.c +++ b/src/modules/drivers/apic/apic_timer.c @@ -1,5 +1,5 @@ -#include -#include +#include +#include #include #include diff --git a/src/modules/drivers/apic/ioapic.c b/src/modules/drivers/apic/ioapic.c index 12fcfd3f7..cc9d9500a 100644 --- a/src/modules/drivers/apic/ioapic.c +++ b/src/modules/drivers/apic/ioapic.c @@ -1,12 +1,12 @@ -#include -#include +#include +#include #include #include #include #include #include -#include +#include #include #include diff --git a/src/modules/drivers/apic/lapic.c b/src/modules/drivers/apic/lapic.c index b59393458..6a6f6915b 100644 --- a/src/modules/drivers/apic/lapic.c +++ b/src/modules/drivers/apic/lapic.c @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/modules/drivers/const/const.c b/src/modules/drivers/const/const.c index 5b6d445da..fa92df1f4 100644 --- a/src/modules/drivers/const/const.c +++ b/src/modules/drivers/const/const.c @@ -12,8 +12,8 @@ /** * @brief Constant devices - * @defgroup modules_drivers_const Constant Devices - * @ingroup modules_drivers + * @defgroup kernel_drivers_const Constant Devices + * @ingroup kernel_drivers * * This module provides the constant devices which provide user space with its primary means of allocating memory and * obtaining constant data. diff --git a/src/modules/drivers/drivers.h b/src/modules/drivers/drivers.h index 5ccaed0bb..4f646dfd2 100644 --- a/src/modules/drivers/drivers.h +++ b/src/modules/drivers/drivers.h @@ -2,6 +2,6 @@ /** * @brief Drivers. - * @defgroup modules_drivers Drivers - * @ingroup modules + * @defgroup kernel_drivers Drivers + * @ingroup kernel */ \ No newline at end of file diff --git a/src/modules/drivers/gop/gop.c b/src/modules/drivers/gop/gop.c index 7614a7a36..ee27c5526 100644 --- a/src/modules/drivers/gop/gop.c +++ b/src/modules/drivers/gop/gop.c @@ -16,8 +16,8 @@ /** * @brief GOP (Graphics Output Protocol) driver. - * @defgroup modules_drivers_gop GOP Driver - * @ingroup modules_drivers + * @defgroup kernel_drivers_gop GOP Driver + * @ingroup kernel_drivers * * This module provides a framebuffer device for the GOP framebuffer provided by the bootloader. * diff --git a/src/modules/drivers/hpet/hpet.c b/src/modules/drivers/hpet/hpet.c index ade98a5eb..f66aef45d 100644 --- a/src/modules/drivers/hpet/hpet.c +++ b/src/modules/drivers/hpet/hpet.c @@ -10,12 +10,12 @@ #include #include #include -#include +#include /** * @brief High Precision Event Timer - * @defgroup modules_drivers_hpet HPET - * @ingroup modules_drivers + * @defgroup kernel_drivers_hpet HPET + * @ingroup kernel_drivers * * @note Since the HPET might be 32bit it could overflow rather quickly, so we implement a system for checking roughly * when it will overflow and accumulate the counter into a 64 bit nanosecond counter. diff --git a/src/modules/drivers/pci/config.c b/src/modules/drivers/pci/config.c index 821e29c3c..11fd9d19a 100644 --- a/src/modules/drivers/pci/config.c +++ b/src/modules/drivers/pci/config.c @@ -1,8 +1,8 @@ -#include +#include #include #include -#include +#include static uint64_t entryCount; static mcfg_t* mcfg; diff --git a/src/modules/drivers/ps2/ps2.c b/src/modules/drivers/ps2/ps2.c index eb363637e..4dde416e0 100644 --- a/src/modules/drivers/ps2/ps2.c +++ b/src/modules/drivers/ps2/ps2.c @@ -9,10 +9,10 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include diff --git a/src/modules/drivers/ps2/ps2.h b/src/modules/drivers/ps2/ps2.h index 8e357cb4c..96ed40ae8 100644 --- a/src/modules/drivers/ps2/ps2.h +++ b/src/modules/drivers/ps2/ps2.h @@ -8,8 +8,8 @@ /** * @brief IBM Personal Computer/2 ports. - * @defgroup modules_drivers_ps2 PS/2 - * @ingroup modules_drivers + * @defgroup kernel_drivers_ps2 PS/2 + * @ingroup kernel_drivers * * @see https://wiki.osdev.org/I8042_PS/2_Controller * @see https://www-ug.eecg.toronto.edu/msl/nios_devices/datasheets/PS2%20Keyboard%20Protocol.htm diff --git a/src/modules/drivers/ps2/ps2_kbd.h b/src/modules/drivers/ps2/ps2_kbd.h index bf0c9b8f8..196271790 100644 --- a/src/modules/drivers/ps2/ps2_kbd.h +++ b/src/modules/drivers/ps2/ps2_kbd.h @@ -7,7 +7,7 @@ /** * @brief PS/2 Keyboard Driver. * @defgroup module_drivers_ps2_kbd PS/2 Keyboard Driver - * @ingroup modules_drivers_ps2 + * @ingroup kernel_drivers_ps2 * * @{ */ diff --git a/src/modules/drivers/ps2/ps2_mouse.h b/src/modules/drivers/ps2/ps2_mouse.h index 9bf94580b..9013610b7 100644 --- a/src/modules/drivers/ps2/ps2_mouse.h +++ b/src/modules/drivers/ps2/ps2_mouse.h @@ -7,7 +7,7 @@ /** * @brief PS/2 Mouse Driver. * @defgroup module_drivers_ps2_mouse PS/2 Mouse Driver - * @ingroup modules_drivers_ps2 + * @ingroup kernel_drivers_ps2 * * @todo Implement scrolling and buttons 4 and 5. * diff --git a/src/modules/drivers/ps2/ps2_scanmap.h b/src/modules/drivers/ps2/ps2_scanmap.h index a09a2714b..304103d22 100644 --- a/src/modules/drivers/ps2/ps2_scanmap.h +++ b/src/modules/drivers/ps2/ps2_scanmap.h @@ -7,7 +7,7 @@ /** * @brief PS/2 Scanmap. * @defgroup module_drivers_ps2_scanmap PS/2 Scanmap - * @ingroup modules_drivers_ps2 + * @ingroup kernel_drivers_ps2 * * @{ */ diff --git a/src/modules/drivers/rtc/rtc.c b/src/modules/drivers/rtc/rtc.c index d9c0797f6..2bdd5c55d 100644 --- a/src/modules/drivers/rtc/rtc.c +++ b/src/modules/drivers/rtc/rtc.c @@ -3,8 +3,8 @@ #include #include #include -#include -#include +#include +#include #include diff --git a/src/modules/fs/9p/9p.c b/src/modules/fs/9p/9p.c index b02116d3c..c2aaa227d 100644 --- a/src/modules/fs/9p/9p.c +++ b/src/modules/fs/9p/9p.c @@ -8,8 +8,8 @@ /** * @brief 9P Filesystems. - * @defgroup modules_fs_9p 9P Filesystem - * @ingroup modules_fs + * @defgroup kernel_fs_9p 9P Filesystem + * @ingroup kernel_fs * * This module provides an implementation of the 9P filesystem protocol where the kernel acts as a client to a 9P * server, allowing the 9P server to be mounted as a filesystem within the kernel's VFS. diff --git a/src/modules/fs/fs.h b/src/modules/fs/fs.h index adc03e657..37c853bfd 100644 --- a/src/modules/fs/fs.h +++ b/src/modules/fs/fs.h @@ -2,6 +2,6 @@ /** * @brief Filesystem Modules. - * @defgroup modules_fs Filesystem Modules - * @ingroup modules + * @defgroup kernel_fs Filesystem Modules + * @ingroup kernel */ \ No newline at end of file diff --git a/src/modules/ipc/ipc.h b/src/modules/ipc/ipc.h index 5f9c7f4fc..3cea58537 100644 --- a/src/modules/ipc/ipc.h +++ b/src/modules/ipc/ipc.h @@ -2,6 +2,6 @@ /** * @brief Inter-Process Communication. - * @defgroup modules_ipc IPC Modules - * @ingroup modules + * @defgroup kernel_ipc IPC Modules + * @ingroup kernel */ \ No newline at end of file diff --git a/src/modules/ipc/pipe/pipe.c b/src/modules/ipc/pipe/pipe.c index a6991061b..322e8a57f 100644 --- a/src/modules/ipc/pipe/pipe.c +++ b/src/modules/ipc/pipe/pipe.c @@ -16,8 +16,8 @@ /** * @brief Pipes. - * @defgroup modules_ipc_pipe Pipes - * @ingroup modules_ipc + * @defgroup kernel_ipc_pipe Pipes + * @ingroup kernel_ipc * * Pipes are exposed in the `/dev/pipe` directory. Pipes are unidirectional communication channels that can be used for * inter-process communication (IPC). diff --git a/src/modules/ipc/shmem/shmem.c b/src/modules/ipc/shmem/shmem.c index 4dcb6a03e..c5cefc11b 100644 --- a/src/modules/ipc/shmem/shmem.c +++ b/src/modules/ipc/shmem/shmem.c @@ -17,8 +17,8 @@ #include /** * @brief Shared Memory - * @defgroup modules_ipc_shmem Shared Memory - * @ingroup modules_ipc + * @defgroup kernel_ipc_shmem Shared Memory + * @ingroup kernel_ipc * * Shared memory is exposed in the `/dev/shmem` directory. Shared memory allows multiple processes to share a section of * memory for inter-process communication (IPC). diff --git a/src/modules/modules.h b/src/modules/modules.h index 3abbacd98..260c85851 100644 --- a/src/modules/modules.h +++ b/src/modules/modules.h @@ -2,6 +2,6 @@ /** * @brief Kernel Modules. - * @defgroup modules Modules + * @defgroup kernel Modules * */ \ No newline at end of file diff --git a/src/modules/net/local/local_listen.h b/src/modules/net/local/local_listen.h index 226eff1e2..ed90161db 100644 --- a/src/modules/net/local/local_listen.h +++ b/src/modules/net/local/local_listen.h @@ -1,6 +1,6 @@ #pragma once -#include <_internal/MAX_PATH.h> +#include <_libstd/MAX_PATH.h> #include #include #include diff --git a/src/modules/net/net.h b/src/modules/net/net.h index 7f93fc181..1cc3e81f0 100644 --- a/src/modules/net/net.h +++ b/src/modules/net/net.h @@ -3,5 +3,5 @@ /** * @brief Networking protocols. * @defgroup module_net Networking Protocols - * @ingroup modules + * @ingroup kernel */ \ No newline at end of file diff --git a/src/modules/smp/smp.c b/src/modules/smp/smp.c index 1115a0252..41c42d918 100644 --- a/src/modules/smp/smp.c +++ b/src/modules/smp/smp.c @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include @@ -15,8 +15,8 @@ /** * @brief Symmetric Multiprocessing support via APIC. - * @defgroup modules_smp SMP - * @ingroup modules + * @defgroup kernel_smp SMP + * @ingroup kernel * * Symmetric Multiprocessing (SMP) support is implemented using the Advanced Programmable Interrupt Controller (APIC) * system. diff --git a/src/modules/smp/trampoline.h b/src/modules/smp/trampoline.h index c3fe1a15d..b22752ba7 100644 --- a/src/modules/smp/trampoline.h +++ b/src/modules/smp/trampoline.h @@ -3,7 +3,7 @@ #ifndef __ASSEMBLER__ #include -#include +#include #include #include @@ -11,8 +11,8 @@ /** * @brief Trampoline for CPU initialization - * @defgroup modules_smp_trampoline Trampoline - * @ingroup modules_smp + * @defgroup kernel_smp_trampoline Trampoline + * @ingroup kernel_smp * * The trampoline is a small piece of code used during the initialization of other CPUs in a multiprocessor system. The * code itself must be position-independent and fit within a single memory page, this is why we do all the weird offset diff --git a/src/programs/core/init/main.c b/src/programs/core/init/main.c index 0f1ac8bff..448080849 100644 --- a/src/programs/core/init/main.c +++ b/src/programs/core/init/main.c @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/programs/core/shell/pipeline.c b/src/programs/core/shell/pipeline.c index 5328e256d..2cc60ad55 100644 --- a/src/programs/core/shell/pipeline.c +++ b/src/programs/core/shell/pipeline.c @@ -1,7 +1,7 @@ #include "pipeline.h" #include "builtin.h" -#include <_internal/MAX_PATH.h> +#include <_libstd/MAX_PATH.h> #include #include #include diff --git a/include/programs/programs.h b/src/programs/programs.h similarity index 100% rename from include/programs/programs.h rename to src/programs/programs.h diff --git a/src/programs/utils/ringtest/main.c b/src/programs/utils/ringtest/main.c index 942079c6a..b4a8cb8ce 100644 --- a/src/programs/utils/ringtest/main.c +++ b/src/programs/utils/ringtest/main.c @@ -20,11 +20,11 @@ int main() memset(&ring.ctrl->regs, -1, sizeof(ring.ctrl->regs)); printf("pushing nop sqe to ring %llu...\n", ring.id); - sqe_t sqe = SQE_CREATE(IORING_NOP, SQE_HARDLINK | (SQE_REG0 << SQE_SAVE), CLOCKS_PER_SEC, 0x1234); + sqe_t sqe = SQE_CREATE(IO_OP_NOP, SQE_HARDLINK | (SQE_REG0 << SQE_SAVE), CLOCKS_PER_SEC, 0x1234); sqe_push(&ring, &sqe); printf("pushing nop sqe to ring %llu...\n", ring.id); - sqe = (sqe_t)SQE_CREATE(IORING_NOP, SQE_LINK, CLOCKS_PER_SEC, 0x5678); + sqe = (sqe_t)SQE_CREATE(IO_OP_NOP, SQE_LINK, CLOCKS_PER_SEC, 0x5678); sqe_push(&ring, &sqe); printf("entering ring...\n"); diff --git a/src/programs/utils/tail/main.c b/src/programs/utils/tail/main.c index 7372349ff..b7a916961 100644 --- a/src/programs/utils/tail/main.c +++ b/src/programs/utils/tail/main.c @@ -1,4 +1,4 @@ -#include <_internal/clock_t.h> +#include <_libstd/clock_t.h> #include #include #include