From be509073b159e6fec8393513c1926cf087edd940 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:35:38 -0700 Subject: [PATCH 1/2] feat: ship python313.dll with Windows C extensions and console IO --- .github/workflows/ci.yml | 27 +- Cargo.lock | 10 + Cargo.toml | 3 + crates/weavepy-capi/Cargo.toml | 6 + .../include/pyconfig/pyconfig-windows.h | 157 ++ crates/weavepy-capi/src/loader.rs | 78 +- crates/weavepy-cli/Cargo.toml | 10 + crates/weavepy-cli/src/lib.rs | 1761 ++++++++++++++++ crates/weavepy-cli/src/main.rs | 1839 ++--------------- crates/weavepy-cli/src/regrtest_cmd.rs | 7 +- crates/weavepy-cli/tests/windows_dll.rs | 211 ++ crates/weavepy-dist/src/main.rs | 252 ++- crates/weavepy-pylib/Cargo.toml | 33 + crates/weavepy-pylib/build.rs | 62 + crates/weavepy-pylib/src/lib.rs | 186 ++ crates/weavepy-vm/build.rs | 10 +- crates/weavepy-vm/src/object.rs | 24 + crates/weavepy-vm/src/stdlib/io.rs | 5 +- crates/weavepy-vm/src/stdlib/io_full.rs | 10 + crates/weavepy-vm/src/stdlib/mod.rs | 9 +- crates/weavepy-vm/src/stdlib/nt_support.rs | 5 +- crates/weavepy-vm/src/stdlib/os.rs | 155 ++ crates/weavepy-vm/src/stdlib/sys.rs | 76 +- crates/weavepy-vm/src/stdlib/win_console.rs | 772 +++++++ crates/weavepy-vm/src/stdlib_tree.rs | 8 +- crates/weavepy/src/lib.rs | 19 +- .../0064-windows-binary-abi-python313-dll.md | 615 ++++++ 27 files changed, 4568 insertions(+), 1782 deletions(-) create mode 100644 crates/weavepy-capi/include/pyconfig/pyconfig-windows.h create mode 100644 crates/weavepy-cli/src/lib.rs create mode 100644 crates/weavepy-cli/tests/windows_dll.rs create mode 100644 crates/weavepy-pylib/Cargo.toml create mode 100644 crates/weavepy-pylib/build.rs create mode 100644 crates/weavepy-pylib/src/lib.rs create mode 100644 crates/weavepy-vm/src/stdlib/win_console.rs create mode 100644 docs/rfcs/0064-windows-binary-abi-python313-dll.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0d2ee6c..48a77a6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,8 +90,12 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - name: Build weavepy CLI - run: cargo build --release -p weavepy-cli + - name: Build weavepy CLI + runtime library + # weavepy-pylib builds python313.dll — on Windows the exe is a + # thin shim over it (RFC 0064 WS1), so the DLL must sit next to + # the exe. Building it on all three OSes keeps the cdylib + # honest everywhere. + run: cargo build --release -p weavepy-cli -p weavepy-pylib - name: Run bundled regrtests (subprocess, parallel) shell: bash run: | @@ -134,8 +138,10 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.13" - - name: Build weavepy CLI + bench harness - run: cargo build --release -p weavepy-cli -p weavepy-bench + - name: Build weavepy CLI + runtime library + bench harness + # weavepy-pylib: python313.dll, the runtime the Windows shim + # exe loads (RFC 0064 WS1). + run: cargo build --release -p weavepy-cli -p weavepy-pylib -p weavepy-bench - name: Run bench gate # Linux and Windows have no committed per-platform baseline # yet, so the gate is advisory there (--allow-missing-baseline @@ -194,8 +200,10 @@ jobs: with: path: target/ecosystem-wheels key: ecosystem-wheels-${{ runner.os }}-${{ hashFiles('tests/ecosystem/manifest.toml') }} - - name: Build weavepy CLI - run: cargo build --release -p weavepy-cli + - name: Build weavepy CLI + runtime library + # weavepy-pylib: python313.dll, the runtime the Windows shim + # exe loads (RFC 0064 WS1). + run: cargo build --release -p weavepy-cli -p weavepy-pylib - name: Fetch ecosystem wheels (no-op on cache hit) # `python`, not `python3`: setup-python exposes both on # ubuntu/macos (same 3.13 interpreter) but only `python` on @@ -234,8 +242,11 @@ jobs: with: path: target/ecosystem-wheels key: ecosystem-wheels-${{ runner.os }}-${{ hashFiles('tests/ecosystem/manifest.toml') }} - - name: Build weavepy CLI - run: cargo build --release -p weavepy-cli + - name: Build weavepy CLI + runtime library + # weavepy-pylib: python313.dll — the Windows artifact ships it + # at the prefix root with libs\python313.lib (RFC 0064 WS3), + # and the builder refuses to package a shim exe without it. + run: cargo build --release -p weavepy-cli -p weavepy-pylib - name: Fetch ecosystem wheels (no-op on cache hit) # `python`, not `python3`: setup-python exposes both on # ubuntu/macos (same 3.13 interpreter) but only `python` on diff --git a/Cargo.lock b/Cargo.lock index 1763b7c3..2bc4e390 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2709,6 +2709,7 @@ dependencies = [ "weavepy", "weavepy-compiler", "weavepy-vm", + "windows-sys 0.61.2", ] [[package]] @@ -2727,6 +2728,7 @@ dependencies = [ "weavepy-conformance", "weavepy-parser", "weavepy-vm", + "windows-sys 0.61.2", ] [[package]] @@ -2794,6 +2796,14 @@ dependencies = [ "weavepy-lexer", ] +[[package]] +name = "weavepy-pylib" +version = "0.0.0" +dependencies = [ + "libc", + "weavepy-cli", +] + [[package]] name = "weavepy-vm" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 039fefd1..2987c36b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/weavepy-jit", "crates/weavepy-lexer", "crates/weavepy-parser", + "crates/weavepy-pylib", "crates/weavepy-vm", "vendor/expat-sys", ] @@ -26,6 +27,7 @@ default-members = [ "crates/weavepy-compiler", "crates/weavepy-lexer", "crates/weavepy-parser", + "crates/weavepy-pylib", "crates/weavepy-vm", ] @@ -49,6 +51,7 @@ categories = ["compilers", "development-tools", "parser-implementations"] # Internal crates (path deps so the workspace builds without publishing). weavepy = { path = "crates/weavepy", version = "0.0.0" } weavepy-capi = { path = "crates/weavepy-capi", version = "0.0.0" } +weavepy-cli = { path = "crates/weavepy-cli", version = "0.0.0" } weavepy-compiler = { path = "crates/weavepy-compiler", version = "0.0.0" } weavepy-conformance = { path = "crates/weavepy-conformance", version = "0.0.0" } weavepy-jit = { path = "crates/weavepy-jit", version = "0.0.0" } diff --git a/crates/weavepy-capi/Cargo.toml b/crates/weavepy-capi/Cargo.toml index 9f56d396..2632486d 100644 --- a/crates/weavepy-capi/Cargo.toml +++ b/crates/weavepy-capi/Cargo.toml @@ -28,6 +28,12 @@ num-bigint = { workspace = true } num-traits = { workspace = true } thiserror = { workspace = true } +# RFC 0064 WS2 — the Windows extension loader calls `LoadLibraryExW` +# directly (CPython's dynload_win.c flag set + `GetLastError` for the +# CPython-shaped ImportError message). +[target.'cfg(windows)'.dependencies] +windows-sys = { workspace = true } + [build-dependencies] cc = "1.0" diff --git a/crates/weavepy-capi/include/pyconfig/pyconfig-windows.h b/crates/weavepy-capi/include/pyconfig/pyconfig-windows.h new file mode 100644 index 00000000..a607be06 --- /dev/null +++ b/crates/weavepy-capi/include/pyconfig/pyconfig-windows.h @@ -0,0 +1,157 @@ +/* pyconfig.h for WeavePy on Windows (RFC 0064 WS3). + * + * CPython ships a hand-maintained `PC/pyconfig.h` on Windows (there + * is no autoconf step); this is WeavePy's equivalent, shaped after + * CPython 3.13's file: the same platform macros, LLP64 type sizes, + * shared-core markers, and — critically — the MSVC autolink pragma + * that makes `cl /LD ext.c /I{Include}` pull `python313.lib` off the + * `/LIBPATH` without the build script naming it. setuptools points + * `/LIBPATH:` at `{sys.base_exec_prefix}\libs`, where the WeavePy + * artifact ships the import library for `python313.dll`. + */ + +#ifndef Py_CONFIG_H +#define Py_CONFIG_H + +/* --- platform identification -------------------------------------- */ + +#define MS_WIN32 /* only support win32 and greater. */ +#define MS_WINDOWS +#ifdef _WIN64 +#define MS_WIN64 +#endif + +#define _Py_STRINGIZE(X) _Py_STRINGIZE1(X) +#define _Py_STRINGIZE1(X) #X + +/* set the COMPILER and support tier (1 for x64, 3 elsewhere; + * WeavePy's shipped target is x86_64-pc-windows-msvc) */ +#ifdef MS_WIN64 +#if defined(_M_X64) || defined(_M_AMD64) +#define COMPILER ("[MSC v." _Py_STRINGIZE(_MSC_VER) " 64 bit (AMD64)]") +#define PY_SUPPORT_TIER 1 +#elif defined(_M_ARM64) +#define COMPILER ("[MSC v." _Py_STRINGIZE(_MSC_VER) " 64 bit (ARM64)]") +#define PY_SUPPORT_TIER 3 +#else +#define COMPILER ("[MSC v." _Py_STRINGIZE(_MSC_VER) " 64 bit (Unknown)]") +#define PY_SUPPORT_TIER 0 +#endif +#endif /* MS_WIN64 */ + +/* Debug builds: MSVC's _DEBUG selects CPython's debug ABI. WeavePy + * does not ship python313_d.dll, so a Debug extension build fails at + * link with a clear missing-python313_d.lib error — the same failure + * a release-only CPython install produces. */ +#ifdef _DEBUG +#define Py_DEBUG 1 +#endif + +/* --- shared core + autolink ---------------------------------------- */ + +/* The Python runtime is a DLL (python313.dll — RFC 0064 WS1). */ +#define MS_COREDLL 1 +#define Py_ENABLE_SHARED 1 + +/* Declspec shaping for the stock headers' PyAPI_FUNC/PyAPI_DATA. */ +#define HAVE_DECLSPEC_DLL + +#ifdef MS_COREDLL +#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN) +/* not building the core — must be an extension or embedder: have + * MSVC pull the import library automatically. */ +#if defined(_MSC_VER) +#if defined(_DEBUG) +#pragma comment(lib, "python313_d.lib") +#elif defined(Py_LIMITED_API) +/* CPython points the limited API at python3.lib (the stable-ABI + * forwarder DLL's import library). WeavePy does not ship the + * forwarder yet (RFC 0064 Future work), so limited-API builds link + * the full runtime library — the resulting .pyd imports + * python313.dll and works on WeavePy 3.13. */ +#pragma comment(lib, "python313.lib") +#else +#pragma comment(lib, "python313.lib") +#endif /* _DEBUG */ +#endif /* _MSC_VER */ +#endif /* Py_BUILD_CORE */ +#endif /* MS_COREDLL */ + +/* --- type sizes (LLP64) -------------------------------------------- */ + +#define SIZEOF_SHORT 2 +#define SIZEOF_INT 4 +#define SIZEOF_LONG 4 +#define SIZEOF_LONG_LONG 8 +#define SIZEOF_FLOAT 4 +#define SIZEOF_DOUBLE 8 +#define SIZEOF_WCHAR_T 2 +#define SIZEOF_FPOS_T 8 +#define SIZEOF_TIME_T 8 +/* off_t is 32 bits on Windows; large files go through fpos_t. */ +#define SIZEOF_OFF_T 4 +#define HAVE_LARGEFILE_SUPPORT 1 + +#ifdef MS_WIN64 +#define SIZEOF_VOID_P 8 +#define SIZEOF_SIZE_T 8 +#define SIZEOF_HKEY 8 +#define SIZEOF_PID_T 4 +#else +#define SIZEOF_VOID_P 4 +#define SIZEOF_SIZE_T 4 +#define SIZEOF_HKEY 4 +#define SIZEOF_PID_T 4 +#endif + +#define WORD_BIT 32 + +/* MSVC provides ssize_t via SSIZE_T (BaseTsd.h); the stock headers + * only need the macro that says the typedef exists once pyport.h has + * mapped it. CPython's PC/pyconfig.h does exactly this. */ +#if defined(MS_WIN64) +typedef __int64 ssize_t; +#else +typedef _W64 int ssize_t; +#endif +#define HAVE_SSIZE_T 1 + +/* --- capabilities the stock headers key off ------------------------ */ + +#define WITH_DOC_STRINGS 1 +#define HAVE_DYNAMIC_LOADING 1 +#define HAVE_STRERROR 1 +#define HAVE_CLOCK 1 +#define HAVE_IO_H 1 +#define HAVE_SYS_UTIME_H 1 +#define HAVE_SYS_TYPES_H 1 +#define HAVE_SYS_STAT_H 1 +#define HAVE_ERRNO_H 1 +#define HAVE_STDDEF_H 1 +#define HAVE_STDINT_H 1 +#define HAVE_WCHAR_H 1 +#define HAVE_FCNTL_H 1 +#define HAVE_DIRECT_H 1 +#define HAVE_PROCESS_H 1 +#define HAVE_SIGNAL_H 1 + +/* Threading: native NT threads, exactly one flavour. */ +#define NT_THREADS 1 +#define WITH_THREAD 1 + +/* IEEE-754 doubles, little-endian (every supported Windows arch). */ +#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1 + +/* IPv6 (Winsock2 has shipped it since XP). */ +#define ENABLE_IPV6 1 + +/* Sockets are real handles on NT. */ +#define USE_SOCKET 1 + +/* Not a debug/free-threaded/tracing build (mirrors + * _weave_sysconfigdata). */ +/* #undef Py_GIL_DISABLED */ +/* #undef Py_TRACE_REFS */ +/* #undef Py_REF_DEBUG */ + +#endif /* !Py_CONFIG_H */ diff --git a/crates/weavepy-capi/src/loader.rs b/crates/weavepy-capi/src/loader.rs index 38a92d29..183f2812 100644 --- a/crates/weavepy-capi/src/loader.rs +++ b/crates/weavepy-capi/src/loader.rs @@ -3,10 +3,17 @@ //! Given a path to a shared library (`.so` / `.dylib` / `.pyd`) //! and a fully-qualified module name, this module: //! -//! 1. Calls [`libloading::Library::new`] to load the library into -//! the process. Symbols the extension imports (everything in -//! `Python.h`) resolve against the host `weavepy` binary, which -//! statically links this crate. +//! 1. Loads the library into the process. On POSIX that is +//! [`libloading::Library::new`] (dlopen), and the extension's +//! C-API imports resolve against the host `weavepy` binary, +//! which statically links this crate. On Windows (RFC 0064 WS2) +//! it is `LoadLibraryExW` with CPython's `dynload_win.c` flags — +//! `LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_ +//! DLL_LOAD_DIR`, so a wheel's `.pyd` resolves vendored +//! dependent DLLs from its own directory and `AddDllDirectory` +//! cookies but never from `PATH`/CWD (bpo-36085) — and the +//! C-API imports resolve against the already-loaded +//! `python313.dll` (RFC 0064 WS1). //! 2. Looks up `PyInit_`. The leaf name is the //! last `.`-delimited component of the module name, matching //! CPython's convention. @@ -43,6 +50,11 @@ pub struct LoadedLibrary { pub enum LoadError { #[error("dlopen failed: {0}")] Dlopen(String), + /// Windows load failure, pre-shaped as CPython's + /// `Python/dynload_win.c` ImportError text (the message tooling + /// and users pattern-match on). The caller surfaces it verbatim. + #[error("DLL load failed while importing {leaf}: {message}")] + DllLoadFailed { leaf: String, message: String }, #[error("missing init symbol {0}")] MissingInit(String), #[error("init function returned NULL{}", .pending.as_deref().map(|s| format!(": {s}")).unwrap_or_default())] @@ -65,10 +77,9 @@ pub fn load_extension_module( if trace_loader { eprintln!("[LOADER] dlopen path={path:?} module={module_name}"); } - let lib = - unsafe { Library::new(path) }.map_err(|e| LoadError::Dlopen(format!("{path:?}: {e}")))?; - let leaf = module_name.rsplit('.').next().unwrap_or(module_name); + let lib = open_extension_library(path, leaf)?; + let init_name = format!("PyInit_{leaf}"); let init: Symbol = unsafe { lib.get(init_name.as_bytes()) @@ -175,6 +186,54 @@ pub fn load_extension_module( Ok(result) } +/// Load the shared library with the platform's CPython semantics. +/// +/// POSIX: plain dlopen (`RTLD_NOW | RTLD_LOCAL`, libloading's +/// default, matching CPython's `dlopenflags` default). +#[cfg(not(windows))] +fn open_extension_library(path: &Path, _leaf: &str) -> Result { + unsafe { Library::new(path) }.map_err(|e| LoadError::Dlopen(format!("{path:?}: {e}"))) +} + +/// Windows: `LoadLibraryExW` with CPython's `dynload_win.c` flag set, +/// and failures shaped as CPython's `ImportError` message with the +/// `FormatMessageW` strerror (via the RFC 0063 error bridge). +#[cfg(windows)] +fn open_extension_library(path: &Path, leaf: &str) -> Result { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Foundation::GetLastError; + use windows_sys::Win32::System::LibraryLoader::{ + LoadLibraryExW, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS, LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR, + }; + let wide: Vec = path + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + // SAFETY: `wide` is a valid NUL-terminated UTF-16 path; the flag + // combination is the one CPython passes for absolute .pyd paths. + let handle = unsafe { + LoadLibraryExW( + wide.as_ptr(), + std::ptr::null_mut(), + LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR, + ) + }; + if handle.is_null() { + // SAFETY: trivially safe; reads this thread's last-error slot. + let code = unsafe { GetLastError() } as i32; + return Err(LoadError::DllLoadFailed { + leaf: leaf.to_owned(), + message: weavepy_vm::stdlib::nt_support::format_message(code), + }); + } + // SAFETY: `handle` is a live HMODULE we own; libloading's Drop + // would FreeLibrary it, but the caller leaks the Library for the + // process lifetime (extension modules are never unloaded). + // (libloading spells HMODULE as `isize`; windows-sys as a pointer.) + Ok(unsafe { libloading::os::windows::Library::from_raw(handle as isize) }.into()) +} + /// Helper used by the higher-level frozen importlib stub. Returns /// `Some(module)` on success; `None` if `path` doesn't exist. pub fn try_load( @@ -231,7 +290,10 @@ pub fn extension_suffixes() -> &'static [&'static str] { ".so", ] } else if cfg!(target_os = "windows") { - &[".pyd", ".dll"] + // The tagged name is what wheels actually install + // (`EXT_SUFFIX` = `.cp313-win_amd64.pyd`); bare `.pyd` and + // `.dll` are the CPython fallbacks (RFC 0064 WS2). + &[".cp313-win_amd64.pyd", ".pyd", ".dll"] } else { &[".so"] } diff --git a/crates/weavepy-cli/Cargo.toml b/crates/weavepy-cli/Cargo.toml index 468e7e05..7e1500f6 100644 --- a/crates/weavepy-cli/Cargo.toml +++ b/crates/weavepy-cli/Cargo.toml @@ -11,6 +11,10 @@ readme.workspace = true keywords.workspace = true categories.workspace = true +[lib] +name = "weavepy_cli" +path = "src/lib.rs" + [[bin]] name = "weavepy" path = "src/main.rs" @@ -29,6 +33,12 @@ dirs = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +# RFC 0064 WS1 — the Windows `weavepy.exe` is a thin shim that +# locates and loads `python313.dll` (built from `weavepy-pylib`) at +# startup; the shim's only platform dependency is the loader API. +[target.'cfg(windows)'.dependencies] +windows-sys = { workspace = true } + [features] default = [] # RFC 0032 — build the `weavepy` binary with the tier-2 JIT compiled in diff --git a/crates/weavepy-cli/src/lib.rs b/crates/weavepy-cli/src/lib.rs new file mode 100644 index 00000000..703f23d6 --- /dev/null +++ b/crates/weavepy-cli/src/lib.rs @@ -0,0 +1,1761 @@ +//! The `weavepy` command-line interpreter driver. +//! +//! Argv-compatible with `python(1)` 3.13: every flag in the CPython +//! manpage is parsed and honoured (those we can't yet act on are +//! accepted and forwarded onto `sys.flags` / `sys._xoptions` so user +//! code that introspects them sees realistic values). Modes: +//! +//! ```text +//! weavepy [flags] [-c command | -m module | script | -] [args ...] +//! weavepy [flags] -- interactive REPL +//! ``` +//! +//! Environment variables (`PYTHON*`) are read after the flag table is +//! parsed and folded in unless `-E` / `-I` says otherwise. +//! +//! Since RFC 0064 this is a *library*: [`cli_main`] is the whole CLI, +//! returning the process exit code. It has two consumers — the +//! `weavepy` bin target (`src/main.rs`), which calls it directly on +//! POSIX (the fully-static binary, unchanged), and `weavepy-pylib`, +//! the `python313` cdylib, which exports it as `weavepy_main` / +//! `Py_Main` / `Py_BytesMain` so the Windows exe can be a thin shim +//! over `python313.dll` (the same split CPython ships on NT). + +mod regrtest_cmd; +mod repl; + +use std::{ + env, fs, + io::{self, Read, Write}, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result}; +use clap::{ArgAction, Parser}; +use tracing_subscriber::EnvFilter; + +use weavepy::{InterpreterFlags, RunOptions}; + +const VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// Recognised subcommands. We thread them through manually instead of +/// using `clap`'s `#[command(subcommand)]` because the bare `weavepy` +/// CLI already overloads the positional `script` slot. Detecting these +/// up front in `main()` keeps the unsugar trivial. +const SUBCOMMANDS: &[&str] = &["regrtest"]; + +/// Run a `weavepy --multiprocessing-fork ` child. The vendored +/// `multiprocessing.popen_spawn_posix`/`popen_forkserver` re-exec us with +/// CPython's frozen command line: `argv == [exe, "--multiprocessing-fork", +/// "tracker_fd=N", "pipe_handle=M", …]`. We must therefore preserve the real +/// argv (so `spawn.is_forking(sys.argv)` holds and the `name=value` kwds are +/// parseable) and hand off to `multiprocessing._run_spawn_child()`, which +/// mirrors CPython's `spawn.spawn_main` POSIX body and *returns* the child's +/// exit code (rather than `sys.exit`-ing, so the Rust bridge controls the +/// process status). +fn run_multiprocessing_child(raw: &[String]) -> i32 { + // `_run_spawn_child` runs the worker target via `spawn._main` and returns + // its exit code; `_multiprocessing._exit(code)` then `std::process::exit`s + // directly, so the `Ok(())` arm is only reached on a clean fall-through. + // CPython's `spawn_main` ends in `sys.exit(exitcode)`, whose interpreter + // finalization runs `atexit` handlers (the worker may register its own, + // e.g. gh-83856 / `test_atexit`, plus `multiprocessing.util._exit_function`). + // Our `_multiprocessing._exit` is a hard `std::process::exit` that bypasses + // the CLI's normal shutdown drain, so run the exit funcs explicitly first. + let snippet = "import multiprocessing, _multiprocessing, atexit as _atexit\n\ + _mp_code = multiprocessing._run_spawn_child()\n\ + _atexit._run_exitfuncs()\n\ + _multiprocessing._exit(int(_mp_code) if _mp_code is not None else 0)\n"; + // The parent's `spawn.get_command_line()` emits + // `[exe, , "--multiprocessing-fork", "name=value", ...]`, + // mirroring CPython so the child inherits `-O`/`-S`/`-E`/`-I`/`-X dev`/… + // (`test_multiprocessing.TestFlags.test_flags`). Split at the + // `--multiprocessing-fork` marker: everything before it is interpreter + // flags we must apply to the child; the marker plus the `name=value` kwds + // become `sys.argv[1:]` so `spawn.is_forking(sys.argv)` still holds. + let exe = raw.first().cloned().unwrap_or_else(|| "weavepy".to_owned()); + let fork_idx = raw + .iter() + .position(|a| a == "--multiprocessing-fork") + .unwrap_or(usize::from(!raw.is_empty())); + let opt_args = if fork_idx > 1 { + &raw[1..fork_idx] + } else { + &[][..] + }; + let tail = if fork_idx < raw.len() { + &raw[fork_idx..] + } else { + &[][..] + }; + let flags = child_flags_from_opts(&exe, opt_args); + let mut argv = vec![exe]; + argv.extend(tail.iter().cloned()); + let opts = RunOptions::new("") + .with_argv(argv) + .with_flags(flags); + match weavepy::run_source_with_options(snippet, &opts) { + Ok(()) => 0, + Err(err) => { + let mut stderr = io::stderr().lock(); + let _ = writeln!(stderr, "{}", err.format(snippet, "")); + 1 + } + } +} + +/// Build the child interpreter flags for a `--multiprocessing-fork` re-exec by +/// re-parsing the interpreter-flag opts the parent placed before the marker +/// (`-O`/`-S`/`-E`/`-I`/`-X dev`/…) through the same clap table + env overrides +/// the normal launch path uses. Falls back to defaults if the opts don't parse +/// (they always should — they come from `_args_from_interpreter_flags()`). +fn child_flags_from_opts(exe: &str, opt_args: &[String]) -> InterpreterFlags { + let parse_argv: Vec = std::iter::once(exe.to_owned()) + .chain(opt_args.iter().cloned()) + .collect(); + match Cli::try_parse_from(&parse_argv) { + Ok(cli) => { + let env = if cli.isolated || cli.ignore_env { + EnvOverrides::ignored() + } else { + EnvOverrides::from_env() + }; + build_flags(&cli, &env) + } + Err(_) => InterpreterFlags::default(), + } +} + +/// CPython 3.13's `python(1)` flag set. +/// +/// Defaults match invoking `python` with no flags. Most of the +/// surface is "accept and propagate" — `sys.flags`, `sys._xoptions`, +/// `sys.warnoptions` reflect the user's choice even when the flag's +/// behaviour is partial. +#[derive(Debug, Parser, Clone, Default)] +#[command( + name = "weavepy", + bin_name = "weavepy", + version = VERSION, + about = "WeavePy: a high-performance, CPython-compatible Python interpreter written in Rust.", + disable_version_flag = true, + disable_help_flag = true, + trailing_var_arg = true, + allow_hyphen_values = true, +)] +struct Cli { + /// Print the version and exit (`python -V` / `--version`). + #[arg(short = 'V', long = "version", action = ArgAction::SetTrue, overrides_with = "version")] + version: bool, + + /// Print this help and exit. + #[arg(short = 'h', long = "help", action = ArgAction::SetTrue, overrides_with = "help")] + help: bool, + + /// Print the help-env summary (which `PYTHON*` vars are honoured) and exit. + #[arg(long = "help-env", action = ArgAction::SetTrue, overrides_with = "help_env")] + help_env: bool, + + /// Print the help-xoptions summary and exit. + #[arg(long = "help-xoptions", action = ArgAction::SetTrue, overrides_with = "help_xoptions")] + help_xoptions: bool, + + /// Optimisation level. `-O` once, `-OO` twice. + #[arg(short = 'O', action = ArgAction::Count)] + optimize: u8, + + /// `bytes`/`str` comparison warnings. `-b` once warns, `-bb` errors. + #[arg(short = 'b', action = ArgAction::Count)] + bytes_warning: u8, + + /// Don't write `.pyc` files. + #[arg(short = 'B', action = ArgAction::SetTrue, overrides_with = "no_bytecode_write")] + no_bytecode_write: bool, + + /// Parser debug output (`sys.flags.debug`; counted like CPython's + /// `-d`, otherwise a no-op stub). + #[arg(short = 'd', action = ArgAction::Count)] + parser_debug: u8, + + /// `-R`: turn on hash randomization (the default; overrides a + /// `PYTHONHASHSEED` fixed seed, like CPython). + #[arg(short = 'R', action = ArgAction::SetTrue, overrides_with = "hash_randomization")] + hash_randomization: bool, + + /// Ignore all `PYTHON*` environment variables. + #[arg(short = 'E', action = ArgAction::SetTrue, overrides_with = "ignore_env")] + ignore_env: bool, + + /// Drop into the REPL after running the script / module / command. + #[arg(short = 'i', action = ArgAction::SetTrue, overrides_with = "inspect_after")] + inspect_after: bool, + + /// Isolated mode: implies `-E -s` and sets `sys.flags.isolated`. + #[arg(short = 'I', action = ArgAction::SetTrue, overrides_with = "isolated")] + isolated: bool, + + /// Don't run `site.main()` on interpreter startup. + #[arg(short = 'S', action = ArgAction::SetTrue, overrides_with = "no_site")] + no_site: bool, + + /// Don't add the user site-packages to `sys.path`. + #[arg(short = 's', action = ArgAction::SetTrue, overrides_with = "no_user_site")] + no_user_site: bool, + + /// Suppress the REPL banner. + #[arg(short = 'q', action = ArgAction::SetTrue, overrides_with = "quiet")] + quiet: bool, + + /// Don't prepend the script dir / cwd to `sys.path`. + #[arg(short = 'P', action = ArgAction::SetTrue, overrides_with = "safe_path")] + safe_path: bool, + + /// Force stdout/stderr unbuffered. + #[arg(short = 'u', action = ArgAction::SetTrue, overrides_with = "unbuffered")] + unbuffered: bool, + + /// Verbose imports. + #[arg(short = 'v', action = ArgAction::Count)] + verbose: u8, + + /// Skip the first source line (shebang trick). + #[arg(short = 'x', action = ArgAction::SetTrue, overrides_with = "skip_first_line")] + skip_first_line: bool, + + /// `-X key[=value]`. Forwarded to `sys._xoptions`. + #[arg(short = 'X', action = ArgAction::Append, value_name = "OPT")] + xoptions: Vec, + + /// `-W filter` warning control. Forwarded to `sys.warnoptions`. + #[arg(short = 'W', action = ArgAction::Append, value_name = "FILTER")] + warnings: Vec, + + /// `--check-hash-based-pycs MODE`. Accepted, ignored (we always + /// use mtime-mode cache invalidation). + #[arg(long = "check-hash-based-pycs", value_name = "MODE")] + check_hash_pycs: Option, + + /// Execute `` as `__main__`. Mirrors `python -c`. + #[arg(short = 'c', value_name = "SOURCE")] + command: Option, + + /// Run library module `` as `__main__`. Mirrors `python -m`. + #[arg(short = 'm', value_name = "MODULE")] + module: Option, + + /// Script path (`script.py`) or `-` for stdin. Optional. + script: Option, + + /// Trailing arguments → `sys.argv[1:]`. + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + args: Vec, +} + +const DIAGNOSTIC_SENTINEL: &str = "exited with diagnostic"; + +const HELP_BODY: &str = "\ +usage: weavepy [option] ... [-c cmd | -m mod | file | -] [arg] ... +Options (and corresponding environment variables): +-b : issue warnings about converting bytes/bytearray to str (-bb: error) +-B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x +-c cmd : program passed in as string (terminates option list) +-d : turn on parser debugging output (for experts only) +-E : ignore PYTHON* environment variables (such as PYTHONPATH) +-h : print this help message and exit (also --help) +-i : inspect interactively after running script; (also PYTHONINSPECT=x) +-I : isolate Python from the user's environment (implies -E and -s) +-m mod : run library module as a script (terminates option list) +-O : remove assert and __debug__-dependent statements; also PYTHONOPTIMIZE=x +-OO : do -O changes and also discard docstrings +-P : don't prepend a potentially unsafe path to sys.path +-q : don't print version and copyright messages on interactive startup +-R : turn on hash randomization; also PYTHONHASHSEED=random (default) +-s : don't add user site directory to sys.path; also PYTHONNOUSERSITE +-S : don't imply 'import site' on initialization +-u : force the stdout and stderr streams to be unbuffered +-v : verbose (trace import statements); also PYTHONVERBOSE=x +-V : print the Python version number and exit (also --version) +-W arg : warning control; arg is action:message:category:module:lineno +-x : skip first line of source, allowing use of non-Unix shebang +-X opt : set implementation-specific option +file : program read from script file +- : program read from stdin (default; interactive mode if a tty) +arg ...: arguments passed to program in sys.argv[1:] +"; + +const HELP_ENV: &str = "\ +Environment variables: +PYTHONHOME : alternate directory (or :). + The default module search path uses /python{X.Y}. +PYTHONPATH : ':'-separated list of directories prefixed to sys.path. +PYTHONSTARTUP : file executed on interactive startup (no default). +PYTHONOPTIMIZE : same as -O option. +PYTHONDEBUG : same as -d option. +PYTHONINSPECT : same as -i option. +PYTHONUNBUFFERED : same as -u option. +PYTHONVERBOSE : same as -v option. +PYTHONNOUSERSITE : same as -s option. +PYTHONHASHSEED : if set to 'random', randomize hash; integer in [0, 4294967295] for repeatable. +PYTHONIOENCODING : Encoding[:errors] used for stdin/stdout/stderr. +PYTHONDONTWRITEBYTECODE: don't write .pyc files (same as -B). +PYTHONWARNINGS : warning control; comma-separated -W filters. +PYTHONBREAKPOINT : override sys.breakpointhook (default 'pdb.set_trace'). +PYTHONUTF8 : force the interpreter into UTF-8 mode. +PYTHONNODEBUGRANGES : disable PEP 657 column-precise tracebacks (no-op today). +PYTHONSAFEPATH : same as -P option. +"; + +const HELP_XOPTIONS: &str = "\ +The following implementation-specific options are available: +-X faulthandler : dump the Python traceback on fatal signals. +-X dev : enable runtime checks helpful for development. +-X utf8 : enable UTF-8 mode for the interpreter. +-X tracemalloc[=N] : start tracing Python memory allocations, keeping N frames. +-X importtime : show how long each import takes (no-op today). +-X showrefcount : output the total reference count (no-op today). +-X frozen_modules=on|off : whether frozen modules should be used. +-X no_debug_ranges : disable PEP 657 ranges (no-op today). +-X pycache_prefix=PATH : redirect __pycache__ to PATH. +-X int_max_str_digits : set sys.int_info.str_digits_check_threshold. +"; + +// Opt-in native crash diagnostics (`WEAVEPY_SEGV_BT`): macOS-only, because +// the raw `siginfo_t`/`ucontext_t` byte offsets below are the Darwin layouts. +#[cfg(target_os = "macos")] +extern "C" { + fn signal(signum: i32, handler: usize) -> usize; + fn sigaction(signum: i32, act: *const SigActionC, old: *mut SigActionC) -> i32; + fn backtrace(array: *mut *mut std::ffi::c_void, size: i32) -> i32; + fn backtrace_symbols_fd(array: *const *mut std::ffi::c_void, size: i32, fd: i32); +} + +/// `struct sigaction` (macOS/BSD layout): an 8-byte handler pointer union, +/// a 4-byte `sigset_t` mask, and a 4-byte flags word. +#[cfg(target_os = "macos")] +#[repr(C)] +struct SigActionC { + sa_sigaction: usize, + sa_mask: u32, + sa_flags: i32, +} + +/// `SA_SIGINFO` — deliver the 3-argument handler so we can read `si_addr`. +#[cfg(target_os = "macos")] +const SA_SIGINFO: i32 = 0x0040; +/// Byte offset of `si_addr` within macOS `siginfo_t` +/// (`si_signo,si_errno,si_code,si_pid,si_uid,si_status` = 24 bytes precede it). +#[cfg(target_os = "macos")] +const SIGINFO_SI_ADDR_OFFSET: usize = 24; + +/// Byte offset of the `mcontext_t` pointer within macOS `ucontext_t` +/// (`uc_onstack,uc_sigmask,uc_stack,uc_link,uc_mcsize` precede it). +#[cfg(all(target_os = "macos", target_arch = "aarch64"))] +const UCONTEXT_MCONTEXT_OFFSET: usize = 48; +/// Byte offset of `__ss` (the ARM thread state) within macOS `mcontext64` +/// — it follows the 16-byte `__es` (ARM exception state). +#[cfg(all(target_os = "macos", target_arch = "aarch64"))] +const MCONTEXT_SS_OFFSET: usize = 16; +/// Byte offset of `tp_name` (a `const char *`) within `PyTypeObject`. +#[cfg(all(target_os = "macos", target_arch = "aarch64"))] +const PYTYPEOBJECT_TP_NAME_OFFSET: usize = 0x18; + +/// Read the C string at `p` (best-effort, capped) for signal-handler +/// diagnostics. Returns a lossy `String`; bails on an obviously-bad pointer +/// so we don't double-fault while already handling a crash. +#[cfg(all(target_os = "macos", target_arch = "aarch64"))] +unsafe fn read_c_str_lossy(p: *const u8, cap: usize) -> String { + if (p as usize) < 0x1000 { + return String::from(""); + } + let mut bytes = Vec::new(); + for i in 0..cap { + let b = unsafe { p.add(i).read() }; + if b == 0 { + break; + } + bytes.push(b); + } + String::from_utf8_lossy(&bytes).into_owned() +} + +#[cfg(target_os = "macos")] +extern "C" fn weavepy_segv_backtrace(sig: i32, info: *const u8, ctx: *mut std::ffi::c_void) { + // `ctx` (the interrupted-thread register file) is only decoded on arm64, + // where the `mcontext64` layout below applies. + #[cfg(not(target_arch = "aarch64"))] + let _ = ctx; + // The faulting memory address (`si_addr`) is the single most useful clue + // for a native crash in a dlopen'd extension: a small value (`0x0`, `0x8`, + // …) is a NULL-based field deref, a huge value a wild pointer. Printing it + // turns an opaque `PyArray_*` frame into an actionable diagnosis. + if !info.is_null() { + let si_addr = unsafe { + info.add(SIGINFO_SI_ADDR_OFFSET) + .cast::() + .read_unaligned() + }; + eprintln!("\n=== WEAVEPY signal {sig} faulting address = 0x{si_addr:x} ==="); + } + // Faulting register file (arm64): `pc` pinpoints the exact instruction and + // `x0` is usually the receiver of a `Py_TYPE(x)->tp_field` chain. When the + // crash is a NULL `tp_mro`/`tp_dict`/… deref, `x0` is still the live type + // pointer, so decoding `x0->tp_name` names the offending type directly. + #[cfg(target_arch = "aarch64")] + if !ctx.is_null() { + unsafe { + let mctx = ctx + .cast::() + .add(UCONTEXT_MCONTEXT_OFFSET) + .cast::<*const u8>() + .read_unaligned(); + if !mctx.is_null() { + let ss = mctx.add(MCONTEXT_SS_OFFSET); + let x = |n: usize| ss.add(n * 8).cast::().read_unaligned(); + let pc = ss.add(256).cast::().read_unaligned(); + eprintln!( + "=== registers: pc=0x{pc:x} x0=0x{:x} x1=0x{:x} x8=0x{:x} x19=0x{:x} x20=0x{:x} ===", + x(0), x(1), x(8), x(19), x(20) + ); + // Heuristic: for a `tp_*` NULL-field crash the type pointer is + // in x0 (and often mirrored in x19/x20). Decode each as a + // candidate `PyTypeObject*` and print its `tp_name`. + for (reg, val) in [("x0", x(0)), ("x19", x(19)), ("x20", x(20))] { + let name_pp = (val as usize + PYTYPEOBJECT_TP_NAME_OFFSET) as *const *const u8; + if (val as usize) > 0x1000 { + let name = read_c_str_lossy(name_pp.read(), 64); + eprintln!("=== {reg} as PyTypeObject* -> tp_name = {name:?} ==="); + } + } + } + } + } + // Native (dladdr-based) backtrace first: it resolves frames inside a + // dlopen'd `.so` (e.g. a Cython extension's static helpers) to their + // real `module + symbol + offset`, which Rust's `std::backtrace` + // mis-attributes to the nearest exported libsystem symbol. + let mut frames: [*mut std::ffi::c_void; 96] = [std::ptr::null_mut(); 96]; + let n = unsafe { backtrace(frames.as_mut_ptr(), 96) }; + eprintln!("=== WEAVEPY signal {sig} native backtrace ==="); + unsafe { backtrace_symbols_fd(frames.as_ptr(), n, 2) }; + eprintln!("=== end native backtrace ==="); + let bt = std::backtrace::Backtrace::force_capture(); + eprintln!("=== WEAVEPY signal {sig} rust backtrace ===\n{bt}\n=== end backtrace ==="); + unsafe { + signal(sig, 0); + } + std::process::abort(); +} + +/// Run the WeavePy CLI against this process's real argv and +/// environment, returning the exit code. This is the entire `weavepy` +/// binary: the bin target's `main` (POSIX) and the `python313` DLL's +/// `weavepy_main` export (Windows) are both one-line calls into here. +pub fn cli_main() -> i32 { + #[cfg(target_os = "macos")] + if std::env::var_os("WEAVEPY_SEGV_BT").is_some() { + // `SA_SIGINFO` so the handler receives `siginfo_t` and can report the + // faulting address; `signal()` alone would only pass the signal number. + let act = SigActionC { + sa_sigaction: weavepy_segv_backtrace as *const () as usize, + sa_mask: 0, + sa_flags: SA_SIGINFO, + }; + unsafe { + sigaction(11, &raw const act, std::ptr::null_mut()); // SIGSEGV + sigaction(10, &raw const act, std::ptr::null_mut()); // SIGBUS + } + } + // Undo Rust's pre-`main` `sanitize_standard_fds` (which re-opens any closed + // std fd onto `/dev/null`) so an inherited-closed stdin/stdout/stderr stays + // closed, matching CPython (`test_posix.test_close_file`). Must run before + // any descriptor work. + weavepy::vm::proc_init::restore_initial_std_fds(); + run_on_large_stack(main_dispatch) +} + +/// [`cli_main`] with an explicit argv (element 0 is the program name), +/// for the `Py_Main` / `Py_BytesMain` embedding entry points exported +/// by `weavepy-pylib`. The override replaces what `os_args_bridged()` +/// would have read from the process for the duration of the run; +/// undecodable bytes should already be PEP 383-bridged by the caller +/// (`weavepy::vm::os_str_bridged`-style) before they get here. +pub fn cli_main_with_args(args: Vec) -> i32 { + *ARGV_OVERRIDE.lock().expect("argv override lock") = Some(args); + cli_main() +} + +/// Explicit-argv override for [`cli_main_with_args`]. A process-global +/// (not a thread-local) because `run_on_large_stack` moves execution +/// onto the dedicated `weavepy-main` thread. +static ARGV_OVERRIDE: std::sync::Mutex>> = std::sync::Mutex::new(None); + +/// The CLI's argv: the [`ARGV_OVERRIDE`] if an embedding entry point +/// installed one, else the process argv through the PEP 383 bridge. +/// The override is read non-destructively — `main_dispatch` and +/// `real_main` both consult argv, and they must agree. +fn bridged_args() -> Vec { + if let Some(args) = ARGV_OVERRIDE.lock().expect("argv override lock").as_ref() { + return args.clone(); + } + weavepy::vm::os_args_bridged() +} + +/// WeavePy evaluates Python by recursive descent, so Python call depth +/// maps onto native (Rust) stack depth (see `crates/weavepy-vm/src/ +/// recursion.rs`). Run the whole interpreter on a thread with a large +/// stack reserve so that `sys.setrecursionlimit` — enforced by the VM's +/// recursion guard (RFC 0037) — is what bounds recursion, rather than +/// the fixed OS main-thread stack (8 MiB on Linux/macOS). This makes the +/// behaviour uniform across platforms *and* build profiles: debug builds +/// have much larger per-activation stack frames than release, so without +/// this a default `setrecursionlimit(1000)` would overflow the native +/// stack in debug before the guard could fire. The reserve is committed +/// lazily by the OS, so it costs address space, not memory. +fn run_on_large_stack(entry: fn() -> i32) -> i32 { + const STACK_BYTES: usize = 1024 * 1024 * 1024; // 1 GiB reserve + + // The interpreter runs on the spawned `weavepy-main` thread, not the + // process's initial OS thread (which only parks in `join()` below). + // Block the asynchronous, process-directed signals (SIGINT, SIGALRM, + // …) on this initial thread *before* spawning so a signal racing in + // during startup can't be stolen by the soon-to-be-parked thread — + // where it would merely trip the pending flag while the VM thread's + // blocking syscall never gets EINTR (CPython's test_io SignalsTest + // would then hang forever). The VM thread re-enables them for itself + // first thing, making it the sole, deterministic delivery target. + weavepy::vm::stdlib::signal_mod::block_async_signals_current_thread(); + + let vm_entry = move || -> i32 { + // Opt-in (`WEAVEPY_CRASH_BT`): register the native crash handler + + // per-thread sigaltstack on the VM thread itself so a stack-overflow + // SIGSEGV can be caught and reported (no-op stub on Windows). + if std::env::var_os("WEAVEPY_CRASH_BT").is_some() { + extern "C" { + fn weavepy_install_crash_handler(); + } + unsafe { weavepy_install_crash_handler() }; + } + weavepy::vm::stdlib::signal_mod::unblock_async_signals_current_thread(); + // Arm SIGINT -> KeyboardInterrupt at startup (CPython does this during + // interpreter init), so even scripts that never `import signal` raise + // KeyboardInterrupt on ^C instead of being killed by the kernel default. + weavepy::vm::stdlib::signal_mod::install_startup_dispositions(); + // Snapshot the OS-thread count *now* — on the VM thread, before any + // user code can spawn `threading` workers or raw pthreads — so that a + // later `os.fork()` can tell "single-threaded" (no warning) from + // "multi-threaded" (CPython's fork `DeprecationWarning`). WeavePy runs + // the interpreter off the parked process-initial thread, so the + // quiescent process already has >1 OS thread; this baseline is what the + // fork-warning check measures additional threads against. + weavepy::vm::stdlib::os_process::capture_thread_baseline(); + entry() + }; + + match std::thread::Builder::new() + .name("weavepy-main".to_owned()) + .stack_size(STACK_BYTES) + .spawn(vm_entry) + { + Ok(handle) => handle.join().unwrap_or(1), + // Extremely unlikely, but if the OS refuses the thread, fall back + // to running on the current thread — restore signal delivery here + // first since we blocked it above. + Err(_) => { + weavepy::vm::stdlib::signal_mod::unblock_async_signals_current_thread(); + weavepy::vm::stdlib::signal_mod::install_startup_dispositions(); + weavepy::vm::stdlib::os_process::capture_thread_baseline(); + entry() + } + } +} + +fn main_dispatch() -> i32 { + init_tracing(); + + // `env::args()` panics on non-UTF-8 argv (bpo-35883's exact repro); + // decode PEP 383-style instead, carrying undecodable bytes in the + // PUA bridge window that `Interpreter::set_argv` maps back to + // lone surrogates (RFC 0050). + let raw: Vec = bridged_args(); + + // Multiprocessing spawn-child entry point. The parent passes + // `--multiprocessing-fork` and an optional payload fd via + // `WEAVEPY_MP_PAYLOAD_FD`; we hand off to + // `multiprocessing._run_spawn_child()` which reads the pickled + // task off the inherited fd and runs it. + if raw.iter().any(|a| a == "--multiprocessing-fork") { + return run_multiprocessing_child(&raw); + } + + // Bare subcommand dispatch (e.g. `weavepy regrtest ...`) — must + // run before clap, which would try to interpret the subcommand as + // a positional `script` and trip on unknown flags after it. + if raw.len() >= 2 && SUBCOMMANDS.contains(&raw[1].as_str()) { + let sub = raw[1].clone(); + let rest: Vec = std::iter::once(format!("weavepy {sub}")) + .chain(raw.into_iter().skip(2)) + .collect(); + return match sub.as_str() { + "regrtest" => match regrtest_cmd::run(rest) { + Ok(code) => code, + Err(err) => { + let mut stderr = io::stderr().lock(); + let _ = writeln!(stderr, "weavepy regrtest: {err:#}"); + 1 + } + }, + _ => unreachable!(), + }; + } + + match real_main() { + Ok(code) => code, + Err(err) => { + if err.to_string() != DIAGNOSTIC_SENTINEL { + let mut stderr = io::stderr().lock(); + let _ = writeln!(stderr, "weavepy: {err:#}"); + } + 1 + } + } +} + +/// Split argv at the first `-c CMD` / `-m MODULE` / `script` / `-` / `--` +/// boundary so flags meant for the child program don't get re-parsed by +/// clap. Returns `(weavepy_args, mode, child_args)`. +/// +/// `mode` is one of: +/// - `Some(("c", ""))` — `-c CMD` was found. +/// - `Some(("m", ""))` — `-m MOD` was found. +/// - `Some(("s", ""))` — a positional script was found. +/// - `Some(("-", ""))` — `-` (stdin) was found. +/// - `None` — interactive mode (no boundary). +fn split_argv(raw: Vec) -> (Vec, Option<(&'static str, String)>, Vec) { + let mut wp: Vec = Vec::with_capacity(raw.len()); + let mut iter = raw.into_iter(); + if let Some(prog) = iter.next() { + wp.push(prog); + } + while let Some(arg) = iter.next() { + if arg == "--" { + return (wp, None, iter.collect()); + } + if arg == "-c" { + let Some(cmd) = iter.next() else { + argument_expected_error('c'); + }; + let rest: Vec = iter.collect(); + return (wp, Some(("c", cmd)), rest); + } + if arg == "-m" { + let Some(m) = iter.next() else { + argument_expected_error('m'); + }; + let rest: Vec = iter.collect(); + return (wp, Some(("m", m)), rest); + } + if arg.starts_with("-c") && arg.len() > 2 { + let cmd = arg[2..].to_owned(); + let rest: Vec = iter.collect(); + return (wp, Some(("c", cmd)), rest); + } + if arg.starts_with("-m") && arg.len() > 2 { + let m = arg[2..].to_owned(); + let rest: Vec = iter.collect(); + return (wp, Some(("m", m)), rest); + } + // Attached `-Xkey[=value]` / `-Wfilter` (CPython's own spelling — + // `test_subprocess.test_encoding_warning` spawns `-Xwarn_default_encoding`): + // normalise to the separate `-X key` form clap parses, so the option + // reaches `sys._xoptions` / `sys.warnoptions`. + if let Some(rest) = arg.strip_prefix("-X").filter(|r| !r.is_empty()) { + wp.push("-X".to_owned()); + wp.push(rest.to_owned()); + continue; + } + if let Some(rest) = arg.strip_prefix("-W").filter(|r| !r.is_empty()) { + wp.push("-W".to_owned()); + wp.push(rest.to_owned()); + continue; + } + // Clustered single-letter options where `-c`/`-m` follows some boolean + // flags, e.g. `-uc CMD` == `-u -c CMD` and `-uIcCMD` == `-u -I -c CMD` + // (CPython accepts this; `test_subprocess` spawns children as `-uc`). + // The `c`/`m` consumes the rest of the cluster as its value, else the + // next argv element. + if arg.starts_with('-') && !arg.starts_with("--") && arg.len() > 2 { + let body: Vec = arg[1..].chars().collect(); + if let Some(pos) = body.iter().position(|&c| c == 'c' || c == 'm') { + const BOOL_SHORT: &[char] = &[ + 'O', 'b', 'B', 'd', 'E', 'i', 'I', 'R', 'S', 's', 'q', 'P', 'u', 'v', 'x', + ]; + if body[..pos].iter().all(|c| BOOL_SHORT.contains(c)) { + for &c in &body[..pos] { + wp.push(format!("-{c}")); + } + let kind = if body[pos] == 'c' { "c" } else { "m" }; + let after: String = body[pos + 1..].iter().collect(); + let value = if after.is_empty() { + iter.next() + .unwrap_or_else(|| argument_expected_error(body[pos])) + } else { + after + }; + let rest: Vec = iter.collect(); + return (wp, Some((kind, value)), rest); + } + } + } + if arg == "-" { + let rest: Vec = iter.collect(); + return (wp, Some(("-", String::new())), rest); + } + // Value-taking flags: consume the following arg too, so it + // isn't mistaken for the positional script (`-X opt script.py`). + if arg == "-X" || arg == "-W" || arg == "--check-hash-based-pycs" { + wp.push(arg); + if let Some(value) = iter.next() { + wp.push(value); + } + continue; + } + if !arg.starts_with('-') { + // Positional script. + let rest: Vec = iter.collect(); + return (wp, Some(("s", arg)), rest); + } + wp.push(arg); + } + (wp, None, Vec::new()) +} + +fn real_main() -> Result { + let raw: Vec = bridged_args(); + let (wp_argv, mode, child_argv) = split_argv(raw); + // Re-parse the WeavePy-only slice with clap. + let mut cli = Cli::parse_from(wp_argv); + // Stuff `mode` back into the parsed Cli so the rest of real_main + // sees a consistent view. + match &mode { + Some(("c", cmd)) => cli.command = Some(decode_command_arg(cmd)), + Some(("m", m)) => cli.module = Some(m.clone()), + // A script path may carry PEP 383-escaped bytes (PUA-bridged by + // `os_args_bridged`); recover the OS-level bytes so the file + // actually opens (RFC 0050). + Some(("s", path)) => cli.script = Some(bridged_arg_to_pathbuf(path)), + Some(("-", _)) => cli.script = Some(PathBuf::from("-")), + _ => {} + } + cli.args = child_argv; + + if cli.help { + print!("{HELP_BODY}"); + return Ok(0); + } + if cli.help_env { + print!("{HELP_ENV}"); + return Ok(0); + } + if cli.help_xoptions { + print!("{HELP_XOPTIONS}"); + return Ok(0); + } + if cli.version { + // PyPy-style: lead with the CPython version the interpreter + // implements (tooling — pyenv, tox, CI matrices, weavepy-dist's + // `version` leg — parses `python -V` for `Python X.Y.Z`), then + // identify the implementation. + let (maj, min, mic) = weavepy_vm::stdlib::sys::PY_VERSION; + println!("Python {maj}.{min}.{mic} (WeavePy {VERSION})"); + return Ok(0); + } + + let env = if cli.isolated || cli.ignore_env { + EnvOverrides::ignored() + } else { + EnvOverrides::from_env() + }; + + let mut flags = build_flags(&cli, &env); + + // Compose pythonpath from env (when honoured) plus -X variants. + let mut extra_path: Vec = env + .pythonpath + .iter() + .filter(|s| !s.is_empty()) + .map(PathBuf::from) + .collect(); + + // `WEAVEPY_CPYTHON_LIB` points at an external stdlib `Lib` directory + // (the vendored CPython tree). Like a real interpreter that finds its + // stdlib relative to the executable, this is part of the *default* + // module search path: it is honoured even under `-I`/`-E` (it is not a + // `PYTHON*` variable, so isolation does not strip it) so child + // interpreters spawned via `sys.executable` — e.g. `assert_python_ok`, + // `multiprocessing` spawn, `subprocess` re-execs — can still import the + // stdlib and the `test` package. Unset in normal use, so this is a + // no-op outside the conformance harness. + if let Some(lib) = env::var_os("WEAVEPY_CPYTHON_LIB") { + for part in env::split_paths(&lib) { + if !part.as_os_str().is_empty() { + extra_path.push(part); + } + } + } + + // getpath's `._pth` layout override (RFC 0062 WS5): a `._pth` + // file next to the binary pins `sys.path` to exactly its entries + // and locks the interpreter down (no PYTHONPATH, no script-dir + // prepend, no site unless the file says `import site`) — CPython's + // embeddable-distribution mechanism, honoured on every platform. + if let Some((entries, import_site)) = read_pth_file() { + flags.pth_paths = Some(entries); + if !import_site { + flags.no_site = true; + } + flags.safe_path = true; + flags.no_user_site = true; + extra_path.clear(); + } + + if let Some(source) = cli.command.clone() { + let mut argv = vec!["-c".to_owned()]; + argv.extend(cli.args.iter().cloned()); + // CPython's `-c` puts the *empty string* at `sys.path[0]` (an + // '' entry means "current directory, resolved at import time"), + // not a materialized cwd path — + // `test_cmd_line_script.test_issue8202_dash_c_file_ignored`. + let opts = RunOptions::new("") + .with_argv(argv) + .with_extra_path(extra_path.drain(..)) + .with_script_dir("") + .with_flags(flags.clone()); + // `-i` is handled inside `run_source_with_options`, which drops + // into a namespace-sharing REPL after the program body. + run_source_with_options(&source, &opts)?; + return Ok(0); + } + + if let Some(module) = cli.module.clone() { + let extra = cli.args.clone(); + run_module(&module, extra, &flags, &extra_path)?; + return Ok(0); + } + + let script = cli.script.clone(); + let trailing = cli.args.clone(); + match script.as_deref() { + Some(path) if path.as_os_str() == "-" => { + run_stdin(trailing.clone(), &flags, &extra_path)?; + Ok(0) + } + Some(path) => { + run_path(path, trailing.clone(), &flags, &extra_path)?; + Ok(0) + } + None => { + // No script. CPython enters the REPL only when stdin is a + // tty (or `-i` forces it); a piped stdin is read to EOF and + // run as a program named `` (`pymain_run_stdin` — + // no banner, no `>>>` prompts, plain tracebacks). + let stdin_is_tty = std::io::IsTerminal::is_terminal(&io::stdin()); + if stdin_is_tty || flags.inspect { + flags.inspect = true; + run_repl(flags, env.startup.as_deref(), trailing)?; + } else { + run_stdin(trailing, &flags, &extra_path)?; + } + Ok(0) + } + } +} + +/// CPython's `pymain_err_print` for an option missing its argument: +/// diagnostics + usage line on stderr, exit status 2. +fn argument_expected_error(opt: char) -> ! { + eprintln!("Argument expected for the -{opt} option"); + eprintln!("usage: weavepy [option] ... [-c cmd | -m mod | file | -] [arg] ..."); + eprintln!("Try `weavepy -h' for more information."); + std::process::exit(2); +} + +/// A startup configuration error CPython reports through +/// `Py_ExitStatusException`: `Fatal Python error: : `, exit 1. +fn config_fatal_error(whence: &str, msg: &str) -> ! { + eprintln!("Fatal Python error: {whence}: {msg}"); + std::process::exit(1); +} + +/// The value of the last `-X name[=value]` occurrence: `None` when the +/// option wasn't given, `Some(None)` for the bare form, `Some(Some(v))` +/// for `-X name=v`. +fn xoption_value<'a>(xoptions: &'a [String], name: &str) -> Option> { + xoptions.iter().rev().find_map(|x| { + if x == name { + Some(None) + } else { + x.strip_prefix(name) + .and_then(|rest| rest.strip_prefix('=')) + .map(Some) + } + }) +} + +/// Parse + validate the PEP 0467 digit cap (`0` or `>= 640`), exiting +/// with CPython's `config_init_int_max_str_digits` fatal error otherwise. +fn parse_int_max_str_digits(value: &str, source: &str) -> i64 { + match value.parse::() { + Ok(n) if n == 0 || n >= 640 => n, + _ => config_fatal_error( + "config_init_int_max_str_digits", + &format!("{source}: invalid limit; must be >= 640 or 0 for unlimited."), + ), + } +} + +/// Locate and parse the `._pth` file governing this executable +/// (CPython getpath: `._pth`, plus the `._pth` spelling on +/// Windows). Returns the absolutized `sys.path` entries and whether an +/// `import site` line re-enables site processing. Comment lines start +/// with `#`; other `import` lines are recognised but only `site` has +/// an effect (matching getpath, which special-cases exactly that). +fn read_pth_file() -> Option<(Vec, bool)> { + let exe = std::env::current_exe().ok()?; + let mut with_suffix = exe.as_os_str().to_owned(); + with_suffix.push("._pth"); + let mut candidates = vec![PathBuf::from(with_suffix)]; + if cfg!(windows) { + candidates.push(exe.with_extension("_pth")); + } + let pth = candidates.into_iter().find(|p| p.is_file())?; + let contents = std::fs::read_to_string(&pth).ok()?; + let exe_dir = exe.parent()?; + let mut entries = Vec::new(); + let mut import_site = false; + for line in contents.lines() { + let line = line.trim_end_matches('\r'); + if line.is_empty() || line.starts_with('#') { + continue; + } + if let Some(rest) = line.strip_prefix("import ") { + if rest.split(',').any(|m| m.trim() == "site") { + import_site = true; + } + continue; + } + entries.push( + lexical_abspath(&exe_dir.join(line)) + .to_string_lossy() + .into_owned(), + ); + } + Some((entries, import_site)) +} + +/// `os.path.abspath` without touching the filesystem: make absolute +/// against the cwd, then collapse `.` and `..` components lexically +/// (symlinks are *not* resolved — the ._pth expectations are computed +/// with `abspath`, which is purely lexical too). +fn lexical_abspath(p: &Path) -> PathBuf { + let abs = if p.is_absolute() { + p.to_path_buf() + } else { + env::current_dir().map_or_else(|_| p.to_path_buf(), |cwd| cwd.join(p)) + }; + let mut out = PathBuf::new(); + for comp in abs.components() { + match comp { + std::path::Component::CurDir => {} + // The joined input is always absolute, so a failed pop can + // only mean we're at the root — where `/..` collapses to + // `/`, exactly like `os.path.normpath`. + std::path::Component::ParentDir => { + out.pop(); + } + other => out.push(other.as_os_str()), + } + } + out +} + +/// Compose the runtime [`InterpreterFlags`] from the CLI table and +/// the environment overrides. `-I` is the trump card. +fn build_flags(cli: &Cli, env: &EnvOverrides) -> InterpreterFlags { + let isolated = cli.isolated; + let ignore_env = cli.ignore_env || isolated; + // Pin the per-process str/bytes hash salt before the interpreter + // hashes anything (PEP 456 / `PYTHONHASHSEED`). `-R` re-enables + // randomization, which is also the default when the var is unset. + if !cli.hash_randomization { + if let Some(seed) = env.hash_seed { + weavepy::vm::object::set_hash_seed(seed); + } + } + // `-X pycache_prefix[=PATH]` beats `PYTHONPYCACHEPREFIX` even when + // given bare / with an empty value (which unsets the env prefix). + let pycache_prefix = match xoption_value(&cli.xoptions, "pycache_prefix") { + Some(v) => v.filter(|p| !p.is_empty()).map(str::to_owned), + None => env.pycache_prefix.clone(), + }; + let int_max_str_digits = match xoption_value(&cli.xoptions, "int_max_str_digits") { + Some(Some(v)) => Some(parse_int_max_str_digits(v, "-X int_max_str_digits")), + Some(None) => config_fatal_error( + "config_init_int_max_str_digits", + "-X int_max_str_digits: invalid limit; must be >= 640 or 0 for unlimited.", + ), + None => env + .int_max_str_digits + .as_deref() + .map(|v| parse_int_max_str_digits(v, "PYTHONINTMAXSTRDIGITS")), + }; + // `-X cpu_count=N|default` / `PYTHON_CPU_COUNT` (gh-109595). + let cpu_count_raw = match xoption_value(&cli.xoptions, "cpu_count") { + Some(Some(v)) => Some(v.to_owned()), + Some(None) => config_fatal_error( + "config_init_cpu_count", + "-X cpu_count=n option: n is missing or invalid", + ), + None => env.cpu_count.clone(), + }; + let cpu_count = cpu_count_raw.and_then(|raw| { + if raw == "default" { + None + } else { + match raw.parse::() { + Ok(n) if n >= 1 => Some(n), + _ => config_fatal_error( + "config_init_cpu_count", + "-X cpu_count=n option: n is missing or invalid", + ), + } + } + }); + // `-X gil` / `PYTHON_GIL` (PEP 703): only "1" is meaningful on a + // build whose GIL can't be disabled; "0" is a startup fatal error. + let gil = match xoption_value(&cli.xoptions, "gil") { + Some(v) => v.map(str::to_owned), + None => env.gil.clone(), + }; + match gil.as_deref() { + None | Some("1") => {} + Some("0") => config_fatal_error( + "config_read_gil", + "Disabling the GIL is not supported by this build", + ), + Some(_) => config_fatal_error( + "config_read_gil", + "PYTHON_GIL / -X gil must be \"0\" or \"1\"", + ), + } + // `-X tracemalloc[=NFRAME]` beats `PYTHONTRACEMALLOC` (CPython + // `config_init_tracemalloc`): a parse failure or negative value is a + // startup fatal error; `0` means disabled; a value beyond + // `_tracemalloc`'s MAX_NFRAME fails at init with the module's own + // ValueError text (`test_tracemalloc.TestCommandLine`). + let tracemalloc_nframe = match xoption_value(&cli.xoptions, "tracemalloc") { + Some(Some(v)) => match v.parse::() { + Ok(n) if n >= 0 => Some(n), + _ => config_fatal_error( + "config_init_tracemalloc", + "-X tracemalloc=NFRAME: invalid number of frames", + ), + }, + // Bare `-X tracemalloc` behaves as `-X tracemalloc=1`. + Some(None) => Some(1), + None => env.tracemalloc.as_deref().map(|v| match v.parse::() { + Ok(n) if n >= 0 => n, + _ => config_fatal_error( + "config_init_tracemalloc", + "PYTHONTRACEMALLOC: invalid number of frames", + ), + }), + }; + let tracemalloc = match tracemalloc_nframe { + None | Some(0) => 0u32, + Some(n) if n > 65535 => { + // CPython surfaces `_PyTraceMalloc_Start`'s ValueError during + // interpreter init. + eprintln!("ValueError: the number of frames must be in range [1; 65535]"); + std::process::exit(1); + } + Some(n) => n as u32, + }; + let mut xoptions = cli.xoptions.clone(); + // `PYTHONDEVMODE` behaves like `-X dev` for `sys.flags.dev_mode` + // (though CPython does *not* mirror it into `sys._xoptions`; the + // duplicate key is harmless for our flag computation). + if env.dev_mode && xoption_value(&xoptions, "dev").is_none() { + xoptions.push("dev".to_owned()); + } + InterpreterFlags { + optimize: cli.optimize.max(env.optimize), + dont_write_bytecode: cli.no_bytecode_write || env.dont_write_bytecode, + inspect: cli.inspect_after || env.inspect, + verbose: cli.verbose.max(env.verbose), + no_site: cli.no_site, + no_user_site: cli.no_user_site || env.no_user_site || isolated, + ignore_environment: ignore_env, + isolated, + quiet: cli.quiet, + unbuffered: cli.unbuffered || env.unbuffered, + skip_first_line: cli.skip_first_line, + bytes_warning: cli.bytes_warning, + safe_path: cli.safe_path || env.safe_path || isolated, + debug: cli.parser_debug.max(env.debug), + xoptions, + warning_filters: { + let mut v = env.warning_filters.clone(); + v.extend(cli.warnings.iter().cloned()); + v + }, + // `-R` re-enables randomization, trumping a fixed seed from + // `PYTHONHASHSEED`. + hash_seed: if cli.hash_randomization { + None + } else { + env.hash_seed + }, + // Filled in by the `._pth` probe in `main` (RFC 0062 WS5). + pth_paths: None, + io_encoding: env.io_encoding.clone(), + io_errors: env.io_errors.clone(), + utf8_mode: env.utf8_mode, + pycache_prefix, + int_max_str_digits, + cpu_count, + tracemalloc, + // `-X faulthandler` beats `PYTHONFAULTHANDLER` only in the sense + // that either one turns it on (CPython `config_init_faulthandler`; + // there is no "off" spelling). + faulthandler: env.faulthandler || xoption_value(&cli.xoptions, "faulthandler").is_some(), + } +} + +/// Subset of `PYTHON*` environment overrides we honour. Materialised +/// once per CLI invocation so each call site reads from a consistent +/// snapshot (env vars don't mutate mid-run). +#[derive(Debug, Default, Clone)] +struct EnvOverrides { + pythonpath: Vec, + startup: Option, + optimize: u8, + dont_write_bytecode: bool, + inspect: bool, + unbuffered: bool, + verbose: u8, + debug: u8, + dev_mode: bool, + no_user_site: bool, + safe_path: bool, + /// `PYTHONPYCACHEPREFIX` (PEP 552), losing to `-X pycache_prefix`. + pycache_prefix: Option, + /// `PYTHONINTMAXSTRDIGITS`, raw (validated during flag composition + /// so `-X int_max_str_digits` precedence applies first). + int_max_str_digits: Option, + /// `PYTHON_CPU_COUNT`, raw (`"default"` or an integer ≥ 1). + cpu_count: Option, + /// `PYTHON_GIL`, raw (`"0"` / `"1"`). + gil: Option, + /// `PYTHONTRACEMALLOC`, raw (validated during flag composition so + /// `-X tracemalloc` precedence applies first). + tracemalloc: Option, + /// `PYTHONFAULTHANDLER` — any non-empty value enables the + /// fatal-signal traceback dumper at startup. + faulthandler: bool, + warning_filters: Vec, + hash_seed: Option, + /// `PYTHONIOENCODING=encoding[:errors]`, split into its halves. Either + /// part may be empty (`:errors` sets only the handler). + io_encoding: Option, + io_errors: Option, + /// `PYTHONUTF8=0|1` (PEP 540). `None` when unset/empty; an invalid + /// value is a startup fatal error (CPython `config_init_utf8_mode`). + utf8_mode: Option, +} + +impl EnvOverrides { + fn from_env() -> Self { + let mut o = Self::default(); + if let Ok(p) = env::var("PYTHONPATH") { + o.pythonpath = p + .split(if cfg!(windows) { ';' } else { ':' }) + .map(str::to_owned) + .collect(); + } + if let Ok(p) = env::var("PYTHONSTARTUP") { + if !p.is_empty() { + o.startup = Some(PathBuf::from(p)); + } + } + // CPython treats a `PYTHON*` variable set to the empty string as + // unset (`config_get_env` / `_Py_GetEnv`); the int-valued ones + // (`PYTHONOPTIMIZE`/`PYTHONVERBOSE`/`PYTHONDEBUG`) parse as an + // integer with any non-numeric value meaning 1 + // (`test_cmd_line.test_sys_flags_set`). + let nonempty = |name: &str| env::var(name).ok().filter(|v| !v.is_empty()); + let env_int = |name: &str| nonempty(name).map(|v| v.parse::().unwrap_or(1)); + if let Some(n) = env_int("PYTHONOPTIMIZE") { + o.optimize = n; + } + o.dont_write_bytecode = nonempty("PYTHONDONTWRITEBYTECODE").is_some(); + o.inspect = nonempty("PYTHONINSPECT").is_some(); + o.unbuffered = nonempty("PYTHONUNBUFFERED").is_some(); + o.verbose = env_int("PYTHONVERBOSE").unwrap_or(0); + // Unlike OPTIMIZE/VERBOSE, `PYTHONDEBUG` is a plain boolean env + // in CPython (`config_get_env`, not the int-parsing variant): + // any non-empty value — including "2" — means 1. + o.debug = u8::from(nonempty("PYTHONDEBUG").is_some()); + o.dev_mode = nonempty("PYTHONDEVMODE").is_some(); + o.no_user_site = nonempty("PYTHONNOUSERSITE").is_some(); + o.safe_path = nonempty("PYTHONSAFEPATH").is_some(); + o.pycache_prefix = nonempty("PYTHONPYCACHEPREFIX"); + o.int_max_str_digits = nonempty("PYTHONINTMAXSTRDIGITS"); + o.cpu_count = nonempty("PYTHON_CPU_COUNT"); + o.gil = nonempty("PYTHON_GIL"); + o.tracemalloc = nonempty("PYTHONTRACEMALLOC"); + o.faulthandler = nonempty("PYTHONFAULTHANDLER").is_some(); + if let Ok(w) = env::var("PYTHONWARNINGS") { + o.warning_filters = w.split(',').map(str::to_owned).collect(); + } + if let Ok(seed) = env::var("PYTHONHASHSEED") { + if seed == "0" { + o.hash_seed = Some(0); + } else if let Ok(n) = seed.parse::() { + o.hash_seed = Some(n); + } + } + // `PYTHONIOENCODING=encoding[:errors]` (CPython): the first `:` + // splits the codec from the error handler; either side may be + // empty (`utf-8`, `:strict`, `ascii:backslashreplace`). + if let Ok(spec) = env::var("PYTHONIOENCODING") { + let (enc, errs) = match spec.split_once(':') { + Some((e, h)) => (e, Some(h)), + None => (spec.as_str(), None), + }; + if !enc.is_empty() { + o.io_encoding = Some(enc.to_owned()); + } + if let Some(h) = errs { + if !h.is_empty() { + o.io_errors = Some(h.to_owned()); + } + } + } + // `PYTHONUTF8` (PEP 540): "1" enables UTF-8 mode, "0" disables it, + // empty means unset; anything else is a startup fatal error + // (CPython's `config_init_utf8_mode`). + if let Ok(v) = env::var("PYTHONUTF8") { + match v.as_str() { + "" => {} + "1" => o.utf8_mode = Some(1), + "0" => o.utf8_mode = Some(0), + other => { + eprintln!( + "Fatal Python error: init_utf8_mode: invalid PYTHONUTF8 environment \ + variable value '{other}'" + ); + std::process::exit(1); + } + } + } + o + } + + fn ignored() -> Self { + Self::default() + } +} + +/// Materialise the `-c` command text from its (possibly PUA-bridged) +/// argv transport, the way CPython's `pymain_run_command` receives it: +/// - clean text (the overwhelmingly common case) passes through; +/// - undecodable bytes under the `C`/`POSIX` locale decode to their +/// byte values (macOS/BSD `_Py_char2wchar` fallback — `test_cmd_line. +/// test_undecodable_code` expects `ascii("\xff")` to print `'\xff'`); +/// - otherwise the command cannot be represented and startup fails with +/// CPython's "Unable to decode the command from the command line". +fn decode_command_arg(cmd: &str) -> String { + use weavepy::vm::object::Object; + match weavepy::vm::argv_str_to_object(cmd) { + Object::WStr(cps) => { + let c_locale = ["LC_ALL", "LC_CTYPE", "LANG"] + .iter() + .find_map(|v| env::var(v).ok().filter(|s| !s.is_empty())) + .is_none_or(|loc| loc == "C" || loc == "POSIX"); + if c_locale { + cps.iter() + .map(|&cp| match cp { + 0xDC80..=0xDCFF => char::from_u32(cp - 0xDC00).unwrap_or('\u{FFFD}'), + other => char::from_u32(other).unwrap_or('\u{FFFD}'), + }) + .collect() + } else { + eprintln!("Unable to decode the command from the command line:"); + std::process::exit(1); + } + } + Object::Str(s) => s.to_string(), + _ => cmd.to_owned(), + } +} + +/// Rebuild a filesystem path from a (possibly PUA-bridged) argv string, +/// recovering the original OS bytes for PEP 383-escaped names. +fn bridged_arg_to_pathbuf(arg: &str) -> PathBuf { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStringExt; + PathBuf::from(std::ffi::OsString::from_vec( + weavepy::vm::bridged_arg_bytes(arg), + )) + } + #[cfg(not(unix))] + { + PathBuf::from(arg) + } +} + +/// Escape a string into a Python single-quoted string literal. +fn quote_py_string(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + out.push('"'); + for c in s.chars() { + match c { + '\\' => out.push_str("\\\\"), + '"' => out.push_str("\\\""), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if (c as u32) < 0x20 => out.push_str(&format!("\\x{:02x}", c as u32)), + c => out.push(c), + } + } + out.push('"'); + out +} + +fn run_module( + name: &str, + args: Vec, + flags: &InterpreterFlags, + extra_path: &[PathBuf], +) -> Result<()> { + // Every `-m` goes through CPython's own entry point, + // `runpy._run_module_as_main`: it imports parent packages (so the + // target's relative imports resolve), redirects a package to its + // `__main__` submodule, executes the target *in* the current + // `__main__` namespace (so `-i -m timeit` leaves `Timer` visible to + // the inspect REPL — `test_cmd_line.test_run_module_bug1764407`), + // and reports a missing module the way CPython does + // (`sys.exit(": Error while finding module specification …")`). + // + // `sys.argv[0]` starts as the literal `'-m'` — CPython's config + // leaves the placeholder in place so code run *during the search* + // (a parent package's `__init__`) sees it + // (`test_cmd_line_script.test_issue8202`); `_run_module_as_main` + // then swaps in the located file path before the target runs. + let mut argv = vec!["-m".to_owned()]; + argv.extend(args.iter().cloned()); + let cwd = env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + let mut bootstrap = String::from("import runpy, sys\n"); + bootstrap.push_str(&format!( + "runpy._run_module_as_main({})\n", + quote_py_string(name) + )); + let opts = RunOptions::new(format!("")) + .with_argv(argv) + .with_extra_path(extra_path.to_vec()) + .with_script_dir(cwd) + .with_flags(flags.clone()); + run_source_with_options(&bootstrap, &opts) +} + +/// Decode a script file's bytes per PEP 263 (BOM + coding cookie, +/// default strict UTF-8). On failure, print CPython's tokenizer-style +/// `SyntaxError` to stderr and exit 1 — like `python bad.py` does. +fn decode_script_source(bytes: &[u8], filename: &str) -> String { + match weavepy::vm::decode_source_bytes(bytes, filename) { + Ok(s) => s, + Err(err) => { + let msg = match &err { + weavepy::vm::RuntimeError::PyException(pe) => pe.message(), + other => other.to_string(), + }; + // A NUL in the source: CPython reports the line the byte sits + // on and echoes that line *truncated at the NUL*, with no + // caret (`test_cmd_line_script.test_syntaxerror_null_bytes`). + if let Some(pos) = bytes.iter().position(|&b| b == 0) { + let line_no = bytes[..pos].iter().filter(|&&b| b == b'\n').count() + 1; + let line_start = bytes[..pos] + .iter() + .rposition(|&b| b == b'\n') + .map_or(0, |i| i + 1); + let line_text = String::from_utf8_lossy(&bytes[line_start..pos]); + eprintln!(" File \"{filename}\", line {line_no}"); + let trimmed = line_text.trim_start(); + if !trimmed.is_empty() { + eprintln!(" {trimmed}"); + } + eprintln!("SyntaxError: {msg}"); + std::process::exit(1); + } + eprintln!(" File \"{filename}\", line 1"); + eprintln!("SyntaxError: {msg}"); + std::process::exit(1); + } + } +} + +fn run_path( + path: &Path, + extra: Vec, + flags: &InterpreterFlags, + extra_path: &[PathBuf], +) -> Result<()> { + // A directory or zipfile argument is executed as a module: CPython's + // `pymain_run_module` adds the path itself to `sys.path[0]` and runs + // `runpy._run_module_as_main("__main__")`, so `/__main__.py` (or the + // zip's top-level `__main__`) becomes the program. (`python ` / + // `python app.zip`.) + if path.is_dir() { + return run_main_module_from_path(path, extra, flags, extra_path); + } + // CPython's `pymain_run_file`: an unopenable script prints + // `: can't open file '': [Errno N] ` + // (no traceback) and exits with status 2. + // + // The file is read exactly *once* and every content sniff (zip + // magic, pyc magic) works off those bytes: a `/dev/fd/N` script + // shares its seek offset with every other descriptor on the same + // open file description, so a probe that consumed 4 magic bytes + // would shear them off the program itself (GH-87235, + // `test_cmd_line_script.test_script_as_dev_fd`). + let bytes = match fs::read(path) { + Ok(b) => b, + Err(e) => { + let abs = std::path::absolute(path).unwrap_or_else(|_| path.to_path_buf()); + let program = env::args().next().unwrap_or_else(|| "weavepy".to_owned()); + let errno = e.raw_os_error().unwrap_or(2); + eprintln!( + "{program}: can't open file '{}': [Errno {errno}] {}", + abs.display(), + errno_message(errno) + ); + std::process::exit(2); + } + }; + // `python app.zip`: the zip's top-level `__main__` becomes the program. + if is_zip_bytes(&bytes) { + return run_main_module_from_path(path, extra, flags, extra_path); + } + // A compiled-bytecode file (`.pyc`) given directly: CPython's + // `pymain_run_file` detects the magic and runs the unmarshalled code + // object as `__main__` (rather than trying to decode it as source). + if is_pyc_bytes(&bytes) { + return run_pyc_as_main(path, extra, flags, extra_path); + } + // CPython absolutizes the script path for `__main__.__file__` / + // `co_filename` (getpath's `abspath(program_full_path)`), while + // `sys.argv[0]` keeps the exact text the user typed + // (`test_cmd_line_script.test_script_abspath`). + let filename = std::path::absolute(path) + .unwrap_or_else(|_| path.to_path_buf()) + .display() + .to_string(); + let source = decode_script_source(&bytes, &filename); + let mut argv = vec![path.display().to_string()]; + argv.extend(extra); + let script_dir = Path::new(&filename) + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .map_or_else(|| PathBuf::from("."), Path::to_path_buf); + let opts = RunOptions::new(filename.clone()) + .with_argv(argv) + .with_extra_path(extra_path.to_vec()) + .with_script_dir(script_dir) + .with_flags(flags.clone()); + run_source_with_options(&source, &opts) +} + +/// The OS `strerror` text for an errno, without the " (os error N)" +/// suffix `std::io::Error`'s Display appends. +fn errno_message(errno: i32) -> String { + let s = io::Error::from_raw_os_error(errno).to_string(); + match s.find(" (os error ") { + Some(i) => s[..i].to_owned(), + None => s, + } +} + +/// CPython's `__pycache__`/legacy-`.pyc` magic (kept in sync with +/// `crates/weavepy-vm/src/pycache.rs` and `importlib.machinery.MAGIC_NUMBER`). +const PYC_MAGIC: [u8; 4] = [0xf3, 0x0d, 0x0d, 0x0a]; + +/// Whether `bytes` begins with the WeavePy bytecode magic + the 16-byte +/// `.pyc` header CPython writes (4 magic, 4 bit-field, 8 mtime/size or hash). +fn is_pyc_bytes(bytes: &[u8]) -> bool { + bytes.len() >= 16 && bytes[..4] == PYC_MAGIC +} + +/// Whether `bytes` begins with a zip signature (local-file/empty/spanned). +/// `python app.zip` runs the zip's top-level `__main__` via `zipimport`. +fn is_zip_bytes(bytes: &[u8]) -> bool { + matches!( + bytes.get(..4), + Some([b'P', b'K', 0x03, 0x04] | [b'P', b'K', 0x05, 0x06] | [b'P', b'K', 0x07, 0x08]) + ) +} + +/// Run a directory or zipfile's top-level `__main__` as the program, with +/// `path` prepended to `sys.path` (CPython's directory/zipapp launch). +fn run_main_module_from_path( + path: &Path, + extra: Vec, + flags: &InterpreterFlags, + extra_path: &[PathBuf], +) -> Result<()> { + let path_str = path.display().to_string(); + let mut argv = vec![path_str.clone()]; + argv.extend(extra); + // `alter_argv=False`: keep `sys.argv[0]` as the dir/zip path (CPython does + // not rewrite it to the located `__main__` for directory/zip execution). + let bootstrap = + String::from("import runpy\nrunpy._run_module_as_main('__main__', alter_argv=False)\n"); + let opts = RunOptions::new(path_str) + .with_argv(argv) + .with_extra_path(extra_path.to_vec()) + .with_script_dir_always(path.to_path_buf()) + .with_flags(flags.clone()); + run_source_with_options(&bootstrap, &opts) +} + +/// Run a `.pyc` file's marshalled code object as `__main__`, mirroring +/// CPython's `run_pyc_file`: `__main__.__file__` is the `.pyc` path and +/// `__spec__` stays `None` (a directly-run file is not an importable module), +/// so `multiprocessing` spawn reconstructs the child via `init_main_from_path`. +fn run_pyc_as_main( + path: &Path, + extra: Vec, + flags: &InterpreterFlags, + extra_path: &[PathBuf], +) -> Result<()> { + let path_str = path.display().to_string(); + let mut argv = vec![path_str.clone()]; + argv.extend(extra); + let script_dir = path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .map_or_else(|| PathBuf::from("."), Path::to_path_buf); + let quoted = quote_py_string(&path_str); + let mut bootstrap = String::from("import sys, marshal\n"); + bootstrap.push_str(&format!("with open({quoted}, 'rb') as _f:\n")); + bootstrap.push_str(" _data = _f.read()\n"); + bootstrap.push_str("_code = marshal.loads(_data[16:])\n"); + bootstrap.push_str("_g = sys.modules['__main__'].__dict__\n"); + bootstrap.push_str(&format!("_g['__file__'] = {quoted}\n")); + bootstrap.push_str("_g['__cached__'] = None\n"); + bootstrap.push_str("_g['__spec__'] = None\n"); + // CPython's `pymain_run_file` on a `.pyc` installs a + // `SourcelessFileLoader` as `__main__.__loader__` + // (`test_cmd_line_script.test_script_compiled`). + bootstrap.push_str("import importlib.machinery as _m\n"); + bootstrap.push_str(&format!( + "_g['__loader__'] = _m.SourcelessFileLoader('__main__', {quoted})\n" + )); + bootstrap.push_str("del _m\n"); + bootstrap.push_str("del sys, marshal, _f, _data\n"); + bootstrap.push_str("exec(_code, _g)\n"); + // The bootstrap gets a synthetic co_filename (the `` + // convention `-m` uses): its frame sits under the pyc's own frames in + // a traceback, and it must not leak the on-disk pyc path — compileall's + // `--strip`/`--prepend` embed a *rewritten* path in the pyc, and + // `test_compileall.test_strip_only` asserts the build dir never + // appears in the traceback of a pyc run directly. + let opts = RunOptions::new("") + .with_argv(argv) + .with_extra_path(extra_path.to_vec()) + .with_script_dir(script_dir) + .with_flags(flags.clone()); + run_source_with_options(&bootstrap, &opts) +} + +fn run_stdin(extra: Vec, flags: &InterpreterFlags, extra_path: &[PathBuf]) -> Result<()> { + let mut buf = String::new(); + io::stdin() + .read_to_string(&mut buf) + .context("failed to read stdin")?; + let mut argv = vec!["-".to_owned()]; + argv.extend(extra); + // Like `-c`: stdin programs get `''` (cwd at import time) as + // `sys.path[0]`, matching CPython's `pymain_run_stdin`. + let opts = RunOptions::new("") + .with_argv(argv) + .with_extra_path(extra_path.to_vec()) + .with_script_dir("") + .with_flags(flags.clone()); + run_source_with_options(&buf, &opts) +} + +fn run_source_with_options(source: &str, opts: &RunOptions) -> Result<()> { + // CLI runs print uncaught exceptions CPython-style, through the + // interpreter's `sys.excepthook` / `traceback` machinery (source + // lines, carets, exception chains) while it is still alive. + let opts = opts.clone().with_print_uncaught(true); + // `-i` / `PYTHONINSPECT`: keep the interpreter alive and drop into + // a REPL that shares the program's `__main__` namespace (CPython's + // `pymain_repl`). An uncaught `SystemExit` is *ignored* — CPython's + // `_Py_HandleSystemExit` says "Don't exit if -i flag was given" + // (so `-i -m timeit`, whose main ends in `sys.exit(...)`, still + // reaches the prompt); any other exception is printed first and + // the prompt appears anyway. + if opts.flags.inspect { + let (interpreter, result) = weavepy::run_source_keep_interpreter(source, &opts); + if let Err(err) = result { + if err.system_exit_code().is_none() && !err.already_printed() { + let mut stderr = io::stderr().lock(); + let diag = err.format(source, &opts.filename); + let _ = stderr.write_all(diag.as_bytes()); + } + } + // No banner in inspect mode (CPython goes straight to `>>>`). + let repl = repl::Repl::new(interpreter, true)?; + return repl.run(None); + } + match weavepy::run_source_with_options(source, &opts) { + Ok(()) => Ok(()), + Err(err) => { + // A `SystemExit` reaching the top level terminates the + // process with its code and prints no traceback — exactly + // like CPython. This is what makes `weavepy -m unittest`, + // `-m test`, and bare `sys.exit()` behave as a drop-in. + if let Some(code) = err.system_exit_code() { + exit_with_system_exit(code); + } + if !err.already_printed() { + let mut stderr = io::stderr().lock(); + let diag = err.format(source, &opts.filename); + let _ = stderr.write_all(diag.as_bytes()); + } + // bpo-1054041: an unhandled KeyboardInterrupt must terminate + // the process *via* SIGINT (so a shell sees death-by-signal, + // returncode == -SIGINT), after the traceback is printed. + // This is CPython's `exit_sigint()` in Modules/main.c. + if err.is_keyboard_interrupt() { + exit_via_sigint(); + } + anyhow::bail!(DIAGNOSTIC_SENTINEL); + } + } +} + +/// Terminate the process the way CPython does when `SystemExit` reaches +/// the top level: `None` → 0, a bool/int → that code (masked to 8 +/// bits), anything else → print `str(code)` to stderr and exit 1. +/// Never prints a traceback. +fn exit_with_system_exit(code: weavepy::vm::object::Object) -> ! { + use weavepy::vm::object::Object; + let _ = io::stdout().flush(); + let status: i32 = match code { + Object::None => 0, + Object::Bool(b) => i32::from(b), + Object::Int(n) => (n & 0xFF) as i32, + // A bare `raise SystemExit` (and `sys.exit()`) carries no + // message; WeavePy models the empty payload as an empty string, + // which means "no error" → exit 0, not a printed message. + Object::Str(s) if s.is_empty() => 0, + // An *int subclass* payload exits with its integer value — + // `sys.exit(pytest.ExitCode.OK)` is an `enum.IntEnum`, and + // CPython's `_Py_HandleSystemExit` does `PyLong_Check(value)` + // which is subclass-inclusive (RFC 0055 WS5). + Object::Instance(ref inst) + if matches!( + inst.native.get(), + Some(Object::Int(_) | Object::Long(_) | Object::Bool(_)) + ) => + { + (code.as_i64().unwrap_or(1) & 0xFF) as i32 + } + // `sys.exit(SomeException('msg'))`: CPython prints `str(code)`. + // The interpreter is already torn down, so mirror + // `BaseException.__str__` from the args tuple directly + // (`test_cmd_line_script.test_issue20500_exit_with_exception_value`). + Object::Instance(inst) => { + // `args` is a real slot on exceptions (RFC 0057); older + // plain instances may still carry it in the dict. + let args = inst.slot_get("args").or_else(|| { + inst.dict + .borrow() + .get(&weavepy::vm::object::DictKey(Object::from_static("args"))) + .cloned() + }); + let text = match args { + Some(Object::Tuple(args)) => match args.len() { + 0 => String::new(), + 1 => args[0].to_str(), + _ => Object::Tuple(args).to_str(), + }, + _ => Object::Instance(inst).to_str(), + }; + let mut stderr = io::stderr().lock(); + let _ = writeln!(stderr, "{text}"); + 1 + } + other => { + let mut stderr = io::stderr().lock(); + let _ = writeln!(stderr, "{}", other.to_str()); + 1 + } + }; + let _ = io::stderr().flush(); + std::process::exit(status); +} + +/// Terminate via `SIGINT` under the default disposition, the way +/// CPython's `exit_sigint()` does when a `KeyboardInterrupt` goes +/// unhandled: reset `SIGINT` to `SIG_DFL` and `kill(getpid(), SIGINT)` +/// so the process dies *by the signal* (`returncode == -SIGINT`), which +/// is what shells and `subprocess` inspect. Falls back to exit code 130 +/// (128 + SIGINT) if, impossibly, the signal doesn't terminate us. +#[cfg(unix)] +fn exit_via_sigint() -> ! { + let _ = io::stdout().flush(); + let _ = io::stderr().flush(); + // Reset SIGINT to SIG_DFL, unblock it on this thread, and raise it + // process-wide so we die *by the signal* (returncode == -SIGINT). + weavepy::vm::stdlib::signal_mod::die_via_sigint(); + // Unreachable in practice; the signal terminates us above. + std::process::exit(130); +} + +#[cfg(not(unix))] +fn exit_via_sigint() -> ! { + let _ = io::stdout().flush(); + let _ = io::stderr().flush(); + std::process::exit(0xC0_00_01_3A_u32 as i32); +} + +fn run_repl(flags: InterpreterFlags, startup: Option<&Path>, argv: Vec) -> Result<()> { + let mut interpreter = weavepy::vm::Interpreter::default(); + interpreter.apply_run_options(&flags); + if !argv.is_empty() { + let mut a = vec![String::new()]; + a.extend(argv); + interpreter.set_argv(a); + } else { + interpreter.set_argv(vec![String::new()]); + } + interpreter.prepend_path(env::current_dir().unwrap_or_else(|_| PathBuf::from("."))); + if !flags.no_site { + let _ = interpreter.run_site(); + } + let repl = repl::Repl::new(interpreter, flags.quiet)?; + repl.run(startup) +} + +fn init_tracing() { + let filter = EnvFilter::try_from_env("WEAVEPY_LOG").unwrap_or_else(|_| EnvFilter::new("warn")); + let _ = tracing_subscriber::fmt() + .with_env_filter(filter) + .with_target(false) + .try_init(); +} diff --git a/crates/weavepy-cli/src/main.rs b/crates/weavepy-cli/src/main.rs index 8889c4be..fadd13d7 100644 --- a/crates/weavepy-cli/src/main.rs +++ b/crates/weavepy-cli/src/main.rs @@ -1,1723 +1,156 @@ -//! The `weavepy` command-line interpreter. +//! The `weavepy` binary. //! -//! Argv-compatible with `python(1)` 3.13: every flag in the CPython -//! manpage is parsed and honoured (those we can't yet act on are -//! accepted and forwarded onto `sys.flags` / `sys._xoptions` so user -//! code that introspects them sees realistic values). Modes: +//! On POSIX this is the fully-static interpreter it has always been: +//! `main` calls straight into the driver library ([`weavepy_cli::cli_main`]) +//! and the C-API symbols stay dlopen-visible in the executable itself +//! (`--export-dynamic` on ELF via `build.rs`, Mach-O default exports +//! on macOS). //! -//! ```text -//! weavepy [flags] [-c command | -m module | script | -] [args ...] -//! weavepy [flags] -- interactive REPL -//! ``` -//! -//! Environment variables (`PYTHON*`) are read after the flag table is -//! parsed and folded in unless `-E` / `-I` says otherwise. - -mod regrtest_cmd; -mod repl; - -use std::{ - env, fs, - io::{self, Read, Write}, - path::{Path, PathBuf}, - process::ExitCode, -}; - -use anyhow::{Context, Result}; -use clap::{ArgAction, Parser}; -use tracing_subscriber::EnvFilter; - -use weavepy::{InterpreterFlags, RunOptions}; - -const VERSION: &str = env!("CARGO_PKG_VERSION"); - -/// Recognised subcommands. We thread them through manually instead of -/// using `clap`'s `#[command(subcommand)]` because the bare `weavepy` -/// CLI already overloads the positional `script` slot. Detecting these -/// up front in `main()` keeps the unsugar trivial. -const SUBCOMMANDS: &[&str] = &["regrtest"]; - -/// Run a `weavepy --multiprocessing-fork ` child. The vendored -/// `multiprocessing.popen_spawn_posix`/`popen_forkserver` re-exec us with -/// CPython's frozen command line: `argv == [exe, "--multiprocessing-fork", -/// "tracker_fd=N", "pipe_handle=M", …]`. We must therefore preserve the real -/// argv (so `spawn.is_forking(sys.argv)` holds and the `name=value` kwds are -/// parseable) and hand off to `multiprocessing._run_spawn_child()`, which -/// mirrors CPython's `spawn.spawn_main` POSIX body and *returns* the child's -/// exit code (rather than `sys.exit`-ing, so the Rust bridge controls the -/// process status). -fn run_multiprocessing_child(raw: &[String]) -> ExitCode { - // `_run_spawn_child` runs the worker target via `spawn._main` and returns - // its exit code; `_multiprocessing._exit(code)` then `std::process::exit`s - // directly, so the `Ok(())` arm is only reached on a clean fall-through. - // CPython's `spawn_main` ends in `sys.exit(exitcode)`, whose interpreter - // finalization runs `atexit` handlers (the worker may register its own, - // e.g. gh-83856 / `test_atexit`, plus `multiprocessing.util._exit_function`). - // Our `_multiprocessing._exit` is a hard `std::process::exit` that bypasses - // the CLI's normal shutdown drain, so run the exit funcs explicitly first. - let snippet = "import multiprocessing, _multiprocessing, atexit as _atexit\n\ - _mp_code = multiprocessing._run_spawn_child()\n\ - _atexit._run_exitfuncs()\n\ - _multiprocessing._exit(int(_mp_code) if _mp_code is not None else 0)\n"; - // The parent's `spawn.get_command_line()` emits - // `[exe, , "--multiprocessing-fork", "name=value", ...]`, - // mirroring CPython so the child inherits `-O`/`-S`/`-E`/`-I`/`-X dev`/… - // (`test_multiprocessing.TestFlags.test_flags`). Split at the - // `--multiprocessing-fork` marker: everything before it is interpreter - // flags we must apply to the child; the marker plus the `name=value` kwds - // become `sys.argv[1:]` so `spawn.is_forking(sys.argv)` still holds. - let exe = raw.first().cloned().unwrap_or_else(|| "weavepy".to_owned()); - let fork_idx = raw - .iter() - .position(|a| a == "--multiprocessing-fork") - .unwrap_or(usize::from(!raw.is_empty())); - let opt_args = if fork_idx > 1 { - &raw[1..fork_idx] - } else { - &[][..] - }; - let tail = if fork_idx < raw.len() { - &raw[fork_idx..] - } else { - &[][..] +//! On Windows (RFC 0064 WS1) the binary is a *thin shim* over +//! `python313.dll`, mirroring CPython's own NT split (`python.exe` → +//! `Py_Main` in the core DLL): extension modules' PE import tables +//! name `python313.dll`, so the interpreter must live in a DLL of +//! that name for `.pyd` imports to resolve in-process. The shim +//! locates the DLL (its own directory first; then the `pyvenv.cfg` +//! `home=` chain, because venvs copy the exe but not the DLL; then +//! the default loader search), loads it, and calls the exported +//! `weavepy_main`. It deliberately references nothing else, so the +//! exe stays shim-sized and every byte of runtime state lives in the +//! DLL image. + +#[cfg(not(windows))] +fn main() { + std::process::exit(weavepy_cli::cli_main()); +} + +#[cfg(windows)] +fn main() { + std::process::exit(shim::run()); +} + +#[cfg(windows)] +mod shim { + use std::ffi::c_void; + use std::os::windows::ffi::OsStrExt; + use std::path::{Path, PathBuf}; + + use windows_sys::Win32::Foundation::HMODULE; + use windows_sys::Win32::System::LibraryLoader::{ + GetProcAddress, LoadLibraryExW, LoadLibraryW, LOAD_WITH_ALTERED_SEARCH_PATH, }; - let flags = child_flags_from_opts(&exe, opt_args); - let mut argv = vec![exe]; - argv.extend(tail.iter().cloned()); - let opts = RunOptions::new("") - .with_argv(argv) - .with_flags(flags); - match weavepy::run_source_with_options(snippet, &opts) { - Ok(()) => ExitCode::SUCCESS, - Err(err) => { - let mut stderr = io::stderr().lock(); - let _ = writeln!(stderr, "{}", err.format(snippet, "")); - ExitCode::from(1) - } - } -} - -/// Build the child interpreter flags for a `--multiprocessing-fork` re-exec by -/// re-parsing the interpreter-flag opts the parent placed before the marker -/// (`-O`/`-S`/`-E`/`-I`/`-X dev`/…) through the same clap table + env overrides -/// the normal launch path uses. Falls back to defaults if the opts don't parse -/// (they always should — they come from `_args_from_interpreter_flags()`). -fn child_flags_from_opts(exe: &str, opt_args: &[String]) -> InterpreterFlags { - let parse_argv: Vec = std::iter::once(exe.to_owned()) - .chain(opt_args.iter().cloned()) - .collect(); - match Cli::try_parse_from(&parse_argv) { - Ok(cli) => { - let env = if cli.isolated || cli.ignore_env { - EnvOverrides::ignored() - } else { - EnvOverrides::from_env() - }; - build_flags(&cli, &env) - } - Err(_) => InterpreterFlags::default(), - } -} - -/// CPython 3.13's `python(1)` flag set. -/// -/// Defaults match invoking `python` with no flags. Most of the -/// surface is "accept and propagate" — `sys.flags`, `sys._xoptions`, -/// `sys.warnoptions` reflect the user's choice even when the flag's -/// behaviour is partial. -#[derive(Debug, Parser, Clone, Default)] -#[command( - name = "weavepy", - bin_name = "weavepy", - version = VERSION, - about = "WeavePy: a high-performance, CPython-compatible Python interpreter written in Rust.", - disable_version_flag = true, - disable_help_flag = true, - trailing_var_arg = true, - allow_hyphen_values = true, -)] -struct Cli { - /// Print the version and exit (`python -V` / `--version`). - #[arg(short = 'V', long = "version", action = ArgAction::SetTrue, overrides_with = "version")] - version: bool, - - /// Print this help and exit. - #[arg(short = 'h', long = "help", action = ArgAction::SetTrue, overrides_with = "help")] - help: bool, - - /// Print the help-env summary (which `PYTHON*` vars are honoured) and exit. - #[arg(long = "help-env", action = ArgAction::SetTrue, overrides_with = "help_env")] - help_env: bool, - - /// Print the help-xoptions summary and exit. - #[arg(long = "help-xoptions", action = ArgAction::SetTrue, overrides_with = "help_xoptions")] - help_xoptions: bool, - - /// Optimisation level. `-O` once, `-OO` twice. - #[arg(short = 'O', action = ArgAction::Count)] - optimize: u8, - - /// `bytes`/`str` comparison warnings. `-b` once warns, `-bb` errors. - #[arg(short = 'b', action = ArgAction::Count)] - bytes_warning: u8, - - /// Don't write `.pyc` files. - #[arg(short = 'B', action = ArgAction::SetTrue, overrides_with = "no_bytecode_write")] - no_bytecode_write: bool, - - /// Parser debug output (`sys.flags.debug`; counted like CPython's - /// `-d`, otherwise a no-op stub). - #[arg(short = 'd', action = ArgAction::Count)] - parser_debug: u8, - - /// `-R`: turn on hash randomization (the default; overrides a - /// `PYTHONHASHSEED` fixed seed, like CPython). - #[arg(short = 'R', action = ArgAction::SetTrue, overrides_with = "hash_randomization")] - hash_randomization: bool, - - /// Ignore all `PYTHON*` environment variables. - #[arg(short = 'E', action = ArgAction::SetTrue, overrides_with = "ignore_env")] - ignore_env: bool, - - /// Drop into the REPL after running the script / module / command. - #[arg(short = 'i', action = ArgAction::SetTrue, overrides_with = "inspect_after")] - inspect_after: bool, - - /// Isolated mode: implies `-E -s` and sets `sys.flags.isolated`. - #[arg(short = 'I', action = ArgAction::SetTrue, overrides_with = "isolated")] - isolated: bool, - - /// Don't run `site.main()` on interpreter startup. - #[arg(short = 'S', action = ArgAction::SetTrue, overrides_with = "no_site")] - no_site: bool, - - /// Don't add the user site-packages to `sys.path`. - #[arg(short = 's', action = ArgAction::SetTrue, overrides_with = "no_user_site")] - no_user_site: bool, - - /// Suppress the REPL banner. - #[arg(short = 'q', action = ArgAction::SetTrue, overrides_with = "quiet")] - quiet: bool, - - /// Don't prepend the script dir / cwd to `sys.path`. - #[arg(short = 'P', action = ArgAction::SetTrue, overrides_with = "safe_path")] - safe_path: bool, - - /// Force stdout/stderr unbuffered. - #[arg(short = 'u', action = ArgAction::SetTrue, overrides_with = "unbuffered")] - unbuffered: bool, - /// Verbose imports. - #[arg(short = 'v', action = ArgAction::Count)] - verbose: u8, + /// The runtime DLL the shim binds — the CPython-compatible ABI + /// name that `.pyd` import tables reference. + const DLL_NAME: &str = "python313.dll"; - /// Skip the first source line (shebang trick). - #[arg(short = 'x', action = ArgAction::SetTrue, overrides_with = "skip_first_line")] - skip_first_line: bool, + /// Exit code when the runtime DLL cannot be found or bound — + /// well clear of Python's 1/2/120 conventions so scripts can + /// tell "the program failed" from "the installation is broken". + const EXIT_NO_RUNTIME: i32 = 103; - /// `-X key[=value]`. Forwarded to `sys._xoptions`. - #[arg(short = 'X', action = ArgAction::Append, value_name = "OPT")] - xoptions: Vec, - - /// `-W filter` warning control. Forwarded to `sys.warnoptions`. - #[arg(short = 'W', action = ArgAction::Append, value_name = "FILTER")] - warnings: Vec, - - /// `--check-hash-based-pycs MODE`. Accepted, ignored (we always - /// use mtime-mode cache invalidation). - #[arg(long = "check-hash-based-pycs", value_name = "MODE")] - check_hash_pycs: Option, - - /// Execute `` as `__main__`. Mirrors `python -c`. - #[arg(short = 'c', value_name = "SOURCE")] - command: Option, - - /// Run library module `` as `__main__`. Mirrors `python -m`. - #[arg(short = 'm', value_name = "MODULE")] - module: Option, - - /// Script path (`script.py`) or `-` for stdin. Optional. - script: Option, - - /// Trailing arguments → `sys.argv[1:]`. - #[arg(trailing_var_arg = true, allow_hyphen_values = true)] - args: Vec, -} - -const DIAGNOSTIC_SENTINEL: &str = "exited with diagnostic"; - -const HELP_BODY: &str = "\ -usage: weavepy [option] ... [-c cmd | -m mod | file | -] [arg] ... -Options (and corresponding environment variables): --b : issue warnings about converting bytes/bytearray to str (-bb: error) --B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x --c cmd : program passed in as string (terminates option list) --d : turn on parser debugging output (for experts only) --E : ignore PYTHON* environment variables (such as PYTHONPATH) --h : print this help message and exit (also --help) --i : inspect interactively after running script; (also PYTHONINSPECT=x) --I : isolate Python from the user's environment (implies -E and -s) --m mod : run library module as a script (terminates option list) --O : remove assert and __debug__-dependent statements; also PYTHONOPTIMIZE=x --OO : do -O changes and also discard docstrings --P : don't prepend a potentially unsafe path to sys.path --q : don't print version and copyright messages on interactive startup --R : turn on hash randomization; also PYTHONHASHSEED=random (default) --s : don't add user site directory to sys.path; also PYTHONNOUSERSITE --S : don't imply 'import site' on initialization --u : force the stdout and stderr streams to be unbuffered --v : verbose (trace import statements); also PYTHONVERBOSE=x --V : print the Python version number and exit (also --version) --W arg : warning control; arg is action:message:category:module:lineno --x : skip first line of source, allowing use of non-Unix shebang --X opt : set implementation-specific option -file : program read from script file -- : program read from stdin (default; interactive mode if a tty) -arg ...: arguments passed to program in sys.argv[1:] -"; - -const HELP_ENV: &str = "\ -Environment variables: -PYTHONHOME : alternate directory (or :). - The default module search path uses /python{X.Y}. -PYTHONPATH : ':'-separated list of directories prefixed to sys.path. -PYTHONSTARTUP : file executed on interactive startup (no default). -PYTHONOPTIMIZE : same as -O option. -PYTHONDEBUG : same as -d option. -PYTHONINSPECT : same as -i option. -PYTHONUNBUFFERED : same as -u option. -PYTHONVERBOSE : same as -v option. -PYTHONNOUSERSITE : same as -s option. -PYTHONHASHSEED : if set to 'random', randomize hash; integer in [0, 4294967295] for repeatable. -PYTHONIOENCODING : Encoding[:errors] used for stdin/stdout/stderr. -PYTHONDONTWRITEBYTECODE: don't write .pyc files (same as -B). -PYTHONWARNINGS : warning control; comma-separated -W filters. -PYTHONBREAKPOINT : override sys.breakpointhook (default 'pdb.set_trace'). -PYTHONUTF8 : force the interpreter into UTF-8 mode. -PYTHONNODEBUGRANGES : disable PEP 657 column-precise tracebacks (no-op today). -PYTHONSAFEPATH : same as -P option. -"; - -const HELP_XOPTIONS: &str = "\ -The following implementation-specific options are available: --X faulthandler : dump the Python traceback on fatal signals. --X dev : enable runtime checks helpful for development. --X utf8 : enable UTF-8 mode for the interpreter. --X tracemalloc[=N] : start tracing Python memory allocations, keeping N frames. --X importtime : show how long each import takes (no-op today). --X showrefcount : output the total reference count (no-op today). --X frozen_modules=on|off : whether frozen modules should be used. --X no_debug_ranges : disable PEP 657 ranges (no-op today). --X pycache_prefix=PATH : redirect __pycache__ to PATH. --X int_max_str_digits : set sys.int_info.str_digits_check_threshold. -"; - -// Opt-in native crash diagnostics (`WEAVEPY_SEGV_BT`): macOS-only, because -// the raw `siginfo_t`/`ucontext_t` byte offsets below are the Darwin layouts. -#[cfg(target_os = "macos")] -extern "C" { - fn signal(signum: i32, handler: usize) -> usize; - fn sigaction(signum: i32, act: *const SigActionC, old: *mut SigActionC) -> i32; - fn backtrace(array: *mut *mut std::ffi::c_void, size: i32) -> i32; - fn backtrace_symbols_fd(array: *const *mut std::ffi::c_void, size: i32, fd: i32); -} - -/// `struct sigaction` (macOS/BSD layout): an 8-byte handler pointer union, -/// a 4-byte `sigset_t` mask, and a 4-byte flags word. -#[cfg(target_os = "macos")] -#[repr(C)] -struct SigActionC { - sa_sigaction: usize, - sa_mask: u32, - sa_flags: i32, -} - -/// `SA_SIGINFO` — deliver the 3-argument handler so we can read `si_addr`. -#[cfg(target_os = "macos")] -const SA_SIGINFO: i32 = 0x0040; -/// Byte offset of `si_addr` within macOS `siginfo_t` -/// (`si_signo,si_errno,si_code,si_pid,si_uid,si_status` = 24 bytes precede it). -#[cfg(target_os = "macos")] -const SIGINFO_SI_ADDR_OFFSET: usize = 24; - -/// Byte offset of the `mcontext_t` pointer within macOS `ucontext_t` -/// (`uc_onstack,uc_sigmask,uc_stack,uc_link,uc_mcsize` precede it). -#[cfg(all(target_os = "macos", target_arch = "aarch64"))] -const UCONTEXT_MCONTEXT_OFFSET: usize = 48; -/// Byte offset of `__ss` (the ARM thread state) within macOS `mcontext64` -/// — it follows the 16-byte `__es` (ARM exception state). -#[cfg(all(target_os = "macos", target_arch = "aarch64"))] -const MCONTEXT_SS_OFFSET: usize = 16; -/// Byte offset of `tp_name` (a `const char *`) within `PyTypeObject`. -#[cfg(all(target_os = "macos", target_arch = "aarch64"))] -const PYTYPEOBJECT_TP_NAME_OFFSET: usize = 0x18; - -/// Read the C string at `p` (best-effort, capped) for signal-handler -/// diagnostics. Returns a lossy `String`; bails on an obviously-bad pointer -/// so we don't double-fault while already handling a crash. -#[cfg(all(target_os = "macos", target_arch = "aarch64"))] -unsafe fn read_c_str_lossy(p: *const u8, cap: usize) -> String { - if (p as usize) < 0x1000 { - return String::from(""); - } - let mut bytes = Vec::new(); - for i in 0..cap { - let b = unsafe { p.add(i).read() }; - if b == 0 { - break; - } - bytes.push(b); - } - String::from_utf8_lossy(&bytes).into_owned() -} - -#[cfg(target_os = "macos")] -extern "C" fn weavepy_segv_backtrace(sig: i32, info: *const u8, ctx: *mut std::ffi::c_void) { - // `ctx` (the interrupted-thread register file) is only decoded on arm64, - // where the `mcontext64` layout below applies. - #[cfg(not(target_arch = "aarch64"))] - let _ = ctx; - // The faulting memory address (`si_addr`) is the single most useful clue - // for a native crash in a dlopen'd extension: a small value (`0x0`, `0x8`, - // …) is a NULL-based field deref, a huge value a wild pointer. Printing it - // turns an opaque `PyArray_*` frame into an actionable diagnosis. - if !info.is_null() { - let si_addr = unsafe { - info.add(SIGINFO_SI_ADDR_OFFSET) - .cast::() - .read_unaligned() - }; - eprintln!("\n=== WEAVEPY signal {sig} faulting address = 0x{si_addr:x} ==="); - } - // Faulting register file (arm64): `pc` pinpoints the exact instruction and - // `x0` is usually the receiver of a `Py_TYPE(x)->tp_field` chain. When the - // crash is a NULL `tp_mro`/`tp_dict`/… deref, `x0` is still the live type - // pointer, so decoding `x0->tp_name` names the offending type directly. - #[cfg(target_arch = "aarch64")] - if !ctx.is_null() { - unsafe { - let mctx = ctx - .cast::() - .add(UCONTEXT_MCONTEXT_OFFSET) - .cast::<*const u8>() - .read_unaligned(); - if !mctx.is_null() { - let ss = mctx.add(MCONTEXT_SS_OFFSET); - let x = |n: usize| ss.add(n * 8).cast::().read_unaligned(); - let pc = ss.add(256).cast::().read_unaligned(); + pub(crate) fn run() -> i32 { + let mut probed: Vec = Vec::new(); + let dll = match locate_and_load(&mut probed) { + Some(dll) => dll, + None => { eprintln!( - "=== registers: pc=0x{pc:x} x0=0x{:x} x1=0x{:x} x8=0x{:x} x19=0x{:x} x20=0x{:x} ===", - x(0), x(1), x(8), x(19), x(20) + "weavepy: {DLL_NAME} not found (probed: {}) — the exe and DLL ship \ + together; reinstall or point PATH at a complete WeavePy distribution", + probed + .iter() + .map(|p| p.display().to_string()) + .collect::>() + .join(", ") ); - // Heuristic: for a `tp_*` NULL-field crash the type pointer is - // in x0 (and often mirrored in x19/x20). Decode each as a - // candidate `PyTypeObject*` and print its `tp_name`. - for (reg, val) in [("x0", x(0)), ("x19", x(19)), ("x20", x(20))] { - let name_pp = (val as usize + PYTYPEOBJECT_TP_NAME_OFFSET) as *const *const u8; - if (val as usize) > 0x1000 { - let name = read_c_str_lossy(name_pp.read(), 64); - eprintln!("=== {reg} as PyTypeObject* -> tp_name = {name:?} ==="); - } - } + return EXIT_NO_RUNTIME; } - } - } - // Native (dladdr-based) backtrace first: it resolves frames inside a - // dlopen'd `.so` (e.g. a Cython extension's static helpers) to their - // real `module + symbol + offset`, which Rust's `std::backtrace` - // mis-attributes to the nearest exported libsystem symbol. - let mut frames: [*mut std::ffi::c_void; 96] = [std::ptr::null_mut(); 96]; - let n = unsafe { backtrace(frames.as_mut_ptr(), 96) }; - eprintln!("=== WEAVEPY signal {sig} native backtrace ==="); - unsafe { backtrace_symbols_fd(frames.as_ptr(), n, 2) }; - eprintln!("=== end native backtrace ==="); - let bt = std::backtrace::Backtrace::force_capture(); - eprintln!("=== WEAVEPY signal {sig} rust backtrace ===\n{bt}\n=== end backtrace ==="); - unsafe { - signal(sig, 0); - } - std::process::abort(); -} - -fn main() -> ExitCode { - #[cfg(target_os = "macos")] - if std::env::var_os("WEAVEPY_SEGV_BT").is_some() { - // `SA_SIGINFO` so the handler receives `siginfo_t` and can report the - // faulting address; `signal()` alone would only pass the signal number. - let act = SigActionC { - sa_sigaction: weavepy_segv_backtrace as *const () as usize, - sa_mask: 0, - sa_flags: SA_SIGINFO, }; - unsafe { - sigaction(11, &raw const act, std::ptr::null_mut()); // SIGSEGV - sigaction(10, &raw const act, std::ptr::null_mut()); // SIGBUS - } - } - // Undo Rust's pre-`main` `sanitize_standard_fds` (which re-opens any closed - // std fd onto `/dev/null`) so an inherited-closed stdin/stdout/stderr stays - // closed, matching CPython (`test_posix.test_close_file`). Must run before - // any descriptor work. - weavepy::vm::proc_init::restore_initial_std_fds(); - run_on_large_stack(main_dispatch) -} - -/// WeavePy evaluates Python by recursive descent, so Python call depth -/// maps onto native (Rust) stack depth (see `crates/weavepy-vm/src/ -/// recursion.rs`). Run the whole interpreter on a thread with a large -/// stack reserve so that `sys.setrecursionlimit` — enforced by the VM's -/// recursion guard (RFC 0037) — is what bounds recursion, rather than -/// the fixed OS main-thread stack (8 MiB on Linux/macOS). This makes the -/// behaviour uniform across platforms *and* build profiles: debug builds -/// have much larger per-activation stack frames than release, so without -/// this a default `setrecursionlimit(1000)` would overflow the native -/// stack in debug before the guard could fire. The reserve is committed -/// lazily by the OS, so it costs address space, not memory. -fn run_on_large_stack(entry: fn() -> ExitCode) -> ExitCode { - const STACK_BYTES: usize = 1024 * 1024 * 1024; // 1 GiB reserve - - // The interpreter runs on the spawned `weavepy-main` thread, not the - // process's initial OS thread (which only parks in `join()` below). - // Block the asynchronous, process-directed signals (SIGINT, SIGALRM, - // …) on this initial thread *before* spawning so a signal racing in - // during startup can't be stolen by the soon-to-be-parked thread — - // where it would merely trip the pending flag while the VM thread's - // blocking syscall never gets EINTR (CPython's test_io SignalsTest - // would then hang forever). The VM thread re-enables them for itself - // first thing, making it the sole, deterministic delivery target. - weavepy::vm::stdlib::signal_mod::block_async_signals_current_thread(); - - let vm_entry = move || -> ExitCode { - // Opt-in (`WEAVEPY_CRASH_BT`): register the native crash handler + - // per-thread sigaltstack on the VM thread itself so a stack-overflow - // SIGSEGV can be caught and reported (no-op stub on Windows). - if std::env::var_os("WEAVEPY_CRASH_BT").is_some() { - extern "C" { - fn weavepy_install_crash_handler(); - } - unsafe { weavepy_install_crash_handler() }; - } - weavepy::vm::stdlib::signal_mod::unblock_async_signals_current_thread(); - // Arm SIGINT -> KeyboardInterrupt at startup (CPython does this during - // interpreter init), so even scripts that never `import signal` raise - // KeyboardInterrupt on ^C instead of being killed by the kernel default. - weavepy::vm::stdlib::signal_mod::install_startup_dispositions(); - // Snapshot the OS-thread count *now* — on the VM thread, before any - // user code can spawn `threading` workers or raw pthreads — so that a - // later `os.fork()` can tell "single-threaded" (no warning) from - // "multi-threaded" (CPython's fork `DeprecationWarning`). WeavePy runs - // the interpreter off the parked process-initial thread, so the - // quiescent process already has >1 OS thread; this baseline is what the - // fork-warning check measures additional threads against. - weavepy::vm::stdlib::os_process::capture_thread_baseline(); - entry() - }; - - match std::thread::Builder::new() - .name("weavepy-main".to_owned()) - .stack_size(STACK_BYTES) - .spawn(vm_entry) - { - Ok(handle) => handle.join().unwrap_or(ExitCode::FAILURE), - // Extremely unlikely, but if the OS refuses the thread, fall back - // to running on the current thread — restore signal delivery here - // first since we blocked it above. - Err(_) => { - weavepy::vm::stdlib::signal_mod::unblock_async_signals_current_thread(); - weavepy::vm::stdlib::signal_mod::install_startup_dispositions(); - weavepy::vm::stdlib::os_process::capture_thread_baseline(); - entry() - } - } -} - -fn main_dispatch() -> ExitCode { - init_tracing(); - - // `env::args()` panics on non-UTF-8 argv (bpo-35883's exact repro); - // decode PEP 383-style instead, carrying undecodable bytes in the - // PUA bridge window that `Interpreter::set_argv` maps back to - // lone surrogates (RFC 0050). - let raw: Vec = weavepy::vm::os_args_bridged(); - - // Multiprocessing spawn-child entry point. The parent passes - // `--multiprocessing-fork` and an optional payload fd via - // `WEAVEPY_MP_PAYLOAD_FD`; we hand off to - // `multiprocessing._run_spawn_child()` which reads the pickled - // task off the inherited fd and runs it. - if raw.iter().any(|a| a == "--multiprocessing-fork") { - return run_multiprocessing_child(&raw); - } - - // Bare subcommand dispatch (e.g. `weavepy regrtest ...`) — must - // run before clap, which would try to interpret the subcommand as - // a positional `script` and trip on unknown flags after it. - if raw.len() >= 2 && SUBCOMMANDS.contains(&raw[1].as_str()) { - let sub = raw[1].clone(); - let rest: Vec = std::iter::once(format!("weavepy {sub}")) - .chain(raw.into_iter().skip(2)) - .collect(); - return match sub.as_str() { - "regrtest" => match regrtest_cmd::run(rest) { - Ok(code) => code, - Err(err) => { - let mut stderr = io::stderr().lock(); - let _ = writeln!(stderr, "weavepy regrtest: {err:#}"); - ExitCode::from(1) - } - }, - _ => unreachable!(), + // SAFETY: `weavepy_main` is exported by `weavepy-pylib` with + // exactly this signature; a DLL that lacks it is not ours. + let entry = unsafe { GetProcAddress(dll, c"weavepy_main".as_ptr().cast()) }; + let Some(entry) = entry else { + eprintln!( + "weavepy: {DLL_NAME} does not export weavepy_main — version-skewed or \ + foreign python313.dll on the search path?" + ); + return EXIT_NO_RUNTIME; }; - } - - match real_main() { - Ok(code) => code, - Err(err) => { - if err.to_string() != DIAGNOSTIC_SENTINEL { - let mut stderr = io::stderr().lock(); - let _ = writeln!(stderr, "weavepy: {err:#}"); - } - ExitCode::from(1) - } - } -} - -/// Split argv at the first `-c CMD` / `-m MODULE` / `script` / `-` / `--` -/// boundary so flags meant for the child program don't get re-parsed by -/// clap. Returns `(weavepy_args, mode, child_args)`. -/// -/// `mode` is one of: -/// - `Some(("c", ""))` — `-c CMD` was found. -/// - `Some(("m", ""))` — `-m MOD` was found. -/// - `Some(("s", ""))` — a positional script was found. -/// - `Some(("-", ""))` — `-` (stdin) was found. -/// - `None` — interactive mode (no boundary). -fn split_argv(raw: Vec) -> (Vec, Option<(&'static str, String)>, Vec) { - let mut wp: Vec = Vec::with_capacity(raw.len()); - let mut iter = raw.into_iter(); - if let Some(prog) = iter.next() { - wp.push(prog); - } - while let Some(arg) = iter.next() { - if arg == "--" { - return (wp, None, iter.collect()); - } - if arg == "-c" { - let Some(cmd) = iter.next() else { - argument_expected_error('c'); - }; - let rest: Vec = iter.collect(); - return (wp, Some(("c", cmd)), rest); - } - if arg == "-m" { - let Some(m) = iter.next() else { - argument_expected_error('m'); - }; - let rest: Vec = iter.collect(); - return (wp, Some(("m", m)), rest); - } - if arg.starts_with("-c") && arg.len() > 2 { - let cmd = arg[2..].to_owned(); - let rest: Vec = iter.collect(); - return (wp, Some(("c", cmd)), rest); - } - if arg.starts_with("-m") && arg.len() > 2 { - let m = arg[2..].to_owned(); - let rest: Vec = iter.collect(); - return (wp, Some(("m", m)), rest); - } - // Attached `-Xkey[=value]` / `-Wfilter` (CPython's own spelling — - // `test_subprocess.test_encoding_warning` spawns `-Xwarn_default_encoding`): - // normalise to the separate `-X key` form clap parses, so the option - // reaches `sys._xoptions` / `sys.warnoptions`. - if let Some(rest) = arg.strip_prefix("-X").filter(|r| !r.is_empty()) { - wp.push("-X".to_owned()); - wp.push(rest.to_owned()); - continue; - } - if let Some(rest) = arg.strip_prefix("-W").filter(|r| !r.is_empty()) { - wp.push("-W".to_owned()); - wp.push(rest.to_owned()); - continue; - } - // Clustered single-letter options where `-c`/`-m` follows some boolean - // flags, e.g. `-uc CMD` == `-u -c CMD` and `-uIcCMD` == `-u -I -c CMD` - // (CPython accepts this; `test_subprocess` spawns children as `-uc`). - // The `c`/`m` consumes the rest of the cluster as its value, else the - // next argv element. - if arg.starts_with('-') && !arg.starts_with("--") && arg.len() > 2 { - let body: Vec = arg[1..].chars().collect(); - if let Some(pos) = body.iter().position(|&c| c == 'c' || c == 'm') { - const BOOL_SHORT: &[char] = &[ - 'O', 'b', 'B', 'd', 'E', 'i', 'I', 'R', 'S', 's', 'q', 'P', 'u', 'v', 'x', - ]; - if body[..pos].iter().all(|c| BOOL_SHORT.contains(c)) { - for &c in &body[..pos] { - wp.push(format!("-{c}")); - } - let kind = if body[pos] == 'c' { "c" } else { "m" }; - let after: String = body[pos + 1..].iter().collect(); - let value = if after.is_empty() { - iter.next() - .unwrap_or_else(|| argument_expected_error(body[pos])) - } else { - after - }; - let rest: Vec = iter.collect(); - return (wp, Some((kind, value)), rest); - } - } - } - if arg == "-" { - let rest: Vec = iter.collect(); - return (wp, Some(("-", String::new())), rest); - } - // Value-taking flags: consume the following arg too, so it - // isn't mistaken for the positional script (`-X opt script.py`). - if arg == "-X" || arg == "-W" || arg == "--check-hash-based-pycs" { - wp.push(arg); - if let Some(value) = iter.next() { - wp.push(value); - } - continue; - } - if !arg.starts_with('-') { - // Positional script. - let rest: Vec = iter.collect(); - return (wp, Some(("s", arg)), rest); - } - wp.push(arg); - } - (wp, None, Vec::new()) -} - -fn real_main() -> Result { - let raw: Vec = weavepy::vm::os_args_bridged(); - let (wp_argv, mode, child_argv) = split_argv(raw); - // Re-parse the WeavePy-only slice with clap. - let mut cli = Cli::parse_from(wp_argv); - // Stuff `mode` back into the parsed Cli so the rest of real_main - // sees a consistent view. - match &mode { - Some(("c", cmd)) => cli.command = Some(decode_command_arg(cmd)), - Some(("m", m)) => cli.module = Some(m.clone()), - // A script path may carry PEP 383-escaped bytes (PUA-bridged by - // `os_args_bridged`); recover the OS-level bytes so the file - // actually opens (RFC 0050). - Some(("s", path)) => cli.script = Some(bridged_arg_to_pathbuf(path)), - Some(("-", _)) => cli.script = Some(PathBuf::from("-")), - _ => {} - } - cli.args = child_argv; - - if cli.help { - print!("{HELP_BODY}"); - return Ok(ExitCode::SUCCESS); - } - if cli.help_env { - print!("{HELP_ENV}"); - return Ok(ExitCode::SUCCESS); - } - if cli.help_xoptions { - print!("{HELP_XOPTIONS}"); - return Ok(ExitCode::SUCCESS); - } - if cli.version { - // PyPy-style: lead with the CPython version the interpreter - // implements (tooling — pyenv, tox, CI matrices, weavepy-dist's - // `version` leg — parses `python -V` for `Python X.Y.Z`), then - // identify the implementation. - let (maj, min, mic) = weavepy_vm::stdlib::sys::PY_VERSION; - println!("Python {maj}.{min}.{mic} (WeavePy {VERSION})"); - return Ok(ExitCode::SUCCESS); - } - - let env = if cli.isolated || cli.ignore_env { - EnvOverrides::ignored() - } else { - EnvOverrides::from_env() - }; - - let mut flags = build_flags(&cli, &env); - - // Compose pythonpath from env (when honoured) plus -X variants. - let mut extra_path: Vec = env - .pythonpath - .iter() - .filter(|s| !s.is_empty()) - .map(PathBuf::from) - .collect(); - - // `WEAVEPY_CPYTHON_LIB` points at an external stdlib `Lib` directory - // (the vendored CPython tree). Like a real interpreter that finds its - // stdlib relative to the executable, this is part of the *default* - // module search path: it is honoured even under `-I`/`-E` (it is not a - // `PYTHON*` variable, so isolation does not strip it) so child - // interpreters spawned via `sys.executable` — e.g. `assert_python_ok`, - // `multiprocessing` spawn, `subprocess` re-execs — can still import the - // stdlib and the `test` package. Unset in normal use, so this is a - // no-op outside the conformance harness. - if let Some(lib) = env::var_os("WEAVEPY_CPYTHON_LIB") { - for part in env::split_paths(&lib) { - if !part.as_os_str().is_empty() { - extra_path.push(part); - } - } - } - - // getpath's `._pth` layout override (RFC 0062 WS5): a `._pth` - // file next to the binary pins `sys.path` to exactly its entries - // and locks the interpreter down (no PYTHONPATH, no script-dir - // prepend, no site unless the file says `import site`) — CPython's - // embeddable-distribution mechanism, honoured on every platform. - if let Some((entries, import_site)) = read_pth_file() { - flags.pth_paths = Some(entries); - if !import_site { - flags.no_site = true; - } - flags.safe_path = true; - flags.no_user_site = true; - extra_path.clear(); - } - - if let Some(source) = cli.command.clone() { - let mut argv = vec!["-c".to_owned()]; - argv.extend(cli.args.iter().cloned()); - // CPython's `-c` puts the *empty string* at `sys.path[0]` (an - // '' entry means "current directory, resolved at import time"), - // not a materialized cwd path — - // `test_cmd_line_script.test_issue8202_dash_c_file_ignored`. - let opts = RunOptions::new("") - .with_argv(argv) - .with_extra_path(extra_path.drain(..)) - .with_script_dir("") - .with_flags(flags.clone()); - // `-i` is handled inside `run_source_with_options`, which drops - // into a namespace-sharing REPL after the program body. - run_source_with_options(&source, &opts)?; - return Ok(ExitCode::SUCCESS); - } - - if let Some(module) = cli.module.clone() { - let extra = cli.args.clone(); - run_module(&module, extra, &flags, &extra_path)?; - return Ok(ExitCode::SUCCESS); - } - - let script = cli.script.clone(); - let trailing = cli.args.clone(); - match script.as_deref() { - Some(path) if path.as_os_str() == "-" => { - run_stdin(trailing.clone(), &flags, &extra_path)?; - Ok(ExitCode::SUCCESS) - } - Some(path) => { - run_path(path, trailing.clone(), &flags, &extra_path)?; - Ok(ExitCode::SUCCESS) - } - None => { - // No script. CPython enters the REPL only when stdin is a - // tty (or `-i` forces it); a piped stdin is read to EOF and - // run as a program named `` (`pymain_run_stdin` — - // no banner, no `>>>` prompts, plain tracebacks). - let stdin_is_tty = std::io::IsTerminal::is_terminal(&io::stdin()); - if stdin_is_tty || flags.inspect { - flags.inspect = true; - run_repl(flags, env.startup.as_deref(), trailing)?; - } else { - run_stdin(trailing, &flags, &extra_path)?; - } - Ok(ExitCode::SUCCESS) - } - } -} - -/// CPython's `pymain_err_print` for an option missing its argument: -/// diagnostics + usage line on stderr, exit status 2. -fn argument_expected_error(opt: char) -> ! { - eprintln!("Argument expected for the -{opt} option"); - eprintln!("usage: weavepy [option] ... [-c cmd | -m mod | file | -] [arg] ..."); - eprintln!("Try `weavepy -h' for more information."); - std::process::exit(2); -} - -/// A startup configuration error CPython reports through -/// `Py_ExitStatusException`: `Fatal Python error: : `, exit 1. -fn config_fatal_error(whence: &str, msg: &str) -> ! { - eprintln!("Fatal Python error: {whence}: {msg}"); - std::process::exit(1); -} - -/// The value of the last `-X name[=value]` occurrence: `None` when the -/// option wasn't given, `Some(None)` for the bare form, `Some(Some(v))` -/// for `-X name=v`. -fn xoption_value<'a>(xoptions: &'a [String], name: &str) -> Option> { - xoptions.iter().rev().find_map(|x| { - if x == name { - Some(None) - } else { - x.strip_prefix(name) - .and_then(|rest| rest.strip_prefix('=')) - .map(Some) - } - }) -} - -/// Parse + validate the PEP 0467 digit cap (`0` or `>= 640`), exiting -/// with CPython's `config_init_int_max_str_digits` fatal error otherwise. -fn parse_int_max_str_digits(value: &str, source: &str) -> i64 { - match value.parse::() { - Ok(n) if n == 0 || n >= 640 => n, - _ => config_fatal_error( - "config_init_int_max_str_digits", - &format!("{source}: invalid limit; must be >= 640 or 0 for unlimited."), - ), - } -} - -/// Locate and parse the `._pth` file governing this executable -/// (CPython getpath: `._pth`, plus the `._pth` spelling on -/// Windows). Returns the absolutized `sys.path` entries and whether an -/// `import site` line re-enables site processing. Comment lines start -/// with `#`; other `import` lines are recognised but only `site` has -/// an effect (matching getpath, which special-cases exactly that). -fn read_pth_file() -> Option<(Vec, bool)> { - let exe = std::env::current_exe().ok()?; - let mut with_suffix = exe.as_os_str().to_owned(); - with_suffix.push("._pth"); - let mut candidates = vec![PathBuf::from(with_suffix)]; - if cfg!(windows) { - candidates.push(exe.with_extension("_pth")); - } - let pth = candidates.into_iter().find(|p| p.is_file())?; - let contents = std::fs::read_to_string(&pth).ok()?; - let exe_dir = exe.parent()?; - let mut entries = Vec::new(); - let mut import_site = false; - for line in contents.lines() { - let line = line.trim_end_matches('\r'); - if line.is_empty() || line.starts_with('#') { - continue; - } - if let Some(rest) = line.strip_prefix("import ") { - if rest.split(',').any(|m| m.trim() == "site") { - import_site = true; - } - continue; - } - entries.push( - lexical_abspath(&exe_dir.join(line)) - .to_string_lossy() - .into_owned(), - ); - } - Some((entries, import_site)) -} - -/// `os.path.abspath` without touching the filesystem: make absolute -/// against the cwd, then collapse `.` and `..` components lexically -/// (symlinks are *not* resolved — the ._pth expectations are computed -/// with `abspath`, which is purely lexical too). -fn lexical_abspath(p: &Path) -> PathBuf { - let abs = if p.is_absolute() { - p.to_path_buf() - } else { - env::current_dir().map_or_else(|_| p.to_path_buf(), |cwd| cwd.join(p)) - }; - let mut out = PathBuf::new(); - for comp in abs.components() { - match comp { - std::path::Component::CurDir => {} - // The joined input is always absolute, so a failed pop can - // only mean we're at the root — where `/..` collapses to - // `/`, exactly like `os.path.normpath`. - std::path::Component::ParentDir => { - out.pop(); - } - other => out.push(other.as_os_str()), - } - } - out -} - -/// Compose the runtime [`InterpreterFlags`] from the CLI table and -/// the environment overrides. `-I` is the trump card. -fn build_flags(cli: &Cli, env: &EnvOverrides) -> InterpreterFlags { - let isolated = cli.isolated; - let ignore_env = cli.ignore_env || isolated; - // Pin the per-process str/bytes hash salt before the interpreter - // hashes anything (PEP 456 / `PYTHONHASHSEED`). `-R` re-enables - // randomization, which is also the default when the var is unset. - if !cli.hash_randomization { - if let Some(seed) = env.hash_seed { - weavepy::vm::object::set_hash_seed(seed); - } - } - // `-X pycache_prefix[=PATH]` beats `PYTHONPYCACHEPREFIX` even when - // given bare / with an empty value (which unsets the env prefix). - let pycache_prefix = match xoption_value(&cli.xoptions, "pycache_prefix") { - Some(v) => v.filter(|p| !p.is_empty()).map(str::to_owned), - None => env.pycache_prefix.clone(), - }; - let int_max_str_digits = match xoption_value(&cli.xoptions, "int_max_str_digits") { - Some(Some(v)) => Some(parse_int_max_str_digits(v, "-X int_max_str_digits")), - Some(None) => config_fatal_error( - "config_init_int_max_str_digits", - "-X int_max_str_digits: invalid limit; must be >= 640 or 0 for unlimited.", - ), - None => env - .int_max_str_digits - .as_deref() - .map(|v| parse_int_max_str_digits(v, "PYTHONINTMAXSTRDIGITS")), - }; - // `-X cpu_count=N|default` / `PYTHON_CPU_COUNT` (gh-109595). - let cpu_count_raw = match xoption_value(&cli.xoptions, "cpu_count") { - Some(Some(v)) => Some(v.to_owned()), - Some(None) => config_fatal_error( - "config_init_cpu_count", - "-X cpu_count=n option: n is missing or invalid", - ), - None => env.cpu_count.clone(), - }; - let cpu_count = cpu_count_raw.and_then(|raw| { - if raw == "default" { - None - } else { - match raw.parse::() { - Ok(n) if n >= 1 => Some(n), - _ => config_fatal_error( - "config_init_cpu_count", - "-X cpu_count=n option: n is missing or invalid", - ), - } - } - }); - // `-X gil` / `PYTHON_GIL` (PEP 703): only "1" is meaningful on a - // build whose GIL can't be disabled; "0" is a startup fatal error. - let gil = match xoption_value(&cli.xoptions, "gil") { - Some(v) => v.map(str::to_owned), - None => env.gil.clone(), - }; - match gil.as_deref() { - None | Some("1") => {} - Some("0") => config_fatal_error( - "config_read_gil", - "Disabling the GIL is not supported by this build", - ), - Some(_) => config_fatal_error( - "config_read_gil", - "PYTHON_GIL / -X gil must be \"0\" or \"1\"", - ), - } - // `-X tracemalloc[=NFRAME]` beats `PYTHONTRACEMALLOC` (CPython - // `config_init_tracemalloc`): a parse failure or negative value is a - // startup fatal error; `0` means disabled; a value beyond - // `_tracemalloc`'s MAX_NFRAME fails at init with the module's own - // ValueError text (`test_tracemalloc.TestCommandLine`). - let tracemalloc_nframe = match xoption_value(&cli.xoptions, "tracemalloc") { - Some(Some(v)) => match v.parse::() { - Ok(n) if n >= 0 => Some(n), - _ => config_fatal_error( - "config_init_tracemalloc", - "-X tracemalloc=NFRAME: invalid number of frames", - ), - }, - // Bare `-X tracemalloc` behaves as `-X tracemalloc=1`. - Some(None) => Some(1), - None => env.tracemalloc.as_deref().map(|v| match v.parse::() { - Ok(n) if n >= 0 => n, - _ => config_fatal_error( - "config_init_tracemalloc", - "PYTHONTRACEMALLOC: invalid number of frames", - ), - }), - }; - let tracemalloc = match tracemalloc_nframe { - None | Some(0) => 0u32, - Some(n) if n > 65535 => { - // CPython surfaces `_PyTraceMalloc_Start`'s ValueError during - // interpreter init. - eprintln!("ValueError: the number of frames must be in range [1; 65535]"); - std::process::exit(1); - } - Some(n) => n as u32, - }; - let mut xoptions = cli.xoptions.clone(); - // `PYTHONDEVMODE` behaves like `-X dev` for `sys.flags.dev_mode` - // (though CPython does *not* mirror it into `sys._xoptions`; the - // duplicate key is harmless for our flag computation). - if env.dev_mode && xoption_value(&xoptions, "dev").is_none() { - xoptions.push("dev".to_owned()); - } - InterpreterFlags { - optimize: cli.optimize.max(env.optimize), - dont_write_bytecode: cli.no_bytecode_write || env.dont_write_bytecode, - inspect: cli.inspect_after || env.inspect, - verbose: cli.verbose.max(env.verbose), - no_site: cli.no_site, - no_user_site: cli.no_user_site || env.no_user_site || isolated, - ignore_environment: ignore_env, - isolated, - quiet: cli.quiet, - unbuffered: cli.unbuffered || env.unbuffered, - skip_first_line: cli.skip_first_line, - bytes_warning: cli.bytes_warning, - safe_path: cli.safe_path || env.safe_path || isolated, - debug: cli.parser_debug.max(env.debug), - xoptions, - warning_filters: { - let mut v = env.warning_filters.clone(); - v.extend(cli.warnings.iter().cloned()); - v - }, - // `-R` re-enables randomization, trumping a fixed seed from - // `PYTHONHASHSEED`. - hash_seed: if cli.hash_randomization { - None - } else { - env.hash_seed - }, - // Filled in by the `._pth` probe in `main` (RFC 0062 WS5). - pth_paths: None, - io_encoding: env.io_encoding.clone(), - io_errors: env.io_errors.clone(), - utf8_mode: env.utf8_mode, - pycache_prefix, - int_max_str_digits, - cpu_count, - tracemalloc, - // `-X faulthandler` beats `PYTHONFAULTHANDLER` only in the sense - // that either one turns it on (CPython `config_init_faulthandler`; - // there is no "off" spelling). - faulthandler: env.faulthandler || xoption_value(&cli.xoptions, "faulthandler").is_some(), - } -} - -/// Subset of `PYTHON*` environment overrides we honour. Materialised -/// once per CLI invocation so each call site reads from a consistent -/// snapshot (env vars don't mutate mid-run). -#[derive(Debug, Default, Clone)] -struct EnvOverrides { - pythonpath: Vec, - startup: Option, - optimize: u8, - dont_write_bytecode: bool, - inspect: bool, - unbuffered: bool, - verbose: u8, - debug: u8, - dev_mode: bool, - no_user_site: bool, - safe_path: bool, - /// `PYTHONPYCACHEPREFIX` (PEP 552), losing to `-X pycache_prefix`. - pycache_prefix: Option, - /// `PYTHONINTMAXSTRDIGITS`, raw (validated during flag composition - /// so `-X int_max_str_digits` precedence applies first). - int_max_str_digits: Option, - /// `PYTHON_CPU_COUNT`, raw (`"default"` or an integer ≥ 1). - cpu_count: Option, - /// `PYTHON_GIL`, raw (`"0"` / `"1"`). - gil: Option, - /// `PYTHONTRACEMALLOC`, raw (validated during flag composition so - /// `-X tracemalloc` precedence applies first). - tracemalloc: Option, - /// `PYTHONFAULTHANDLER` — any non-empty value enables the - /// fatal-signal traceback dumper at startup. - faulthandler: bool, - warning_filters: Vec, - hash_seed: Option, - /// `PYTHONIOENCODING=encoding[:errors]`, split into its halves. Either - /// part may be empty (`:errors` sets only the handler). - io_encoding: Option, - io_errors: Option, - /// `PYTHONUTF8=0|1` (PEP 540). `None` when unset/empty; an invalid - /// value is a startup fatal error (CPython `config_init_utf8_mode`). - utf8_mode: Option, -} - -impl EnvOverrides { - fn from_env() -> Self { - let mut o = Self::default(); - if let Ok(p) = env::var("PYTHONPATH") { - o.pythonpath = p - .split(if cfg!(windows) { ';' } else { ':' }) - .map(str::to_owned) - .collect(); - } - if let Ok(p) = env::var("PYTHONSTARTUP") { - if !p.is_empty() { - o.startup = Some(PathBuf::from(p)); - } - } - // CPython treats a `PYTHON*` variable set to the empty string as - // unset (`config_get_env` / `_Py_GetEnv`); the int-valued ones - // (`PYTHONOPTIMIZE`/`PYTHONVERBOSE`/`PYTHONDEBUG`) parse as an - // integer with any non-numeric value meaning 1 - // (`test_cmd_line.test_sys_flags_set`). - let nonempty = |name: &str| env::var(name).ok().filter(|v| !v.is_empty()); - let env_int = |name: &str| nonempty(name).map(|v| v.parse::().unwrap_or(1)); - if let Some(n) = env_int("PYTHONOPTIMIZE") { - o.optimize = n; - } - o.dont_write_bytecode = nonempty("PYTHONDONTWRITEBYTECODE").is_some(); - o.inspect = nonempty("PYTHONINSPECT").is_some(); - o.unbuffered = nonempty("PYTHONUNBUFFERED").is_some(); - o.verbose = env_int("PYTHONVERBOSE").unwrap_or(0); - // Unlike OPTIMIZE/VERBOSE, `PYTHONDEBUG` is a plain boolean env - // in CPython (`config_get_env`, not the int-parsing variant): - // any non-empty value — including "2" — means 1. - o.debug = u8::from(nonempty("PYTHONDEBUG").is_some()); - o.dev_mode = nonempty("PYTHONDEVMODE").is_some(); - o.no_user_site = nonempty("PYTHONNOUSERSITE").is_some(); - o.safe_path = nonempty("PYTHONSAFEPATH").is_some(); - o.pycache_prefix = nonempty("PYTHONPYCACHEPREFIX"); - o.int_max_str_digits = nonempty("PYTHONINTMAXSTRDIGITS"); - o.cpu_count = nonempty("PYTHON_CPU_COUNT"); - o.gil = nonempty("PYTHON_GIL"); - o.tracemalloc = nonempty("PYTHONTRACEMALLOC"); - o.faulthandler = nonempty("PYTHONFAULTHANDLER").is_some(); - if let Ok(w) = env::var("PYTHONWARNINGS") { - o.warning_filters = w.split(',').map(str::to_owned).collect(); - } - if let Ok(seed) = env::var("PYTHONHASHSEED") { - if seed == "0" { - o.hash_seed = Some(0); - } else if let Ok(n) = seed.parse::() { - o.hash_seed = Some(n); - } - } - // `PYTHONIOENCODING=encoding[:errors]` (CPython): the first `:` - // splits the codec from the error handler; either side may be - // empty (`utf-8`, `:strict`, `ascii:backslashreplace`). - if let Ok(spec) = env::var("PYTHONIOENCODING") { - let (enc, errs) = match spec.split_once(':') { - Some((e, h)) => (e, Some(h)), - None => (spec.as_str(), None), - }; - if !enc.is_empty() { - o.io_encoding = Some(enc.to_owned()); - } - if let Some(h) = errs { - if !h.is_empty() { - o.io_errors = Some(h.to_owned()); + let entry: unsafe extern "C" fn() -> i32 = unsafe { std::mem::transmute(entry) }; + unsafe { entry() } + } + + /// Probe order (RFC 0064 WS1): the exe's own directory (the + /// distribution layout — DLL beside the exes at the prefix + /// root, and cargo's `target//` during development); + /// the `pyvenv.cfg` `home=` directory (venvs copy the exe, not + /// the DLL); finally the loader's default search. + fn locate_and_load(probed: &mut Vec) -> Option { + let exe_dir = std::env::current_exe() + .ok() + .and_then(|p| p.parent().map(Path::to_path_buf)); + if let Some(dir) = &exe_dir { + let candidate = dir.join(DLL_NAME); + if let Some(dll) = load_at(&candidate) { + return Some(dll); + } + probed.push(candidate); + if let Some(home) = pyvenv_home(dir) { + let candidate = home.join(DLL_NAME); + if let Some(dll) = load_at(&candidate) { + return Some(dll); } - } - } - // `PYTHONUTF8` (PEP 540): "1" enables UTF-8 mode, "0" disables it, - // empty means unset; anything else is a startup fatal error - // (CPython's `config_init_utf8_mode`). - if let Ok(v) = env::var("PYTHONUTF8") { - match v.as_str() { - "" => {} - "1" => o.utf8_mode = Some(1), - "0" => o.utf8_mode = Some(0), - other => { - eprintln!( - "Fatal Python error: init_utf8_mode: invalid PYTHONUTF8 environment \ - variable value '{other}'" - ); - std::process::exit(1); - } - } - } - o - } - - fn ignored() -> Self { - Self::default() - } -} - -/// Materialise the `-c` command text from its (possibly PUA-bridged) -/// argv transport, the way CPython's `pymain_run_command` receives it: -/// - clean text (the overwhelmingly common case) passes through; -/// - undecodable bytes under the `C`/`POSIX` locale decode to their -/// byte values (macOS/BSD `_Py_char2wchar` fallback — `test_cmd_line. -/// test_undecodable_code` expects `ascii("\xff")` to print `'\xff'`); -/// - otherwise the command cannot be represented and startup fails with -/// CPython's "Unable to decode the command from the command line". -fn decode_command_arg(cmd: &str) -> String { - use weavepy::vm::object::Object; - match weavepy::vm::argv_str_to_object(cmd) { - Object::WStr(cps) => { - let c_locale = ["LC_ALL", "LC_CTYPE", "LANG"] - .iter() - .find_map(|v| env::var(v).ok().filter(|s| !s.is_empty())) - .is_none_or(|loc| loc == "C" || loc == "POSIX"); - if c_locale { - cps.iter() - .map(|&cp| match cp { - 0xDC80..=0xDCFF => char::from_u32(cp - 0xDC00).unwrap_or('\u{FFFD}'), - other => char::from_u32(other).unwrap_or('\u{FFFD}'), - }) - .collect() - } else { - eprintln!("Unable to decode the command from the command line:"); - std::process::exit(1); - } - } - Object::Str(s) => s.to_string(), - _ => cmd.to_owned(), - } -} - -/// Rebuild a filesystem path from a (possibly PUA-bridged) argv string, -/// recovering the original OS bytes for PEP 383-escaped names. -fn bridged_arg_to_pathbuf(arg: &str) -> PathBuf { - #[cfg(unix)] - { - use std::os::unix::ffi::OsStringExt; - PathBuf::from(std::ffi::OsString::from_vec( - weavepy::vm::bridged_arg_bytes(arg), - )) - } - #[cfg(not(unix))] - { - PathBuf::from(arg) - } -} - -/// Escape a string into a Python single-quoted string literal. -fn quote_py_string(s: &str) -> String { - let mut out = String::with_capacity(s.len() + 2); - out.push('"'); - for c in s.chars() { - match c { - '\\' => out.push_str("\\\\"), - '"' => out.push_str("\\\""), - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - '\t' => out.push_str("\\t"), - c if (c as u32) < 0x20 => out.push_str(&format!("\\x{:02x}", c as u32)), - c => out.push(c), - } - } - out.push('"'); - out -} - -fn run_module( - name: &str, - args: Vec, - flags: &InterpreterFlags, - extra_path: &[PathBuf], -) -> Result<()> { - // Every `-m` goes through CPython's own entry point, - // `runpy._run_module_as_main`: it imports parent packages (so the - // target's relative imports resolve), redirects a package to its - // `__main__` submodule, executes the target *in* the current - // `__main__` namespace (so `-i -m timeit` leaves `Timer` visible to - // the inspect REPL — `test_cmd_line.test_run_module_bug1764407`), - // and reports a missing module the way CPython does - // (`sys.exit(": Error while finding module specification …")`). - // - // `sys.argv[0]` starts as the literal `'-m'` — CPython's config - // leaves the placeholder in place so code run *during the search* - // (a parent package's `__init__`) sees it - // (`test_cmd_line_script.test_issue8202`); `_run_module_as_main` - // then swaps in the located file path before the target runs. - let mut argv = vec!["-m".to_owned()]; - argv.extend(args.iter().cloned()); - let cwd = env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); - let mut bootstrap = String::from("import runpy, sys\n"); - bootstrap.push_str(&format!( - "runpy._run_module_as_main({})\n", - quote_py_string(name) - )); - let opts = RunOptions::new(format!("")) - .with_argv(argv) - .with_extra_path(extra_path.to_vec()) - .with_script_dir(cwd) - .with_flags(flags.clone()); - run_source_with_options(&bootstrap, &opts) -} - -/// Decode a script file's bytes per PEP 263 (BOM + coding cookie, -/// default strict UTF-8). On failure, print CPython's tokenizer-style -/// `SyntaxError` to stderr and exit 1 — like `python bad.py` does. -fn decode_script_source(bytes: &[u8], filename: &str) -> String { - match weavepy::vm::decode_source_bytes(bytes, filename) { - Ok(s) => s, - Err(err) => { - let msg = match &err { - weavepy::vm::RuntimeError::PyException(pe) => pe.message(), - other => other.to_string(), + probed.push(candidate); + } + } + probed.push(PathBuf::from(DLL_NAME)); + let wide = to_wide(std::ffi::OsStr::new(DLL_NAME)); + // SAFETY: `wide` is a NUL-terminated UTF-16 string. + let dll = unsafe { LoadLibraryW(wide.as_ptr()) }; + (!dll.is_null()).then_some(dll) + } + + /// `LoadLibraryExW` with an absolute path; `None` when the file + /// is absent or refuses to load. + fn load_at(path: &Path) -> Option { + if !path.is_file() { + return None; + } + let wide = to_wide(path.as_os_str()); + // SAFETY: `wide` is a NUL-terminated UTF-16 path; + // LOAD_WITH_ALTERED_SEARCH_PATH resolves the DLL's own + // (static) imports relative to its location, matching how + // CPython's shim binds its core DLL. + let dll = unsafe { + LoadLibraryExW( + wide.as_ptr(), + std::ptr::null_mut::(), + LOAD_WITH_ALTERED_SEARCH_PATH, + ) + }; + (!dll.is_null()).then_some(dll) + } + + /// The `home` key of `{venv}/pyvenv.cfg` when the exe sits in a + /// venv's `Scripts\` directory — the base prefix, where the real + /// DLL lives. Whitespace-tolerant like CPython's getpath. + fn pyvenv_home(exe_dir: &Path) -> Option { + let cfg = exe_dir.parent()?.join("pyvenv.cfg"); + let text = std::fs::read_to_string(cfg).ok()?; + for line in text.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; }; - // A NUL in the source: CPython reports the line the byte sits - // on and echoes that line *truncated at the NUL*, with no - // caret (`test_cmd_line_script.test_syntaxerror_null_bytes`). - if let Some(pos) = bytes.iter().position(|&b| b == 0) { - let line_no = bytes[..pos].iter().filter(|&&b| b == b'\n').count() + 1; - let line_start = bytes[..pos] - .iter() - .rposition(|&b| b == b'\n') - .map_or(0, |i| i + 1); - let line_text = String::from_utf8_lossy(&bytes[line_start..pos]); - eprintln!(" File \"{filename}\", line {line_no}"); - let trimmed = line_text.trim_start(); - if !trimmed.is_empty() { - eprintln!(" {trimmed}"); + if key.trim().eq_ignore_ascii_case("home") { + let value = value.trim(); + if !value.is_empty() { + return Some(PathBuf::from(value)); } - eprintln!("SyntaxError: {msg}"); - std::process::exit(1); - } - eprintln!(" File \"{filename}\", line 1"); - eprintln!("SyntaxError: {msg}"); - std::process::exit(1); - } - } -} - -fn run_path( - path: &Path, - extra: Vec, - flags: &InterpreterFlags, - extra_path: &[PathBuf], -) -> Result<()> { - // A directory or zipfile argument is executed as a module: CPython's - // `pymain_run_module` adds the path itself to `sys.path[0]` and runs - // `runpy._run_module_as_main("__main__")`, so `/__main__.py` (or the - // zip's top-level `__main__`) becomes the program. (`python ` / - // `python app.zip`.) - if path.is_dir() { - return run_main_module_from_path(path, extra, flags, extra_path); - } - // CPython's `pymain_run_file`: an unopenable script prints - // `: can't open file '': [Errno N] ` - // (no traceback) and exits with status 2. - // - // The file is read exactly *once* and every content sniff (zip - // magic, pyc magic) works off those bytes: a `/dev/fd/N` script - // shares its seek offset with every other descriptor on the same - // open file description, so a probe that consumed 4 magic bytes - // would shear them off the program itself (GH-87235, - // `test_cmd_line_script.test_script_as_dev_fd`). - let bytes = match fs::read(path) { - Ok(b) => b, - Err(e) => { - let abs = std::path::absolute(path).unwrap_or_else(|_| path.to_path_buf()); - let program = env::args().next().unwrap_or_else(|| "weavepy".to_owned()); - let errno = e.raw_os_error().unwrap_or(2); - eprintln!( - "{program}: can't open file '{}': [Errno {errno}] {}", - abs.display(), - errno_message(errno) - ); - std::process::exit(2); - } - }; - // `python app.zip`: the zip's top-level `__main__` becomes the program. - if is_zip_bytes(&bytes) { - return run_main_module_from_path(path, extra, flags, extra_path); - } - // A compiled-bytecode file (`.pyc`) given directly: CPython's - // `pymain_run_file` detects the magic and runs the unmarshalled code - // object as `__main__` (rather than trying to decode it as source). - if is_pyc_bytes(&bytes) { - return run_pyc_as_main(path, extra, flags, extra_path); - } - // CPython absolutizes the script path for `__main__.__file__` / - // `co_filename` (getpath's `abspath(program_full_path)`), while - // `sys.argv[0]` keeps the exact text the user typed - // (`test_cmd_line_script.test_script_abspath`). - let filename = std::path::absolute(path) - .unwrap_or_else(|_| path.to_path_buf()) - .display() - .to_string(); - let source = decode_script_source(&bytes, &filename); - let mut argv = vec![path.display().to_string()]; - argv.extend(extra); - let script_dir = Path::new(&filename) - .parent() - .filter(|p| !p.as_os_str().is_empty()) - .map_or_else(|| PathBuf::from("."), Path::to_path_buf); - let opts = RunOptions::new(filename.clone()) - .with_argv(argv) - .with_extra_path(extra_path.to_vec()) - .with_script_dir(script_dir) - .with_flags(flags.clone()); - run_source_with_options(&source, &opts) -} - -/// The OS `strerror` text for an errno, without the " (os error N)" -/// suffix `std::io::Error`'s Display appends. -fn errno_message(errno: i32) -> String { - let s = io::Error::from_raw_os_error(errno).to_string(); - match s.find(" (os error ") { - Some(i) => s[..i].to_owned(), - None => s, - } -} - -/// CPython's `__pycache__`/legacy-`.pyc` magic (kept in sync with -/// `crates/weavepy-vm/src/pycache.rs` and `importlib.machinery.MAGIC_NUMBER`). -const PYC_MAGIC: [u8; 4] = [0xf3, 0x0d, 0x0d, 0x0a]; - -/// Whether `bytes` begins with the WeavePy bytecode magic + the 16-byte -/// `.pyc` header CPython writes (4 magic, 4 bit-field, 8 mtime/size or hash). -fn is_pyc_bytes(bytes: &[u8]) -> bool { - bytes.len() >= 16 && bytes[..4] == PYC_MAGIC -} - -/// Whether `bytes` begins with a zip signature (local-file/empty/spanned). -/// `python app.zip` runs the zip's top-level `__main__` via `zipimport`. -fn is_zip_bytes(bytes: &[u8]) -> bool { - matches!( - bytes.get(..4), - Some([b'P', b'K', 0x03, 0x04] | [b'P', b'K', 0x05, 0x06] | [b'P', b'K', 0x07, 0x08]) - ) -} - -/// Run a directory or zipfile's top-level `__main__` as the program, with -/// `path` prepended to `sys.path` (CPython's directory/zipapp launch). -fn run_main_module_from_path( - path: &Path, - extra: Vec, - flags: &InterpreterFlags, - extra_path: &[PathBuf], -) -> Result<()> { - let path_str = path.display().to_string(); - let mut argv = vec![path_str.clone()]; - argv.extend(extra); - // `alter_argv=False`: keep `sys.argv[0]` as the dir/zip path (CPython does - // not rewrite it to the located `__main__` for directory/zip execution). - let bootstrap = - String::from("import runpy\nrunpy._run_module_as_main('__main__', alter_argv=False)\n"); - let opts = RunOptions::new(path_str) - .with_argv(argv) - .with_extra_path(extra_path.to_vec()) - .with_script_dir_always(path.to_path_buf()) - .with_flags(flags.clone()); - run_source_with_options(&bootstrap, &opts) -} - -/// Run a `.pyc` file's marshalled code object as `__main__`, mirroring -/// CPython's `run_pyc_file`: `__main__.__file__` is the `.pyc` path and -/// `__spec__` stays `None` (a directly-run file is not an importable module), -/// so `multiprocessing` spawn reconstructs the child via `init_main_from_path`. -fn run_pyc_as_main( - path: &Path, - extra: Vec, - flags: &InterpreterFlags, - extra_path: &[PathBuf], -) -> Result<()> { - let path_str = path.display().to_string(); - let mut argv = vec![path_str.clone()]; - argv.extend(extra); - let script_dir = path - .parent() - .filter(|p| !p.as_os_str().is_empty()) - .map_or_else(|| PathBuf::from("."), Path::to_path_buf); - let quoted = quote_py_string(&path_str); - let mut bootstrap = String::from("import sys, marshal\n"); - bootstrap.push_str(&format!("with open({quoted}, 'rb') as _f:\n")); - bootstrap.push_str(" _data = _f.read()\n"); - bootstrap.push_str("_code = marshal.loads(_data[16:])\n"); - bootstrap.push_str("_g = sys.modules['__main__'].__dict__\n"); - bootstrap.push_str(&format!("_g['__file__'] = {quoted}\n")); - bootstrap.push_str("_g['__cached__'] = None\n"); - bootstrap.push_str("_g['__spec__'] = None\n"); - // CPython's `pymain_run_file` on a `.pyc` installs a - // `SourcelessFileLoader` as `__main__.__loader__` - // (`test_cmd_line_script.test_script_compiled`). - bootstrap.push_str("import importlib.machinery as _m\n"); - bootstrap.push_str(&format!( - "_g['__loader__'] = _m.SourcelessFileLoader('__main__', {quoted})\n" - )); - bootstrap.push_str("del _m\n"); - bootstrap.push_str("del sys, marshal, _f, _data\n"); - bootstrap.push_str("exec(_code, _g)\n"); - // The bootstrap gets a synthetic co_filename (the `` - // convention `-m` uses): its frame sits under the pyc's own frames in - // a traceback, and it must not leak the on-disk pyc path — compileall's - // `--strip`/`--prepend` embed a *rewritten* path in the pyc, and - // `test_compileall.test_strip_only` asserts the build dir never - // appears in the traceback of a pyc run directly. - let opts = RunOptions::new("") - .with_argv(argv) - .with_extra_path(extra_path.to_vec()) - .with_script_dir(script_dir) - .with_flags(flags.clone()); - run_source_with_options(&bootstrap, &opts) -} - -fn run_stdin(extra: Vec, flags: &InterpreterFlags, extra_path: &[PathBuf]) -> Result<()> { - let mut buf = String::new(); - io::stdin() - .read_to_string(&mut buf) - .context("failed to read stdin")?; - let mut argv = vec!["-".to_owned()]; - argv.extend(extra); - // Like `-c`: stdin programs get `''` (cwd at import time) as - // `sys.path[0]`, matching CPython's `pymain_run_stdin`. - let opts = RunOptions::new("") - .with_argv(argv) - .with_extra_path(extra_path.to_vec()) - .with_script_dir("") - .with_flags(flags.clone()); - run_source_with_options(&buf, &opts) -} - -fn run_source_with_options(source: &str, opts: &RunOptions) -> Result<()> { - // CLI runs print uncaught exceptions CPython-style, through the - // interpreter's `sys.excepthook` / `traceback` machinery (source - // lines, carets, exception chains) while it is still alive. - let opts = opts.clone().with_print_uncaught(true); - // `-i` / `PYTHONINSPECT`: keep the interpreter alive and drop into - // a REPL that shares the program's `__main__` namespace (CPython's - // `pymain_repl`). An uncaught `SystemExit` is *ignored* — CPython's - // `_Py_HandleSystemExit` says "Don't exit if -i flag was given" - // (so `-i -m timeit`, whose main ends in `sys.exit(...)`, still - // reaches the prompt); any other exception is printed first and - // the prompt appears anyway. - if opts.flags.inspect { - let (interpreter, result) = weavepy::run_source_keep_interpreter(source, &opts); - if let Err(err) = result { - if err.system_exit_code().is_none() && !err.already_printed() { - let mut stderr = io::stderr().lock(); - let diag = err.format(source, &opts.filename); - let _ = stderr.write_all(diag.as_bytes()); } } - // No banner in inspect mode (CPython goes straight to `>>>`). - let repl = repl::Repl::new(interpreter, true)?; - return repl.run(None); + None } - match weavepy::run_source_with_options(source, &opts) { - Ok(()) => Ok(()), - Err(err) => { - // A `SystemExit` reaching the top level terminates the - // process with its code and prints no traceback — exactly - // like CPython. This is what makes `weavepy -m unittest`, - // `-m test`, and bare `sys.exit()` behave as a drop-in. - if let Some(code) = err.system_exit_code() { - exit_with_system_exit(code); - } - if !err.already_printed() { - let mut stderr = io::stderr().lock(); - let diag = err.format(source, &opts.filename); - let _ = stderr.write_all(diag.as_bytes()); - } - // bpo-1054041: an unhandled KeyboardInterrupt must terminate - // the process *via* SIGINT (so a shell sees death-by-signal, - // returncode == -SIGINT), after the traceback is printed. - // This is CPython's `exit_sigint()` in Modules/main.c. - if err.is_keyboard_interrupt() { - exit_via_sigint(); - } - anyhow::bail!(DIAGNOSTIC_SENTINEL); - } - } -} - -/// Terminate the process the way CPython does when `SystemExit` reaches -/// the top level: `None` → 0, a bool/int → that code (masked to 8 -/// bits), anything else → print `str(code)` to stderr and exit 1. -/// Never prints a traceback. -fn exit_with_system_exit(code: weavepy::vm::object::Object) -> ! { - use weavepy::vm::object::Object; - let _ = io::stdout().flush(); - let status: i32 = match code { - Object::None => 0, - Object::Bool(b) => i32::from(b), - Object::Int(n) => (n & 0xFF) as i32, - // A bare `raise SystemExit` (and `sys.exit()`) carries no - // message; WeavePy models the empty payload as an empty string, - // which means "no error" → exit 0, not a printed message. - Object::Str(s) if s.is_empty() => 0, - // An *int subclass* payload exits with its integer value — - // `sys.exit(pytest.ExitCode.OK)` is an `enum.IntEnum`, and - // CPython's `_Py_HandleSystemExit` does `PyLong_Check(value)` - // which is subclass-inclusive (RFC 0055 WS5). - Object::Instance(ref inst) - if matches!( - inst.native.get(), - Some(Object::Int(_) | Object::Long(_) | Object::Bool(_)) - ) => - { - (code.as_i64().unwrap_or(1) & 0xFF) as i32 - } - // `sys.exit(SomeException('msg'))`: CPython prints `str(code)`. - // The interpreter is already torn down, so mirror - // `BaseException.__str__` from the args tuple directly - // (`test_cmd_line_script.test_issue20500_exit_with_exception_value`). - Object::Instance(inst) => { - // `args` is a real slot on exceptions (RFC 0057); older - // plain instances may still carry it in the dict. - let args = inst.slot_get("args").or_else(|| { - inst.dict - .borrow() - .get(&weavepy::vm::object::DictKey(Object::from_static("args"))) - .cloned() - }); - let text = match args { - Some(Object::Tuple(args)) => match args.len() { - 0 => String::new(), - 1 => args[0].to_str(), - _ => Object::Tuple(args).to_str(), - }, - _ => Object::Instance(inst).to_str(), - }; - let mut stderr = io::stderr().lock(); - let _ = writeln!(stderr, "{text}"); - 1 - } - other => { - let mut stderr = io::stderr().lock(); - let _ = writeln!(stderr, "{}", other.to_str()); - 1 - } - }; - let _ = io::stderr().flush(); - std::process::exit(status); -} - -/// Terminate via `SIGINT` under the default disposition, the way -/// CPython's `exit_sigint()` does when a `KeyboardInterrupt` goes -/// unhandled: reset `SIGINT` to `SIG_DFL` and `kill(getpid(), SIGINT)` -/// so the process dies *by the signal* (`returncode == -SIGINT`), which -/// is what shells and `subprocess` inspect. Falls back to exit code 130 -/// (128 + SIGINT) if, impossibly, the signal doesn't terminate us. -#[cfg(unix)] -fn exit_via_sigint() -> ! { - let _ = io::stdout().flush(); - let _ = io::stderr().flush(); - // Reset SIGINT to SIG_DFL, unblock it on this thread, and raise it - // process-wide so we die *by the signal* (returncode == -SIGINT). - weavepy::vm::stdlib::signal_mod::die_via_sigint(); - // Unreachable in practice; the signal terminates us above. - std::process::exit(130); -} - -#[cfg(not(unix))] -fn exit_via_sigint() -> ! { - let _ = io::stdout().flush(); - let _ = io::stderr().flush(); - std::process::exit(0xC0_00_01_3A_u32 as i32); -} -fn run_repl(flags: InterpreterFlags, startup: Option<&Path>, argv: Vec) -> Result<()> { - let mut interpreter = weavepy::vm::Interpreter::default(); - interpreter.apply_run_options(&flags); - if !argv.is_empty() { - let mut a = vec![String::new()]; - a.extend(argv); - interpreter.set_argv(a); - } else { - interpreter.set_argv(vec![String::new()]); - } - interpreter.prepend_path(env::current_dir().unwrap_or_else(|_| PathBuf::from("."))); - if !flags.no_site { - let _ = interpreter.run_site(); + fn to_wide(s: &std::ffi::OsStr) -> Vec { + s.encode_wide().chain(std::iter::once(0)).collect() } - let repl = repl::Repl::new(interpreter, flags.quiet)?; - repl.run(startup) -} - -fn init_tracing() { - let filter = EnvFilter::try_from_env("WEAVEPY_LOG").unwrap_or_else(|_| EnvFilter::new("warn")); - let _ = tracing_subscriber::fmt() - .with_env_filter(filter) - .with_target(false) - .try_init(); } diff --git a/crates/weavepy-cli/src/regrtest_cmd.rs b/crates/weavepy-cli/src/regrtest_cmd.rs index 84784eab..405d4454 100644 --- a/crates/weavepy-cli/src/regrtest_cmd.rs +++ b/crates/weavepy-cli/src/regrtest_cmd.rs @@ -25,7 +25,6 @@ //! `/vendor/cpython-tests/`) when present. use std::path::{Path, PathBuf}; -use std::process::ExitCode; use std::time::Duration; use anyhow::{Context, Result}; @@ -122,7 +121,7 @@ struct Cli { stream: bool, } -pub(crate) fn run(argv: Vec) -> Result { +pub(crate) fn run(argv: Vec) -> Result { let cli = Cli::parse_from(argv); let workspace = resolve_workspace(cli.workspace.as_deref())?; let report_dir = cli @@ -209,9 +208,9 @@ pub(crate) fn run(argv: Vec) -> Result { // stamp the gate is advisory — the helper prints the NOTE line and // returns false, so the run exits 0 with the reports still written. if !cli.no_check && weavepy_conformance::regrtest::strict_gate_blocks(&expectations, &summary) { - return Ok(ExitCode::from(1)); + return Ok(1); } - Ok(ExitCode::SUCCESS) + Ok(0) } fn resolve_workspace(explicit: Option<&Path>) -> Result { diff --git a/crates/weavepy-cli/tests/windows_dll.rs b/crates/weavepy-cli/tests/windows_dll.rs new file mode 100644 index 00000000..ec105239 --- /dev/null +++ b/crates/weavepy-cli/tests/windows_dll.rs @@ -0,0 +1,211 @@ +//! The `python313.dll` contract, end to end (RFC 0064 WS5). +//! +//! On Windows `weavepy.exe` is a thin shim over `python313.dll` — the +//! runtime, and the import target every `.pyd`'s PE header names. This +//! battery is the smoke half of the POSIX `force_link_completeness` +//! contract, adapted to the PE world: +//! +//! 1. the DLL exists next to the exe and loads; +//! 2. `GetProcAddress` resolves the embedding entry points and a +//! curated sample spanning the export families — including the +//! `varargs.c` symbols that need explicit `/EXPORT`s (a regression +//! here means the MSVC export plumbing in `weavepy-pylib/build.rs` +//! broke); +//! 3. the shim runs Python *through* the DLL (`sys.dllhandle` is the +//! real HMODULE); +//! 4. `os.add_dll_directory` round-trips (WS2); +//! 5. a broken `.pyd` raises CPython's exact +//! `ImportError: DLL load failed while importing …` shape. +//! +//! CI builds `-p weavepy-pylib` alongside the workspace, so the DLL is +//! always present there; a missing DLL fails loudly with the build +//! command rather than skipping. +#![cfg(windows)] + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn exe_path() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_weavepy")) +} + +/// `python313.dll` sits next to the exe (both land in +/// `target//`). +fn dll_path() -> PathBuf { + let dll = exe_path() + .parent() + .expect("exe path has a parent") + .join("python313.dll"); + assert!( + dll.is_file(), + "python313.dll not found at {} — build it with \ + `cargo build -p weavepy-pylib` (same profile as this test)", + dll.display() + ); + dll +} + +fn load_dll(path: &Path) -> *mut core::ffi::c_void { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::System::LibraryLoader::LoadLibraryExW; + let wide: Vec = path + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let handle = unsafe { LoadLibraryExW(wide.as_ptr(), std::ptr::null_mut(), 0) }; + assert!( + !handle.is_null(), + "LoadLibraryExW({}) failed with Win32 error {}", + path.display(), + unsafe { windows_sys::Win32::Foundation::GetLastError() } + ); + handle +} + +/// The export families, sampled: numbers, strings, containers, +/// modules, errors, abstract, types, capsules, GIL/lifecycle, +/// singletons (a `#[no_mangle] static`), the embedding entry points, +/// and the `varargs.c` set that rides `/EXPORT` linker args. +const SYMBOL_SAMPLE: &[&str] = &[ + // embedding entry points (weavepy-pylib itself) + "weavepy_main", + "Py_Main", + "Py_BytesMain", + // lifecycle / GIL + "Py_Initialize", + "Py_IsInitialized", + "PyGILState_Ensure", + "PyGILState_Release", + "PyEval_SaveThread", + "PyEval_RestoreThread", + // numbers + "PyLong_FromLong", + "PyLong_AsLong", + "PyFloat_FromDouble", + // strings / bytes + "PyUnicode_FromString", + "PyBytes_FromStringAndSize", + // containers + "PyTuple_New", + "PyList_New", + "PyList_Append", + "PyDict_New", + "PyDict_SetItemString", + // modules / import + "PyModule_Create2", + "PyModule_GetDict", + "PyImport_ImportModule", + // errors + "PyErr_SetString", + "PyErr_Occurred", + "PyErr_Clear", + // abstract / types / capsules + "PyObject_GetAttrString", + "PyObject_CallObject", + "PyType_FromSpec", + "PyType_Ready", + "PyCapsule_New", + "PyCapsule_GetPointer", + // a #[no_mangle] static (data export, not a function) + "_Py_NoneStruct", + // varargs.c — native-archive symbols needing explicit /EXPORT + "PyArg_ParseTuple", + "PyArg_ParseTupleAndKeywords", + "Py_BuildValue", + "PyErr_Format", + "PyObject_CallMethod", + "PyUnicode_FromFormat", +]; + +#[test] +fn dll_loads_and_exports_resolve() { + use windows_sys::Win32::System::LibraryLoader::GetProcAddress; + let handle = load_dll(&dll_path()); + let mut missing = Vec::new(); + for name in SYMBOL_SAMPLE { + let cname = std::ffi::CString::new(*name).unwrap(); + let addr = unsafe { GetProcAddress(handle, cname.as_ptr().cast()) }; + if addr.is_none() { + missing.push(*name); + } + } + assert!( + missing.is_empty(), + "python313.dll is missing exports: {missing:?} — if these are \ + varargs.c symbols, check the /EXPORT list in weavepy-pylib/build.rs" + ); +} + +/// Run the shim exe with `-c code` and return `(success, stdout, stderr)`. +fn run_c(code: &str) -> (bool, String, String) { + let out = Command::new(exe_path()) + .arg("-c") + .arg(code) + .output() + .expect("failed to spawn weavepy.exe"); + ( + out.status.success(), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +#[test] +fn shim_runs_python_through_the_dll() { + // The DLL loaded into the process is what sys.dllhandle reports. + let (ok, stdout, stderr) = + run_c("import sys; assert sys.dllhandle != 0, sys.dllhandle; print('ok')"); + assert!( + ok, + "sys.dllhandle probe failed\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + assert_eq!(stdout.trim(), "ok"); +} + +#[test] +fn add_dll_directory_round_trips() { + let code = r#" +import os, tempfile +d = tempfile.mkdtemp() +h = os.add_dll_directory(d) +r = repr(h) +assert r.startswith(""), r +h.close() +assert repr(h) == "", repr(h) +with os.add_dll_directory(d) as ctx: + assert repr(ctx).startswith(") -> Result }; if !path.is_file() { bail!( - "weavepy binary not found at {} — build it with `cargo build --release -p weavepy-cli` \ + "weavepy binary not found at {} — build it with `cargo build --release -p weavepy-cli{}` \ or pass --weavepy", - path.display() + path.display(), + if cfg!(windows) { " -p weavepy-pylib" } else { "" }, ); } Ok(path) } +/// The runtime DLL and its MSVC import library, which must sit next +/// to the exe being packaged (cargo writes all three into +/// `target//` — RFC 0064 WS1/WS3). The exe is a thin shim +/// over the DLL, so a Windows artifact without it would not even +/// start; the import library is what `pip install` of a C sdist +/// links (`{prefix}\libs\python313.lib`, setuptools' convention). +#[cfg(windows)] +fn resolve_windows_runtime(weavepy: &Path) -> Result<(PathBuf, PathBuf)> { + let dir = weavepy + .parent() + .context("weavepy binary path has no parent directory")?; + let dll = dir.join("python313.dll"); + // rustc names the cdylib's import library `python313.dll.lib`. + let implib = dir.join("python313.dll.lib"); + if !dll.is_file() || !implib.is_file() { + bail!( + "python313.dll / python313.dll.lib not found next to {} — the Windows exe is a \ + shim over the runtime DLL (RFC 0064); build both with \ + `cargo build --release -p weavepy-cli -p weavepy-pylib`", + weavepy.display() + ); + } + Ok((dll, implib)) +} + fn exe_name(base: &str) -> String { if cfg!(windows) { format!("{base}.exe") @@ -352,6 +382,32 @@ fn build_artifact(workspace: &Path, weavepy: &Path, out: &Path, format: Format) format!("failed to copy {} to {}", weavepy.display(), dest.display()) })?; } + // RFC 0064 WS3 — the binary ABI. `python313.dll` sits beside + // the exes at the prefix root (the shim's first probe and + // where a `.pyd`'s PE import resolves from), and the MSVC + // import library ships as `libs\python313.lib` — the exact + // path setuptools' `library_dirs` convention + // (`{sys.base_exec_prefix}\libs`) and pyconfig.h's autolink + // pragma expect. + #[cfg(windows)] + { + let (dll, implib) = resolve_windows_runtime(weavepy)?; + let dll_dest = staging.join("python313.dll"); + std::fs::copy(&dll, &dll_dest).with_context(|| { + format!("failed to copy {} to {}", dll.display(), dll_dest.display()) + })?; + let libs_dir = staging.join("libs"); + std::fs::create_dir_all(&libs_dir) + .with_context(|| format!("failed to create {}", libs_dir.display()))?; + let implib_dest = libs_dir.join("python313.lib"); + std::fs::copy(&implib, &implib_dest).with_context(|| { + format!( + "failed to copy {} to {}", + implib.display(), + implib_dest.display() + ) + })?; + } } else { // bin/weavepy + the PEP 394-ish shim names. let bin_dir = staging.join("bin"); @@ -492,9 +548,9 @@ fn artifact_readme(name: &str) -> String { .\\python3.exe\n\ ```\n\ \n\ - `weavepy.exe` is the real binary; `python.exe`, `python3.exe`, and\n\ - `python3.13.exe` are copies of it at the artifact root — the CPython\n\ - Windows convention (POSIX artifacts use `bin/` symlinks instead).\n\ + The exes are thin shims over `python313.dll` at the artifact root —\n\ + the runtime itself, and what C extensions link against (the CPython\n\ + Windows convention; POSIX artifacts use `bin/` symlinks instead).\n\ The layout is self-locating — no environment variables are required.\n\ \n\ ## Packaging\n\ @@ -507,10 +563,10 @@ fn artifact_readme(name: &str) -> String { .venv\\Scripts\\python.exe -m pip install \n\ ```\n\ \n\ - The CPython 3.13 C header set ships under `Include\\` (the CPython\n\ - Windows convention), but building or loading C extensions on\n\ - Windows is not supported yet (it needs a `python313.dll` for\n\ - extensions to link against).\n\ + Building C extensions from source needs MSVC (Visual Studio Build\n\ + Tools): the CPython 3.13 header set ships under `Include\\` and the\n\ + import library under `libs\\python313.lib`, the paths setuptools\n\ + uses by convention.\n\ \n\ ## License\n\ \n\ @@ -779,25 +835,18 @@ fn run_check( (Some(wheels), true) => leg_pip(&venv_python, wheels, &env), }); - // Leg 6: C-extension build (unix only, needs a C compiler). - legs.push(if cfg!(unix) { - if which("cc").is_some() { - leg_cext(&python3, scratch, &env) - } else { - Leg { - name: "cext", - status: LegStatus::Skip, - detail: "no `cc` on PATH".to_owned(), - } - } - } else { + // Leg 6: C-extension build (needs a C toolchain: `cc` on unix, + // MSVC on Windows — the Windows script discovers MSVC itself via + // `cl` on PATH or vswhere/vcvars64 and exits 2 when there is + // none, RFC 0064 WS3). + legs.push(if cfg!(unix) && which("cc").is_none() { Leg { name: "cext", status: LegStatus::Skip, - // A static exe has nothing for a .pyd's PE import table to - // resolve against; C builds await the python313.dll wave. - detail: "C builds are a Windows non-goal (RFC 0063)".to_owned(), + detail: "no `cc` on PATH".to_owned(), } + } else { + leg_cext(&python3, scratch, &env) }); // Leg 7: the decoy cache must still be empty — anything in it means @@ -1046,8 +1095,13 @@ fn leg_cext(python3: &Path, scratch: &Path, env: &[(OsString, OsString)]) -> Leg detail: format!("failed to create {}: {err}", cext_dir.display()), }; } + let script = if cfg!(windows) { + CEXT_SCRIPT_NT + } else { + CEXT_SCRIPT + }; let script_path = scratch.join("cext_build_check.py"); - if let Err(err) = std::fs::write(&script_path, CEXT_SCRIPT) { + if let Err(err) = std::fs::write(&script_path, script) { return Leg { name: "cext", status: LegStatus::Fail, @@ -1060,7 +1114,25 @@ fn leg_cext(python3: &Path, scratch: &Path, env: &[(OsString, OsString)]) -> Leg cext_dir.as_os_str().to_owned(), )); let script_arg = script_path.display().to_string(); - grade_output("cext", run_captured(python3, &[&script_arg], &env, None)) + let result = run_captured(python3, &[&script_arg], &env, None); + // Exit 2 is the Windows script's "no MSVC toolchain" sentinel — + // a machine without Visual Studio skips the leg rather than + // failing the whole check. + if let Ok(out) = &result { + if out.status.code() == Some(2) { + let stdout = String::from_utf8_lossy(&out.stdout).trim().to_owned(); + return Leg { + name: "cext", + status: LegStatus::Skip, + detail: stdout + .lines() + .last() + .unwrap_or("no MSVC toolchain") + .to_owned(), + }; + } + } + grade_output("cext", result) } fn leg_decoy(decoy: &Path) -> Leg { @@ -1344,3 +1416,117 @@ import _weavepy_dist_cext assert _weavepy_dist_cext.add(20, 22) == 42 print("cext ok:", mod_path) "#; + +/// The Windows twin of [`CEXT_SCRIPT`] (RFC 0064 WS3): same inline +/// module, but built with MSVC `cl /LD` against the shipped +/// `Include\` headers and linked against `libs\python313.lib` (found +/// via the pyconfig.h autolink pragma + `/LIBPATH`, exactly what +/// setuptools does). MSVC is discovered the way setuptools' msvc +/// module does it — `cl` already on PATH, else vswhere → +/// `vcvars64.bat` env capture. Exit 2 = no toolchain (the leg SKIPs). +const CEXT_SCRIPT_NT: &str = r#" +import os, subprocess, sys, sysconfig + +scratch = os.environ["WEAVEPY_DIST_CEXT_DIR"] +includepy = sysconfig.get_config_var("INCLUDEPY") +ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".pyd" +libs = os.path.join(sys.base_exec_prefix, "libs") +assert includepy and os.path.isfile(os.path.join(includepy, "Python.h")), ( + f"no Python.h under INCLUDEPY={includepy!r}" +) +assert os.path.isfile(os.path.join(libs, "python313.lib")), ( + f"no python313.lib under {libs!r}" +) + + +def msvc_env(): + """Env with cl.exe reachable, or None if no MSVC install exists.""" + from shutil import which + if which("cl"): + return dict(os.environ) + vswhere = os.path.join( + os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"), + "Microsoft Visual Studio", "Installer", "vswhere.exe", + ) + if not os.path.isfile(vswhere): + return None + proc = subprocess.run( + [vswhere, "-latest", "-products", "*", + "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", "installationPath"], + capture_output=True, text=True, + ) + lines = [l.strip() for l in proc.stdout.splitlines() if l.strip()] + if proc.returncode != 0 or not lines: + return None + vcvars = os.path.join(lines[0], "VC", "Auxiliary", "Build", "vcvars64.bat") + if not os.path.isfile(vcvars): + return None + # Capture the env vcvars64 sets up (setuptools' _get_vc_env trick). + probe = subprocess.run( + ["cmd", "/S", "/C", f'"{vcvars}" >NUL 2>&1 && set'], + capture_output=True, text=True, + ) + if probe.returncode != 0: + return None + env = {} + for line in probe.stdout.splitlines(): + key, sep, value = line.partition("=") + if sep: + env[key] = value + return env if env else None + + +env = msvc_env() +if env is None: + print("no MSVC toolchain (cl not on PATH, vswhere found no VC tools)") + sys.exit(2) + +SOURCE = r''' +#define PY_SSIZE_T_CLEAN +#include + +static PyObject * +add(PyObject *self, PyObject *args) +{ + Py_ssize_t a, b; + if (!PyArg_ParseTuple(args, "nn", &a, &b)) + return NULL; + return PyLong_FromSsize_t(a + b); +} + +static PyMethodDef methods[] = { + {"add", add, METH_VARARGS, "add two ints"}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, "_weavepy_dist_cext", NULL, -1, methods +}; + +PyMODINIT_FUNC +PyInit__weavepy_dist_cext(void) +{ + return PyModule_Create(&module); +} +''' + +src = os.path.join(scratch, "_weavepy_dist_cext.c") +with open(src, "w") as f: + f.write(SOURCE) + +mod_path = os.path.join(scratch, "_weavepy_dist_cext" + ext_suffix) +cmd = [ + "cl", "/nologo", "/LD", "/O2", "/W3", + "/I", includepy, src, + "/link", "/LIBPATH:" + libs, "/OUT:" + mod_path, +] +proc = subprocess.run(cmd, capture_output=True, text=True, cwd=scratch, env=env) +assert proc.returncode == 0, "%r failed:\n%s\n%s" % (cmd, proc.stdout, proc.stderr) + +sys.path.insert(0, scratch) +import _weavepy_dist_cext + +assert _weavepy_dist_cext.add(20, 22) == 42 +print("cext ok:", mod_path) +"#; diff --git a/crates/weavepy-pylib/Cargo.toml b/crates/weavepy-pylib/Cargo.toml new file mode 100644 index 00000000..38b31efd --- /dev/null +++ b/crates/weavepy-pylib/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "weavepy-pylib" +description = "The WeavePy runtime as a CPython-ABI shared library (python313.dll on Windows)." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true + +# RFC 0064 WS1: the library artifact is named for the CPython ABI it +# implements — `python313.dll` on Windows (what every `.pyd`'s PE +# import table references), `libpython313.{so,dylib}` elsewhere +# (built for workspace honesty, not shipped — see the RFC Non-goals). +[lib] +name = "python313" +crate-type = ["cdylib"] + +[dependencies] +weavepy-cli = { workspace = true } +libc = { workspace = true } + +[features] +default = [] +# Mirror of the CLI's `jit` feature so a JIT-enabled distribution can +# build a JIT-enabled DLL. +jit = ["weavepy-cli/jit"] + +[lints] +workspace = true diff --git a/crates/weavepy-pylib/build.rs b/crates/weavepy-pylib/build.rs new file mode 100644 index 00000000..e517ac2f --- /dev/null +++ b/crates/weavepy-pylib/build.rs @@ -0,0 +1,62 @@ +//! Build helper for the `python313` cdylib (RFC 0064 WS1). +//! +//! rustc derives a cdylib's export table from the reachable +//! `#[no_mangle]` surface of the whole crate graph, which covers the +//! ~682 Rust-defined C-API symbols in `weavepy-capi` and this crate's +//! own entry points. The one blind spot is `weavepy-capi/src/varargs.c`: +//! its variadic helpers (`PyArg_ParseTuple`, `Py_BuildValue`, …) are +//! compiled by `cc` into a native static archive, and native-archive +//! symbols are *not* part of rustc's export list. On MSVC each one +//! needs an explicit `/EXPORT` (plus `/INCLUDE` so the archive member +//! is pulled even if the Rust side's force-link table were ever +//! reorganised away). +//! +//! The list below is the complete set of public definitions in +//! `varargs.c`; `src/lib.rs` carries a unit test that re-derives the +//! set from the C source and fails if the two drift. + +use std::env; + +/// Public symbols defined in `weavepy-capi/src/varargs.c` that a +/// `.pyd` may import and that rustc's cdylib export machinery cannot +/// see. Keep in sync with the C file (enforced by the unit test in +/// `src/lib.rs`). +pub const VARARGS_C_EXPORTS: &[&str] = &[ + "PyArg_Parse", + "PyArg_ParseTuple", + "PyArg_ParseTupleAndKeywords", + "PyArg_UnpackTuple", + "PyArg_VaParse", + "PyArg_VaParseTupleAndKeywords", + "PyBytes_FromFormat", + "PyBytes_FromFormatV", + "PyErr_Format", + "PyErr_FormatUnraisable", + "PyErr_FormatV", + "PyErr_WarnFormat", + "PyOS_snprintf", + "PyObject_CallFunction", + "PyObject_CallFunctionObjArgs", + "PyObject_CallMethod", + "PyObject_CallMethodObjArgs", + "PyTuple_Pack", + "PyUnicode_FromFormat", + "PyUnicode_FromFormatV", + "Py_BuildValue", + "Py_VaBuildValue", + "_PyErr_FormatFromCause", +]; + +fn main() { + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); + if target_os == "windows" && target_env == "msvc" { + for sym in VARARGS_C_EXPORTS { + // `/INCLUDE` forces the symbol (and so its archive + // member) into the link; `/EXPORT` adds it to the DLL's + // export table alongside the rustc-derived set. + println!("cargo:rustc-link-arg-cdylib=/INCLUDE:{sym}"); + println!("cargo:rustc-link-arg-cdylib=/EXPORT:{sym}"); + } + } +} diff --git a/crates/weavepy-pylib/src/lib.rs b/crates/weavepy-pylib/src/lib.rs new file mode 100644 index 00000000..f6336aa7 --- /dev/null +++ b/crates/weavepy-pylib/src/lib.rs @@ -0,0 +1,186 @@ +//! The WeavePy runtime as a CPython-ABI shared library (RFC 0064 WS1). +//! +//! On Windows this crate builds `python313.dll` — the module name +//! every CPython-3.13 extension's PE import table references. The +//! whole interpreter lives here: the `weavepy.exe` shim +//! (`weavepy-cli/src/main.rs`) loads this DLL and calls +//! [`weavepy_main`], and a `.pyd` loaded later binds its +//! `python313.dll` imports to this already-loaded module, so there is +//! exactly one runtime in the process. +//! +//! The C-API itself needs no code in this crate: the ~682 +//! `#[no_mangle]` symbols defined in `weavepy-capi` (linked +//! transitively through `weavepy-cli` → `weavepy`) are exported by +//! rustc's cdylib machinery, and the variadic C helpers from +//! `varargs.c` are exported via `/EXPORT` link args emitted by this +//! crate's `build.rs`. What lives here are the *entry points*: +//! [`weavepy_main`] for the shim, and the CPython embedding twins +//! [`Py_Main`] / [`Py_BytesMain`] that stock `pylifecycle.h` declares. +//! +//! On POSIX the same crate builds `libpython313.{so,dylib}`; it is +//! compiled everywhere (keeping the export surface honest on every +//! `cargo test --workspace`) but only the Windows artifact ships — +//! the POSIX distribution keeps its fully-static binary (RFC 0064 +//! Non-goals). + +use std::ffi::{c_char, c_int, CStr}; + +/// Run the WeavePy CLI against the process's real argv and +/// environment, returning the exit code. The `weavepy.exe` shim's +/// whole job is `GetProcAddress(dll, "weavepy_main")` + call. +#[no_mangle] +pub extern "C" fn weavepy_main() -> c_int { + weavepy_cli::cli_main() +} + +/// CPython's wide-argv embedding entry point. +/// +/// Decodes `argv` (UTF-16 on Windows, UTF-32 elsewhere — `wchar_t`'s +/// platform width) and runs the CLI with it. Ill-formed sequences +/// decode lossily (U+FFFD), matching how WeavePy's own Windows argv +/// path treats non-Unicode argv today. +/// +/// # Safety +/// +/// `argv` must point to `argc` valid NUL-terminated `wchar_t` +/// strings, per the CPython contract. +#[no_mangle] +pub unsafe extern "C" fn Py_Main(argc: c_int, argv: *mut *mut libc::wchar_t) -> c_int { + let mut args: Vec = Vec::new(); + if !argv.is_null() { + for i in 0..usize::try_from(argc.max(0)).unwrap_or(0) { + // SAFETY: caller guarantees `argc` valid entries. + let arg = unsafe { *argv.add(i) }; + if arg.is_null() { + break; + } + // SAFETY: caller guarantees NUL termination. + args.push(unsafe { decode_wide_arg(arg) }); + } + } + weavepy_cli::cli_main_with_args(args) +} + +/// CPython's byte-argv embedding entry point (PEP 587's +/// `Py_BytesMain`). Bytes are decoded as UTF-8; undecodable bytes +/// decode lossily (U+FFFD) — the embedding twin of the CLI's +/// Windows argv posture. (The POSIX CLI's PEP 383 surrogateescape +/// bridge applies to *process* argv; embedders passing non-UTF-8 +/// argv through this entry point get the lossy decode, documented +/// here rather than silently diverging per platform.) +/// +/// # Safety +/// +/// `argv` must point to `argc` valid NUL-terminated C strings, per +/// the CPython contract. +#[no_mangle] +pub unsafe extern "C" fn Py_BytesMain(argc: c_int, argv: *mut *mut c_char) -> c_int { + let mut args: Vec = Vec::new(); + if !argv.is_null() { + for i in 0..usize::try_from(argc.max(0)).unwrap_or(0) { + // SAFETY: caller guarantees `argc` valid entries. + let arg = unsafe { *argv.add(i) }; + if arg.is_null() { + break; + } + // SAFETY: caller guarantees NUL termination. + let bytes = unsafe { CStr::from_ptr(arg) }.to_bytes(); + args.push(String::from_utf8_lossy(bytes).into_owned()); + } + } + weavepy_cli::cli_main_with_args(args) +} + +/// Decode one NUL-terminated `wchar_t` string: UTF-16 where +/// `wchar_t` is 2 bytes (Windows), UTF-32 where it is 4 (POSIX); +/// lossy on ill-formed input. +/// +/// # Safety +/// +/// `ptr` must point to a valid NUL-terminated `wchar_t` string. +// `wchar_t` is u16 on Windows and i32 on POSIX; `as u32` is the one +// portable bridge (a negative POSIX unit wraps and `from_u32` rejects +// it as REPLACEMENT_CHARACTER, which is the lossy contract anyway). +#[allow(clippy::cast_lossless, clippy::cast_sign_loss)] +unsafe fn decode_wide_arg(ptr: *const libc::wchar_t) -> String { + let mut len = 0usize; + // SAFETY: caller guarantees NUL termination. + while unsafe { *ptr.add(len) } != 0 { + len += 1; + } + // SAFETY: `len` counted valid elements above. + let units = unsafe { std::slice::from_raw_parts(ptr, len) }; + if std::mem::size_of::() == 2 { + let units16: Vec = units.iter().map(|&u| u as u16).collect(); + String::from_utf16_lossy(&units16) + } else { + units + .iter() + .map(|&u| char::from_u32(u as u32).unwrap_or(char::REPLACEMENT_CHARACTER)) + .collect() + } +} + +#[cfg(test)] +mod tests { + /// The `/EXPORT` list in `build.rs` must exactly match the public + /// definitions in `weavepy-capi/src/varargs.c` — a drifted list + /// means a `.pyd` importing a variadic helper gets an unresolved + /// import on Windows. + #[test] + fn varargs_export_list_matches_c_source() { + let manifest = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let c_src = std::fs::read_to_string(manifest.join("../weavepy-capi/src/varargs.c")) + .expect("varargs.c readable"); + let build_rs = + std::fs::read_to_string(manifest.join("build.rs")).expect("build.rs readable"); + + // Public definitions: ` Name(` at column 0. The + // crash-handler helper is internal (the driver references it + // at DLL link time; no extension imports it). + let mut defined: Vec = Vec::new(); + for line in c_src.lines() { + let Some(rest) = line + .strip_prefix("PyObject *") + .or_else(|| line.strip_prefix("int ")) + .or_else(|| line.strip_prefix("void ")) + else { + continue; + }; + let Some(paren) = rest.find('(') else { + continue; + }; + let name = rest[..paren].trim(); + if name.is_empty() || !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + continue; + } + if name == "weavepy_install_crash_handler" { + continue; + } + defined.push(name.to_owned()); + } + defined.sort(); + defined.dedup(); + assert!( + defined.len() >= 20, + "suspiciously few public definitions parsed from varargs.c: {defined:?}" + ); + for name in &defined { + assert!( + build_rs.contains(&format!("\"{name}\"")), + "varargs.c defines `{name}` but build.rs does not export it" + ); + } + // And nothing exported that the C file doesn't define. + for line in build_rs.lines() { + let line = line.trim(); + let Some(name) = line.strip_prefix('"').and_then(|l| l.strip_suffix("\",")) else { + continue; + }; + assert!( + defined.iter().any(|d| d == name), + "build.rs exports `{name}` but varargs.c does not define it" + ); + } + } +} diff --git a/crates/weavepy-vm/build.rs b/crates/weavepy-vm/build.rs index b85d18b1..c24f6a8e 100644 --- a/crates/weavepy-vm/build.rs +++ b/crates/weavepy-vm/build.rs @@ -90,16 +90,18 @@ fn main() { } src.push_str("];\n"); - // The one generated-per-platform file in a real install. Windows - // ships CPython's static `PC/pyconfig.h`, which is out of scope - // this wave (RFC 0062 non-goal); the empty string means "keep the - // pre-0062 stub". + // The one generated-per-platform file in a real install. POSIX + // variants are real autoconf outputs; the Windows variant mirrors + // CPython's hand-maintained `PC/pyconfig.h` (no autoconf on NT), + // including the `python313.lib` MSVC autolink pragma (RFC 0064 + // WS3). `None` (other platforms) keeps the pre-0062 stub. let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); let pyconfig = match target_os.as_str() { "macos" => Some(capi_include.join("pyconfig").join("pyconfig-macos.h")), "linux" | "freebsd" | "android" => { Some(capi_include.join("pyconfig").join("pyconfig-linux.h")) } + "windows" => Some(capi_include.join("pyconfig").join("pyconfig-windows.h")), _ => None, }; match pyconfig { diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 6dd40313..b01201d8 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -5685,6 +5685,14 @@ impl PyFile { return Err(os_error("not readable")); } (FileBackend::Stdin, None) => { + // RFC 0064 WS4 — a real console reads through the + // UTF-16 bridge (`ReadConsoleW` + Ctrl-Z EOF), so + // interactive input carries the full Unicode range + // regardless of the console codepage. + #[cfg(windows)] + if let Some(result) = crate::stdlib::win_console::stdin_console_read(None) { + return result; + } // Raw bytes (not `read_to_string`, which would reject non-UTF-8 // binary data piped to `sys.stdin.buffer`). Text-mode callers // decode the result via `decode_text`. @@ -5693,6 +5701,10 @@ impl PyFile { .map_err(|e| os_error(format!("read: {e}")))?; } (FileBackend::Stdin, Some(n)) => { + #[cfg(windows)] + if let Some(result) = crate::stdlib::win_console::stdin_console_read(Some(n)) { + return result; + } // Read up to `n` bytes, looping past short reads (pipes deliver // data in fragments) until we have `n` or hit EOF — matching // `BufferedReader.read(n)`. Honouring `n` (instead of draining @@ -6020,6 +6032,14 @@ impl PyFile { s.len() } FileBackend::Stdout(sink) => { + // RFC 0064 WS4 — a real console gets the UTF-16 bridge + // (`WriteConsoleW`), so the full Unicode range renders + // regardless of the console codepage; redirected fds + // keep the byte sink below. + #[cfg(windows)] + if let Some(result) = crate::stdlib::win_console::console_write(1, data) { + return result; + } let mut s = sink.borrow_mut(); let n = s .write(data) @@ -6035,6 +6055,10 @@ impl PyFile { n } FileBackend::Stderr(sink) => { + #[cfg(windows)] + if let Some(result) = crate::stdlib::win_console::console_write(2, data) { + return result; + } let mut s = sink.borrow_mut(); let n = s .write(data) diff --git a/crates/weavepy-vm/src/stdlib/io.rs b/crates/weavepy-vm/src/stdlib/io.rs index 269e3f3c..d889d052 100644 --- a/crates/weavepy-vm/src/stdlib/io.rs +++ b/crates/weavepy-vm/src/stdlib/io.rs @@ -2286,8 +2286,9 @@ fn iobase_readlines(args: &[Object]) -> Result { /// Extract a writable byte buffer (`bytearray` or a writable, contiguous /// `memoryview` over one) as `(storage, start, capacity)` — the argument shape -/// CPython's `readinto`/`readinto1` accept. -fn readinto_writable_buffer( +/// CPython's `readinto`/`readinto1` accept. `pub(crate)`: +/// `_WindowsConsoleIO.readinto` (RFC 0064 WS4) shares it. +pub(crate) fn readinto_writable_buffer( arg: Option<&Object>, ) -> Result<(Rc>>, usize, usize), RuntimeError> { match arg { diff --git a/crates/weavepy-vm/src/stdlib/io_full.rs b/crates/weavepy-vm/src/stdlib/io_full.rs index eb25ddb3..576960b3 100644 --- a/crates/weavepy-vm/src/stdlib/io_full.rs +++ b/crates/weavepy-vm/src/stdlib/io_full.rs @@ -108,6 +108,16 @@ pub fn build(cache: &ModuleCache) -> Rc { ); } + // `_WindowsConsoleIO` (RFC 0064 WS4) — Windows only, exactly as + // CPython's `_io` omits it elsewhere (`hasattr(_io, + // '_WindowsConsoleIO')` is how `_pyio` and test_winconsoleio + // detect the platform). + #[cfg(windows)] + d.insert( + DictKey(Object::from_static("_WindowsConsoleIO")), + Object::Type(crate::stdlib::win_console::windows_console_io_type()), + ); + // CPython exposes the buffer-size default and a couple of // module-level constants. Keep parity for code that reads // `_io.DEFAULT_BUFFER_SIZE`. diff --git a/crates/weavepy-vm/src/stdlib/mod.rs b/crates/weavepy-vm/src/stdlib/mod.rs index 0fb1fc3f..fa111a0c 100644 --- a/crates/weavepy-vm/src/stdlib/mod.rs +++ b/crates/weavepy-vm/src/stdlib/mod.rs @@ -49,8 +49,13 @@ pub mod math; // stdlib consumes. #[cfg(windows)] pub mod msvcrt_mod; +// `pub` (not `pub(crate)`): `weavepy-capi`'s extension loader reuses +// the `format_message` strerror source for CPython's "DLL load failed +// while importing …" ImportError shape (RFC 0064 WS2). #[cfg(windows)] -pub(crate) mod nt_support; +pub mod nt_support; +// `_WindowsConsoleIO` + the ReadConsoleW/WriteConsoleW byte bridge +// the PyFile stdio monolith reroutes through (RFC 0064 WS4). pub mod operator_accel; pub mod os; pub mod os_process; @@ -89,6 +94,8 @@ pub mod unicodedata_mod; pub mod weakref_mod; pub mod weave_frame_mod; #[cfg(windows)] +pub(crate) mod win_console; +#[cfg(windows)] pub mod winapi_mod; #[cfg(windows)] pub mod winreg_mod; diff --git a/crates/weavepy-vm/src/stdlib/nt_support.rs b/crates/weavepy-vm/src/stdlib/nt_support.rs index 30778737..edbee73a 100644 --- a/crates/weavepy-vm/src/stdlib/nt_support.rs +++ b/crates/weavepy-vm/src/stdlib/nt_support.rs @@ -222,8 +222,9 @@ pub(crate) fn winerror_to_errno(winerror: i32) -> i32 { /// `FormatMessageW` for a Win32 (or Winsock) error code, with /// CPython's trims: trailing CR/LF/dot whitespace removed. Falls back -/// to the CPython shape for unknown codes. -pub(crate) fn format_message(winerror: i32) -> String { +/// to the CPython shape for unknown codes. `pub`: also the strerror +/// source for the extension loader's `ImportError` (RFC 0064 WS2). +pub fn format_message(winerror: i32) -> String { use windows_sys::Win32::System::Diagnostics::Debug::{ FormatMessageW, FORMAT_MESSAGE_FROM_SYSTEM, FORMAT_MESSAGE_IGNORE_INSERTS, }; diff --git a/crates/weavepy-vm/src/stdlib/os.rs b/crates/weavepy-vm/src/stdlib/os.rs index dd48e9ef..df3ef7df 100644 --- a/crates/weavepy-vm/src/stdlib/os.rs +++ b/crates/weavepy-vm/src/stdlib/os.rs @@ -764,6 +764,16 @@ pub fn build(cache: &ModuleCache) -> Rc { DictKey(Object::from_static("_path_splitroot_ex")), builtin("_path_splitroot_ex", nt_path_splitroot_ex), ); + // RFC 0064 WS2 — `os.add_dll_directory` (PEP 578-audited + // `AddDllDirectory`). Binary wheels' `__init__` shims call it + // to make vendored dependent DLLs resolvable by the loader + // flags the extension loader passes (`LOAD_LIBRARY_SEARCH_ + // DEFAULT_DIRS` honours these cookies; `PATH`/CWD are not + // searched — bpo-36085). + d.insert( + DictKey(Object::from_static("add_dll_directory")), + builtin_kw("add_dll_directory", os_add_dll_directory), + ); } // `os.supports_follow_symlinks` must hold the *function objects* that @@ -5829,6 +5839,151 @@ fn os_fsync(args: &[Object]) -> Result { Ok(Object::None) } +/// `os.add_dll_directory(path)` — RFC 0064 WS2, CPython's +/// `Lib/os.py` + `nt._add_dll_directory`: fire the PEP 578 +/// `os.add_dll_directory` audit event, register the directory with +/// the loader (`AddDllDirectory`), and hand back an +/// `_AddedDllDirectory` whose `close()` (also `__exit__`) removes it +/// again. The API itself rejects relative/nonexistent paths, which +/// surfaces as the CPython-shaped `OSError`. +#[cfg(windows)] +fn os_add_dll_directory( + args: &[Object], + kwargs: &[(String, Object)], +) -> Result { + use crate::stdlib::nt_support::wide; + use windows_sys::Win32::System::LibraryLoader::AddDllDirectory; + let path = path_arg_or_kw(args, 0, "path", kwargs, "add_dll_directory")?; + crate::stdlib::sys::audit_event("os.add_dll_directory", &[Object::from_str(path.clone())])?; + let wpath = wide(&path); + // SAFETY: `wpath` is NUL-terminated UTF-16 and outlives the call. + let cookie = unsafe { AddDllDirectory(wpath.as_ptr()) }; + if cookie.is_null() { + return Err(crate::stdlib::nt_support::last_win32_error_to_py(Some( + &path, + ))); + } + Ok(build_added_dll_directory(path, cookie as usize as i64)) +} + +/// The shared `_AddedDllDirectory` type: `close()`, context-manager +/// protocol, and CPython's repr (``, +/// `` once closed). State lives on the instance +/// (`_path`, `_cookie`); `close()` mirrors CPython in calling +/// `RemoveDllDirectory` unconditionally, so a double close raises +/// `OSError` exactly as a stale cookie does there. +#[cfg(windows)] +fn added_dll_directory_type() -> Rc { + use crate::object::BuiltinFn; + use crate::types::TypeObject; + thread_local! { + static CLS: RefCell>> = const { RefCell::new(None) }; + } + fn self_dict(args: &[Object]) -> Option>> { + match args.first() { + Some(Object::Instance(i)) => Some(i.dict.clone()), + _ => None, + } + } + fn close_impl(args: &[Object]) -> Result { + use windows_sys::Win32::System::LibraryLoader::RemoveDllDirectory; + let dict = self_dict(args) + .ok_or_else(|| type_error("close() requires an _AddedDllDirectory instance"))?; + let cookie = dict + .borrow() + .get(&DictKey(Object::from_static("_cookie"))) + .and_then(Object::as_i64) + .unwrap_or(0); + // SAFETY: the cookie came from `AddDllDirectory`; the API + // validates it and fails on anything stale. + let ok = unsafe { RemoveDllDirectory(cookie as usize as *mut std::ffi::c_void) }; + if ok == 0 { + return Err(crate::stdlib::nt_support::last_win32_error_to_py(None)); + } + dict.borrow_mut() + .insert(DictKey(Object::from_static("_path")), Object::None); + Ok(Object::None) + } + CLS.with(|slot| { + if let Some(c) = slot.borrow().as_ref() { + return c.clone(); + } + let bt = crate::builtin_types::builtin_types(); + let mut dict = DictData::default(); + dict.insert( + DictKey(Object::from_static("close")), + Object::Builtin(Rc::new(BuiltinFn { + name: "close", + binds_instance: true, + call: Box::new(close_impl), + call_kw: None, + })), + ); + dict.insert( + DictKey(Object::from_static("__enter__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__enter__", + binds_instance: true, + call: Box::new(|args| Ok(args.first().cloned().unwrap_or(Object::None))), + call_kw: None, + })), + ); + dict.insert( + DictKey(Object::from_static("__exit__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__exit__", + binds_instance: true, + call: Box::new(|args| { + close_impl(args)?; + Ok(Object::Bool(false)) + }), + call_kw: None, + })), + ); + dict.insert( + DictKey(Object::from_static("__repr__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__repr__", + binds_instance: true, + call: Box::new(|args| { + let path = self_dict(args) + .and_then(|d| { + d.borrow() + .get(&DictKey(Object::from_static("_path"))) + .cloned() + }) + .unwrap_or(Object::None); + Ok(match path { + Object::None => Object::from_static(""), + p => Object::from_str(format!("", p.repr())), + }) + }), + call_kw: None, + })), + ); + let cls = TypeObject::new_user("_AddedDllDirectory", vec![bt.object_.clone()], dict) + .expect("_AddedDllDirectory type"); + *slot.borrow_mut() = Some(cls.clone()); + cls + }) +} + +/// Mint one `_AddedDllDirectory` instance for [`os_add_dll_directory`]. +#[cfg(windows)] +fn build_added_dll_directory(path: String, cookie: i64) -> Object { + use crate::types::PyInstance; + let inst = Rc::new(PyInstance::new(added_dll_directory_type())); + { + let mut d = inst.dict.borrow_mut(); + d.insert( + DictKey(Object::from_static("_path")), + Object::from_str(path), + ); + d.insert(DictKey(Object::from_static("_cookie")), Object::Int(cookie)); + } + Object::Instance(inst) +} + /// Resolve an NT path helper argument preserving the `str`/`bytes` flavour /// (these mirror CPython's `path_t`-converted `nt._get*` helpers, which /// return the same type they were given). diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs index 807265f4..53ac42fe 100644 --- a/crates/weavepy-vm/src/stdlib/sys.rs +++ b/crates/weavepy-vm/src/stdlib/sys.rs @@ -470,11 +470,20 @@ pub fn build(cache: &ModuleCache) -> Rc { // compiler tokens are tagged `WeavePy`; `python_implementation()` // still reports `CPython` (no PyPy/Jython/IronPython marker), so // implementation-gated stdlib tests behave as on CPython. + // + // On Windows the compiler token also carries CPython's MSC arch + // tag (`64 bit (AMD64)` / `64 bit (ARM64)`): that substring is + // what `sysconfig.get_platform()` sniffs to answer `win-amd64` + // — the value setuptools bakes into wheel tags and build dirs + // (RFC 0064 WS3). Without it the platform reads as `win32`. d.insert( DictKey(Object::from_static("version")), Object::from_str(format!( - "{}.{}.{} (WeavePy) [WeavePy]", - PY_VERSION.0, PY_VERSION.1, PY_VERSION.2 + "{}.{}.{} (WeavePy) [WeavePy{}]", + PY_VERSION.0, + PY_VERSION.1, + PY_VERSION.2, + version_arch_tag() )), ); d.insert( @@ -495,11 +504,18 @@ pub fn build(cache: &ModuleCache) -> Rc { DictKey(Object::from_static("winver")), Object::from_static("3.13"), ); - // CPython publishes the HMODULE of python3xx.dll here. WeavePy - // is a static executable with no python DLL (RFC 0063 - // Non-goals: the `python313.dll` restructure is its own wave), - // so the handle is 0. - d.insert(DictKey(Object::from_static("dllhandle")), Object::Int(0)); + // CPython publishes the HMODULE of python3xx.dll here. Since + // RFC 0064 the runtime ships as a real `python313.dll` loaded + // by the `weavepy.exe` shim, so the handle is the module's: + // nonzero whenever this interpreter is running out of the DLL + // (the shipped configuration), 0 in a statically-linked + // embedder (e.g. Rust test harnesses) — the truthful answer + // for a process with no Python DLL. `ctypes.pythonapi` + // constructs against this handle. + d.insert( + DictKey(Object::from_static("dllhandle")), + Object::Int(python_dll_handle()), + ); d.insert( DictKey(Object::from_static("getwindowsversion")), builtin("getwindowsversion", sys_getwindowsversion), @@ -796,6 +812,22 @@ pub fn build(cache: &ModuleCache) -> Rc { }) } +/// CPython's NT compiler-bracket arch suffix (`[MSC v.19xx 64 bit +/// (AMD64)]`), reduced to the part `sysconfig.get_platform()` and +/// `platform.architecture()` actually sniff. Empty off Windows — +/// POSIX `get_platform()` reads `os.uname()`, not `sys.version`. +const fn version_arch_tag() -> &'static str { + if cfg!(all(windows, target_arch = "x86_64")) { + " 64 bit (AMD64)" + } else if cfg!(all(windows, target_arch = "aarch64")) { + " 64 bit (ARM64)" + } else if cfg!(all(windows, target_arch = "x86")) { + " 32 bit (Intel)" + } else { + "" + } +} + fn host_platform() -> &'static str { if cfg!(target_os = "linux") { "linux" @@ -1570,6 +1602,36 @@ const VERSION_INFO_FIELDS: &[&str] = &["major", "minor", "micro", "releaselevel" #[cfg(windows)] const WINDOWS_VERSION_VISIBLE: [&str; 5] = ["major", "minor", "build", "platform", "service_pack"]; +/// The `HMODULE` of `python313.dll` when this interpreter is running +/// out of the runtime DLL (RFC 0064 WS1: the shipped exe is a shim +/// that loads it), 0 when statically linked (embedder test harnesses). +/// `GetModuleHandleW` peeks at the process's loaded-module list +/// without loading anything and without taking a reference. +#[cfg(windows)] +fn python_dll_handle() -> i64 { + use windows_sys::Win32::System::LibraryLoader::GetModuleHandleW; + // "python313.dll" as static UTF-16, NUL-terminated. + const NAME: &[u16] = &[ + b'p' as u16, + b'y' as u16, + b't' as u16, + b'h' as u16, + b'o' as u16, + b'n' as u16, + b'3' as u16, + b'1' as u16, + b'3' as u16, + b'.' as u16, + b'd' as u16, + b'l' as u16, + b'l' as u16, + 0, + ]; + // SAFETY: NAME is a valid NUL-terminated UTF-16 string. + let handle = unsafe { GetModuleHandleW(NAME.as_ptr()) }; + handle as usize as i64 +} + /// `sys.getwindowsversion()` — the 10-member struct sequence of /// `Python/sysmodule.c`'s `sys_getwindowsversion_impl`. Sourced from /// ntdll's `RtlGetVersion` rather than `GetVersionExW`: the latter lies diff --git a/crates/weavepy-vm/src/stdlib/win_console.rs b/crates/weavepy-vm/src/stdlib/win_console.rs new file mode 100644 index 00000000..436a438e --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/win_console.rs @@ -0,0 +1,772 @@ +//! `_WindowsConsoleIO` and the console byte bridge (RFC 0064 WS4). +//! +//! CPython's `Modules/_io/winconsoleio.c` exists because the Windows +//! console is not a byte stream: bytes written with `WriteFile` are +//! interpreted in the console *codepage* (usually not UTF-8), so +//! anything outside it mojibakes. The fix is to talk to the console in +//! UTF-16 — `ReadConsoleW`/`WriteConsoleW` — and present UTF-8 at the +//! Python-visible edge. +//! +//! Two consumers share the bridge here: +//! +//! 1. **`_io._WindowsConsoleIO`** — the raw-io type itself (reachable +//! the CPython way, `_io._WindowsConsoleIO(fd_or_path, mode)`), +//! registered by `io_full::build` on Windows only. +//! 2. **The native `PyFile` stdio monolith** — WeavePy's std streams +//! are one native object, not CPython's three-layer stack (a +//! documented RFC 0050/0053 divergence). `object.rs` routes +//! `Stdin`/`Stdout`/`Stderr` backends through +//! [`stdin_console_read`]/[`console_write`] when the fd is a real +//! console, so interactive I/O round-trips the full Unicode range +//! regardless of codepage — CPython-faithful *behavior* through +//! WeavePy-shaped plumbing. Redirected/piped fds (everything CI +//! sees) fail the `GetConsoleMode` probe and keep the RFC 0063 +//! byte paths untouched. + +use std::ffi::c_void; +use std::sync::Mutex; + +use crate::error::{type_error, value_error, RuntimeError}; +use crate::object::{BuiltinFn, DictData, DictKey, Object, StrKey}; +use crate::stdlib::nt_support::{crt, last_win32_error_to_py, win32_error_to_py}; +use crate::sync::Rc; +use crate::sync::RefCell; +use crate::types::{PyInstance, TypeFlags, TypeObject}; + +use windows_sys::Win32::Foundation::{ + GetLastError, ERROR_OPERATION_ABORTED, GENERIC_READ, GENERIC_WRITE, INVALID_HANDLE_VALUE, +}; +use windows_sys::Win32::System::Console::{ + GetConsoleMode, GetNumberOfConsoleInputEvents, ReadConsoleW, WriteConsoleW, +}; + +// --------------------------------------------------------------------------- +// Console detection. +// --------------------------------------------------------------------------- + +/// The OS handle behind a CRT fd when that fd is a *real* console +/// (`GetConsoleMode` succeeds), `None` for anything redirected — the +/// probe CPython's `_PyIO_get_console_type` builds on. Checked per +/// operation, not cached: `os.dup2` can repoint a std fd at a file +/// mid-process and the byte path must follow it. +pub(crate) fn console_handle(fd: i32) -> Option { + let handle = unsafe { crt::_get_osfhandle(fd) }; + if handle == -1 || handle == -2 { + return None; + } + let mut mode = 0u32; + (unsafe { GetConsoleMode(handle as *mut c_void, &raw mut mode) } != 0).then_some(handle) +} + +/// Classify a console handle as input (`'r'`) or screen (`'w'`) — +/// CPython's trick: only input handles answer +/// `GetNumberOfConsoleInputEvents`. +fn console_kind(handle: isize) -> Option { + let mut mode = 0u32; + if unsafe { GetConsoleMode(handle as *mut c_void, &raw mut mode) } == 0 { + return None; + } + let mut events = 0u32; + if unsafe { GetNumberOfConsoleInputEvents(handle as *mut c_void, &raw mut events) } != 0 { + Some('r') + } else { + Some('w') + } +} + +/// The console type a *path* names (`_PyIO_get_console_type`): +/// `CONIN$` is input, `CONOUT$` is screen, `CON` is either (resolved +/// from the opened handle). Accepts the `\\.\` device prefix. +fn path_console_kind(path: &str) -> Option { + let leaf = path + .strip_prefix("\\\\.\\") + .or_else(|| path.strip_prefix("//./")) + .unwrap_or(path); + if leaf.eq_ignore_ascii_case("CONIN$") { + Some('r') + } else if leaf.eq_ignore_ascii_case("CONOUT$") { + Some('w') + } else if leaf.eq_ignore_ascii_case("CON") { + Some('x') + } else { + None + } +} + +// --------------------------------------------------------------------------- +// The UTF-16 bridge: write. +// --------------------------------------------------------------------------- + +/// CPython's per-call ceiling (winconsoleio.c `BUFMAX` rationale): +/// `WriteConsoleW` over ~32766 wchars can fail with not-enough-memory. +const WCHAR_CHUNK: usize = 32766; + +/// Bytes of an incomplete UTF-8 sequence dangling at the end of `data` +/// (0 when the tail is complete). CPython's `_find_last_utf8_boundary`: +/// a `BufferedWriter` chunk may split a character, and the split tail +/// must stay unconsumed for the next write rather than mojibake. +fn trailing_incomplete_utf8(data: &[u8]) -> usize { + let n = data.len(); + for back in 1..=n.min(3) { + let b = data[n - back]; + if b & 0xC0 == 0x80 { + continue; // continuation byte — keep scanning for the lead + } + let need = if b >= 0xF0 { + 4 + } else if b >= 0xE0 { + 3 + } else if b >= 0xC0 { + 2 + } else { + 1 + }; + return if need > back { back } else { 0 }; + } + 0 +} + +/// Write `data` to a console handle via `WriteConsoleW`, returning the +/// count of *bytes consumed*. Whole characters only: a trailing split +/// UTF-8 sequence is left for the caller's next write (unless the +/// buffer holds nothing else — then it is written with U+FFFD, exactly +/// what `MultiByteToWideChar` without `MB_ERR_INVALID_CHARS` does with +/// invalid bytes mid-buffer too). +pub(crate) fn write_console(handle: isize, data: &[u8]) -> Result { + if data.is_empty() { + return Ok(0); + } + let mut consume = data.len() - trailing_incomplete_utf8(data); + if consume == 0 { + consume = data.len(); + } + let text = String::from_utf8_lossy(&data[..consume]); + let wide: Vec = text.encode_utf16().collect(); + let mut off = 0usize; + while off < wide.len() { + let chunk = (wide.len() - off).min(WCHAR_CHUNK); + let mut written = 0u32; + let ok = unsafe { + WriteConsoleW( + handle as *mut c_void, + wide[off..].as_ptr().cast(), + chunk as u32, + &raw mut written, + std::ptr::null(), + ) + }; + if ok == 0 { + return Err(win32_error_to_py(unsafe { GetLastError() } as i32, None)); + } + if written == 0 { + break; + } + off += written as usize; + } + Ok(consume) +} + +/// Route a `PyFile` `Stdout`/`Stderr` write through the console bridge +/// when the fd is a real console; `None` keeps the ordinary sink path. +pub(crate) fn console_write(fd: i32, data: &[u8]) -> Option> { + console_handle(fd).map(|handle| write_console(handle, data)) +} + +// --------------------------------------------------------------------------- +// The UTF-16 bridge: read. +// --------------------------------------------------------------------------- + +/// wchars per `ReadConsoleW` request for unbounded reads. +const READ_WCHARS: usize = 8192; + +/// Run any tripped Python signal handler on the main thread (the +/// `os`/`socket` blocking-call pattern): a `ReadConsoleW` aborted by +/// Ctrl-C surfaces the handler's `KeyboardInterrupt` here. +fn service_pending_signals() -> Result<(), RuntimeError> { + if !crate::gil::is_main_thread() || !crate::stdlib::signal_mod::signals_pending() { + return Ok(()); + } + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by the active builtin call on this (main) + // thread; the interpreter outlives this synchronous call. + let interp = unsafe { &mut *ptr }; + interp.run_pending_signals_public()?; + } + Ok(()) +} + +/// One `ReadConsoleW` request, transcoded to UTF-8. Empty means EOF +/// (Ctrl-Z at the start of the read, winconsoleio.c) or a Ctrl-C whose +/// handler chose not to raise. A Ctrl-C aborting the read runs the +/// Python handler after CPython's 100ms grace sleep — the default +/// SIGINT handler raises `KeyboardInterrupt` out of here. +fn read_chunk(handle: isize, nwchars: usize) -> Result, RuntimeError> { + let mut wbuf = vec![0u16; nwchars.clamp(1, READ_WCHARS)]; + let mut read = 0u32; + let ok = unsafe { + ReadConsoleW( + handle as *mut c_void, + wbuf.as_mut_ptr().cast(), + wbuf.len() as u32, + &raw mut read, + std::ptr::null(), + ) + }; + if ok == 0 { + let err = unsafe { GetLastError() }; + if err == ERROR_OPERATION_ABORTED { + std::thread::sleep(std::time::Duration::from_millis(100)); + service_pending_signals()?; + return Ok(Vec::new()); + } + return Err(win32_error_to_py(err as i32, None)); + } + let wchars = &wbuf[..read as usize]; + if wchars.first() == Some(&0x1a) { + return Ok(Vec::new()); // Ctrl-Z: EOF + } + Ok(String::from_utf16_lossy(wchars).into_bytes()) +} + +/// Read console bytes with a caller-owned carry buffer (`pending` +/// holds UTF-8 spill: one wchar can decode to more bytes than the +/// caller asked for). `None` reads to EOF (Ctrl-Z); `Some(n)` returns +/// up to `n` bytes, stopping at a completed line — the console cooks +/// input per line, and blocking past Enter would hang `read(n)` on an +/// interactive prompt. +pub(crate) fn read_console( + handle: isize, + pending: &mut Vec, + n: Option, +) -> Result, RuntimeError> { + match n { + Some(n) => { + while pending.len() < n { + let want = n - pending.len(); + let chunk = read_chunk(handle, want)?; + if chunk.is_empty() { + break; + } + pending.extend_from_slice(&chunk); + if pending.ends_with(b"\n") { + break; + } + } + let take = n.min(pending.len()); + Ok(pending.drain(..take).collect()) + } + None => { + loop { + let chunk = read_chunk(handle, READ_WCHARS)?; + if chunk.is_empty() { + break; + } + pending.extend_from_slice(&chunk); + } + Ok(std::mem::take(pending)) + } + } +} + +/// Carry buffer for the `PyFile` `Stdin` bridge (fd 0 is a singleton; +/// the spill must survive across `read(1)` probes from +/// `readline_unbounded`). +static STDIN_PENDING: Mutex> = Mutex::new(Vec::new()); + +/// Route a `PyFile` `Stdin` read through the console bridge when fd 0 +/// is a real console; `None` keeps the ordinary byte path. +pub(crate) fn stdin_console_read(n: Option) -> Option, RuntimeError>> { + let handle = console_handle(0)?; + let mut pending = STDIN_PENDING.lock().unwrap_or_else(|e| e.into_inner()); + Some(read_console(handle, &mut pending, n)) +} + +// --------------------------------------------------------------------------- +// The `_io._WindowsConsoleIO` type. +// --------------------------------------------------------------------------- + +fn wcio_self(args: &[Object]) -> Result, RuntimeError> { + match args.first() { + Some(Object::Instance(i)) => Ok(i.clone()), + _ => Err(type_error( + "unbound method _WindowsConsoleIO requires a _WindowsConsoleIO instance", + )), + } +} + +fn wcio_get(inst: &PyInstance, name: &str) -> Option { + inst.dict.borrow().get(&StrKey(name)).cloned() +} + +fn wcio_set(inst: &PyInstance, name: &'static str, value: Object) { + inst.dict + .borrow_mut() + .insert(DictKey(Object::from_static(name)), value); +} + +fn wcio_fd(inst: &PyInstance) -> i64 { + wcio_get(inst, "_fd").and_then(|o| o.as_i64()).unwrap_or(-1) +} + +fn wcio_flag(inst: &PyInstance, name: &str) -> bool { + matches!(wcio_get(inst, name), Some(Object::Bool(true))) +} + +fn wcio_check_open(inst: &PyInstance) -> Result { + let fd = wcio_fd(inst); + if fd < 0 { + return Err(value_error("I/O operation on closed file.")); + } + Ok(fd as i32) +} + +fn wcio_console_handle_checked(inst: &PyInstance) -> Result { + let fd = wcio_check_open(inst)?; + console_handle(fd).ok_or_else(|| value_error("Cannot open non-console file")) +} + +/// `__init__(file, mode='r', closefd=True, opener=None)` — CPython's +/// `_io__WindowsConsoleIO___init___impl`. +fn wcio_init(args: &[Object], kwargs: &[(String, Object)]) -> Result { + let inst = wcio_self(args)?; + let positional = &args[1..]; + let kw = |name: &str| { + kwargs + .iter() + .find(|(k, _)| k == name) + .map(|(_, v)| v.clone()) + }; + let file = positional + .first() + .cloned() + .or_else(|| kw("file")) + .ok_or_else(|| { + type_error("_WindowsConsoleIO() missing required argument 'file' (pos 1)") + })?; + let mode = match positional.get(1).cloned().or_else(|| kw("mode")) { + Some(Object::Str(s)) => s.to_string(), + None => "r".to_owned(), + Some(other) => { + return Err(type_error(format!( + "argument 2 must be str, not {}", + other.type_name() + ))) + } + }; + let closefd = match positional.get(2).cloned().or_else(|| kw("closefd")) { + Some(v) => v.is_truthy(), + None => true, + }; + let opener = positional.get(3).cloned().or_else(|| kw("opener")); + + // Mode chars: 'b' is a no-op, exactly one of 'r'/'w' picks the + // direction, anything else is CPython's ValueError. + let mut readable = false; + let mut writable = false; + for c in mode.chars() { + match c { + 'b' => {} + 'r' => readable = true, + 'w' | 'a' | 'x' => writable = true, + _ => return Err(value_error(format!("invalid mode: {mode}"))), + } + } + if readable == writable { + return Err(value_error("Console buffer must be readable or writable")); + } + let wanted = if readable { 'r' } else { 'w' }; + + crate::stdlib::sys::audit_event( + "open", + &[ + file.clone(), + Object::from_str(mode.clone()), + Object::Int(i64::from(closefd)), + ], + )?; + + let (fd, kind) = match &file { + Object::Int(fd) => { + let fd = *fd as i32; + let handle = unsafe { crt::_get_osfhandle(fd) }; + if handle == -1 || handle == -2 { + return Err(crate::stdlib::nt_support::crt_error_to_py( + crate::py_errno::EBADF, + None, + )); + } + let kind = + console_kind(handle).ok_or_else(|| value_error("Cannot open non-console file"))?; + (fd, kind) + } + Object::Str(path) => { + if !closefd { + return Err(value_error("Cannot use closefd=False with file name")); + } + let path = path.to_string(); + let named_kind = path_console_kind(&path); + let fd = match &opener { + Some(op) if !matches!(op, Object::None) => { + // CPython honors a custom opener even here; it must + // return a console fd (validated below). + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| value_error("no running interpreter for opener call"))?; + // SAFETY: published by the enclosing VM frame on this thread. + let interp = unsafe { &mut *ptr }; + let flags = if readable { + crt::O_RDONLY | crt::O_BINARY + } else { + crt::O_WRONLY | crt::O_BINARY + }; + let result = interp.call_object( + op.clone(), + &[file.clone(), Object::Int(i64::from(flags))], + &[], + )?; + match result.as_i64() { + Some(fd) if fd >= 0 => fd as i32, + Some(_) => { + return Err(value_error("opener returned a negative file descriptor")) + } + None => return Err(type_error("expected integer from opener")), + } + } + _ => { + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_EXISTING, + }; + let wpath = crate::stdlib::nt_support::wide(&path); + // CPython opens read+write first (a console screen + // buffer wants both for mode probing), falling back + // to the mode's own access. + let mut handle = unsafe { + CreateFileW( + wpath.as_ptr(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + std::ptr::null(), + OPEN_EXISTING, + 0, + std::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + let access = if readable { + GENERIC_READ + } else { + GENERIC_WRITE + }; + handle = unsafe { + CreateFileW( + wpath.as_ptr(), + access, + FILE_SHARE_READ | FILE_SHARE_WRITE, + std::ptr::null(), + OPEN_EXISTING, + 0, + std::ptr::null_mut(), + ) + }; + } + if handle == INVALID_HANDLE_VALUE { + return Err(last_win32_error_to_py(Some(&path))); + } + let fd = + unsafe { crt::_open_osfhandle(handle as crt::intptr_t, crt::O_BINARY) }; + if fd < 0 { + unsafe { + windows_sys::Win32::Foundation::CloseHandle(handle); + } + return Err(crate::stdlib::nt_support::last_crt_error_to_py(Some(&path))); + } + fd + } + }; + let handle = unsafe { crt::_get_osfhandle(fd) }; + let kind = named_kind + .filter(|k| *k != 'x') + .or_else(|| console_kind(handle)); + let Some(kind) = kind else { + if closefd { + unsafe { + crt::_close(fd); + } + } + return Err(value_error("Cannot open non-console file")); + }; + (fd, kind) + } + other => { + return Err(type_error(format!( + "expected int or str, not {}", + other.type_name(), + ))) + } + }; + + // Direction mismatch is CPython's exact pair of messages. + if kind == 'r' && wanted == 'w' { + return Err(value_error("Cannot open console input buffer for writing")); + } + if kind == 'w' && wanted == 'r' { + return Err(value_error("Cannot open console output buffer for reading")); + } + + wcio_set(&inst, "_fd", Object::Int(i64::from(fd))); + wcio_set(&inst, "_readable", Object::Bool(readable)); + wcio_set(&inst, "_writable", Object::Bool(writable)); + wcio_set(&inst, "_closefd", Object::Bool(closefd)); + wcio_set(&inst, "name", file); + wcio_set(&inst, "_pending", Object::new_bytes(Vec::new())); + Ok(Object::None) +} + +fn wcio_close(args: &[Object]) -> Result { + let inst = wcio_self(args)?; + let fd = wcio_fd(inst.as_ref()); + if fd >= 0 { + if wcio_flag(&inst, "_closefd") { + // CPython's internal_close ignores the CRT result too. + unsafe { + crt::_close(fd as i32); + } + } + wcio_set(&inst, "_fd", Object::Int(-1)); + } + Ok(Object::None) +} + +fn wcio_pending(inst: &PyInstance) -> Vec { + wcio_get(inst, "_pending") + .and_then(|o| o.as_bytes_view()) + .unwrap_or_default() +} + +fn wcio_read_impl(inst: &Rc, n: Option) -> Result { + if !wcio_flag(inst, "_readable") { + return Err(crate::stdlib::io::unsupported_op( + "File not open for reading", + )); + } + let handle = wcio_console_handle_checked(inst)?; + let mut pending = wcio_pending(inst); + let result = read_console(handle, &mut pending, n); + wcio_set(inst, "_pending", Object::new_bytes(pending)); + Ok(Object::new_bytes(result?)) +} + +fn wcio_read(args: &[Object]) -> Result { + let inst = wcio_self(args)?; + let n = match args.get(1) { + None | Some(Object::None) => None, + Some(v) => match v.as_i64() { + Some(n) if n < 0 => None, + Some(n) => Some(n as usize), + None => return Err(type_error("argument should be integer or None")), + }, + }; + wcio_read_impl(&inst, n) +} + +fn wcio_readall(args: &[Object]) -> Result { + let inst = wcio_self(args)?; + wcio_read_impl(&inst, None) +} + +fn wcio_readinto(args: &[Object]) -> Result { + let inst = wcio_self(args)?; + let (dst, start, cap) = crate::stdlib::io::readinto_writable_buffer(args.get(1))?; + let data = wcio_read_impl(&inst, Some(cap))?; + let bytes = data.as_bytes_view().expect("read returns bytes"); + let n = bytes.len().min(cap); + dst.borrow_mut()[start..start + n].copy_from_slice(&bytes[..n]); + Ok(Object::Int(n as i64)) +} + +fn wcio_write(args: &[Object]) -> Result { + let inst = wcio_self(args)?; + if !wcio_flag(&inst, "_writable") { + return Err(crate::stdlib::io::unsupported_op( + "File not open for writing", + )); + } + let handle = wcio_console_handle_checked(&inst)?; + let data = args.get(1).and_then(|o| o.as_bytes_view()).ok_or_else(|| { + type_error(format!( + "a bytes-like object is required, not '{}'", + args.get(1).map_or("NoneType", |o| o.type_name()) + )) + })?; + let n = write_console(handle, &data)?; + Ok(Object::Int(n as i64)) +} + +/// Build the `_io._WindowsConsoleIO` type (memoised — one identity per +/// process, like the rest of the `_io` family). Bases on the shared +/// `RawIOBase` so the IOBase mixins (`__enter__`, `readline`, …) come +/// along, exactly as CPython's type inherits them. +pub(crate) fn windows_console_io_type() -> Rc { + use crate::object::MethodWrapper; + thread_local! { + static CLS: RefCell>> = const { RefCell::new(None) }; + } + CLS.with(|slot| { + if let Some(c) = slot.borrow().as_ref() { + return c.clone(); + } + let raw_base = crate::stdlib::io::build_iobase_family().raw.clone(); + let mut dict = DictData::default(); + let mut method = |n: &'static str, body: fn(&[Object]) -> Result| { + dict.insert( + DictKey(Object::from_static(n)), + Object::Builtin(Rc::new(BuiltinFn { + name: n, + binds_instance: true, + call: Box::new(body), + call_kw: None, + })), + ); + }; + method("read", wcio_read); + method("readall", wcio_readall); + method("readinto", wcio_readinto); + method("write", wcio_write); + method("close", wcio_close); + method("fileno", |args| { + let inst = wcio_self(args)?; + Ok(Object::Int(i64::from(wcio_check_open(&inst)?))) + }); + method("isatty", |args| { + let inst = wcio_self(args)?; + wcio_check_open(&inst)?; + Ok(Object::Bool(true)) + }); + method("readable", |args| { + let inst = wcio_self(args)?; + wcio_check_open(&inst)?; + Ok(Object::Bool(wcio_flag(&inst, "_readable"))) + }); + method("writable", |args| { + let inst = wcio_self(args)?; + wcio_check_open(&inst)?; + Ok(Object::Bool(wcio_flag(&inst, "_writable"))) + }); + method("seekable", |args| { + let inst = wcio_self(args)?; + wcio_check_open(&inst)?; + Ok(Object::Bool(false)) + }); + method("flush", |args| { + let inst = wcio_self(args)?; + wcio_check_open(&inst)?; + Ok(Object::None) + }); + method("__repr__", |args| { + let inst = wcio_self(args)?; + let mode = if wcio_flag(&inst, "_readable") { + "rb" + } else { + "wb" + }; + Ok(Object::from_str(format!( + "<_io._WindowsConsoleIO mode='{mode}' closefd={}>", + if wcio_flag(&inst, "_closefd") { + "True" + } else { + "False" + } + ))) + }); + dict.insert( + DictKey(Object::from_static("__new__")), + Object::StaticMethod(MethodWrapper::new(Object::Builtin(Rc::new(BuiltinFn { + name: "__new__", + binds_instance: false, + call: Box::new(wcio_new), + call_kw: Some(Box::new(|a, _kw| wcio_new(a))), + })))), + ); + dict.insert( + DictKey(Object::from_static("__init__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__init__", + binds_instance: true, + call: Box::new(|a| wcio_init(a, &[])), + call_kw: Some(Box::new(wcio_init)), + })), + ); + let ty = TypeObject::new_with_flags( + "_WindowsConsoleIO", + vec![raw_base], + dict, + TypeFlags { + is_exception: false, + is_builtin: true, + }, + ) + .expect("_WindowsConsoleIO type"); + // `closed` and `mode` are getset descriptors on the C type. + let getset = |name: &'static str, + body: fn(&[Object]) -> Result, + doc: &'static str| { + let prop = Object::Property(Rc::new(crate::object::PyProperty::new( + Object::Builtin(Rc::new(BuiltinFn { + name, + binds_instance: true, + call: Box::new(body), + call_kw: None, + })), + Object::None, + Object::None, + Object::from_static(doc), + ))); + crate::descr_registry::register( + &prop, + crate::descr_registry::DescrKind::GetSet, + ty.clone(), + name, + None, + ); + ty.dict + .borrow_mut() + .insert(DictKey(Object::from_static(name)), prop); + }; + getset( + "closed", + |args| { + let inst = wcio_self(args)?; + Ok(Object::Bool(wcio_fd(&inst) < 0)) + }, + "True if the file is closed", + ); + getset( + "mode", + |args| { + let inst = wcio_self(args)?; + Ok(Object::from_static(if wcio_flag(&inst, "_readable") { + "rb" + } else { + "wb" + })) + }, + "String giving the file mode", + ); + crate::stdlib::io::set_type_module(&ty, "_io"); + *slot.borrow_mut() = Some(ty.clone()); + ty + }) +} + +fn wcio_new(args: &[Object]) -> Result { + let cls = match args.first() { + Some(Object::Type(t)) => t.clone(), + _ => { + return Err(type_error( + "_WindowsConsoleIO.__new__(X): X is not a type object", + )) + } + }; + let inst = Object::Instance(Rc::new(PyInstance::new(cls))); + crate::gc_trace::track(inst.clone()); + Ok(inst) +} diff --git a/crates/weavepy-vm/src/stdlib_tree.rs b/crates/weavepy-vm/src/stdlib_tree.rs index 29eab906..442879f0 100644 --- a/crates/weavepy-vm/src/stdlib_tree.rs +++ b/crates/weavepy-vm/src/stdlib_tree.rs @@ -607,10 +607,10 @@ fn materialize(prefix: &Path) -> bool { std::fs::write( include_dir.join("pyconfig.h"), crate::cpython_headers::PYCONFIG_H.unwrap_or( - // Windows keeps the pre-0062 stub: C builds there - // await the python313.dll wave (RFC 0063 - // Non-goals — a static exe has nothing for a - // .pyd's PE import table to resolve against). + // Platforms without a real generated pyconfig + // (not macOS/Linux/Windows — those all embed one, + // Windows since RFC 0064 WS3) keep the pre-0062 + // stub. "/* Generated by WeavePy (RFC 0055); mirrors _sysconfigdata. */\n\ #define PY_VERSION_HEX 0x030d00f0\n\ #define SIZEOF_VOID_P 8\n\ diff --git a/crates/weavepy/src/lib.rs b/crates/weavepy/src/lib.rs index 5b4ba2c8..d35533fd 100644 --- a/crates/weavepy/src/lib.rs +++ b/crates/weavepy/src/lib.rs @@ -47,12 +47,19 @@ fn load_extension( let interp_ptr: *mut vm::Interpreter = interp; match capi::load_extension_module(interp_ptr, &path, full_name) { Ok(module) => Ok(Some(module)), - Err(err) => Err(vm::RuntimeError::PyException( - vm::PyException::from_builtin( - "ImportError", - format!("could not load extension '{full_name}': {err}"), - ), - )), + Err(err) => { + // The Windows load-failure variant is already CPython's + // exact ImportError text ("DLL load failed while + // importing X: …" — RFC 0064 WS2); surface it verbatim. + // Other failures keep the WeavePy-prefixed shape. + let message = match &err { + capi::loader::LoadError::DllLoadFailed { .. } => err.to_string(), + _ => format!("could not load extension '{full_name}': {err}"), + }; + Err(vm::RuntimeError::PyException( + vm::PyException::from_builtin("ImportError", message), + )) + } } } diff --git a/docs/rfcs/0064-windows-binary-abi-python313-dll.md b/docs/rfcs/0064-windows-binary-abi-python313-dll.md new file mode 100644 index 00000000..a2ae9f0b --- /dev/null +++ b/docs/rfcs/0064-windows-binary-abi-python313-dll.md @@ -0,0 +1,615 @@ +# RFC 0064: The python313.dll wave — Windows binary extensions, the runtime cdylib, MSVC builds, and console Unicode + +- **Status**: Accepted +- **Authors**: WeavePy authors +- **Created**: 2026-08-12 +- **Tracking issue**: TBD +- **Builds on**: RFC 0063 (the NT-native runtime this wave gives a + linkable ABI: CRT fds, `_winapi`/`msvcrt`/`winreg`/`_overlapped`, + the zip artifact, the advisory Windows lanes and the `measured_os` + stamp), RFC 0062 (the installable header tree, compiler-truthful + sysconfig, and the dist check matrix whose cext leg this wave + un-skips on NT), RFC 0043–0047 (the binary ABI: layout-faithful + mirrors, `PyType_FromSpec`, inline storage, real numpy/Cython — + everything a `.pyd` calls once its imports resolve), RFC 0022 + (the C-API foundation and its force-link discipline). + +## Summary + +RFC 0063 made WeavePy a real pure-Python interpreter on Windows and +drew one hard boundary: no C extensions, because a `.pyd` built for +CPython carries a PE import table naming `python313.dll`, and WeavePy +was a static executable with nothing for the loader to bind against. +This wave erases that boundary. The runtime moves into a +**`python313` cdylib** — a new `weavepy-pylib` crate whose Windows +artifact is a real `python313.dll` exporting the full ~682-symbol +C-API surface (the same `#[no_mangle]` set the RFC 0022 force-link +table already enumerates) plus a `weavepy_main` entry point — and +`weavepy.exe` becomes a **thin shim** that locates the DLL (its own +directory, then the `pyvenv.cfg` `home=` chain for venv copies), +loads it, and calls `weavepy_main`. POSIX keeps today's fully-static +binary and `--export-dynamic`/`dynamic_lookup` story, unchanged. +On top of the DLL land the consumers: `ExtensionFileLoader` gains +CPython's `LoadLibraryExW` search semantics and its +"`DLL load failed while importing X`" `ImportError` shape, +`os.add_dll_directory` arrives with the `_AddedDllDirectory` +context manager, the artifact ships `libs\python313.lib` (the +MSVC import library rustc already produces) plus a pyconfig.h that +autolinks it — so `pip install` of both **binary wheels** (numpy, +pandas: the PE import now resolves) and **C sdists** (setuptools → +MSVC → link against `libs\`) works mechanically — and the dist +`check` cext leg un-skips on Windows, compiling and importing a +`.pyd` end-to-end when MSVC is present. `_WindowsConsoleIO` +completes RFC 0063's deferred console-Unicode story with +`ReadConsoleW`/`WriteConsoleW`-backed interactive stdio. The lanes +stay advisory-until-measured exactly as RFC 0063 left them; the +flip-to-blocking baseline transplant remains the named follow-up, +now with the cext legs included in what it measures. + +## Motivation + +1. **The drop-in claim on Windows currently excludes the packages + people drop in for.** The RFC 0055/0056/0060 ecosystem story — + numpy, pandas, cryptography, orjson, charset_normalizer's mypyc + `.so` — is what "daily driver" means, and none of it can load on + Windows: `import numpy` finds the binary wheel's + `_multiarray_umath.cp313-win_amd64.pyd`, the loader maps it, and + the PE import of `python313.dll` fails before `PyInit_*` is ever + reachable. Every wave that grows the POSIX ecosystem matrix + widens the gap on the platform with the largest desktop install + base. + +2. **The boundary was drawn deliberately, and its other side was + prepared.** RFC 0063's Non-goals named this exact wave: "the + honest fix (restructuring the workspace so a `python313.dll` + cdylib exports the C-API and the exe links it) is its own wave." + The prerequisites are all landed: the header tree installs + (RFC 0062 WS2), `EXT_SUFFIX` is already truthful + (`.cp313-win_amd64.pyd`), the NT runtime beneath the C-API is + proven by the RFC 0063 test battery, and the `measured_os` + advisory machinery is sitting there waiting for the cext story + to become measurable. + +3. **The export mechanism already exists — it is just aimed at the + wrong binary format.** The C-API is ~682 `#[no_mangle]` symbols + kept alive by the `#[used]` `FORCE_LINK` table (RFC 0022) and + made dlopen-visible by `--export-dynamic` on Linux and Mach-O + default-export on macOS. PE is the one format where an + executable's symbols are invisible to the loader by default — + the same symbol set compiled into a cdylib is exported with no + new per-symbol work, because rustc builds a cdylib's export + list from exactly the reachable `#[no_mangle]` surface. + +4. **A static-exe Windows Python mis-signals toolchains.** setuptools + on Windows unconditionally links extensions against + `{base_exec_prefix}\libs\python313.lib`; the directory not + existing fails builds with a linker error users cannot act on. + `sys.dllhandle == 0` tells `ctypes.pythonapi` consumers there is + no Python DLL. Truthful signals require the DLL to exist. + +## CPython reference + +- **The DLL split**: on Windows, `python.exe` is a ~30 KB shim + (`Programs/python.c`) whose `wmain` calls `Py_Main` in + `python313.dll`; every C-API symbol lives in the DLL + (`PC/pyconfig.h` defines `MS_COREDLL`/`Py_ENABLE_SHARED`). + Extensions import `python313.dll` by name; the loader resolves it + to the already-loaded module in-process. +- **Import-library autolink**: `PC/pyconfig.h` emits + `#pragma comment(lib,"python313.lib")` when building non-core + code, so an extension's link step pulls the import library off + the `/LIBPATH` that distutils/setuptools point at + `{sys.base_exec_prefix}\libs` (`Lib/distutils/command/build_ext.py`, + preserved by setuptools' `_distutils`). +- **Extension loading**: `Python/dynload_win.c` calls + `LoadLibraryExW(path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | + LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR)` — dependent DLLs resolve from + the `.pyd`'s own directory, `AddDllDirectory` cookies, System32, + and application dir; **not** `PATH`, **not** CWD (CPython 3.8+ + behavior, bpo-36085). Failure raises + `ImportError("DLL load failed while importing {name}: {strerror}")` + with `name` set. +- **`os.add_dll_directory`**: `Lib/os.py` defines + `_AddedDllDirectory` (with `close()`, `__enter__`/`__exit__`, + `repr` as ``) over + `nt._add_dll_directory`/`nt._remove_dll_directory` + (`AddDllDirectory`/`RemoveDllDirectory` in + `Modules/posixmodule.c`); raises the `os.add_dll_directory` + audit event. +- **`sys.dllhandle`**: the `HMODULE` of `python313.dll` + (`PC/getpathp.c` era; today `PC/python_ver_rc.h` sibling code in + `Python/sysmodule.c` gated on `MS_COREDLL`). `ctypes.pythonapi` + is `PyDLL(None)` on POSIX but `PyDLL("python dll", handle= + sys.dllhandle)` on Windows (`Lib/ctypes/__init__.py`). +- **Venv resolution**: CPython venvs use `venvlauncher.exe`; + python-build-standalone instead copies the base exe and relies on + `pyvenv.cfg` `home=` pointing at the base prefix — the model + WeavePy adopted in RFC 0063 WS6 and this wave's shim honours. +- **`_WindowsConsoleIO`**: `Modules/_io/winconsoleio.c` — raw io + over console handles; `read` via `ReadConsoleW` then + `WideCharToMultiByte(CP_UTF8)`, `write` via + `MultiByteToWideChar` then `WriteConsoleW` (chunked; CPython + caps at 32766 wchars per call), Ctrl-C surfacing as + `ERROR_OPERATION_ABORTED` → `KeyboardInterrupt` via the signal + machinery, Ctrl-Z (`\x1a`) as EOF at the start of a read, + `fileno()` returning the CRT fd, `isatty()` always true. + `Lib/io.py`/`_pyio.py` route `open()` of console paths + (`CONIN$`/`CONOUT$`/`CON`) and interactive std streams through it + when `sys.platform == 'win32'`. + +## Detailed design + +Five workstreams. WS1 is the restructure everything else consumes; +WS2–WS4 are the consumers (import system, build system, console); +WS5 is verification. The verification channel matches RFC 0063: +`cargo check --target x86_64-pc-windows-msvc` must stay clean +locally for every touched crate (compilation without linking), the +blocking `windows-latest` `cargo test` job grows integration tests +that exercise the DLL for real, and the macOS/Linux gates +(regrtest `--check`, ecosystem `--check`, bench, `weavepy-dist +check`) must hold unchanged. + +### WS1 — the runtime cdylib and the thin exe + +**The crate split.** `weavepy-cli` today is a single `main.rs` +(~1.7K lines) plus `repl.rs`/`regrtest_cmd.rs`, bin-only. It gains a +**lib target**: the driver logic moves verbatim into +`weavepy-cli/src/lib.rs` behind one public entry point, + +```rust +/// Run the WeavePy CLI against this process's real argv/env. +/// Returns the process exit code. +pub fn cli_main() -> i32 +``` + +and the bin `main.rs` shrinks to a platform switch. A new crate +**`crates/weavepy-pylib`** owns the shared library: + +```toml +[lib] +name = "python313" +crate-type = ["cdylib"] + +[dependencies] +weavepy-cli = { workspace = true } +``` + +Its `lib.rs` exports the embedding entry points: + +- `#[no_mangle] pub extern "C" fn weavepy_main() -> c_int` — calls + `weavepy_cli::cli_main()`. Argv/env come from the process (on + Windows, `std::env::args_os` reads `GetCommandLineW`, which is + process-global and DLL-safe). +- `#[no_mangle] pub unsafe extern "C" fn Py_Main(argc, argv: + *mut *mut wchar_t) -> c_int` and `Py_BytesMain(argc, argv: + *mut *mut c_char) -> c_int` — the CPython embedding twins, + decoding their argv (UTF-16 on Windows, UTF-32 elsewhere; WTF-8 + tolerant like the RFC 0060 `sys.orig_argv` bridge) and calling a + `cli_main_with_args(Vec)` variant. Stock + `pylifecycle.h` already declares both. + +The ~682 C-API symbols need no enumeration: they are `#[no_mangle] +pub extern "C"` items in `weavepy-capi`, which `weavepy-pylib` +links transitively (cli → weavepy umbrella → capi), and rustc +derives a cdylib's PE export table from the reachable `#[no_mangle]` +surface of the whole crate graph. The `#[used]` `FORCE_LINK` table +(RFC 0022) guarantees reachability, exactly as it does for the +static exe today. `weavepy-pylib` calls `weavepy::install_capi_loader()` +inside `weavepy_main` before delegating, same as the CLI does today +(the call is already inside `run_source_with_options_impl`, so this +is belt-and-braces, not new behavior). + +**The thin shim.** `weavepy-cli`'s dependency table splits by +target: on `cfg(not(windows))` the bin keeps the full static link +(`fn main` calls `weavepy_cli::cli_main()` from the lib — POSIX +behavior, size, and the `--export-dynamic` build.rs contract are +all byte-identical to today). On `cfg(windows)` the bin does **not** +reference the lib; `main` is a loader: + +1. `GetModuleFileNameW` → the exe's own directory; try + `{exe_dir}\python313.dll` via `LoadLibraryExW(abs_path, NULL, + LOAD_WITH_ALTERED_SEARCH_PATH)`. +2. If absent (the venv case — RFC 0063 venvs copy the base exe as + `Scripts\python.exe` and do *not* copy the DLL): read + `{exe_dir}\..\pyvenv.cfg`, take the `home =` value, and load + `{home}\python313.dll`. +3. Failing both, a plain `LoadLibraryW(L"python313.dll")` (default + search order) as the last resort — this is what makes + `cargo run -p weavepy-cli` work from a target dir where cargo + placed the DLL, and what a user who split the files across + `PATH` gets. +4. On failure: a clear two-line error naming the paths probed and + the wave's contract ("weavepy.exe requires python313.dll from + the same distribution"), exit code 103 (well clear of Python's + 1/2/120 conventions). +5. `GetProcAddress(dll, "weavepy_main")` → call → `exit(code)`. + +Because the Windows bin never touches the runtime crates, the MSVC +linker's archive semantics leave the shim at shim size; the +`[target.'cfg(not(windows))'.dependencies]` split in `Cargo.toml` +makes the independence structural rather than an artifact of +dead-code elimination. `windows-sys` (already a workspace dep) is +the shim's only Windows dependency. + +**Process-global state and the DLL boundary.** All interpreter +state lives in the DLL's image (thread-locals, the GIL, the GC +registries); the shim owns nothing but the loader call, so there is +exactly one runtime in the process and a `.pyd`'s +`python313.dll` import binds to the already-loaded module by name — +the same in-process resolution CPython relies on. The RFC 0063 +`SetConsoleCtrlHandler` registration happens inside +`weavepy_main`'s init path (it already does — `install_startup_dispositions` +is called from the driver, which now lives in the DLL), so signal +delivery is unchanged. + +**`sys.dllhandle`.** The RFC 0063 hardcoded `0` becomes truthful: +`sys::build` on Windows calls +`GetModuleHandleW(w!("python313.dll"))` and publishes the `HMODULE` +as an int — nonzero through the shim (and through any embedder that +loaded the DLL), 0 in statically-linked Rust test harnesses, which +is the honest answer for a process with no Python DLL. +`ctypes.pythonapi` then constructs against the real handle. + +**POSIX.** Nothing ships differently: the cdylib crate builds a +`libpython313.so`/`.dylib` as a workspace member (useful for future +embedding work and for keeping the crate honest under +`cargo test --workspace`), but the artifact layout, the static +`bin/weavepy`, and the export mechanics are untouched. Packaging a +POSIX shared library is explicitly out of scope (Non-goals). + +### WS2 — `.pyd` loading: search semantics, error shape, `os.add_dll_directory` + +**Loader flags.** `weavepy-capi/src/loader.rs` currently opens every +extension via `libloading::Library::new`. That is right on POSIX and +wrong on Windows (it inherits the legacy default search order, +including CWD and `PATH`). The Windows arm switches to +`libloading::os::windows::Library::load_with_flags(path, +LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR)` +— CPython's exact `dynload_win.c` flags, so a wheel's `.pyd` can +resolve its vendored dependent DLLs from its own directory and from +`AddDllDirectory` cookies, and *cannot* pick DLLs off `PATH`/CWD +(the bpo-36085 hardening; delocate/`.libs` layouts depend on the +former, security posture on the latter). + +**Error shape.** A failed load on Windows raises +`ImportError("DLL load failed while importing {leaf}: {strerror}")` +with `name`/`path` set — the message tooling and Stack Overflow +muscle memory both match on. `strerror` comes through the RFC 0063 +`FormatMessageW` path (trailing CRLF trimmed). POSIX keeps its +dlerror-based message. + +**`os.add_dll_directory`.** Rust-native in `os.rs`'s existing +`#[cfg(windows)]` block (WeavePy's `os` is Rust-owned; the frozen +`nt` shim re-exports): + +- `os.add_dll_directory(path)` — validates the path is absolute and + a directory (CPython delegates both to the API's + `ERROR_INVALID_PARAMETER`), fires the `os.add_dll_directory` + audit event (PEP 578 machinery from RFC 0031/0060), calls + `AddDllDirectory`, and returns an `_AddedDllDirectory` instance. +- `_AddedDllDirectory`: `close()` (idempotent; calls + `RemoveDllDirectory` with the stored cookie), `__enter__` + returning self, `__exit__` closing, and CPython's repr — + ``, `` once + closed. Implemented as a small native type; the cookie is a + `DLL_DIRECTORY_COOKIE` held as a pointer-sized int. + +`Win32_System_LibraryLoader` is already in the workspace +`windows-sys` feature set (RFC 0063), so no dependency motion. + +### WS3 — the MSVC build surface: import library, autolink, dist + +**The import library.** Linking a cdylib on `*-pc-windows-msvc` +already produces `python313.dll.lib` beside the DLL — rustc emits +it; nothing new is compiled. The dist builder learns to carry both: + +- `{prefix}\python313.dll` — beside the exes at the prefix root + (the loader's first probe, and CPython's own layout). +- `{prefix}\libs\python313.lib` — the import library, renamed from + rustc's `python313.dll.lib` to the name MSVC's `/DEFAULTLIB` + and setuptools' `library_dirs` convention expect. setuptools + computes `{sys.base_exec_prefix}\libs` on its own; shipping the + file at that path is the entire integration. + +`build_artifact` locates both next to the packaged binary (they are +siblings in `target/release/`), fails the build if the DLL is +missing on Windows (a Windows artifact without the DLL is not an +artifact), and the `weavepy` binary resolution error message grows +the `-p weavepy-pylib` build hint. + +**pyconfig.h autolink.** The Windows `pyconfig.h` the stdlib +materializer writes (RFC 0062 WS2 stub, kept by RFC 0063) becomes +CPython-shaped where build systems can see it: `MS_WINDOWS`, +`Py_ENABLE_SHARED`, `MS_COREDLL`, the `Py_BUILD_CORE`-guarded + +```c +#pragma comment(lib,"python313.lib") +``` + +autolink, and the `HAVE_DECLSPEC_DLL`/`PyMODINIT_FUNC` export +shaping stock headers key off. A `.pyd` compiled with `cl /LD +ext.c /Ipath\to\include /link /LIBPATH:path\to\libs` — or through +setuptools, which passes exactly those — then binds `python313.lib` +without the build script naming it. + +**The dist check cext leg un-skips on Windows.** The +`cfg!(unix)`-gated SKIP becomes a real leg: a `CEXT_SCRIPT_WINDOWS` +that discovers MSVC (in order: `cl.exe` already on `PATH`, then +`vswhere.exe` at its fixed `%ProgramFiles(x86)%\Microsoft Visual +Studio\Installer` home → newest VC tools → run the compile under +`VsDevCmd.bat -arch=x64`), compiles the same minimal module with +`cl /LD`, `/I` at the shipped `Include\`, `/LIBPATH:` at the +shipped `libs\`, names it `_weavepy_dist_cext.cp313-win_amd64.pyd`, +imports it, and calls it. No MSVC found → SKIP with the discovery +trail in the detail (truthful skip, not a silent one). The venv +leg's interpreter (a copied shim) exercises the `pyvenv.cfg` +`home=` DLL probe by construction, so the WS1 fallback is covered +by the existing matrix without a new leg. + +**sysconfig residuals.** `EXT_SUFFIX`, `EXE`, and `VERSION` landed in +RFC 0063 WS6. `get_platform() == "win-amd64"` lands *here*: the frozen +`sysconfig` sniffs `'amd64' in sys.version.lower()` (CPython's own +detection), so `sys.version`'s compiler bracket gains CPython's NT +arch tag (`[WeavePy 64 bit (AMD64)]`) on Windows — without it the +platform read as `win32` and setuptools would tag wheels wrong. +No new config vars: CPython's NT `sysconfig` table carries no +`LIBRARY`/`LDLIBRARY`/`Py_ENABLE_SHARED` (those are POSIX Makefile +surface — `_init_non_posix` plus the native `_sysconfig` module +never emit them), and build tools locate the import library by the +`{sys.base_exec_prefix}\libs` convention instead. Adding them would +be a divergence, not a compatibility win. `INCLUDEPY`'s NT value +keeps pointing at the artifact `Include\` from RFC 0063. + +### WS4 — `_WindowsConsoleIO`: the deferred console-Unicode story + +A new Windows-gated raw-io type on `_io`, following the RFC 0063 +module pattern (`io_full::build` inserts it next to `FileIO`; +absent on POSIX exactly as CPython's `_io` omits it there): + +- **Construction** from a CRT fd or a console path + (`CONIN$`/`CONOUT$`/`CON`), deciding readable/writable from + `GetConsoleMode` on the underlying handle; non-console handles + raise `ValueError` like CPython. +- **`read`/`readinto`/`readall`**: `ReadConsoleW` into a wchar + buffer, transcoded with `WideCharToMultiByte(CP_UTF8)`; a + leading `\x1a` (Ctrl-Z) at the start of a read is EOF; a read + interrupted by Ctrl-C surfaces `ERROR_OPERATION_ABORTED`, which + maps to the RFC 0063 signal trip → `KeyboardInterrupt` after the + handler runs (the eval-breaker check the dispatcher already + performs). +- **`write`**: `MultiByteToWideChar(CP_UTF8)` then `WriteConsoleW`, + chunked at CPython's 32766-wchar ceiling; partial-write + accounting returns the consumed *byte* count of whole characters, + per winconsoleio.c. +- **Surface**: `fileno()`, `isatty()` (always `True`), + `readable()`/`writable()`/`seekable()` (`False`), `close()` + through the CRT fd owner from RFC 0063 WS1, `name`, `mode`. + +**stdio wiring.** WeavePy's std streams are a monolithic native +`PyFile` (not CPython's three-layer stack), a documented +architectural divergence that RFC 0050/0053 built the WTF-8 stdio +contract on. This wave keeps the monolith and reroutes its byte +transport: at stream-construction time on Windows, if the CRT fd is +a real console (`GetConsoleMode` succeeds), the `PyFile` backend +reads/writes through the same `ReadConsoleW`/`WriteConsoleW` bridge +`_WindowsConsoleIO` uses, so interactive I/O round-trips the full +Unicode range regardless of the console codepage — CPython-faithful +*behavior* through WeavePy-shaped plumbing. Redirected/piped +streams (everything CI sees) keep the RFC 0063 CRT-fd path +unchanged. `sys.stdin.isatty()` etc. already answer correctly via +`_isatty`. + +### WS5 — verification: what blocks now, what the flip measures later + +**Blocking, this wave, on `windows-latest` `cargo test`** (the +job builds `-p weavepy-pylib` before testing so the DLL exists in +`target/debug/`): + +1. `weavepy-cli/tests/windows_dll.rs` (Windows-gated): + - the DLL loads from `target/debug/python313.dll`; + - `GetProcAddress` resolves `weavepy_main`, `Py_Main`, + `Py_BytesMain`, and a curated ~30-symbol C-API sample + spanning the export families (`PyLong_FromLong`, + `PyModule_Create2`, `PyErr_SetString`, `PyType_FromSpec`, + `_Py_NoneStruct`, `PyCapsule_New`, …) — the smoke half of + the POSIX `force_link_completeness` contract; + - the shim exe runs Python through the DLL: + `weavepy.exe -c "import sys; assert sys.dllhandle != 0"`, + plus an `os.add_dll_directory` round-trip (add → repr → + close → closed repr) and the `ImportError` message-shape + probe against a nonexistent `.pyd`. +2. The RFC 0063 integration battery keeps passing (the driver + moved crates; its behavior must not). + +**Blocking, this wave, on macOS/ubuntu**: everything already +blocking, unchanged — regrtest `--check` `unexpected 0`, ecosystem +`--check` all rows, bench gate, `weavepy-dist check` all legs, +`cargo test --workspace` (which now also compiles `weavepy-pylib` +everywhere, keeping the cdylib honest on all three OSes). + +**Advisory, unchanged mechanism**: the Windows regrtest/ecosystem/ +bench/dist-check lanes keep running and uploading measured +artifacts. The dist-check lane now exercises the DLL layout and +(runner images carry MSVC) the un-skipped cext leg. The ecosystem +lane's numpy/pandas rows become *mechanically possible* on Windows +for the first time; their first measured results ride the existing +artifact upload. The flip-to-blocking commit — transplanting +measured `status_windows` rows, `bench-windows-x86_64.json`, and +`measured_os += ["windows"]` — remains the named first follow-up, +exactly as RFC 0063 WS7 specified; this wave adds no new flip +mechanism because RFC 0063 already landed it. + +### Non-goals + +- **Shipping a POSIX shared library.** `libpython313.so`/`.dylib` + builds as a side effect of the crate split but is not packaged; + the POSIX artifact, exe, and export story are unchanged. A + `python3-config`/embedding wave can pick it up later (RFC 0062 + future work). +- **`python313_d.dll` debug builds, `pythonw.exe`** (the + GUI-subsystem exe), the `py.exe` launcher, MSI/Store packaging, + and code signing. +- **ARM64 Windows** — same posture as RFC 0063 (builds, ctypes + `SUPPORTED=false`, no lanes). +- **Flipping the Windows lanes to blocking in this commit** — the + flip requires CI-measured artifacts by definition (RFC 0063 WS7); + this wave widens what those artifacts measure. +- **The stable ABI's version-crossing promises** (`python3.dll` + forwarding for abi3 wheels built against other minors): abi3 + wheels tagged for 3.13 load through `python313.dll` like any + other; a `python3.dll` forwarder DLL is deferred until a concrete + consumer demands it. + +### Acceptance criteria + +1. **The restructure is invisible on POSIX**: `cargo fmt`, + `clippy -D warnings`, `cargo test --workspace`, regrtest + `--check` (`unexpected 0`), ecosystem `--check` (all rows), + bench gate, and `weavepy-dist check` (all 7 legs) green on + macOS, with the ubuntu twins green in CI. +2. **Windows compiles clean from the cross-check**: + `cargo check --target x86_64-pc-windows-msvc --workspace` + passes locally (no linking; the link is proven on the runner). +3. **The DLL is real and complete**: the `windows-latest` test job + loads `python313.dll`, resolves the entry points and the C-API + symbol sample, and runs Python end-to-end through the shim — + all blocking. +4. **`sys.dllhandle` is truthful**, `os.add_dll_directory` matches + CPython's surface (context manager, repr, audit event, absolute- + path validation), and extension-load failures raise CPython's + `ImportError` shape on Windows. +5. **The artifact carries the ABI**: `weavepy-dist build` on + Windows places `python313.dll` at the prefix root and + `libs\python313.lib` beside `Include\`; `weavepy-dist check` + passes with the cext leg PASS (MSVC present) or a truthful + discovery-trail SKIP — no unconditional SKIP remains. +6. **Console Unicode round-trips**: the `_WindowsConsoleIO` type + registers on `_io` (Windows), the console-backed stdio bridge + passes its Windows-gated unit tests (UTF-8 supplementary-plane + round-trip through `WriteConsoleW`/`ReadConsoleW` mocks at the + CRT layer where a real console is absent in CI, plus behavior + tests under a real console handle when available). +7. **The RFC 0063 battery keeps passing unmodified** — the driver + relocation is behavior-neutral. + +## Drawbacks + +- **The exe/DLL split is a second distribution identity to keep + honest.** A version-skewed pair (old exe, new DLL) is a new + failure class that the static exe could not have. Mitigation: the + shim and DLL ship from one build; the shim's failure message + names both paths; `weavepy-dist check`'s identity leg runs + through the shim and would surface skew as a version mismatch. +- **`GetProcAddress`-based dispatch hides link errors until + runtime.** A typo'd entry-point name fails at shim startup, not + at build. Mitigation: the blocking Windows test calls the real + entry points on every PR. +- **rustc's cdylib export behavior is now load-bearing.** If a + future rustc narrows default exports (e.g. under fat LTO), the + DLL could silently thin. Mitigation: the ~30-symbol + `GetProcAddress` sample in the blocking test turns "silently + thin" into "red PR"; the `FORCE_LINK` table keeps reachability + explicit. +- **The console bridge adds a third stdio transport** (POSIX fd, + NT CRT fd, NT console-W). The monolithic `PyFile` keeps the + surface area contained, but it is one more arm in every stdio + bugfix. +- **Iteration on Windows-only failures is still a CI round-trip** + (RFC 0063's standing drawback). The cross-check target and + fine-grained test battery are the standing mitigation. + +## Alternatives + +- **Export the C-API from the exe and ship a forwarder/stub + `python313.dll`** (trampolines resolved via + `GetProcAddress(GetModuleHandle(NULL))` at `DllMain`): keeps the + static exe, but PE export forwarders cannot target an exe by + name, so every one of ~682 symbols needs a generated jump thunk; + the DLL would be unloadable outside a WeavePy process (embedders, + tools that `LoadLibrary` the Python DLL directly); and the exe + needs 682 `/EXPORT` args anyway. Strictly more machinery than + moving the code into the DLL, with a worse compatibility story. +- **Link the shim against the import library at build time** + (CPython's literal shape) instead of `LoadLibraryW` at startup: + cargo cannot express "bin links a sibling crate's cdylib + artifact" on stable (artifact dependencies are unstable), so the + link would need a build.rs racing the cdylib's build for the + `.lib`'s existence. Runtime loading is order-independent, keeps + `cargo build --workspace` correct by construction, and costs one + `LoadLibrary` + `GetProcAddress` at startup. +- **Let the Windows bin keep its static runtime and also ship the + DLL**: two copies of the interpreter in one process (the exe's + dead static copy plus the DLL the `.pyd`s bind), ~4× artifact + bloat across the four exe copies, and a state-split footgun if + any exe-side code ever runs. Rejected outright. +- **Copy `python313.dll` into every venv's `Scripts\`** instead of + teaching the shim `pyvenv.cfg`: burns ~35 MB per venv and leaves + stale-DLL venvs behind on upgrade; the `home=` probe is four + lines and matches how the landmark walk already resolves venvs. +- **Build `_WindowsConsoleIO` as the full CPython three-layer stdio + stack** (raw + buffered + text wrapper for std streams): the + faithful shape, but it would rebuild WeavePy's stdio architecture + in one wave for no user-visible delta over rerouting the + monolith's transport; the RFC 0050 WTF-8 contract tests pin the + observable behavior either way. Deferred, not rejected — + revisited if `sys.stdout.buffer.raw`-introspecting code appears + in the ecosystem lane. + +## Prior art + +- **CPython** (`PC/`, `Programs/python.c`, `Python/dynload_win.c`): + the shim-exe + core-DLL split, the `libs\` import-library + convention, the autolink pragma, and the `LoadLibraryExW` flag + set are transcribed, not invented. +- **python-build-standalone**: ships exactly this shape + (`python.exe` + `python313.dll` + `libs\python313.lib`) built + outside MSBuild, and its venvs rely on `pyvenv.cfg` `home=` — + the direct precedent for the shim's probe order. +- **PyPy on Windows**: `libpypy3.9-c.dll` beside a thin exe; + its documented lesson (extensions and embedders need the DLL's + directory discoverable) shaped the exe-dir-first probe. +- **Rust cdylib C-API precedents**: the `#[no_mangle]`-graph export + model is how PyO3's `abi3` builds and wasm component crates ship + multi-crate C surfaces; the `FORCE_LINK` table predates this wave + (RFC 0022) and was designed for exactly this reuse. +- **RustPython**: still exe-only on Windows and cannot load CPython + wheels — the counterexample this wave graduates past. + +## Unresolved questions + +- Whether `Py_Main`/`Py_BytesMain` should tear down and return + (CPython returns the exit code and the caller may re-init) or + behave like `weavepy_main` (single-shot). This wave implements + return-the-code without re-init support, matching WeavePy's + existing `Py_Finalize` no-op posture; revisit with the embedding + wave. +- Whether the ecosystem lane's Windows wheel fetch should pin + `win_amd64` wheels in `tools/ecosystem_fetch.py` now or at the + flip commit — answered at the flip, when the lane's first + measured numpy/pandas rows exist. +- Whether `ctypes.CDLL(sys.executable)`-style self-loads (rare, but + real) need the exe to re-export anything. Believed no (consumers + use `sys.dllhandle`); the flip's measured `test_ctypes` rows will + answer. + +## Future work + +- The flip-to-blocking baseline commit (measured `status_windows` + rows including the cext-dependent files, ecosystem + `status_windows` for the binary-wheel rows, + `bench-windows-x86_64.json`, `measured_os += ["windows"]`) — + first follow-up, unchanged from RFC 0063's naming. +- A `python3.dll` stable-ABI forwarder once an abi3 consumer that + needs it appears in the ecosystem lane. +- `pythonw.exe` (GUI subsystem) and `venvlauncher`-style script + shims. +- Packaging the POSIX shared library + `python3-config` (RFC 0062 + future work, now cheaper: the cdylib exists). +- The full three-layer stdio stack if `buffer`/`raw` introspection + surfaces as a real-world blocker. + +## Results + +*(To be filled in at landing, per repo convention: the Windows CI +battery outcomes, the first advisory-lane sweeps over the DLL +layout, and the unchanged macOS/Linux baselines.)* From cc6e9a5f66a22c9714f16c45e1b1789e8a3564d7 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:39:25 -0700 Subject: [PATCH 2/2] ci: build python313.dll before the workspace test job --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48a77a6f..df5b9290 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,11 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 + # `cargo test` compiles weavepy-pylib's source into a unit-test + # harness but never emits the cdylib artifact, so build it + # explicitly: the windows_dll integration battery (RFC 0064 WS5) + # needs target/debug/python313.dll next to the shim exe. + - run: cargo build -p weavepy-pylib - run: cargo test --workspace --all-targets --all-features - run: cargo test --workspace --doc