From a3c7d7755047faa720aa1e9b556a367a1391f836 Mon Sep 17 00:00:00 2001 From: jiminu Date: Tue, 15 Sep 2026 17:52:31 +0900 Subject: [PATCH] Add repeatable prompt and diagnostic benchmarks --- docs/PERFORMANCE.md | 43 +++++++++++- scripts/benchmark-prompt.py | 127 ++++++++++++++++++++++++++++++++++++ scripts/benchmark.sh | 115 ++++++++++++++++++++++++++++++-- tests/benchmark_test.bash | 74 +++++++++++++++++++++ 4 files changed, 350 insertions(+), 9 deletions(-) create mode 100644 scripts/benchmark-prompt.py diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 79cd4c2..967e20e 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -21,9 +21,9 @@ SELFISHELL_BENCHMARK_PROFILE=full bash scripts/benchmark.sh Each metric reports the mean, median (`p50`), 95th percentile (`p95`), and maximum duration in milliseconds. `interactive-cached` loads the platform `.zshrc` and exits; it does not measure a visible prompt, command-to-prompt -latency, or deferred plugin readiness. Measure those separately in a terminal. +latency, or deferred plugin readiness. Use `--prompt` for PTY prompt timing. -### Base mode +## Base mode Base mode is the default. It measures Selfishell's own startup cost independent of external integrations: Starship, @@ -36,7 +36,7 @@ cannot reintroduce host tools. A normal local run therefore does not read the developer's mise configuration or execute or modify the developer's plugin checkout. -### Full-environment mode +## Full-environment mode Full mode provisions pinned mise, Starship, fzf, zoxide, Zinit, and its Zsh plugins through the production installers into the temporary `HOME`. It uses @@ -45,6 +45,43 @@ include the managed shell integrations without changing the developer's tools or plugin checkouts. Provisioning requires network access; run it locally when needed, outside CI and the network-free test suite. +## Prompt and diagnostic measurements + +```sh +bash scripts/benchmark.sh --mode full --prompt --diagnostics +``` + +Both options are independent and also work in network-free `base` mode. +They use the same iteration count (`SELFISHELL_BENCHMARK_ITERATIONS`, default +30) and append metrics to `SELFISHELL_BENCHMARK_RESULTS_FILE` when set. +They have no enforced timing budgets. + +`--prompt` requires Python 3, using only its standard library. It measures +`prompt-first-{empty,repository}` and `prompt-command-{empty,repository}` in +a 160-column PTY with the managed Starship configuration. Each scenario warms +one shell before collecting samples. Every measured shell runs three `:` +commands after a fixed 100 ms settling interval. A numbered prompt marker +distinguishes a new command cycle from an editing redraw. Timings include shell +process creation for the first prompt and run until the marker reaches the PTY; +they exclude GUI terminal rendering. The settling interval does not prove +deferred plugins are ready. Repository timings depend on this checkout's size +and working-tree state; language projects may have different costs. + +`--diagnostics` applies configuration with `install --skip-packages --yes` in +a separate temporary HOME, then measures `cli-status` and `cli-doctor` after +one warm-up invocation each. It prints that invocation's output and exit code +to identify the measured state; a changed exit code during sampling fails the +benchmark. Exit 1 is expected when required tools are missing. Both modes query +available system package inventories. Base mode uses `/usr/bin:/bin`; full mode +also shares the temporary pinned shell tools and Zinit plugins and includes +the caller's PATH tools and package managers. +It does not install the remaining developer tools or system packages, so these +results describe a partially provisioned environment, not a complete setup. +HOME, XDG directories, and mise data/cache/state paths point into the temporary tree. +Prompt and diagnostic measurements also bound mise's ancestor configuration +search with `MISE_CEILING_PATHS`, excluding settings above the measured directory +or source root. + ## Startup caches Interactive startup audits completion directories on first use and once daily. diff --git a/scripts/benchmark-prompt.py b/scripts/benchmark-prompt.py new file mode 100644 index 0000000..dbf2588 --- /dev/null +++ b/scripts/benchmark-prompt.py @@ -0,0 +1,127 @@ +"""PTY measurement helper for benchmark.sh; uses its disposable HOME and ZDOTDIR.""" + +import fcntl +import math +import os +from pathlib import Path +import pty +import select +import signal +import struct +import sys +import termios +import time + + +def wait_for_prompt(fd, count, timeout=15): + # Editing may redraw the previous prompt: only a new precmd counts. + marker = f"__SFS_READY_{count}__".encode() + deadline = time.monotonic() + timeout + data = b"" + while marker not in data: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise RuntimeError(f"Prompt {count} timed out: {data[-2000:]!r}") + if select.select([fd], [], [], remaining)[0]: + chunk = os.read(fd, 65536) + if not chunk: + raise RuntimeError(f"Shell exited before prompt {count}: {data[-2000:]!r}") + data = (data + chunk)[-65536:] + return time.perf_counter() + + +def finish_shell(pid, fd): + # Let Zsh release history locks before another sample starts. + os.write(fd, b"exit\n") + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + exited, status = os.waitpid(pid, os.WNOHANG) + if exited: + if status: + raise RuntimeError(f"Shell exited with wait status {status}") + return + if select.select([fd], [], [], 0.05)[0]: + try: + os.read(fd, 65536) + except OSError: + pass # Linux PTYs may report EIO while the child exits. + raise RuntimeError("Shell did not exit after measurement") + + +def measure(cwd, env, iterations): + # Keep mise from discovering personal configuration above either scenario. + env = dict(env, MISE_CEILING_PATHS=str(Path(cwd).resolve())) + starts, commands = [], [] + for iteration in range(iterations + 1): + start = time.perf_counter() + pid, fd = pty.fork() + if pid == 0: + try: + fcntl.ioctl(0, termios.TIOCSWINSZ, struct.pack("HHHH", 32, 160, 0, 0)) + os.chdir(cwd) + os.execve("/bin/zsh", ["zsh", "-d", "-i"], env) + finally: + os._exit(1) + try: + elapsed = (wait_for_prompt(fd, 1) - start) * 1000 + if iteration: + starts.append(elapsed) + # Fixed settling time; this does not assert deferred plugin readiness. + time.sleep(0.1) + for count in range(2, 5): + start = time.perf_counter() + os.write(fd, b":\n") + elapsed = (wait_for_prompt(fd, count) - start) * 1000 + if iteration: + commands.append(elapsed) + finish_shell(pid, fd) + finally: + os.close(fd) + try: + exited, _ = os.waitpid(pid, os.WNOHANG) + if not exited: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + except ChildProcessError: + pass + return starts, commands + + +def report(label, samples): + ordered = sorted(samples) + values = [sum(samples) / len(samples)] + values.extend(ordered[math.ceil(len(samples) * q) - 1] for q in (0.5, 0.95, 1)) + print(label + "\t" + "\t".join(f"{value:.3f}" for value in values)) + + +def prompt_environment(root, home): + # Keep project and host integration settings out of the measurement environment. + env = {"HOME": str(home), "ZDOTDIR": os.environ["ZDOTDIR"], + "LANG": os.environ.get("LANG", "en_US.UTF-8")} + env.update( + XDG_CONFIG_HOME=str(home / ".config"), XDG_DATA_HOME=str(home / ".local/share"), + XDG_STATE_HOME=str(home / ".local/state"), XDG_CACHE_HOME=str(home / ".cache"), + MISE_DATA_DIR=str(home / ".local/share/mise"), MISE_CACHE_DIR=str(home / ".cache/mise"), + MISE_STATE_DIR=str(home / ".local/state/mise"), + MISE_GLOBAL_CONFIG_FILE=str(home / ".config/mise/config.toml"), MISE_OFFLINE="1", + STARSHIP_CONFIG=str(root / "config/shared/starship.toml"), SELFISHELL_UPDATE_NOTICE="0", + SELFISHELL_BENCHMARK_PLATFORM_CONFIG=os.environ["SELFISHELL_BENCHMARK_PLATFORM_CONFIG"], + PATH=os.environ["SELFISHELL_BENCHMARK_PATH"], SHELL="/bin/zsh", TERM="xterm-256color", + ) + if "WSL_DISTRO_NAME" in os.environ: + env["WSL_DISTRO_NAME"] = os.environ["WSL_DISTRO_NAME"] + return env + + +def main(): + iterations, root = int(sys.argv[1]), Path(sys.argv[2]) + home = Path(os.environ["HOME"]) + env = prompt_environment(root, home) + for scenario, cwd in (("empty", home), ("repository", root)): + starts, commands = measure(cwd, env, iterations) + report(f"prompt-first-{scenario}", starts) + report(f"prompt-command-{scenario}", commands) + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 1b12968..b63259c 100644 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -2,16 +2,18 @@ set -euo pipefail -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" ITERATIONS="${SELFISHELL_BENCHMARK_ITERATIONS:-30}" ENFORCE_BUDGETS="${SELFISHELL_BENCHMARK_ENFORCE:-0}" PROFILE_MODE="${SELFISHELL_BENCHMARK_PROFILE:-base}" RESULTS_FILE="${SELFISHELL_BENCHMARK_RESULTS_FILE:-}" ZPROF_FILE="${SELFISHELL_BENCHMARK_ZPROF_FILE:-}" +MEASURE_PROMPT=0 +MEASURE_DIAGNOSTICS=0 usage() { cat <<'EOF' -Usage: scripts/benchmark.sh [--mode base|full] +Usage: scripts/benchmark.sh [--mode base|full] [--prompt] [--diagnostics] base Selfishell's own startup cost, independent of external integrations (mise/starship/zinit/fzf/zoxide are excluded). This is the default. @@ -20,7 +22,12 @@ Usage: scripts/benchmark.sh [--mode base|full] plugins) into an isolated HOME before measuring, so the interactive-cached metric reflects a real full-environment startup. Starship, fzf, and zoxide are installed through mise. - This script does not invoke Apt/Homebrew. + This script does not install Apt/Homebrew packages. + + --prompt Measure first and command-to-prompt latency using a PTY + in an empty directory and this repository (requires python3). + --diagnostics Measure status and doctor after isolated configuration setup. + Full mode includes the caller's PATH tools and package managers. SELFISHELL_BENCHMARK_PROFILE=base|full is equivalent to --mode. EOF @@ -37,6 +44,8 @@ while (("$#" > 0)); do fi PROFILE_MODE="$1" ;; + --prompt) MEASURE_PROMPT=1 ;; + --diagnostics) MEASURE_DIAGNOSTICS=1 ;; --help | -h) usage exit 0 @@ -58,6 +67,11 @@ case "$PROFILE_MODE" in ;; esac +if [[ "$MEASURE_PROMPT" == 1 ]] && ! command -v python3 >/dev/null 2>&1; then + printf '%s\n' '--prompt requires python3 (standard library only).' >&2 + exit 1 +fi + # Validate arguments before creating any temporary files. TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/selfishell-benchmark.XXXXXX")" TEST_HOME="$TEST_ROOT/home" @@ -82,6 +96,7 @@ fi ln -s "$ROOT_DIR/config/shared/zsh/common.zsh" "$TEST_HOME/.config/selfishell/zsh/common.zsh" ln -s "$ROOT_DIR/config/shared/zsh/runtime.zsh" "$TEST_HOME/.config/selfishell/zsh/runtime.zsh" +ln -s "$ROOT_DIR/config/shared/zsh/history.zsh" "$TEST_HOME/.config/selfishell/zsh/history.zsh" ln -s "$ROOT_DIR/config/shared/zsh/completion.zsh" "$TEST_HOME/.config/selfishell/zsh/completion.zsh" ln -s "$ROOT_DIR/config/shared/zsh/interactive.zsh" "$TEST_HOME/.config/selfishell/zsh/interactive.zsh" ln -s "$ROOT_DIR/config/shared/zsh/update-notice.zsh" "$TEST_HOME/.config/selfishell/zsh/update-notice.zsh" @@ -102,6 +117,7 @@ install_full_integrations() ( export HOME="$TEST_HOME" XDG_CONFIG_HOME="$TEST_HOME/.config" export XDG_DATA_HOME="$TEST_DATA_HOME" XDG_STATE_HOME="$TEST_HOME/.local/state" export XDG_CACHE_HOME="$TEST_HOME/.cache" SELFISHELL_ROOT="$ROOT_DIR" + export MISE_CEILING_PATHS="$ROOT_DIR" cd "$TEST_HOME" source "$ROOT_DIR/lib/common.sh" source "$ROOT_DIR/lib/paths.sh" @@ -206,6 +222,85 @@ record_result() { fi } +run_prompt_benchmark() { + local prompt_results result + + verify_full_integrations "$ROOT_DIR" + mkdir "$TEST_ROOT/prompt" + cat >"$TEST_ROOT/prompt/.zshrc" <<'EOF' +source "$SELFISHELL_BENCHMARK_PLATFORM_CONFIG" +autoload -Uz add-zsh-hook +setopt promptsubst +typeset -gi _selfishell_benchmark_prompt=0 +_selfishell_benchmark_precmd() { (( ++_selfishell_benchmark_prompt )); } +add-zsh-hook precmd _selfishell_benchmark_precmd +RPROMPT+='__SFS_READY_${_selfishell_benchmark_prompt}__' +EOF + prompt_results="$(HOME="$TEST_HOME" ZDOTDIR="$TEST_ROOT/prompt" \ + SELFISHELL_BENCHMARK_PLATFORM_CONFIG="$PLATFORM_CONFIG" \ + SELFISHELL_BENCHMARK_PATH="$INTERACTIVE_PATH" \ + python3 "$ROOT_DIR/scripts/benchmark-prompt.py" "$ITERATIONS" "$ROOT_DIR")" || return + while IFS= read -r result; do + record_result "$result" + done <<<"$prompt_results" +} + +run_diagnostic_command() ( + local diagnostic_shell + diagnostic_shell="$(PATH="$DIAGNOSTIC_PATH" command -v zsh)" + cd "$DIAGNOSTIC_HOME" + env -i HOME="$DIAGNOSTIC_HOME" PATH="$DIAGNOSTIC_PATH" SHELL="$diagnostic_shell" \ + TMPDIR="$TEST_ROOT" TERM=dumb NO_COLOR=1 \ + XDG_CONFIG_HOME="$DIAGNOSTIC_HOME/.config" XDG_DATA_HOME="$DIAGNOSTIC_HOME/.local/share" \ + XDG_STATE_HOME="$DIAGNOSTIC_HOME/.local/state" XDG_CACHE_HOME="$DIAGNOSTIC_HOME/.cache" \ + MISE_DATA_DIR="$TEST_DATA_HOME/mise" MISE_CACHE_DIR="$DIAGNOSTIC_HOME/.cache/mise" \ + MISE_STATE_DIR="$DIAGNOSTIC_HOME/.local/state/mise" MISE_OFFLINE=1 MISE_CEILING_PATHS="$ROOT_DIR" \ + HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_ANALYTICS=1 \ + /bin/bash "$ROOT_DIR/bin/selfishell" "$@" +) + +run_diagnostic_sample() { + local command="$1" expected_status="$2" status=0 + run_diagnostic_command "$command" >/dev/null 2>&1 || status=$? + if [[ "$status" != "$expected_status" ]]; then + printf 'Diagnostic %s exit changed from %s to %s.\n' "$command" "$expected_status" "$status" >&2 + return 1 + fi +} + +run_diagnostic_benchmark() { + local command status output result + export DIAGNOSTIC_HOME="$TEST_ROOT/diagnostics-home" + export DIAGNOSTIC_PATH=/usr/bin:/bin + [[ "$PROFILE_MODE" != full ]] || DIAGNOSTIC_PATH="$TEST_HOME/.local/bin:$PATH" + mkdir -p "$DIAGNOSTIC_HOME/.local/share" + if [[ -d "$TEST_DATA_HOME/zinit" ]]; then + ln -s "$TEST_DATA_HOME/zinit" "$DIAGNOSTIC_HOME/.local/share/zinit" + fi + run_diagnostic_command install --skip-packages --yes >"$TEST_ROOT/diagnostics-setup.log" 2>&1 || { + cat "$TEST_ROOT/diagnostics-setup.log" >&2 + return 1 + } + record_result '# Diagnostics: configured HOME; no system packages installed; missing tools may yield exit 1.' + export -f run_diagnostic_command run_diagnostic_sample + for command in status doctor; do + status=0 + output="$(run_diagnostic_command "$command" 2>&1)" || status=$? + case "$status" in + 0 | 1) ;; + *) + printf '%s\n' "$output" >&2 + return "$status" + ;; + esac + record_result "# cli-$command exit=$status" + printf '%s\n' "$output" + # shellcheck disable=SC2016 # Run the exported function in each timed child. + result="$(benchmark "cli-$command" "$ITERATIONS" bash -c 'run_diagnostic_sample "$@"' _ "$command" "$status")" || return + record_result "$result" + done +} + run_common_zsh() { # In full mode, $TEST_HOME/.local/bin holds the pinned integrations. Base # mode uses it only for the benchmark-only macOS brew barrier. @@ -227,13 +322,14 @@ run_interactive_zsh() { } verify_full_integrations() { + local directory="${1:-$TEST_HOME}" [[ "$PROFILE_MODE" == full ]] || return 0 ( - cd "$TEST_HOME" + cd "$directory" HOME="$TEST_HOME" ZDOTDIR="$TEST_HOME" XDG_CONFIG_HOME="$TEST_HOME/.config" \ XDG_DATA_HOME="$TEST_DATA_HOME" XDG_CACHE_HOME="$TEST_HOME/.cache" \ MISE_GLOBAL_CONFIG_FILE="$TEST_HOME/.config/mise/config.toml" MISE_SHELL='' \ - PATH="$INTERACTIVE_PATH" TERM=xterm-256color MISE_OFFLINE=1 \ + PATH="$INTERACTIVE_PATH" TERM=xterm-256color MISE_OFFLINE=1 MISE_CEILING_PATHS="$ROOT_DIR" \ /bin/zsh -d -i -c ' for tool in starship fzf zoxide; do [[ "${commands[$tool]}" == "$MISE_DATA_DIR/installs/"* ]] || exit 1 @@ -301,7 +397,7 @@ record_result "$baseline_result" # The first run creates the completion dump. Following measurements represent # the cached common configuration used during ordinary startup. export -f run_common_zsh run_interactive_zsh -export ROOT_DIR TEST_HOME TEST_DATA_HOME COMMON_PATH INTERACTIVE_PATH +export ROOT_DIR TEST_ROOT TEST_HOME TEST_DATA_HOME COMMON_PATH INTERACTIVE_PATH record_result "$(benchmark common-first 1 bash -c 'run_common_zsh')" common_result="$(benchmark common-cached "$ITERATIONS" bash -c 'run_common_zsh')" record_result "$common_result" @@ -332,3 +428,10 @@ check_budget cli-help "$(printf '%s\n' "$help_result" | awk -F '\t' '{ print $4 if [[ -n "$ZPROF_FILE" ]]; then profile_interactive_zsh fi + +if [[ "$MEASURE_PROMPT" == 1 ]]; then + run_prompt_benchmark +fi +if [[ "$MEASURE_DIAGNOSTICS" == 1 ]]; then + run_diagnostic_benchmark +fi diff --git a/tests/benchmark_test.bash b/tests/benchmark_test.bash index 00f9988..1c9ed1d 100755 --- a/tests/benchmark_test.bash +++ b/tests/benchmark_test.bash @@ -56,6 +56,79 @@ test_benchmark_base_mode_runs_without_network() { fail "Base-mode benchmark did not report the expected metrics: $output" } +test_benchmark_measures_prompts_and_configured_diagnostics() { + local output results + + setup_test_home + results="$TEST_ROOT/results.tsv" + output="$(SELFISHELL_BENCHMARK_ITERATIONS=1 SELFISHELL_BENCHMARK_RESULTS_FILE="$results" \ + bash "$ROOT_DIR/scripts/benchmark.sh" --mode base --prompt --diagnostics)" + + for metric in prompt-first-empty prompt-command-empty prompt-first-repository prompt-command-repository cli-status cli-doctor; do + awk -F '\t' -v metric="$metric" '$4 == metric && NF == 8 && $6 > 0 { found = 1 } END { exit !found }' "$results" || + fail "Missing numeric result for $metric: $output" + done + [[ "$output" == *'Diagnostics: configured HOME'* && "$output" == *'cli-status exit='* && "$output" == *'[SUMMARY] Managed paths:'* ]] || + fail "Diagnostics did not describe the measured installation: $output" +} + +test_prompt_probe_validates_context_and_prompt_cycles() { + PYTHONDONTWRITEBYTECODE=1 python3 - "$ROOT_DIR/scripts" <<'PY' +import importlib.util +import os +from pathlib import Path +import sys +import tempfile + +spec = importlib.util.spec_from_file_location("probe", sys.argv[1] + "/benchmark-prompt.py") +probe = importlib.util.module_from_spec(spec) +spec.loader.exec_module(probe) +os.environ.update(WSL_DISTRO_NAME="Ubuntu", ZDOTDIR="/temporary/prompt", + SELFISHELL_BENCHMARK_PLATFORM_CONFIG="/fixture/zshrc", + SELFISHELL_BENCHMARK_PATH="/usr/bin:/mnt/c/Windows", + MISE_DATA_DIR="/ambient/mise", VIRTUAL_ENV="/ambient/venv") +env = probe.prompt_environment(Path("/fixture"), Path("/temporary/home")) +assert env["WSL_DISTRO_NAME"] == "Ubuntu", "WSL startup optimization was disabled" +assert env["MISE_DATA_DIR"] == "/temporary/home/.local/share/mise" +assert "VIRTUAL_ENV" not in env +reader, writer = os.pipe() +try: + os.write(writer, b"__SFS_READY_1__") + try: + probe.wait_for_prompt(reader, 2, timeout=0.01) + except RuntimeError as error: + assert "timed out" in str(error) + else: + raise AssertionError("An editing redraw counted as the next prompt") + os.write(writer, b"__SFS_READY_2__") + probe.wait_for_prompt(reader, 2, timeout=0.1) +finally: + os.close(writer) +try: + probe.wait_for_prompt(reader, 3, timeout=0.1) +except RuntimeError as error: + assert "Shell exited" in str(error) +else: + raise AssertionError("A closed shell produced a successful measurement") +finally: + os.close(reader) +with tempfile.TemporaryDirectory(prefix="selfishell-prompt-test-") as directory: + home = Path(directory) + (home / ".zshrc").write_text(''' +setopt promptsubst +typeset -gi count=0 +precmd() { (( ++count )); } +RPROMPT='__SFS_READY_${count}__' +zshexit() { print finished >> "$HOME/finished"; } +[[ "$MISE_CEILING_PATHS" == "$PWD" ]] || exit 2 +''') + probe.measure(home.resolve(), {"HOME": directory, "ZDOTDIR": directory, + "PATH": "/usr/bin:/bin", "TERM": "xterm-256color"}, 1) + assert (home / "finished").exists(), "Probe killed the shell before its exit hooks" + assert len((home / "finished").read_text().splitlines()) == 2 +PY +} + test_benchmark_rejects_missing_retained_zsh_module() { local checkout output status=0 @@ -138,6 +211,7 @@ test_benchmark_writes_opt_in_zprof_report() { [[ -s "$profile_file" ]] || fail "Benchmark did not write the requested zprof report" grep -Fq 'num calls' "$profile_file" || fail "Benchmark output is not a zprof report" + ! grep -Fq 'no such file or directory' "$profile_file" || fail "Benchmark omitted a sourced module" teardown_test_home }