diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c35edc4..7f1c27a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: - run: ruff check . - run: ruff format --check . - name: prose rules - run: python -m tools.prosecheck README.md CONTRIBUTING.md LICENSE.md GLOSSARY.md docs lessons blueprints containers corpora/programs + run: python -m tools.prosecheck README.md CONTRIBUTING.md LICENSE.md GLOSSARY.md docs lessons blueprints containers corpora/programs gxplug # The table in containers/README.md is generated from containers/matrix.toml, and a # hand edit to either one is how the two start describing different compilers. @@ -243,9 +243,21 @@ jobs: sudo apt-get install -y gcc-16 gcc-16 --version + # The plugin headers, so the tests marked needs_gcc_plugin run here rather than + # skipping, and g++-16 with them. A plugin is C++, and the gcc driver will + # happily compile a .cc file right up to the point where it tries to exec a + # cc1plus that only the g++ package ships. + # + # Tolerated if the PPA has not built them for this snapshot: those tests are + # gated properly by the plugin workflow, which builds gxplug against four real + # channels, and a missing package in a trunk PPA should not turn this job red + # for a reason that has nothing to do with the change under review. + sudo apt-get install -y g++-16 gcc-16-plugin-dev \ + || echo "the PPA is missing g++-16 or the plugin headers, the gxplug tests will skip" + - run: pip install -e ".[dev]" - name: the tests that need a compiler - run: python -m pytest -q -m needs_gcc + run: python -m pytest -q -m "needs_gcc or needs_gcc_plugin" # A drift signal rather than an assertion. If the parser stops understanding a dump, # the count moves and the log says by how much. diff --git a/.github/workflows/plugin.yml b/.github/workflows/plugin.yml new file mode 100644 index 0000000..90d9dec --- /dev/null +++ b/.github/workflows/plugin.yml @@ -0,0 +1,196 @@ +name: plugin + +# gxplug, built against every GCC a reader might plausibly have, and checked against the +# one promise the whole design rests on: adding -fplugin=gxplug.so changes nothing about +# the code GCC produces. +# +# A plugin is the most fragile thing in this project. It compiles against GCC's private +# headers, it is loaded by dlopen into a compiler it was not linked against, and it fails +# by segfaulting in the middle of somebody's lesson rather than by refusing to build. The +# only defence is to keep building it in as many places as possible, on both architectures, +# and to check the guarantee every time rather than asserting it in a document. +# +# Four channels, which is the answer to open question 4 in the design notes: +# +# debian a distribution package, gcc-16 with gcc-16-plugin-dev +# fedora a different distribution, packaged differently, gcc-plugin-devel +# homebrew macOS, where the plugin has to be linked with -undefined dynamic_lookup and +# gmp.h is not on the default include path +# source the compiler this project builds itself, pulled from the build matrix +# +# The four disagree about where the headers are, what the compiler is called, and how a +# shared object gets its undefined symbols resolved. Every one of those differences is +# probed for in gxplug/Makefile, and this is what proves the probes work. + +on: + push: + branches: [main] + paths: + - "gxplug/**" + - "gxray/plug.py" + - "tests/test_plug.py" + - ".github/workflows/plugin.yml" + pull_request: + paths: + - "gxplug/**" + - "gxray/plug.py" + - "tests/test_plug.py" + - ".github/workflows/plugin.yml" + workflow_dispatch: + +# The source channel pulls an image the build matrix published, and a package in GHCR is +# private until somebody makes it otherwise. +permissions: + contents: read + packages: read + +jobs: + build: + name: ${{ matrix.channel }} on ${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container }} + timeout-minutes: 15 + + # fail-fast off because the interesting result is which channels broke, not that one + # did. A change that breaks Fedora and nothing else is a different bug from one that + # breaks everything, and cancelling the rest hides the difference. + strategy: + fail-fast: false + matrix: + include: + - channel: debian + arch: amd64 + runner: ubuntu-24.04 + container: debian:sid + - channel: debian + arch: arm64 + runner: ubuntu-24.04-arm + container: debian:sid + - channel: fedora + arch: amd64 + runner: ubuntu-24.04 + container: fedora:rawhide + - channel: fedora + arch: arm64 + runner: ubuntu-24.04-arm + container: fedora:rawhide + # GitHub's macOS runners are arm64 only now, so Intel Homebrew cannot be built + # here. The one thing that differs between the two is the prefix, /usr/local + # against /opt/homebrew, and the Makefile asks `brew --prefix` rather than + # naming either. test_plug.py asserts that it keeps asking. + - channel: homebrew + arch: arm64 + runner: macos-15 + container: "" + - channel: source + arch: amd64 + runner: ubuntu-24.04 + container: ghcr.io/tamnd/gcc-internals/rel:amd64 + - channel: source + arch: arm64 + runner: ubuntu-24.04-arm + container: ghcr.io/tamnd/gcc-internals/rel:arm64 + + steps: + # Before the checkout, not after. A bare debian:sid has no git, and actions/checkout + # without one silently falls back to downloading a tarball over the REST API, which + # works until the day it does not. + - name: install a compiler and its plugin headers + shell: bash + run: | + set -eux + case "${{ matrix.channel }}" in + debian) + apt-get update + apt-get install --no-install-recommends -y \ + ca-certificates diffutils g++-16 gcc-16 gcc-16-plugin-dev git make python3 + echo "GXPLUG_GCC=gcc-16" >> "$GITHUB_ENV" + ;; + fedora) + dnf install -y --setopt=install_weak_deps=False \ + diffutils gcc gcc-c++ gcc-plugin-devel git make python3 + echo "GXPLUG_GCC=gcc" >> "$GITHUB_ENV" + ;; + homebrew) + brew install gcc + # Whichever major version the formula is on today. The plugin ABI only has + # to match the compiler that loads it, and that is the same one either way, + # so pinning a version here would break the job for no benefit. + found=$(ls "$(brew --prefix)"/bin/gcc-1* 2>/dev/null | sort -V | tail -1) + test -n "$found" + echo "GXPLUG_GCC=$found" >> "$GITHUB_ENV" + ;; + source) + # The image is ubuntu:24.04 with our own GCC ahead of everything on PATH, + # and its plugin headers went in with make install. It has no python3, + # because nothing a compiler image does needs one. + apt-get update + apt-get install --no-install-recommends -y python3 + rm -rf /var/lib/apt/lists/* + echo "GXPLUG_GCC=gcc" >> "$GITHUB_ENV" + ;; + esac + + - uses: actions/checkout@v4 + + # Printed on every run, passing or not. When a channel breaks it is nearly always + # because a probe resolved somewhere unexpected, and the first question is always + # which compiler and which headers this job actually found. + - name: what the probes decided + shell: bash + run: | + "$GXPLUG_GCC" --version | head -1 + make -C gxplug probe GCC="$GXPLUG_GCC" + + - name: build + shell: bash + run: make -C gxplug GCC="$GXPLUG_GCC" + + # The guarantee. Same source, compiled twice, assembly compared byte for byte. + - name: the plugin changes nothing about the generated code + shell: bash + run: make -C gxplug check GCC="$GXPLUG_GCC" + + # A build that produces no events is a plugin that loaded and did nothing, which the + # byte comparison above would happily call a pass. + - name: the stream is real, and reaches RTL + shell: bash + run: | + set -eux + printf 'int f(int n){int s=0;for(int i=0;i t.c + "$GXPLUG_GCC" -O2 -S -fplugin=./gxplug/gxplug.so \ + -fplugin-arg-gxplug-out=events.ndjson t.c -o t.s + wc -l events.ndjson + grep -q '"pass":"expand"' events.ndjson + grep -q '"insns":[0-9]' events.ndjson + grep -q '"statements":[0-9]' events.ndjson + + # The reader for the stream, run against what this channel's compiler just emitted + # rather than against a recording. Loaded by path rather than as `from gxray import + # plug`, because importing the package pulls in every other module and their + # dependencies, and this step is meant to prove the reader works, not that pip does. + - name: the reader understands this channel's stream + shell: bash + run: | + python3 - <<'PY' + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location("plug", "gxray/plug.py") + plug = importlib.util.module_from_spec(spec) + # Registered before it is executed, not after. @dataclass looks its own class up + # in sys.modules while the module body is still running, and on Python 3.14 a + # module that is not there yet fails with an AttributeError about NoneType that + # says nothing at all about the real cause. + sys.modules["plug"] = plug + spec.loader.exec_module(plug) + + stream = plug.load("events.ndjson") + runs = stream.runs + changed = [r for r in runs if r.changed] + print(f"{len(stream.events)} events, {len(runs)} runs, {len(changed)} changed something") + assert stream.functions == ["f"], stream.functions + assert any(r.name == "expand" for r in runs), "never reached RTL" + assert all(r.end is not None for r in runs), "a pass started and never finished" + assert 0 < len(changed) < len(runs) + PY diff --git a/containers/Dockerfile.plug b/containers/Dockerfile.plug index deb439d..9de27cb 100644 --- a/containers/Dockerfile.plug +++ b/containers/Dockerfile.plug @@ -39,7 +39,11 @@ WORKDIR /usr/src/gxplug # The plugin builds here rather than being copied in, because a plugin binary is only valid # against the compiler it was compiled against and an image that shipped one built anywhere # else would be shipping a segfault. -RUN if [ -f Makefile ]; then make; else echo "gxplug has no Makefile yet, image has the toolchain only"; fi +# +# `make check` and not `make`. The image is what a reader pulls to follow a lesson, so it +# has to ship a plugin that has been shown not to change the code, not one that merely +# compiled. It costs a couple of seconds on a five minute build. +RUN make && make check LABEL org.opencontainers.image.source="https://github.com/tamnd/gcc-internals" LABEL org.opencontainers.image.licenses="GPL-3.0-or-later" diff --git a/gxplug/Makefile b/gxplug/Makefile new file mode 100644 index 0000000..84f87fb --- /dev/null +++ b/gxplug/Makefile @@ -0,0 +1,121 @@ +# Build gxplug against whichever GCC you point it at. +# +# The reader should never have to know about -Wl,-undefined,dynamic_lookup or about where +# Homebrew keeps gmp.h. Both are probed for here. Set GCC to pick a compiler: +# +# make # uses gcc +# make GCC=gcc-16 # Homebrew, Debian, anything with a -N suffix +# make check # builds it, then proves it changes no generated code +# +# The plugin has to be built by the same GCC it will be loaded into. The ABI is sensitive +# to version and to configure options, and plugin_init refuses a mismatch rather than +# crashing later, so a wrong GCC here is a clear error and not a mystery. + +GCC ?= gcc +CXX_FOR_PLUGIN ?= $(GCC) + +# Where this GCC keeps its plugin headers. Empty means the plugin development package is +# not installed, which is the single most common reason a first attempt fails. +PLUGIN_DIR := $(shell $(GCC) -print-file-name=plugin 2>/dev/null) +PLUGIN_INCLUDE := $(PLUGIN_DIR)/include + +UNAME_S := $(shell uname -s) + +# GCC itself is built without RTTI, and a plugin that disagrees gets link errors about +# typeinfo that say nothing useful about the cause. +CXXFLAGS ?= -O2 -std=c++17 -fno-rtti -fPIC -Wall -Wextra -Wno-unused-parameter + +# Ours, kept separate from CPPFLAGS and placed ahead of it. A developer machine often has +# a CPPFLAGS in the environment pointing at Homebrew LLVM, and an LLVM header found before +# GCC's own is a failure that reads like a compiler bug. Appending to CPPFLAGS would put +# that first; this puts it second and still lets someone add to it deliberately. +GXPLUG_CPPFLAGS := -I$(PLUGIN_INCLUDE) + +# gmp.h is included by GCC's own headers and on macOS it lives under the Homebrew prefix, +# which is not on the default include path. Probe rather than hardcode, because the prefix +# differs between Apple silicon and Intel and because Linux does not need this at all. +GMP_PROBE := $(shell $(GCC) $(GXPLUG_CPPFLAGS) -fsyntax-only probe/gmp.c >/dev/null 2>&1 && echo yes || echo no) +ifeq ($(GMP_PROBE),no) + BREW_PREFIX := $(shell brew --prefix 2>/dev/null) + ifneq ($(BREW_PREFIX),) + GXPLUG_CPPFLAGS += -I$(BREW_PREFIX)/include + endif +endif + +# A plugin is C++, and the `gcc` driver knows what to do with a .cc file, so this looks +# like it works right up to the point where it tries to exec a cc1plus that only the g++ +# package ships. Asking here turns "No such file or directory" into a sentence that names +# the package. +CXX_PROBE := $(shell echo 'int main(){}' | $(CXX_FOR_PLUGIN) -x c++ -fsyntax-only - >/dev/null 2>&1 && echo yes || echo no) + +# Darwin resolves a plugin's undefined symbols out of cc1 at dlopen time, and the linker +# has to be told that is acceptable. ELF platforms do this by default and passing the flag +# there is an error, so it is conditional. +LDFLAGS_PLUGIN := +ifeq ($(UNAME_S),Darwin) + LDFLAGS_PLUGIN += -Wl,-undefined,dynamic_lookup +endif + +# .so on both platforms, not .dylib. GCC's -fplugin= wants a shared object and Darwin is +# happy to produce one under that name. +PLUGIN := gxplug.so +SRC := src/gxplug.cc + +.PHONY: all check probe clean + +all: $(PLUGIN) + +$(PLUGIN): $(SRC) + @test -f "$(PLUGIN_INCLUDE)/gcc-plugin.h" || { \ + echo "gxplug: no plugin headers under $(PLUGIN_INCLUDE)"; \ + echo "gxplug: install the plugin development package for this compiler"; \ + echo "gxplug: Debian/Ubuntu apt install gcc-16-plugin-dev"; \ + echo "gxplug: Fedora dnf install gcc-plugin-devel"; \ + echo "gxplug: Homebrew already included with the gcc formula"; \ + exit 1; \ + } + @test "$(CXX_PROBE)" = yes || { \ + echo "gxplug: $(CXX_FOR_PLUGIN) cannot compile C++, and a plugin is C++"; \ + echo "gxplug: install the matching C++ compiler, not another C one"; \ + echo "gxplug: Debian/Ubuntu apt install g++-16"; \ + echo "gxplug: Fedora dnf install gcc-c++"; \ + exit 1; \ + } + $(CXX_FOR_PLUGIN) $(GXPLUG_CPPFLAGS) $(CPPFLAGS) $(CXXFLAGS) -shared $(LDFLAGS_PLUGIN) -o $@ $(SRC) + +# What the probes decided, for when a build fails somewhere unfamiliar and the first +# question is which compiler and which headers it actually found. +probe: + @echo "GCC $(GCC)" + @echo "version $$($(GCC) -dumpfullversion 2>/dev/null)" + @echo "plugin headers $(PLUGIN_INCLUDE)" + @echo "gcc-plugin.h $$(test -f '$(PLUGIN_INCLUDE)/gcc-plugin.h' && echo present || echo MISSING)" + @echo "uname $(UNAME_S)" + @echo "compiles c++ $(CXX_PROBE)" + @echo "gmp.h on path $(GMP_PROBE)" + @echo "CPPFLAGS $(GXPLUG_CPPFLAGS) $(CPPFLAGS)" + @echo "link flags $(LDFLAGS_PLUGIN)" + +# The guarantee, checked rather than asserted. Compile something twice, once with the +# plugin and once without, and require the assembly to be identical byte for byte. If this +# ever fails then the plugin is not a passive observer any more and no lesson built on it +# can be trusted. +check: $(PLUGIN) + @set -e; \ + tmp=$$(mktemp -d); \ + trap 'rm -rf "$$tmp"' EXIT; \ + printf 'int f(int n){int s=0;for(int i=0;i "$$tmp/t.c"; \ + $(GCC) -O2 -S "$$tmp/t.c" -o "$$tmp/without.s"; \ + $(GCC) -O2 -S -fplugin=./$(PLUGIN) -fplugin-arg-gxplug-out="$$tmp/events.ndjson" \ + "$$tmp/t.c" -o "$$tmp/with.s"; \ + if ! cmp -s "$$tmp/without.s" "$$tmp/with.s"; then \ + echo "gxplug: FAIL, the plugin changed the generated code"; \ + diff -u "$$tmp/without.s" "$$tmp/with.s" || true; \ + exit 1; \ + fi; \ + events=$$(wc -l < "$$tmp/events.ndjson" | tr -d ' '); \ + test "$$events" -gt 0 || { echo "gxplug: FAIL, no events emitted"; exit 1; }; \ + echo "gxplug: assembly identical with and without the plugin, $$events events" + +clean: + rm -f $(PLUGIN) diff --git a/gxplug/README.md b/gxplug/README.md new file mode 100644 index 0000000..8f21e3b --- /dev/null +++ b/gxplug/README.md @@ -0,0 +1,101 @@ +# gxplug + +A GCC plugin that watches the pass manager and writes down what it saw. + +The text dumps tell you what the IR looked like at a few chosen points. They do not tell you which passes actually ran, in what order, on which function, how long each took, or what the compiler believed about the IR at that moment. `gxplug` emits that as one JSON object per line and changes nothing else. + +## The one rule + +**`gxplug` never changes what GCC compiles.** + +A reader has to be able to add `-fplugin=gxplug.so` to any command in this book and get byte identical code out, plus a stream of events. Nothing here registers a pass, overrides a gate, or modifies a tree. Everything it touches is read only. + +That is a promise a document cannot keep, so it is checked instead. `make check` compiles the same program twice, once with the plugin and once without, and compares the assembly byte for byte. The `plugin` workflow runs the same check against four different compilers on every push that touches this directory. If it ever fails then no lesson built on the plugin is trustworthy, and the failure is the most important thing in the repository that day. + +## Building it + +```sh +make # against whatever `gcc` means here +make GCC=gcc-16 # against a specific compiler +make probe # what the build decided, and where it looked +make check # build it, then prove it changes no generated code +``` + +The plugin has to be built by the same GCC that will load it. The plugin ABI depends on the version and on the configure options, and `plugin_init` refuses a mismatch rather than crashing somewhere later, so pointing `GCC` at the wrong compiler gives a clear error instead of a mystery. + +The one prerequisite is the plugin headers, which most distributions ship separately from the compiler: + +| Channel | What to install | +|---|---| +| Debian | `gcc-16` and `gcc-16-plugin-dev` | +| Fedora | `gcc` and `gcc-plugin-devel` | +| Homebrew | `gcc`, which already includes them | +| source | nothing, `make install` puts them under the prefix | + +If `make` cannot find `gcc-plugin.h` it says so and prints that table, because a missing plugin development package is what goes wrong on almost every first attempt. + +Two other things differ between those four channels, and both are probed for rather than assumed. On macOS a shared object has to be linked with `-undefined dynamic_lookup`, because its undefined symbols are resolved out of `cc1` at `dlopen` time, and passing that flag on an ELF platform is an error, so it is conditional on `uname`. Also on macOS, `gmp.h` comes from Homebrew and is not on the default include path, so the build compiles `probe/gmp.c` first and adds `$(brew --prefix)/include` only when that fails. + +## Running it + +```sh +gcc-16 -O2 -S -fplugin=./gxplug.so l1.c -o l1.s +``` + +With no arguments the stream goes to stderr, so that adding the flag and nothing else shows you something. Standard output is deliberately left alone, because `-S` writes assembly there and the whole point is that it comes out unchanged. + +| Argument | What it does | +|---|---| +| `-fplugin-arg-gxplug-out=PATH` | write the stream to a file | +| `-fplugin-arg-gxplug-fd=N` | write it to an already open descriptor | + +The second is how a notebook reads the stream without a temporary file. Giving both, or giving an argument that is not one of these, is an error and stops the compilation. A typo in a `-fplugin-arg-` name is otherwise the sort of thing that quietly does nothing for an hour. + +## What a record looks like + +Two records per pass. A `pass-start` when the pass manager is about to run it, and a `pass-end` carrying the duration and the state the pass left behind. + +```json +{"seq":41,"event":"pass-end","pass":"cfg","pass_number":20,"function":"f", + "properties":84239,"statements":8,"insns":null,"blocks":6,"seconds":0.000065} +``` + +| Field | Meaning | +|---|---| +| `seq` | position in the stream, so ordering does not depend on timestamps | +| `event` | `pass-start` or `pass-end` | +| `pass` | the pass name, the same one `-fdump-passes` prints | +| `pass_number` | GCC's static pass number, `-1` for a pass that has none | +| `function` | the function being compiled, `null` during an IPA pass | +| `properties` | `cfun->curr_properties`, the IR property bits, as an integer | +| `statements` | GIMPLE statements, or `null` once the function is RTL | +| `insns` | RTL insns, or `null` before `expand` | +| `blocks` | basic blocks, `null` once the CFG has been freed | +| `seconds` | how long the pass took, on `pass-end` only | + +A count that does not apply is `null` and not `0`, so a reader can tell "there was nothing to count here" from "counted, and the answer was zero". + +`statements` and `insns` are never both set. A basic block holds either a GIMPLE sequence or an RTL insn chain, in a union, and walking it as the wrong one does not fail a check. It reads whichever pointer happens to be there and segfaults, which is what the first version of this plugin did. + +## Reading the stream + +`gxray.plug` turns a stream into something a widget can hold. + +```python +from gxray import plug + +stream = plug.load("events.ndjson") +runs = stream.runs +changed = [r for r in runs if r.changed] +print(len(runs), "passes ran,", len(changed), "left a mark") +``` + +`changed` is deliberately narrow. It means the statement count, the insn count, the block count, or the property bits moved. A pass that rewrote an expression without changing any of those reports `False`, so the prose has to say what the number means rather than let a reader conclude that most passes do nothing. What it is good for is the honest ratio in T04: how many passes ran, and how many left a mark you can see from outside. + +A `pass-start` with no matching `pass-end` becomes a run whose `end` is `None`, rather than being dropped. It means the compilation died inside that pass, which is the single most useful thing the stream can tell you. + +## Why not read the dumps + +`gxray.passes` already parses `-fdump-passes`, and that answers a different question: which passes this compiler knows about, and whether they are switched on. It is a static list. It does not know that `ccp` ran three times, that two of those changed nothing, or that the one expensive pass in the run was somewhere you were not looking. + +Nothing in `gxray.plug` parses text, which is the point. The plugin emits JSON precisely so that the reading side does not have to be a parser. diff --git a/gxplug/probe/gmp.c b/gxplug/probe/gmp.c new file mode 100644 index 0000000..2cde255 --- /dev/null +++ b/gxplug/probe/gmp.c @@ -0,0 +1,14 @@ +/* Does this compiler find gmp.h without help? + + GCC's own headers include it, so a plugin cannot build without it. On macOS it comes + from Homebrew and is not on the default include path, and on Linux it is. This file + exists as a file rather than as a printf inside the Makefile because make treats a + `#` as the start of a comment even in the middle of a $(shell ...) call, which eats + the rest of the line and produces an error about an unterminated call. */ +#include + +int +main (void) +{ + return 0; +} diff --git a/gxplug/src/gxplug.cc b/gxplug/src/gxplug.cc new file mode 100644 index 0000000..3926f96 --- /dev/null +++ b/gxplug/src/gxplug.cc @@ -0,0 +1,348 @@ +/* gxplug — a GCC plugin that watches and never touches. + + The text dumps tell you what the IR looked like. They do not tell you which pass ran, + in what order, how long it took, or what the compiler believed about the IR at that + moment. This plugin emits that as newline-delimited JSON and changes nothing else. + + The one rule, from 03-architecture.md: gxplug never changes what GCC compiles. A reader + must be able to add -fplugin=gxplug.so to any command in the book and get byte identical + code out, plus a stream of events. Everything here is read only for that reason. There + is no pass registered, no gate overridden and no tree modified. + + Copyright the gcc-internals authors. + SPDX-License-Identifier: GPL-3.0-or-later */ + +#include "gcc-plugin.h" +#include "config.h" +#include "system.h" +#include "coretypes.h" +/* error() lives here, and it is the right way for a plugin to complain: it goes through + GCC's diagnostic machinery, counts towards the error count, and makes the compilation + fail rather than printing to stderr and carrying on. */ +#include "diagnostic-core.h" +#include "tm.h" +#include "tree.h" +#include "tree-pass.h" +#include "context.h" +#include "function.h" +#include "basic-block.h" +#include "gimple.h" +#include "gimple-iterator.h" +#include "cgraph.h" +/* For get_insns and NEXT_INSN, which are how the size of a function is measured once it + is no longer GIMPLE. */ +#include "rtl.h" +/* emit-rtl.h declares need_atomic_barrier_p taking an `enum memmodel`, and does not + include the header that defines it. GCC's own sources always include memmodel.h first, + so the omission never shows up in tree; out of tree it is an error about an enum + without a previous declaration, pointing at a line that has nothing to do with you. */ +#include "memmodel.h" +#include "emit-rtl.h" +#include "plugin-version.h" + +/* Without this GCC refuses to load the plugin at all, which is the licence check and not + a formality. */ +int plugin_is_GPL_compatible; + +namespace { + +/* Where the stream goes. Default is stderr, so that a reader who adds -fplugin= and + nothing else sees something. stdout is left alone on purpose: -S writes assembly there + and the whole point is that it comes out unchanged. */ +FILE *stream = nullptr; +bool close_stream_at_end = false; + +/* Sequence number across the whole compilation, so a consumer can order records without + trusting timestamps. */ +unsigned long seq = 0; + +/* The pass we last announced. PLUGIN_PASS_EXECUTION fires before a pass runs and there is + no matching event for after it, so a pass is closed by the arrival of the next one and + the state recorded then is that pass's result. The last pass of a function is closed by + PLUGIN_ALL_PASSES_END. */ +const char *open_pass = nullptr; +int open_pass_number = -1; +double open_started = 0.0; + +double +now_seconds () +{ + struct timespec ts; + /* CLOCK_MONOTONIC rather than the wall clock, because the interesting number is a + duration and a clock that can step backwards makes negative ones. */ + if (clock_gettime (CLOCK_MONOTONIC, &ts) != 0) + return 0.0; + return (double) ts.tv_sec + (double) ts.tv_nsec / 1e9; +} + +void +emit_json_string (const char *s) +{ + fputc ('"', stream); + for (const unsigned char *p = (const unsigned char *) s; *p; p++) + switch (*p) + { + case '"': fputs ("\\\"", stream); break; + case '\\': fputs ("\\\\", stream); break; + case '\n': fputs ("\\n", stream); break; + case '\r': fputs ("\\r", stream); break; + case '\t': fputs ("\\t", stream); break; + default: + if (*p < 0x20) + fprintf (stream, "\\u%04x", *p); + else + fputc (*p, stream); + } + fputc ('"', stream); +} + +/* Is the current function in RTL yet? This matters more than it looks. A basic block + holds either a GIMPLE sequence or an RTL insn chain, in a union, and walking it as the + wrong one does not fail a check, it reads whichever pointer happens to be there and + segfaults in *rest_of_compilation. Which is exactly what the first version of this + plugin did. */ +bool +in_rtl () +{ + return cfun && (cfun->curr_properties & PROP_rtl) != 0; +} + +/* GIMPLE statements in the current function, or -1 when there are none to count: no + function at all, which is every IPA pass, no CFG yet, or the function has already been + lowered to RTL. */ +long +statement_count () +{ + if (!cfun || !cfun->cfg || in_rtl ()) + return -1; + long n = 0; + basic_block bb; + FOR_EACH_BB_FN (bb, cfun) + for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) + n++; + return n; +} + +/* The other half of the same question, for after the function becomes RTL. Counted off + the insn chain rather than per block, because the chain is intact even in the late + passes that have thrown the CFG away. */ +long +insn_count () +{ + if (!in_rtl ()) + return -1; + long n = 0; + for (rtx_insn *insn = get_insns (); insn; insn = NEXT_INSN (insn)) + n++; + return n; +} + +/* Basic blocks, which is the one count that means the same thing on both sides of the + GIMPLE to RTL boundary. Null once pass_free_cfg has run. */ +long +block_count () +{ + if (!cfun || !cfun->cfg) + return -1; + return n_basic_blocks_for_fn (cfun); +} + +const char * +current_function_name_or_null () +{ + if (!cfun || !cfun->decl) + return nullptr; + return IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (cfun->decl)); +} + +void +emit_count (const char *key, long value) +{ + if (value < 0) + fprintf (stream, ",\"%s\":null", key); + else + fprintf (stream, ",\"%s\":%ld", key, value); +} + +/* One record. `event` is what happened, `pass` is the pass it happened to. */ +void +emit_event (const char *event, const char *pass, int pass_number, double duration) +{ + if (!stream) + return; + + fprintf (stream, "{\"seq\":%lu,\"event\":", seq++); + emit_json_string (event); + + fputs (",\"pass\":", stream); + if (pass) + emit_json_string (pass); + else + fputs ("null", stream); + + fprintf (stream, ",\"pass_number\":%d", pass_number); + + fputs (",\"function\":", stream); + const char *fn = current_function_name_or_null (); + if (fn) + emit_json_string (fn); + else + fputs ("null", stream); + + /* What the compiler currently believes is true of this function's IR. This is the + dynamic property set, not the pass's declared one, which is the difference between + what is and what was asked for. */ + if (cfun) + fprintf (stream, ",\"properties\":%u", (unsigned) cfun->curr_properties); + else + fputs (",\"properties\":null", stream); + + /* Null rather than -1 for a count that does not apply here, so a consumer can tell + "nothing to count" from "counted, and it was zero". */ + emit_count ("statements", statement_count ()); + emit_count ("insns", insn_count ()); + emit_count ("blocks", block_count ()); + + if (duration >= 0.0) + fprintf (stream, ",\"seconds\":%.9f", duration); + else + fputs (",\"seconds\":null", stream); + + fputs ("}\n", stream); +} + +/* Close whichever pass is open, attributing to it the state as it is now. Called when the + next pass starts and when the pass manager finishes. */ +void +close_open_pass () +{ + if (!open_pass) + return; + emit_event ("pass-end", open_pass, open_pass_number, now_seconds () - open_started); + open_pass = nullptr; + open_pass_number = -1; +} + +void +on_pass_execution (void *gcc_data, void * /*user_data*/) +{ + opt_pass *pass = (opt_pass *) gcc_data; + if (!pass) + return; + + close_open_pass (); + + emit_event ("pass-start", pass->name, pass->static_pass_number, -1.0); + open_pass = pass->name; + open_pass_number = pass->static_pass_number; + open_started = now_seconds (); +} + +void +on_all_passes_end (void * /*gcc_data*/, void * /*user_data*/) +{ + close_open_pass (); +} + +void +on_finish_unit (void * /*gcc_data*/, void * /*user_data*/) +{ + close_open_pass (); + if (stream) + { + fflush (stream); + if (close_stream_at_end) + fclose (stream); + stream = nullptr; + } +} + +/* -fplugin-arg-gxplug-out=PATH writes to a file, -fplugin-arg-gxplug-fd=N to an already + open descriptor, which is how a notebook reads the stream without a temporary file. + Neither given means stderr. */ +bool +open_stream (plugin_name_args *info) +{ + const char *path = nullptr; + const char *fd_arg = nullptr; + + for (int i = 0; i < info->argc; i++) + { + if (strcmp (info->argv[i].key, "out") == 0) + path = info->argv[i].value; + else if (strcmp (info->argv[i].key, "fd") == 0) + fd_arg = info->argv[i].value; + else + { + error ("gxplug: unknown argument %qs", info->argv[i].key); + return false; + } + } + + if (path && fd_arg) + { + error ("gxplug: give %qs or %qs, not both", "out", "fd"); + return false; + } + + if (path) + { + stream = fopen (path, "w"); + if (!stream) + { + error ("gxplug: cannot write to %qs", path); + return false; + } + close_stream_at_end = true; + } + else if (fd_arg) + { + char *end = nullptr; + long fd = strtol (fd_arg, &end, 10); + if (!end || *end || fd < 0 || fd > 1024) + { + error ("gxplug: fd= wants a small non-negative integer, got %qs", fd_arg); + return false; + } + stream = fdopen ((int) fd, "w"); + if (!stream) + { + error ("gxplug: cannot write to file descriptor %ld", fd); + return false; + } + close_stream_at_end = true; + } + else + stream = stderr; + + return true; +} + +} /* anonymous namespace */ + +int +plugin_init (plugin_name_args *info, plugin_gcc_version *version) +{ + /* A plugin built against one GCC and loaded into another is undefined behaviour that + usually shows up as a crash in the middle of a lesson. Refusing here is the whole + reason the plug image builds the plugin rather than shipping one. */ + if (!plugin_default_version_check (version, &gcc_version)) + { + error ("gxplug: built against GCC %s, loaded into GCC %s", gcc_version.basever, + version->basever); + return 1; + } + + if (!open_stream (info)) + return 1; + + static plugin_info about + = { "0.1", "gxplug: pass events as newline delimited JSON, and no change to codegen" }; + register_callback (info->base_name, PLUGIN_INFO, nullptr, &about); + + register_callback (info->base_name, PLUGIN_PASS_EXECUTION, on_pass_execution, nullptr); + register_callback (info->base_name, PLUGIN_ALL_PASSES_END, on_all_passes_end, nullptr); + register_callback (info->base_name, PLUGIN_FINISH_UNIT, on_finish_unit, nullptr); + + return 0; +} diff --git a/gxray/__init__.py b/gxray/__init__.py index d7a2378..1a73fe3 100644 --- a/gxray/__init__.py +++ b/gxray/__init__.py @@ -28,6 +28,7 @@ mdesc, options, passes, + plug, regalloc, rtl, source, @@ -74,6 +75,7 @@ "mdesc", "options", "passes", + "plug", "regalloc", "rtl", "source", diff --git a/gxray/plug.py b/gxray/plug.py new file mode 100644 index 0000000..d1595a9 --- /dev/null +++ b/gxray/plug.py @@ -0,0 +1,252 @@ +"""Read what `gxplug` emitted. + +The plugin writes one JSON object per line, two per pass: a `pass-start` when the pass +manager is about to run it and a `pass-end` carrying the duration and the state the pass +left behind. `gxplug/README.md` describes the record; this module turns a stream of them +into something a widget can hold. + +Why this exists when `gxray.passes` already reads `-fdump-passes`: that tells you which +passes GCC *knows about* and whether they are on. This tells you which ones actually ran, +on which function, in what order, and what changed. At `-O2` on `l1.c` the first is 395 +lines and the second is a few hundred events of which most changed nothing, and the gap +between those two numbers is the point of T04. + +Nothing here parses text. The plugin emits JSON precisely so that this file does not have +to be a parser, which is the rule in 03-architecture.md about preferring structured output +over scraping a dump. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Iterator +from dataclasses import dataclass, field +from pathlib import Path + +# GCC's IR property bits, from `gcc/tree-pass.h`. A record carries `properties` as a plain +# integer because that is what `cfun->curr_properties` is; naming the bits is this side's +# job. +# +# These are duplicated from a GCC header, which is normally how a project goes stale. It is +# safe here only because `test_plug.py` reads `vendor/gcc/gcc/tree-pass.h` at the pinned tag +# and fails if this table and that header disagree, on any bit, in either direction. +PROPERTIES: dict[str, int] = { + "gimple_any": 1 << 0, + "gimple_lcf": 1 << 1, + "gimple_leh": 1 << 2, + "cfg": 1 << 3, + "objsz": 1 << 4, + "ssa": 1 << 5, + "no_crit_edges": 1 << 6, + "rtl": 1 << 7, + "gimple_lomp": 1 << 8, + "cfglayout": 1 << 9, + "gimple_lcx": 1 << 10, + "loops": 1 << 11, + "gimple_lvec": 1 << 12, + "gimple_eomp": 1 << 13, + "gimple_lva": 1 << 14, + "gimple_opt_math": 1 << 15, + "gimple_lomp_dev": 1 << 16, + "rtl_split_insns": 1 << 17, + "loop_opts_done": 1 << 18, + "assumptions_done": 1 << 19, + "gimple_lbitint": 1 << 20, + "last_full_fold": 1 << 21, +} + + +def property_names(bits: int | None) -> list[str]: + """The properties set in `bits`, in bit order. + + An unknown bit is reported as `bit23` rather than dropped. A newer GCC adding a + property should show up as something a reader can see and ask about, not as silence. + """ + if bits is None: + return [] + named = {value: name for name, value in PROPERTIES.items()} + out = [] + for i in range(bits.bit_length()): + bit = 1 << i + if bits & bit: + out.append(named.get(bit, f"bit{i}")) + return out + + +@dataclass(frozen=True) +class Event: + """One line of the stream.""" + + seq: int + event: str + pass_name: str | None + pass_number: int + function: str | None + properties: int | None + statements: int | None + insns: int | None + blocks: int | None + seconds: float | None + + @property + def property_names(self) -> list[str]: + return property_names(self.properties) + + @property + def size(self) -> int | None: + """Statements before the function becomes RTL, insns after. + + A pass tape wants one number per cell and does not care which side of `expand` it + is on, but it must not pretend a GIMPLE statement and an RTL insn are the same + unit. They are not, and the jump at `expand` is real and worth seeing. + """ + return self.statements if self.statements is not None else self.insns + + +@dataclass(frozen=True) +class Run: + """A pass that ran: its start event, its end event, and what happened in between.""" + + name: str | None + number: int + function: str | None + start: Event + end: Event | None + + @property + def seconds(self) -> float | None: + return self.end.seconds if self.end else None + + @property + def changed(self) -> bool: + """Did anything measurable change? + + Deliberately narrow: this is size and properties, not semantics. A pass that + rewrote an expression without changing the statement count reports False, and the + prose has to say so rather than letting a reader infer that most passes do + nothing. What it is good for is the honest ratio in T04: how many passes ran, and + how many left a mark you can see from outside. + """ + if self.end is None: + return False + return ( + self.start.statements != self.end.statements + or self.start.insns != self.end.insns + or self.start.blocks != self.end.blocks + or self.start.properties != self.end.properties + ) + + +@dataclass +class Stream: + """Every event from one compilation, and the runs assembled out of them.""" + + events: list[Event] = field(default_factory=list) + + @property + def runs(self) -> list[Run]: + """Pair each `pass-start` with the `pass-end` that follows it. + + The plugin guarantees the two alternate, because it closes the open pass before + opening another and closes the last one at `PLUGIN_ALL_PASSES_END`. A start with + no end still becomes a `Run` with `end=None` rather than being dropped: it means + the compilation died in that pass, which is the single most interesting thing the + stream can tell you. + """ + out: list[Run] = [] + open_start: Event | None = None + for event in self.events: + if event.event == "pass-start": + if open_start is not None: + out.append( + Run( + open_start.pass_name, + open_start.pass_number, + open_start.function, + open_start, + None, + ) + ) + open_start = event + elif event.event == "pass-end" and open_start is not None: + out.append( + Run( + open_start.pass_name, + open_start.pass_number, + open_start.function, + open_start, + event, + ) + ) + open_start = None + if open_start is not None: + out.append( + Run( + open_start.pass_name, + open_start.pass_number, + open_start.function, + open_start, + None, + ) + ) + return out + + @property + def functions(self) -> list[str]: + """Every function named in the stream, in the order it first appears.""" + seen: dict[str, None] = {} + for event in self.events: + if event.function is not None: + seen.setdefault(event.function, None) + return list(seen) + + def for_function(self, name: str) -> Stream: + return Stream([e for e in self.events if e.function == name]) + + @property + def seconds(self) -> float: + return sum(e.seconds for e in self.events if e.seconds is not None) + + +def _event(record: dict) -> Event: + """One record to one Event, tolerating fields we do not know about. + + A newer plugin adding a key must not break an older reader, so unknown keys are + ignored rather than rejected. A missing key is a different matter and raises, because + that means the two really are out of step. + """ + try: + return Event( + seq=record["seq"], + event=record["event"], + pass_name=record["pass"], + pass_number=record["pass_number"], + function=record["function"], + properties=record["properties"], + statements=record["statements"], + insns=record["insns"], + blocks=record["blocks"], + seconds=record["seconds"], + ) + except KeyError as exc: + raise ValueError(f"gxplug record is missing {exc.args[0]}: {record!r}") from exc + + +def parse_lines(lines: Iterable[str]) -> Iterator[Event]: + for number, line in enumerate(lines, 1): + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"gxplug line {number} is not JSON: {line!r}") from exc + yield _event(record) + + +def parse(text: str) -> Stream: + return Stream(list(parse_lines(text.splitlines()))) + + +def load(path: str | Path) -> Stream: + return parse(Path(path).read_text(encoding="utf-8")) diff --git a/justfile b/justfile index 73d3492..6087954 100644 --- a/justfile +++ b/justfile @@ -23,7 +23,7 @@ lint: ruff format --check . prose: - {{py}} -m tools.prosecheck README.md CONTRIBUTING.md LICENSE.md GLOSSARY.md docs lessons blueprints containers corpora/programs + {{py}} -m tools.prosecheck README.md CONTRIBUTING.md LICENSE.md GLOSSARY.md docs lessons blueprints containers corpora/programs gxplug test: {{py}} -m pytest -q @@ -281,6 +281,43 @@ image id: --build-arg EXTRA_PACKAGES="$pkgs" \ -t "gcc-internals/{{id}}" . +# Build gxplug against {{gcc}}. Needs that compiler's plugin headers, which most +# distributions package separately. See gxplug/README.md. +plugin: + make -C gxplug GCC={{gcc}} + +# Which compiler, which headers, and what the two portability probes decided. The first +# thing to run when a build fails somewhere unfamiliar. +plugin-probe: + make -C gxplug probe GCC={{gcc}} + +# The guarantee: same program, compiled with and without the plugin, assembly compared byte +# for byte. If this fails, nothing built on gxplug can be trusted. +plugin-check: + make -C gxplug check GCC={{gcc}} + +# Write a stream for one file and say what is in it. `just plugin-run corpora/programs/l1.c` +plugin-run FILE OPT="-O2": plugin + #!/usr/bin/env bash + set -euo pipefail + out=$(mktemp -t gxplug.XXXXXX) + trap 'rm -f "$out" "$out.s"' EXIT + {{gcc}} {{OPT}} -S -fplugin=./gxplug/gxplug.so \ + -fplugin-arg-gxplug-out="$out" {{FILE}} -o "$out.s" + {{py}} - "$out" <<'PY' + import sys + from gxray import plug + + stream = plug.load(sys.argv[1]) + runs = stream.runs + changed = [r for r in runs if r.changed] + print(f"{len(stream.events)} events, {len(runs)} passes ran, {len(changed)} left a mark") + print("functions:", ", ".join(stream.functions)) + print(f"{stream.seconds:.4f} seconds inside passes") + for run in sorted(changed, key=lambda r: r.seconds or 0, reverse=True)[:10]: + print(f" {run.seconds or 0:8.6f} {run.name}") + PY + # The pinned GCC tree, about 1.3 GB shallow. Only refcheck needs it. gcc-src: git submodule update --init --depth 1 diff --git a/pyproject.toml b/pyproject.toml index 83b55a6..b585398 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ testpaths = ["tests"] filterwarnings = ["error"] markers = [ "needs_gcc: drives a real gcc-16 and is skipped when there is not one on PATH", + "needs_gcc_plugin: builds gxplug, so it needs a gcc-16 with its plugin headers as well", "needs_network: talks to Compiler Explorer, never run in CI", "needs_marimo: builds a real island and needs the site extra installed", "needs_nbclient: runs a lesson in a real kernel and needs the lessons extra installed", diff --git a/tests/conftest.py b/tests/conftest.py index cc2b049..abb36d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ import importlib.util import shutil +import subprocess from pathlib import Path import pytest @@ -23,6 +24,51 @@ HAVE_NBCLIENT = importlib.util.find_spec("nbclient") is not None +def _gcc_with_plugin_headers() -> str | None: + """A GCC that can build a plugin, or None. + + Three conditions, and they are separate. + + A GCC of any version other than 16 has the wrong ABI. `plugin_default_version_check` + would refuse the result at load time, so building against it proves nothing. + + A GCC 16 without its plugin development package has no `gcc-plugin.h` to compile + against, and that is the ordinary case on a machine that installed a compiler and + nothing else. + + A GCC 16 that cannot compile C++ is the one that looks fine until it is not. A plugin + is C++, the `gcc` driver knows perfectly well what to do with a `.cc` file, and it + goes all the way to trying to exec a `cc1plus` that only the `g++` package ships. + """ + for name in ("gcc-16", "gcc"): + found = shutil.which(name) + if not found: + continue + version = subprocess.run( + [found, "-dumpfullversion"], capture_output=True, text=True, check=False + ) + if not version.stdout.startswith("16."): + continue + where = subprocess.run( + [found, "-print-file-name=plugin"], capture_output=True, text=True, check=False + ) + if not (Path(where.stdout.strip()) / "include" / "gcc-plugin.h").exists(): + continue + cxx = subprocess.run( + [found, "-x", "c++", "-fsyntax-only", "-"], + input="int main(){}\n", + capture_output=True, + text=True, + check=False, + ) + if cxx.returncode == 0: + return found + return None + + +GCC_PLUGIN = _gcc_with_plugin_headers() + + LESSONS = Path(__file__).resolve().parent.parent / "lessons" @@ -78,13 +124,28 @@ def setjmp_graph() -> str: return fixture("setjmp-O1-optimized-graph.dot") +@pytest.fixture(scope="session") +def gcc_plugin() -> str: + """The compiler the plugin tests build against. + + Only reached by a test marked `needs_gcc_plugin`, which has already been skipped when + there is no such compiler, so the assert is a statement about the marker and not a + thing that can fire in a normal run. + """ + assert GCC_PLUGIN is not None + return GCC_PLUGIN + + def pytest_collection_modifyitems(config, items): no_gcc = pytest.mark.skip(reason="no gcc-16 on PATH") + no_plugin = pytest.mark.skip(reason="no gcc-16 with plugin headers, try gcc-16-plugin-dev") no_marimo = pytest.mark.skip(reason="marimo is not installed, try pip install -e '.[site]'") no_kernel = pytest.mark.skip(reason="nbclient is missing, try pip install -e '.[lessons]'") for item in items: if "needs_gcc" in item.keywords and not HAVE_GCC: item.add_marker(no_gcc) + if "needs_gcc_plugin" in item.keywords and GCC_PLUGIN is None: + item.add_marker(no_plugin) if "needs_marimo" in item.keywords and not HAVE_MARIMO: item.add_marker(no_marimo) if "needs_nbclient" in item.keywords and not HAVE_NBCLIENT: diff --git a/tests/test_matrix.py b/tests/test_matrix.py index 2ffa646..822d5fa 100644 --- a/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -217,9 +217,34 @@ def test_a_readme_with_no_markers_says_so_rather_than_writing_nothing(): matrix.rewrite("# no markers here\n", "table") -def test_the_lockfile_is_read_even_when_it_is_empty(): - """It is empty until the first full matrix run publishes, and that is not a crash.""" - assert matrix.digests() == {} +def test_the_lockfile_is_read_even_when_it_is_empty(tmp_path): + """A lockfile with no images in it is a state, not a crash. + + It is what a fresh clone of this project would have before the first matrix run + publishes anything, and `digests` has to return an empty mapping rather than raise. + Written against a temporary file and not the committed one, because the committed one + stopped being empty the moment the matrix first published and a test that reads it is + a test that expires. + """ + empty = tmp_path / "images.lock.json" + empty.write_text('{"images": {}}', encoding="utf-8") + assert matrix.digests(empty) == {} + + +def test_a_lockfile_that_is_not_there_at_all_is_also_not_a_crash(tmp_path): + assert matrix.digests(tmp_path / "nothing.json") == {} + + +def test_the_committed_lockfile_is_readable_and_pins_by_digest(): + """Whatever is in it, every entry has to be a digest. + + This is the half of the lockfile check that can run without knowing whether a full + weekly run has happened yet. `lock_problems` is the half that knows, and issue #61 is + about making it blocking once all twelve images exist. + """ + for name, digest in matrix.digests().items(): + assert digest.startswith("sha256:"), name + assert len(digest) == len("sha256:") + 64, name def test_an_image_the_matrix_no_longer_builds_is_the_dangerous_one(): diff --git a/tests/test_plug.py b/tests/test_plug.py new file mode 100644 index 0000000..5913f93 --- /dev/null +++ b/tests/test_plug.py @@ -0,0 +1,394 @@ +"""gxplug, and the reader for what it emits. + +Two kinds of test here. Most run anywhere and work on a recorded stream. A few build the +plugin against a real GCC 16 and run it, and those skip when there is no such compiler, +because a contributor on a laptop with GCC 13 should still get a green suite. + +The one that matters most is `test_the_plugin_does_not_change_the_generated_code`. Every +lesson that uses gxplug rests on the plugin being a passive observer, and an assertion in +a design document is not a guarantee. +""" + +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path + +import pytest + +from gxray import plug + +REPO = Path(__file__).resolve().parent.parent +GXPLUG = REPO / "gxplug" +TREE_PASS_H = REPO / "vendor" / "gcc" / "gcc" / "tree-pass.h" + + +# A stream is easier to reason about as a literal than as a fixture built by a compiler, +# and these are real lines, trimmed out of a run of `gcc-16 -O2 -S` on the loop in +# `make check`. +STREAM = "\n".join( + json.dumps(record) + for record in [ + { + "seq": 0, + "event": "pass-start", + "pass": "cfg", + "pass_number": 20, + "function": "f", + "properties": 16385, + "statements": None, + "insns": None, + "blocks": None, + "seconds": None, + }, + { + "seq": 1, + "event": "pass-end", + "pass": "cfg", + "pass_number": 20, + "function": "f", + "properties": 84239, + "statements": 8, + "insns": None, + "blocks": 6, + "seconds": 0.000065, + }, + { + "seq": 2, + "event": "pass-start", + "pass": "*warn_function_return", + "pass_number": -1, + "function": "f", + "properties": 84239, + "statements": 8, + "insns": None, + "blocks": 6, + "seconds": None, + }, + { + "seq": 3, + "event": "pass-end", + "pass": "*warn_function_return", + "pass_number": -1, + "function": "f", + "properties": 84239, + "statements": 8, + "insns": None, + "blocks": 6, + "seconds": 0.000001, + }, + { + "seq": 4, + "event": "pass-start", + "pass": "expand", + "pass_number": 180, + "function": "f", + "properties": 84239, + "statements": 8, + "insns": None, + "blocks": 6, + "seconds": None, + }, + { + "seq": 5, + "event": "pass-end", + "pass": "expand", + "pass_number": 180, + "function": "f", + "properties": 3538136, + "statements": None, + "insns": 26, + "blocks": 8, + "seconds": 0.000211, + }, + ] +) + + +def test_a_stream_parses_into_events(): + stream = plug.parse(STREAM) + assert len(stream.events) == 6 + assert stream.events[0].pass_name == "cfg" + assert stream.events[1].statements == 8 + + +def test_blank_lines_are_not_events(): + assert len(plug.parse(f"\n\n{STREAM}\n\n").events) == 6 + + +def test_a_line_that_is_not_json_says_which_line(): + with pytest.raises(ValueError, match="line 2"): + plug.parse( + '{"seq":0,"event":"pass-start","pass":null,"pass_number":0,' + '"function":null,"properties":null,"statements":null,"insns":null,' + '"blocks":null,"seconds":null}\nnot json\n' + ) + + +def test_a_record_missing_a_field_says_which_field(): + with pytest.raises(ValueError, match="missing blocks"): + plug.parse( + '{"seq":0,"event":"pass-start","pass":null,"pass_number":0,' + '"function":null,"properties":null,"statements":null,"insns":null,' + '"seconds":null}' + ) + + +def test_an_unknown_field_is_ignored_rather_than_rejected(): + """A newer plugin must not break an older reader.""" + record = json.loads(STREAM.splitlines()[0]) + record["something_new"] = 42 + assert plug.parse(json.dumps(record)).events[0].pass_name == "cfg" + + +def test_starts_and_ends_pair_into_runs(): + runs = plug.parse(STREAM).runs + assert [r.name for r in runs] == ["cfg", "*warn_function_return", "expand"] + assert runs[0].seconds == 0.000065 + + +def test_a_pass_with_no_end_is_kept_because_it_means_the_compiler_died_there(): + """Dropping it would hide the one thing the stream is best placed to tell you.""" + truncated = "\n".join(STREAM.splitlines()[:5]) + runs = plug.parse(truncated).runs + assert runs[-1].name == "expand" + assert runs[-1].end is None + assert runs[-1].seconds is None + + +def test_a_pass_that_changed_nothing_says_so(): + runs = {r.name: r for r in plug.parse(STREAM).runs} + assert runs["*warn_function_return"].changed is False + assert runs["cfg"].changed is True + assert runs["expand"].changed is True + + +def test_size_follows_the_ir_across_expand(): + """Statements before expand, insns after, and never both at once.""" + events = plug.parse(STREAM).events + assert events[1].size == 8 + assert events[5].size == 26 + for event in events: + assert event.statements is None or event.insns is None + + +def test_properties_decode_to_names(): + stream = plug.parse(STREAM) + after_cfg = stream.events[1].property_names + assert "cfg" in after_cfg + assert "rtl" not in after_cfg + assert "rtl" in stream.events[5].property_names + + +def test_an_unknown_property_bit_is_shown_rather_than_dropped(): + """A GCC that adds a property should produce a visible question, not silence.""" + assert plug.property_names(1 << 30) == ["bit30"] + + +def test_no_properties_at_all_is_an_empty_list(): + assert plug.property_names(None) == [] + + +def test_functions_come_out_in_the_order_they_appear(): + assert plug.parse(STREAM).functions == ["f"] + + +def test_the_property_table_matches_the_pinned_gcc_header(): + """The one duplication in `plug.py`, held honest here. + + `PROPERTIES` restates bit values from `gcc/tree-pass.h`. That is exactly the kind of + copy that rots, so it is compared against the header in the pinned tree, in both + directions: a bit we invented and a bit GCC added both fail. + """ + if not TREE_PASS_H.exists(): + pytest.skip("vendor/gcc is not checked out, try just gcc-src") + + pattern = re.compile(r"^#define\s+PROP_(\w+)\s+\(1 << (\d+)\)", re.MULTILINE) + from_gcc = { + name: 1 << int(shift) + for name, shift in pattern.findall(TREE_PASS_H.read_text(encoding="utf-8")) + } + assert from_gcc, "no PROP_ bits found, has tree-pass.h changed shape?" + assert plug.PROPERTIES == from_gcc + + +def test_the_makefile_asks_for_the_homebrew_prefix_rather_than_naming_one(): + """Intel Homebrew installs under /usr/local and Apple silicon under /opt/homebrew. + + GitHub retired its Intel macOS runners, so the plugin workflow can only build the + arm64 half. The only thing that actually differs between the two is the prefix, and + the way to be right on a machine CI cannot reach is to ask rather than to guess. This + is the cheap standing check that nobody replaces the question with an answer. + """ + makefile = (GXPLUG / "Makefile").read_text(encoding="utf-8") + assert "brew --prefix" in makefile + assert "/opt/homebrew" not in makefile + assert "/usr/local" not in makefile + + +# --- the tests that need a compiler --------------------------------------------------- + +needs_plugin_gcc = pytest.mark.needs_gcc_plugin + + +@pytest.fixture(scope="module") +def built_plugin(gcc_plugin) -> Path: + """Build the plugin once for every test that needs it.""" + # The developer's environment is left intact on purpose. An inherited CPPFLAGS + # pointing at another toolchain's headers is a real failure mode, and one the Makefile + # is written to survive, so building under it is the more honest test. + # + # The output is captured so a passing run stays quiet, and printed in full when the + # build fails. `check=True` on its own reports an exit status and throws away the + # compiler error that explains it, which turns a one line diagnosis into a round trip + # through CI. + built = subprocess.run( + ["make", "-C", str(GXPLUG), f"GCC={gcc_plugin}"], capture_output=True, text=True + ) + if built.returncode != 0: + pytest.fail(f"building gxplug failed\n{built.stdout}\n{built.stderr}") + return GXPLUG / "gxplug.so" + + +@needs_plugin_gcc +def test_the_plugin_builds(built_plugin): + assert built_plugin.exists() + + +@needs_plugin_gcc +def test_the_plugin_does_not_change_the_generated_code(built_plugin, gcc_plugin, tmp_path): + """The guarantee the whole design rests on. + + `gxplug never changes what GCC compiles`, from 03-architecture.md. Compile the same + program twice, with and without, and require the assembly to match byte for byte. + """ + gcc = gcc_plugin + source = tmp_path / "t.c" + source.write_text( + "int f(int n){int s=0;for(int i=0;i 0 + + +@needs_plugin_gcc +def test_a_real_stream_parses_and_covers_both_irs(built_plugin, gcc_plugin, tmp_path): + """Not just valid JSON: the run has to reach RTL and report both kinds of size.""" + gcc = gcc_plugin + source = tmp_path / "t.c" + source.write_text("int f(int n){int s=0;for(int i=0;i 100 + assert stream.functions == ["f"] + + names = [r.name for r in stream.runs] + assert "expand" in names, "the run never reached RTL" + + assert any(e.statements is not None for e in stream.events), "no GIMPLE was measured" + assert any(e.insns is not None for e in stream.events), "no RTL was measured" + + # The honest ratio T04 is about: many passes run, few leave a visible mark. + runs = stream.runs + changed = [r for r in runs if r.changed] + assert 0 < len(changed) < len(runs) + + +@needs_plugin_gcc +def test_every_start_is_closed(built_plugin, gcc_plugin, tmp_path): + """A dangling start would mean the plugin lost track, or the compiler died.""" + gcc = gcc_plugin + source = tmp_path / "t.c" + source.write_text("int f(int n){return n*n;}\nint g(void){return f(7);}\n") + events = tmp_path / "events.ndjson" + subprocess.run( + [ + gcc, + "-O2", + "-S", + f"-fplugin={built_plugin}", + f"-fplugin-arg-gxplug-out={events}", + str(source), + "-o", + str(tmp_path / "t.s"), + ], + check=True, + ) + stream = plug.load(events) + assert all(run.end is not None for run in stream.runs) + starts = [e for e in stream.events if e.event == "pass-start"] + ends = [e for e in stream.events if e.event == "pass-end"] + assert len(starts) == len(ends) + + +@needs_plugin_gcc +def test_an_unknown_argument_is_an_error_and_not_a_shrug(built_plugin, gcc_plugin, tmp_path): + """A typo in -fplugin-arg- should fail loudly rather than silently do nothing.""" + gcc = gcc_plugin + source = tmp_path / "t.c" + source.write_text("int main(void){return 0;}\n") + result = subprocess.run( + [ + gcc, + "-O2", + "-S", + f"-fplugin={built_plugin}", + "-fplugin-arg-gxplug-outt=/dev/null", + str(source), + "-o", + str(tmp_path / "t.s"), + ], + capture_output=True, + text=True, + ) + assert result.returncode != 0 + assert "unknown argument" in result.stderr + + +@needs_plugin_gcc +def test_the_makefile_check_target_passes(built_plugin, gcc_plugin): + """The same guarantee, from the entry point a reader would actually type.""" + gcc = gcc_plugin + result = subprocess.run( + ["make", "-C", str(GXPLUG), "check", f"GCC={gcc}"], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "assembly identical" in result.stdout