A plugin that watches GCC and changes nothing

.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
       - run: ruff check .
       - run: ruff format --check .
       - name: prose rules
-        run: python -m tools.prosecheck README.md CONTRIBUTING.md LICENSE.md GLOSSARY.md docs lessons blueprints containers corpora/programs
+        run: python -m tools.prosecheck README.md CONTRIBUTING.md LICENSE.md GLOSSARY.md docs lessons blueprints containers corpora/programs gxplug
 
       # The table in containers/README.md is generated from containers/matrix.toml, and a
       # hand edit to either one is how the two start describing different compilers.
@@ -243,9 +243,21 @@ jobs:
           sudo apt-get install -y gcc-16
           gcc-16 --version
 
+          # The plugin headers, so the tests marked needs_gcc_plugin run here rather than
+          # skipping, and g++-16 with them. A plugin is C++, and the gcc driver will
+          # happily compile a .cc file right up to the point where it tries to exec a
+          # cc1plus that only the g++ package ships.
+          #
+          # Tolerated if the PPA has not built them for this snapshot: those tests are
+          # gated properly by the plugin workflow, which builds gxplug against four real
+          # channels, and a missing package in a trunk PPA should not turn this job red
+          # for a reason that has nothing to do with the change under review.
+          sudo apt-get install -y g++-16 gcc-16-plugin-dev \
+            || echo "the PPA is missing g++-16 or the plugin headers, the gxplug tests will skip"
+
       - run: pip install -e ".[dev]"
       - name: the tests that need a compiler
-        run: python -m pytest -q -m needs_gcc
+        run: python -m pytest -q -m "needs_gcc or needs_gcc_plugin"
 
       # A drift signal rather than an assertion. If the parser stops understanding a dump,
       # the count moves and the log says by how much.

.github/workflows/plugin.yml

@@ -0,0 +1,196 @@
+name: plugin
+
+# gxplug, built against every GCC a reader might plausibly have, and checked against the
+# one promise the whole design rests on: adding -fplugin=gxplug.so changes nothing about
+# the code GCC produces.
+#
+# A plugin is the most fragile thing in this project. It compiles against GCC's private
+# headers, it is loaded by dlopen into a compiler it was not linked against, and it fails
+# by segfaulting in the middle of somebody's lesson rather than by refusing to build. The
+# only defence is to keep building it in as many places as possible, on both architectures,
+# and to check the guarantee every time rather than asserting it in a document.
+#
+# Four channels, which is the answer to open question 4 in the design notes:
+#
+#   debian     a distribution package, gcc-16 with gcc-16-plugin-dev
+#   fedora     a different distribution, packaged differently, gcc-plugin-devel
+#   homebrew   macOS, where the plugin has to be linked with -undefined dynamic_lookup and
+#              gmp.h is not on the default include path
+#   source     the compiler this project builds itself, pulled from the build matrix
+#
+# The four disagree about where the headers are, what the compiler is called, and how a
+# shared object gets its undefined symbols resolved. Every one of those differences is
+# probed for in gxplug/Makefile, and this is what proves the probes work.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "gxplug/**"
+      - "gxray/plug.py"
+      - "tests/test_plug.py"
+      - ".github/workflows/plugin.yml"
+  pull_request:
+    paths:
+      - "gxplug/**"
+      - "gxray/plug.py"
+      - "tests/test_plug.py"
+      - ".github/workflows/plugin.yml"
+  workflow_dispatch:
+
+# The source channel pulls an image the build matrix published, and a package in GHCR is
+# private until somebody makes it otherwise.
+permissions:
+  contents: read
+  packages: read
+
+jobs:
+  build:
+    name: ${{ matrix.channel }} on ${{ matrix.arch }}
+    runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.container }}
+    timeout-minutes: 15
+
+    # fail-fast off because the interesting result is which channels broke, not that one
+    # did. A change that breaks Fedora and nothing else is a different bug from one that
+    # breaks everything, and cancelling the rest hides the difference.
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - channel: debian
+            arch: amd64
+            runner: ubuntu-24.04
+            container: debian:sid
+          - channel: debian
+            arch: arm64
+            runner: ubuntu-24.04-arm
+            container: debian:sid
+          - channel: fedora
+            arch: amd64
+            runner: ubuntu-24.04
+            container: fedora:rawhide
+          - channel: fedora
+            arch: arm64
+            runner: ubuntu-24.04-arm
+            container: fedora:rawhide
+          # GitHub's macOS runners are arm64 only now, so Intel Homebrew cannot be built
+          # here. The one thing that differs between the two is the prefix, /usr/local
+          # against /opt/homebrew, and the Makefile asks `brew --prefix` rather than
+          # naming either. test_plug.py asserts that it keeps asking.
+          - channel: homebrew
+            arch: arm64
+            runner: macos-15
+            container: ""
+          - channel: source
+            arch: amd64
+            runner: ubuntu-24.04
+            container: ghcr.io/tamnd/gcc-internals/rel:amd64
+          - channel: source
+            arch: arm64
+            runner: ubuntu-24.04-arm
+            container: ghcr.io/tamnd/gcc-internals/rel:arm64
+
+    steps:
+      # Before the checkout, not after. A bare debian:sid has no git, and actions/checkout
+      # without one silently falls back to downloading a tarball over the REST API, which
+      # works until the day it does not.
+      - name: install a compiler and its plugin headers
+        shell: bash
+        run: |
+          set -eux
+          case "${{ matrix.channel }}" in
+            debian)
+              apt-get update
+              apt-get install --no-install-recommends -y \
+                ca-certificates diffutils g++-16 gcc-16 gcc-16-plugin-dev git make python3
+              echo "GXPLUG_GCC=gcc-16" >> "$GITHUB_ENV"
+              ;;
+            fedora)
+              dnf install -y --setopt=install_weak_deps=False \
+                diffutils gcc gcc-c++ gcc-plugin-devel git make python3
+              echo "GXPLUG_GCC=gcc" >> "$GITHUB_ENV"
+              ;;
+            homebrew)
+              brew install gcc
+              # Whichever major version the formula is on today. The plugin ABI only has
+              # to match the compiler that loads it, and that is the same one either way,
+              # so pinning a version here would break the job for no benefit.
+              found=$(ls "$(brew --prefix)"/bin/gcc-1* 2>/dev/null | sort -V | tail -1)
+              test -n "$found"
+              echo "GXPLUG_GCC=$found" >> "$GITHUB_ENV"
+              ;;
+            source)
+              # The image is ubuntu:24.04 with our own GCC ahead of everything on PATH,
+              # and its plugin headers went in with make install. It has no python3,
+              # because nothing a compiler image does needs one.
+              apt-get update
+              apt-get install --no-install-recommends -y python3
+              rm -rf /var/lib/apt/lists/*
+              echo "GXPLUG_GCC=gcc" >> "$GITHUB_ENV"
+              ;;
+          esac
+
+      - uses: actions/checkout@v4
+
+      # Printed on every run, passing or not. When a channel breaks it is nearly always
+      # because a probe resolved somewhere unexpected, and the first question is always
+      # which compiler and which headers this job actually found.
+      - name: what the probes decided
+        shell: bash
+        run: |
+          "$GXPLUG_GCC" --version | head -1
+          make -C gxplug probe GCC="$GXPLUG_GCC"
+
+      - name: build
+        shell: bash
+        run: make -C gxplug GCC="$GXPLUG_GCC"
+
+      # The guarantee. Same source, compiled twice, assembly compared byte for byte.
+      - name: the plugin changes nothing about the generated code
+        shell: bash
+        run: make -C gxplug check GCC="$GXPLUG_GCC"
+
+      # A build that produces no events is a plugin that loaded and did nothing, which the
+      # byte comparison above would happily call a pass.
+      - name: the stream is real, and reaches RTL
+        shell: bash
+        run: |
+          set -eux
+          printf 'int f(int n){int s=0;for(int i=0;i<n;i++)s+=i*i;return s;}\n' > t.c
+          "$GXPLUG_GCC" -O2 -S -fplugin=./gxplug/gxplug.so \
+            -fplugin-arg-gxplug-out=events.ndjson t.c -o t.s
+          wc -l events.ndjson
+          grep -q '"pass":"expand"' events.ndjson
+          grep -q '"insns":[0-9]' events.ndjson
+          grep -q '"statements":[0-9]' events.ndjson
+
+      # The reader for the stream, run against what this channel's compiler just emitted
+      # rather than against a recording. Loaded by path rather than as `from gxray import
+      # plug`, because importing the package pulls in every other module and their
+      # dependencies, and this step is meant to prove the reader works, not that pip does.
+      - name: the reader understands this channel's stream
+        shell: bash
+        run: |
+          python3 - <<'PY'
+          import importlib.util
+          import sys
+
+          spec = importlib.util.spec_from_file_location("plug", "gxray/plug.py")
+          plug = importlib.util.module_from_spec(spec)
+          # Registered before it is executed, not after. @dataclass looks its own class up
+          # in sys.modules while the module body is still running, and on Python 3.14 a
+          # module that is not there yet fails with an AttributeError about NoneType that
+          # says nothing at all about the real cause.
+          sys.modules["plug"] = plug
+          spec.loader.exec_module(plug)
+
+          stream = plug.load("events.ndjson")
+          runs = stream.runs
+          changed = [r for r in runs if r.changed]
+          print(f"{len(stream.events)} events, {len(runs)} runs, {len(changed)} changed something")
+          assert stream.functions == ["f"], stream.functions
+          assert any(r.name == "expand" for r in runs), "never reached RTL"
+          assert all(r.end is not None for r in runs), "a pass started and never finished"
+          assert 0 < len(changed) < len(runs)
+          PY

containers/Dockerfile.plug

@@ -39,7 +39,11 @@ WORKDIR /usr/src/gxplug
 # The plugin builds here rather than being copied in, because a plugin binary is only valid
 # against the compiler it was compiled against and an image that shipped one built anywhere
 # else would be shipping a segfault.
-RUN if [ -f Makefile ]; then make; else echo "gxplug has no Makefile yet, image has the toolchain only"; fi
+#
+# `make check` and not `make`. The image is what a reader pulls to follow a lesson, so it
+# has to ship a plugin that has been shown not to change the code, not one that merely
+# compiled. It costs a couple of seconds on a five minute build.
+RUN make && make check
 
 LABEL org.opencontainers.image.source="https://github.com/tamnd/gcc-internals"
 LABEL org.opencontainers.image.licenses="GPL-3.0-or-later"

gxplug/Makefile

@@ -0,0 +1,121 @@
+# Build gxplug against whichever GCC you point it at.
+#
+# The reader should never have to know about -Wl,-undefined,dynamic_lookup or about where
+# Homebrew keeps gmp.h. Both are probed for here. Set GCC to pick a compiler:
+#
+#     make                    # uses gcc
+#     make GCC=gcc-16         # Homebrew, Debian, anything with a -N suffix
+#     make check              # builds it, then proves it changes no generated code
+#
+# The plugin has to be built by the same GCC it will be loaded into. The ABI is sensitive
+# to version and to configure options, and plugin_init refuses a mismatch rather than
+# crashing later, so a wrong GCC here is a clear error and not a mystery.
+
+GCC ?= gcc
+CXX_FOR_PLUGIN ?= $(GCC)
+
+# Where this GCC keeps its plugin headers. Empty means the plugin development package is
+# not installed, which is the single most common reason a first attempt fails.
+PLUGIN_DIR := $(shell $(GCC) -print-file-name=plugin 2>/dev/null)
+PLUGIN_INCLUDE := $(PLUGIN_DIR)/include
+
+UNAME_S := $(shell uname -s)
+
+# GCC itself is built without RTTI, and a plugin that disagrees gets link errors about
+# typeinfo that say nothing useful about the cause.
+CXXFLAGS ?= -O2 -std=c++17 -fno-rtti -fPIC -Wall -Wextra -Wno-unused-parameter
+
+# Ours, kept separate from CPPFLAGS and placed ahead of it. A developer machine often has
+# a CPPFLAGS in the environment pointing at Homebrew LLVM, and an LLVM header found before
+# GCC's own is a failure that reads like a compiler bug. Appending to CPPFLAGS would put
+# that first; this puts it second and still lets someone add to it deliberately.
+GXPLUG_CPPFLAGS := -I$(PLUGIN_INCLUDE)
+
+# gmp.h is included by GCC's own headers and on macOS it lives under the Homebrew prefix,
+# which is not on the default include path. Probe rather than hardcode, because the prefix
+# differs between Apple silicon and Intel and because Linux does not need this at all.
+GMP_PROBE := $(shell $(GCC) $(GXPLUG_CPPFLAGS) -fsyntax-only probe/gmp.c >/dev/null 2>&1 && echo yes || echo no)
+ifeq ($(GMP_PROBE),no)
+  BREW_PREFIX := $(shell brew --prefix 2>/dev/null)
+  ifneq ($(BREW_PREFIX),)
+    GXPLUG_CPPFLAGS += -I$(BREW_PREFIX)/include
+  endif
+endif
+
+# A plugin is C++, and the `gcc` driver knows what to do with a .cc file, so this looks
+# like it works right up to the point where it tries to exec a cc1plus that only the g++
+# package ships. Asking here turns "No such file or directory" into a sentence that names
+# the package.
+CXX_PROBE := $(shell echo 'int main(){}' | $(CXX_FOR_PLUGIN) -x c++ -fsyntax-only - >/dev/null 2>&1 && echo yes || echo no)
+
+# Darwin resolves a plugin's undefined symbols out of cc1 at dlopen time, and the linker
+# has to be told that is acceptable. ELF platforms do this by default and passing the flag
+# there is an error, so it is conditional.
+LDFLAGS_PLUGIN :=
+ifeq ($(UNAME_S),Darwin)
+  LDFLAGS_PLUGIN += -Wl,-undefined,dynamic_lookup
+endif
+
+# .so on both platforms, not .dylib. GCC's -fplugin= wants a shared object and Darwin is
+# happy to produce one under that name.
+PLUGIN := gxplug.so
+SRC := src/gxplug.cc
+
+.PHONY: all check probe clean
+
+all: $(PLUGIN)
+
+$(PLUGIN): $(SRC)
+	@test -f "$(PLUGIN_INCLUDE)/gcc-plugin.h" || { \
+	  echo "gxplug: no plugin headers under $(PLUGIN_INCLUDE)"; \
+	  echo "gxplug: install the plugin development package for this compiler"; \
+	  echo "gxplug:   Debian/Ubuntu   apt install gcc-16-plugin-dev"; \
+	  echo "gxplug:   Fedora          dnf install gcc-plugin-devel"; \
+	  echo "gxplug:   Homebrew        already included with the gcc formula"; \
+	  exit 1; \
+	}
+	@test "$(CXX_PROBE)" = yes || { \
+	  echo "gxplug: $(CXX_FOR_PLUGIN) cannot compile C++, and a plugin is C++"; \
+	  echo "gxplug: install the matching C++ compiler, not another C one"; \
+	  echo "gxplug:   Debian/Ubuntu   apt install g++-16"; \
+	  echo "gxplug:   Fedora          dnf install gcc-c++"; \
+	  exit 1; \
+	}
+	$(CXX_FOR_PLUGIN) $(GXPLUG_CPPFLAGS) $(CPPFLAGS) $(CXXFLAGS) -shared $(LDFLAGS_PLUGIN) -o $@ $(SRC)
+
+# What the probes decided, for when a build fails somewhere unfamiliar and the first
+# question is which compiler and which headers it actually found.
+probe:
+	@echo "GCC             $(GCC)"
+	@echo "version         $$($(GCC) -dumpfullversion 2>/dev/null)"
+	@echo "plugin headers  $(PLUGIN_INCLUDE)"
+	@echo "gcc-plugin.h    $$(test -f '$(PLUGIN_INCLUDE)/gcc-plugin.h' && echo present || echo MISSING)"
+	@echo "uname           $(UNAME_S)"
+	@echo "compiles c++    $(CXX_PROBE)"
+	@echo "gmp.h on path   $(GMP_PROBE)"
+	@echo "CPPFLAGS        $(GXPLUG_CPPFLAGS) $(CPPFLAGS)"
+	@echo "link flags      $(LDFLAGS_PLUGIN)"
+
+# The guarantee, checked rather than asserted. Compile something twice, once with the
+# plugin and once without, and require the assembly to be identical byte for byte. If this
+# ever fails then the plugin is not a passive observer any more and no lesson built on it
+# can be trusted.
+check: $(PLUGIN)
+	@set -e; \
+	tmp=$$(mktemp -d); \
+	trap 'rm -rf "$$tmp"' EXIT; \
+	printf 'int f(int n){int s=0;for(int i=0;i<n;i++)s+=i*i;return s;}\n' > "$$tmp/t.c"; \
+	$(GCC) -O2 -S "$$tmp/t.c" -o "$$tmp/without.s"; \
+	$(GCC) -O2 -S -fplugin=./$(PLUGIN) -fplugin-arg-gxplug-out="$$tmp/events.ndjson" \
+	  "$$tmp/t.c" -o "$$tmp/with.s"; \
+	if ! cmp -s "$$tmp/without.s" "$$tmp/with.s"; then \
+	  echo "gxplug: FAIL, the plugin changed the generated code"; \
+	  diff -u "$$tmp/without.s" "$$tmp/with.s" || true; \
+	  exit 1; \
+	fi; \
+	events=$$(wc -l < "$$tmp/events.ndjson" | tr -d ' '); \
+	test "$$events" -gt 0 || { echo "gxplug: FAIL, no events emitted"; exit 1; }; \
+	echo "gxplug: assembly identical with and without the plugin, $$events events"
+
+clean:
+	rm -f $(PLUGIN)