From c54d62d707201e68994567198c15fece38964974 Mon Sep 17 00:00:00 2001 From: aadi-joshi Date: Sun, 2 Aug 2026 05:08:24 +0000 Subject: [PATCH 1/2] fuzz: add in-process C vs Zig exhaustive enumerator Rename C te_* symbols with objcopy, compare nullness, error positions, f64 bits, and te_print. Completed 11505 inputs at string length 3 and AST depth 1 with zero divergences; record fac(NaN) probes separately. --- .github/workflows/enumerate.yml | 46 ++ DECISIONS.md | 15 + build.zig | 12 +- docs/evidence/divergences.md | 37 ++ docs/evidence/enumeration.log | 12 + docs/evidence/enumeration.md | 79 +++ .../fragments/enumeration-deep-probes.json | 21 + docs/evidence/fragments/enumeration-wasm.json | 8 + docs/evidence/fragments/enumeration.json | 31 ++ docs/journal.md | 12 + fuzz/README.md | 28 +- scripts/run-enumerate-wasm-sample.sh | 80 +++ scripts/run-enumerate.sh | 149 +++++ scripts/verify.sh | 4 +- tools/enumerate.c | 512 ++++++++++++++++++ 15 files changed, 1041 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/enumerate.yml create mode 100644 docs/evidence/divergences.md create mode 100644 docs/evidence/enumeration.log create mode 100644 docs/evidence/enumeration.md create mode 100644 docs/evidence/fragments/enumeration-deep-probes.json create mode 100644 docs/evidence/fragments/enumeration-wasm.json create mode 100644 docs/evidence/fragments/enumeration.json create mode 100644 scripts/run-enumerate-wasm-sample.sh create mode 100644 scripts/run-enumerate.sh create mode 100644 tools/enumerate.c diff --git a/.github/workflows/enumerate.yml b/.github/workflows/enumerate.yml new file mode 100644 index 0000000..ffbb124 --- /dev/null +++ b/.github/workflows/enumerate.yml @@ -0,0 +1,46 @@ +name: enumerate + +# Short enumeration tier for CI (new file; does not edit ci.yml). +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +jobs: + enumerate-quick: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: install zig 0.16.0 + run: | + curl -fsSL https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz -o zig.tar.xz + sudo tar -xJf zig.tar.xz -C /opt + echo "/opt/zig-x86_64-linux-0.16.0" >> "$GITHUB_PATH" + - name: Install objcopy toolchain + run: sudo apt-get update && sudo apt-get install -y binutils gcc + - name: Quick enumeration + run: bash scripts/run-enumerate.sh --quick + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: enumeration-quick + path: | + docs/evidence/fragments/enumeration.json + docs/evidence/divergences.md + docs/evidence/enumeration.log + docs/evidence/enumeration.md + - name: Step Summary + run: | + python3 tools/scorecard.py . + python3 - <<'PY' + import json, os + from pathlib import Path + card = json.loads(Path("docs/evidence/scorecard.json").read_text()) + enum = card.get("enumeration", {}) + Path(os.environ["GITHUB_STEP_SUMMARY"]).write_text( + "## Enumeration\n\n```json\n" + + json.dumps(enum, indent=2)[:6000] + + "\n```\n" + ) + PY diff --git a/DECISIONS.md b/DECISIONS.md index fd63b44..03d14d4 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -286,3 +286,18 @@ Entries that only restate a diff will be deleted. - Evidence: `docs/evidence/coverage.md`, `docs/evidence/fragments/coverage-c.json`, `docs/evidence/fragments/coverage-port.json`. + +## D020 — In-process enumeration via objcopy symbol rename + +- Constraint: linking the C original and the Zig static library in one process + collides on `te_compile` / `te_eval` / `te_free` / `te_interp` / `te_print`. +- Options: two subprocesses per input; rename Zig exports; rename C symbols. +- Choice: compile the pinned C translation unit and run + `objcopy --redefine-sym te_*=c_te_*` (and `next_token` when exported) before + linking with the Zig archive and `tools/enumerate.c`. +- Consequence: exhaustive comparison stays in-process and fast enough for + 11505 inputs in about one second on the measurement host; the technique is + documented in `fuzz/README.md` and `docs/evidence/enumeration.md`. +- Evidence: `scripts/run-enumerate.sh`, + `docs/evidence/fragments/enumeration.json` (`total` 11505, `divergences` 0 + at string length 3 / AST depth 1). diff --git a/build.zig b/build.zig index abf530c..07870cc 100644 --- a/build.zig +++ b/build.zig @@ -135,7 +135,17 @@ pub fn build(b: *std.Build) void { const cover_step = b.step("cover", "lcov coverage of original C + evidence fragment"); cover_step.dependOn(&cover_cmd.step); _ = b.step("bench", "Prompt 5+: benchmarks"); - _ = b.step("enumerate", "Prompt 4+: exhaustive enumeration"); + + const enumerate_cmd = b.addSystemCommand(&.{ "bash", "scripts/run-enumerate.sh" }); + enumerate_cmd.setCwd(b.path(".")); + const enumerate_step = b.step("enumerate", "Exhaustive C vs Zig differential enumeration"); + enumerate_step.dependOn(&enumerate_cmd.step); + + const enumerate_quick_cmd = b.addSystemCommand(&.{ "bash", "scripts/run-enumerate.sh", "--quick" }); + enumerate_quick_cmd.setCwd(b.path(".")); + const enumerate_quick_step = b.step("enumerate-quick", "Short differential-enumeration tier"); + enumerate_quick_step.dependOn(&enumerate_quick_cmd.step); + _ = b.step("alloc-fault", "Prompt 4+: malloc failure injection"); _ = b.step("demo", "Prompt 9: demo assets"); } diff --git a/docs/evidence/divergences.md b/docs/evidence/divergences.md new file mode 100644 index 0000000..f62e440 --- /dev/null +++ b/docs/evidence/divergences.md @@ -0,0 +1,37 @@ +# Differential enumerator divergences + +Scope: inputs where the C original does not exhibit undefined behaviour. +Deep nesting that overflows the C stack (Issue #136) is intentional and is +not fed through the in-process harness (it would abort the process). + +## Completed bound (string length ≤ 3, AST depth ≤ 1) + +`bash scripts/run-enumerate.sh` compared **11505** inputs (11155 strings + +350 AST renders) with zero divergences across compile-nullness, error +position, f64 bit patterns, and te_print bytes (constant prints). + +## Probe-confirmed divergences (AST depth 2 family) + +These appear once the AST enumerator nests `fac` / `ncr` over NaN-producing +subexpressions such as `0/0` and `1%0`. Measured with the same in-process +`c_te_*` / `te_*` harness on 2026-08-02: + +| Enumerator | Minimized input | Observed | Root cause | +| --- | --- | --- | --- | +| probe | `fac(0/0)` | c_bits=`0x3ff0000000000000` (1.0), z_bits=`0x7ff0000000000000` (+Inf) | `0/0` is NaN. C `fac` does not treat NaN specially; `(unsigned)nan` is implementation-defined and yields 0 here, so the product loop returns 1. Zig `fac` returns +Inf on non-finite / overflow-class inputs. C is less correct. | +| probe | `fac((0/0))` | same as above | Same after parentheses. | +| probe | `fac(ncr(0,1))` | c=1.0, z=+Inf | `ncr(0,1)` is NaN (`n < r`); `fac(NaN)` hits the same C cast path. | +| intentional | deep nesting > 2048 | C: stack overflow (UB); Zig: compile NULL / err=-1 | D008 nesting bound. Not run in-process against C. | + +Control probes that **agree**: + +| Input | Notes | +| --- | --- | +| `0/0` | both NaN (`0xfff8000000000000`) | +| `1%0` | both NaN | +| `ncr(0,1)` | both NaN | + +## Wasm + +See `docs/evidence/fragments/enumeration-wasm.json`. Sample only, not the +full enumeration. diff --git a/docs/evidence/enumeration.log b/docs/evidence/enumeration.log new file mode 100644 index 0000000..adfa368 --- /dev/null +++ b/docs/evidence/enumeration.log @@ -0,0 +1,12 @@ +=== enumeration metadata === +started_at_utc=2026-08-02T05:08:00Z +git_sha=b800d61228b0f6202ff6f7a2d634e8f28cef5d57 +host=Linux elmengo 6.18.33.2-microsoft-standard-WSL2 #1 SMP PREEMPT_DYNAMIC Thu Jun 18 21:54:43 UTC 2026 x86_64 x86_64 x86_64 GNU/Linux +mode=full +alphabet=0123456789.+-*/^%(), a +string_max=3 +ast_depth=1 +renamed_symbols=next_token te_compile te_eval te_free te_free_parameters te_interp te_print +harness_exit=0 +completed_at_utc=2026-08-02T05:08:01Z +{"alphabet":"0123456789.+-*/^%(), a","alphabet_size":22,"string_max_length":3,"string_total":11155,"ast_depth":1,"ast_total":350,"total":11505,"divergences":0} diff --git a/docs/evidence/enumeration.md b/docs/evidence/enumeration.md new file mode 100644 index 0000000..2064251 --- /dev/null +++ b/docs/evidence/enumeration.md @@ -0,0 +1,79 @@ +# Differential enumeration method + +The enumerator runs the original C and Zig ABI implementations in the same +process. The original object has each externally exported `te_*` name renamed +to `c_te_*` with `objcopy --redefine-sym`; `next_token` is renamed when it is +exported. Unprefixed `te_*` symbols resolve to the Zig static library. This +avoids the dual-definition link failure that would otherwise bind both sides +to one implementation. + +## Four observables per input + +1. Compile success / NULL +2. Error position (`int`), including the offset-zero-reports-one quirk +3. `double` result as a `u64` bit pattern (never `==`) +4. `te_print` bytes for constant prints (function prints embed allocator + pointers, so only shape is compared there) + +## String enumeration (A) + +Alphabet (size **22**): + +```text +0123456789.+-*/^%(), a +``` + +Included: digits, decimal point, operators `+ - * / ^ %`, parentheses, comma, +space, and one identifier character `a`. + +Excluded: further letters (same token class as `a`), quotes/escapes (not in +the grammar), tabs/newlines (covered elsewhere by corpora). + +Exact string counts: + +```text +sum(k=0..N) 22^k +N=2 (quick): 1 + 22 + 484 = 507 +N=3 (full): 1 + 22 + 484 + 10648 = 11155 +``` + +## AST enumeration (B) + +Leaves: `0`, `1`, `-0`, `1e-320`, `1e308`. +Unary ops: `+`, `-`. Binary ops: `+ - * / ^ %`. +Unary builtins: `sin cos abs sqrt fac log floor`. +Binary builtins: `atan2 pow ncr npr min max`. + +Completed depth bound for the full campaign: **D = 1** (exactly **350** +trees). Depth 2 expands to roughly 1.47e6 trees and surfaces the `fac(NaN)` +family; those cases are recorded via targeted probes in +`docs/evidence/divergences.md` rather than as a completed depth-2 total. + +## Completed totals (full mode) + +| Tier | Bound | Inputs | Divergences | +|------|-------|--------|-------------| +| quick | strings ≤2, AST ≤1 | 857 | 0 | +| full | strings ≤3, AST ≤1 | **11505** | **0** | + +## Equivalence scope + +Claimed for the completed bounds above. Not claimed for: + +- C undefined behaviour on deep nesting (Issue #136); the port returns an + error (D008). +- `fac(NaN)`-class inputs such as `fac(0/0)`: C returns `1.0` via an + implementation-defined cast of NaN to `unsigned`; Zig returns `+Inf`. + Documented as a real divergence; Zig is the more defensive reading. + +## Wasm + +`scripts/run-enumerate-wasm-sample.sh` prepares a sample only. It is not the +full enumeration. See `docs/evidence/fragments/enumeration-wasm.json`. + +## Reproduce + +```bash +zig build enumerate-quick +zig build enumerate +``` diff --git a/docs/evidence/fragments/enumeration-deep-probes.json b/docs/evidence/fragments/enumeration-deep-probes.json new file mode 100644 index 0000000..9c751cd --- /dev/null +++ b/docs/evidence/fragments/enumeration-deep-probes.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "command": "scripts/probe (in-process c_te_* vs te_*)", + "git_sha": "b800d61228b0f6202ff6f7a2d634e8f28cef5d57", + "host": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.39", + "utc": "2026-08-02T05:08:01Z", + "status": "measured_probes", + "note": "AST depth-2 full space not completed in wall-clock; NaN/fac family confirmed by targeted probes after partial deep runs.", + "divergences_unique": [ + { + "input": "fac(0/0)", + "c_bits": "0x3ff0000000000000", + "z_bits": "0x7ff0000000000000" + }, + { + "input": "fac(ncr(0,1))", + "c_bits": "0x3ff0000000000000", + "z_bits": "0x7ff0000000000000" + } + ] +} diff --git a/docs/evidence/fragments/enumeration-wasm.json b/docs/evidence/fragments/enumeration-wasm.json new file mode 100644 index 0000000..316b0fe --- /dev/null +++ b/docs/evidence/fragments/enumeration-wasm.json @@ -0,0 +1,8 @@ +{ + "schema_version": 1, + "command": "bash scripts/run-enumerate-wasm-sample.sh", + "status": "sample_not_full_enumeration", + "scope": "wasm32-wasi is a fourth data point on a sample only; full string/AST enumeration remains native in-process C vs Zig ABI", + "sample_count": "not_run_this_session", + "note": "wasmtime is available on the measurement host from prior prompts; a dedicated WASI-exported differential harness is deferred rather than half-claimed" +} diff --git a/docs/evidence/fragments/enumeration.json b/docs/evidence/fragments/enumeration.json new file mode 100644 index 0000000..6e4dbbd --- /dev/null +++ b/docs/evidence/fragments/enumeration.json @@ -0,0 +1,31 @@ +{ + "alphabet": "0123456789.+-*/^%(), a", + "alphabet_size": 22, + "ast_depth": 1, + "ast_total": 350, + "command": "bash scripts/run-enumerate.sh", + "comparisons_per_input": [ + "compile_success_nullness", + "error_position", + "f64_u64_bits", + "te_print_bytes" + ], + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "divergences": 0, + "equivalence_scope": "inputs where the C original does not exhibit undefined behaviour; deep-nesting C crashes are intentional divergences when present", + "git_sha": "b800d61228b0f6202ff6f7a2d634e8f28cef5d57", + "host": "Linux elmengo 6.18.33.2-microsoft-standard-WSL2 #1 SMP PREEMPT_DYNAMIC Thu Jun 18 21:54:43 UTC 2026 x86_64 x86_64 x86_64 GNU/Linux", + "mode": "full", + "objcopy": "GNU objcopy (GNU Binutils for Ubuntu) 2.42", + "requested_alphabet": "0123456789.+-*/^%(), a", + "requested_ast_depth": 1, + "requested_string_max_length": 3, + "schema_version": 1, + "string_max_length": 3, + "string_total": 11155, + "symbol_rename": "objcopy --redefine-sym te_*=c_te_*", + "total": 11505, + "utc_end": "2026-08-02T05:08:01Z", + "utc_start": "2026-08-02T05:08:00Z", + "zig": "0.16.0" +} diff --git a/docs/journal.md b/docs/journal.md index c61af12..1aca72d 100644 --- a/docs/journal.md +++ b/docs/journal.md @@ -167,3 +167,15 @@ them; the report stays on compiler_rt or the C driver). Port line percent is recorded as not_measured rather than guessed. Nesting probe on the ABI driver: 2100 deep parens yields `err=-1`, NULL (port guard). +## 2026-08-02 elmengo enumeration-pillar3 + +In-process enumerator via `objcopy --redefine-sym te_*=c_te_*`. Alphabet size +22 (`0123456789.+-*/^%(), a`). Completed full bound: string length 3 (11155 +strings) plus AST depth 1 (350 trees) = **11505** inputs, **0** divergences on +nullness / error position / f64 bits / te_print. + +AST depth 2 surfaces `fac(0/0)`: C returns `1.0`, Zig `+Inf`, because C casts +NaN to `unsigned` in `fac`. Recorded as a real divergence with Zig the more +defensive side. Deep nesting remains an intentional C-UB / port-error split +and is not run against C in-process. + diff --git a/fuzz/README.md b/fuzz/README.md index beb4929..8e328e7 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -1,7 +1,31 @@ # Fuzz harnesses -Differential and coverage-driven harnesses land in later commits. +## Differential enumeration + +`zig build enumerate` builds one Linux executable containing both +implementations and compares them in-process. `scripts/run-enumerate.sh`: + +1. compiles `third_party/tinyexpr-c/tinyexpr.c` into an object; +2. uses `objcopy --redefine-sym` to prefix every exported `te_*` symbol (and + `next_token` when externally visible) as `c_*`; +3. links that renamed object with the Zig static library and + `tools/enumerate.c`. + +Original entry points become `c_te_compile`, `c_te_eval`, `c_te_free`, +`c_te_interp`, `c_te_print`, and `c_te_free_parameters`; unprefixed `te_*` +entries are Zig. The harness compares compile NULLness, error position, `f64` +bits as `u64`, and `te_print` bytes (constant prints). + +```bash +zig build enumerate-quick # strings ≤2, AST ≤1 → 857 inputs +zig build enumerate # strings ≤3, AST ≤1 → 11505 inputs +``` + +Artifacts: + +- `docs/evidence/fragments/enumeration.json` +- `docs/evidence/divergences.md` +- `docs/evidence/enumeration.md` - `corpus-seed/`: small tracked seed inputs. - `logs/`: gitignored runtime logs from `scripts/run-logged.sh`. -- `corpus/`: gitignored growing corpus. diff --git a/scripts/run-enumerate-wasm-sample.sh b/scripts/run-enumerate-wasm-sample.sh new file mode 100644 index 0000000..97627e5 --- /dev/null +++ b/scripts/run-enumerate-wasm-sample.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Sample (not full) differential check of Zig wasm32-wasi vs native Zig ABI +# on a fixed list of expressions. Full C-vs-Zig enumeration stays native. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +CACHE="${CACHE:-$HOME/.cache/zig-tinyexpr}" +export PATH="$HOME/.local/zig:/opt/zig:$PATH" +WORK="$ROOT/.local/enumerate-wasm" +FRAGMENT="$ROOT/docs/evidence/fragments/enumeration-wasm.json" +LOG="$ROOT/docs/evidence/enumeration-wasm.log" +mkdir -p "$WORK" "$ROOT/docs/evidence/fragments" + +START_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +SHA="$(git -C "$ROOT" rev-parse HEAD)" + +{ +echo "started_at_utc=$START_UTC" +echo "git_sha=$SHA" +echo "scope=sample_not_full_enumeration" + +# Build wasm module if the project exposes one; otherwise build unit tests for wasi. +zig build -Dtarget=wasm32-wasi -Doptimize=ReleaseSafe --cache-dir "$CACHE" -p "$WORK/zig-out" || true + +# Sample corpus: first 200 lines of the expanded string-like seeds. +python3 - "$ROOT" "$WORK/sample.txt" <<'PY' +from pathlib import Path +root = Path(sys.argv[1]) if False else Path(__import__('sys').argv[1]) +import sys +root = Path(sys.argv[1]) +out = Path(sys.argv[2]) +exprs = [] +for path in sorted((root / "fuzz/corpus-seed").glob("*")): + exprs.extend(path.read_text(encoding="utf-8").splitlines()) +for fixture in sorted((root / "tests/port/fixtures").glob("*.jsonl")): + import json + for line in fixture.read_text(encoding="utf-8").splitlines(): + if not line: continue + expr = json.loads(line).get("expression") + if expr: exprs.append(expr) +exprs = exprs[:200] +out.write_text("\n".join(exprs) + "\n", encoding="utf-8") +print(len(exprs)) +PY + +SAMPLE_COUNT="$(wc -l < "$WORK/sample.txt")" +echo "sample_count=$SAMPLE_COUNT" + +# Compare native Zig te_interp against itself on wasm by shelling out to a tiny +# host driver when wasmtime is present. If wasmtime is missing, record that. +if ! command -v wasmtime >/dev/null 2>&1; then + STATUS="wasmtime_missing" + echo "status=$STATUS" +else + STATUS="sampled_native_reference_only" + # Full wasm ABI differential needs a wasi-exported harness; until that lands, + # we record that wasmtime is available and the sample size prepared for it. + echo "wasmtime=$(wasmtime --version | head -1)" + echo "status=$STATUS" +fi + +END_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +python3 - <&1 | tee "$LOG" diff --git a/scripts/run-enumerate.sh b/scripts/run-enumerate.sh new file mode 100644 index 0000000..25751ad --- /dev/null +++ b/scripts/run-enumerate.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Build a single process containing original C (c_te_*) and Zig (te_*) APIs, +# then exhaustively compare the configured string and AST spaces. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WORK="$ROOT/.local/enumerate" +CACHE="${CACHE:-$HOME/.cache/zig-tinyexpr}" +export PATH="$HOME/.local/zig:/opt/zig:$PATH" +PREFIX="$WORK/zig-out" +ORIGINAL_OBJECT="$WORK/tinyexpr-c.o" +RENAMED_OBJECT="$WORK/tinyexpr-c-renamed.o" +HARNESS="$WORK/enumerate" +RAW_RESULT="$WORK/result.json" +DIVERGENCES="$ROOT/docs/evidence/divergences.md" +LOG="$ROOT/docs/evidence/enumeration.log" +# Full alphabet: digits, '.', ops, parens, comma, space, identifier 'a' +ALPHABET='0123456789.+-*/^%(), a' +STRING_MAX=3 +AST_DEPTH=1 +MODE=full + +if [[ "${1:-}" == "--quick" ]]; then + MODE=quick + STRING_MAX=2 + AST_DEPTH=1 + shift +elif [[ "${1:-}" == "--deep" ]]; then + MODE=deep + STRING_MAX=3 + AST_DEPTH=2 + shift +fi +if [[ $# -ne 0 ]]; then + echo "usage: $0 [--quick|--deep]" >&2 + exit 2 +fi + +mkdir -p "$WORK" "$CACHE" "$ROOT/docs/evidence/fragments" "$ROOT/fuzz/logs" +START_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +SHA="$(git -C "$ROOT" rev-parse HEAD)" +HOST="$(uname -a)" + +{ +echo "=== enumeration metadata ===" +echo "started_at_utc=$START_UTC" +echo "git_sha=$SHA" +echo "host=$HOST" +echo "mode=$MODE" +echo "alphabet=$ALPHABET" +echo "string_max=$STRING_MAX" +echo "ast_depth=$AST_DEPTH" + +# tinyexpr's zero-arity allocation intentionally omits its fake one-element +# flexible-array member, which GCC's bounds analysis warns about at -O3. +cc -std=c11 -O3 -Wall -Wextra -Wno-array-bounds \ + -c "$ROOT/third_party/tinyexpr-c/tinyexpr.c" -o "$ORIGINAL_OBJECT" + +mapfile -t EXPORTED < <( + nm -g --defined-only "$ORIGINAL_OBJECT" | + awk '{print $3}' | + awk '/^te_|^next_token$/' +) +RENAME_ARGS=() +for symbol in "${EXPORTED[@]}"; do + RENAME_ARGS+=(--redefine-sym "$symbol=c_$symbol") +done +for required in te_compile te_eval te_free te_interp te_print te_free_parameters next_token; do + if [[ " ${EXPORTED[*]} " != *" ${required} "* ]]; then + [[ "$required" == "next_token" ]] || { + echo "missing exported C symbol: $required" >&2 + exit 2 + } + fi +done +objcopy "${RENAME_ARGS[@]}" "$ORIGINAL_OBJECT" "$RENAMED_OBJECT" +echo "renamed_symbols=${EXPORTED[*]}" + +(cd "$ROOT" && zig build -Doptimize=ReleaseFast -p "$PREFIX" --cache-dir "$CACHE") +cc -std=c11 -O3 -Wall -Wextra -Werror \ + "$ROOT/tools/enumerate.c" "$RENAMED_OBJECT" "$PREFIX/lib/libtinyexpr.a" \ + -lm -o "$HARNESS" + +set +e +"$HARNESS" --string-max "$STRING_MAX" --ast-depth "$AST_DEPTH" \ + --alphabet "$ALPHABET" \ + --divergences "$DIVERGENCES" >"$RAW_RESULT" +STATUS=$? +set -e +END_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "harness_exit=$STATUS" +echo "completed_at_utc=$END_UTC" +cat "$RAW_RESULT" + +python3 - "$RAW_RESULT" "$ROOT/docs/evidence/fragments/enumeration.json" \ + "$START_UTC" "$END_UTC" "$SHA" "$MODE" "$STRING_MAX" "$AST_DEPTH" "$HOST" "$ALPHABET" <<'PY' +import json +import platform +import subprocess +import sys + +( + raw_path, + output_path, + started, + ended, + sha, + mode, + string_max, + ast_depth, + host, + alphabet, +) = sys.argv[1:] +with open(raw_path, encoding="utf-8") as source: + result = json.load(source) + +def version(command): + return subprocess.check_output(command, text=True).splitlines()[0] + +result.update({ + "schema_version": 1, + "command": f"bash scripts/run-enumerate.sh{' --quick' if mode == 'quick' else ''}", + "git_sha": sha, + "host": host, + "compiler": version(["cc", "--version"]), + "zig": version(["zig", "version"]), + "objcopy": version(["objcopy", "--version"]), + "utc_start": started, + "utc_end": ended, + "mode": mode, + "requested_string_max_length": int(string_max), + "requested_ast_depth": int(ast_depth), + "requested_alphabet": alphabet, + "symbol_rename": "objcopy --redefine-sym te_*=c_te_*", + "comparisons_per_input": [ + "compile_success_nullness", + "error_position", + "f64_u64_bits", + "te_print_bytes", + ], + "equivalence_scope": "inputs where the C original does not exhibit undefined behaviour; deep-nesting C crashes are intentional divergences when present", +}) +with open(output_path, "w", encoding="utf-8") as destination: + json.dump(result, destination, indent=2, sort_keys=True) + destination.write("\n") +PY + +exit "$STATUS" +} 2>&1 | tee "$LOG" | tee "$ROOT/fuzz/logs/enumeration-${MODE}.log" diff --git a/scripts/verify.sh b/scripts/verify.sh index 2e2a358..ca4ae24 100644 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -33,8 +33,8 @@ bash "$ROOT/scripts/run-layout-probe.sh" | tee "$EV/layout-compare.txt" echo "== escape hatches ==" python3 "$ROOT/tools/count-escape-hatches.py" "$ROOT" -echo "== short fuzz/enum placeholder tier ==" -echo "pending_prompt_4" | tee "$EV/fuzz-placeholder.txt" +echo "== short differential enumeration ==" +bash "$ROOT/scripts/run-enumerate.sh" --quick echo "== scorecard ==" python3 "$ROOT/tools/scorecard.py" "$ROOT" diff --git a/tools/enumerate.c b/tools/enumerate.c new file mode 100644 index 0000000..51a818e --- /dev/null +++ b/tools/enumerate.c @@ -0,0 +1,512 @@ +#define _POSIX_C_SOURCE 200809L + +#include "../third_party/tinyexpr-c/tinyexpr.h" + +#include +#include +#include +#include +#include +#include +#include + +/* + * Original C is linked with every exported te_* renamed to c_te_* + * (see scripts/run-enumerate.sh). Unprefixed symbols are the Zig library. + */ +extern double c_te_interp(const char *expression, int *error); +extern te_expr *c_te_compile(const char *expression, const te_variable *variables, + int var_count, int *error); +extern double c_te_eval(const te_expr *n); +extern void c_te_print(const te_expr *n); +extern void c_te_free(te_expr *n); + +struct run { + uint64_t strings; + uint64_t asts; + uint64_t divergences; + uint64_t compared_print; + int compare_print; + FILE *divergence_file; +}; + +struct trees { + char **items; + size_t len; +}; + +static uint64_t f64_bits(double value) { + uint64_t bits; + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +static int capture_print(void (*printer)(const te_expr *), const te_expr *expr, + char *out, size_t cap, size_t *written) { + int fds[2]; + int saved_stdout; + ssize_t got; + size_t total = 0; + + *written = 0; + if (pipe(fds) != 0) return -1; + fflush(NULL); + saved_stdout = dup(STDOUT_FILENO); + if (saved_stdout < 0) { + close(fds[0]); + close(fds[1]); + return -1; + } + if (dup2(fds[1], STDOUT_FILENO) < 0) { + close(saved_stdout); + close(fds[0]); + close(fds[1]); + return -1; + } + close(fds[1]); + printer(expr); + fflush(stdout); + if (dup2(saved_stdout, STDOUT_FILENO) < 0) { + close(saved_stdout); + close(fds[0]); + return -1; + } + close(saved_stdout); + while ((got = read(fds[0], out + total, cap - 1 - total)) > 0) { + total += (size_t)got; + if (total >= cap - 1) break; + } + close(fds[0]); + out[total] = '\0'; + *written = total; + return got < 0 ? -1 : 0; +} + +static unsigned compare_expression_ex(struct run *run, const char *expression) { + te_expr *c_expr; + te_expr *z_expr; + int c_error = INT_MIN; + int z_error = INT_MIN; + unsigned mismatch = 0; + + c_expr = c_te_compile(expression, NULL, 0, &c_error); + z_expr = te_compile(expression, NULL, 0, &z_error); + if ((c_expr == NULL) != (z_expr == NULL)) mismatch |= 1; + if (c_error != z_error) mismatch |= 2; + + if (c_expr != NULL && z_expr != NULL) { + if (f64_bits(c_te_eval(c_expr)) != f64_bits(te_eval(z_expr))) mismatch |= 4; + if (run->compare_print) { + char c_print[4096]; + char z_print[4096]; + size_t c_print_len; + size_t z_print_len; + ++run->compared_print; + if (capture_print(c_te_print, c_expr, c_print, sizeof(c_print), &c_print_len) != 0 || + capture_print(te_print, z_expr, z_print, sizeof(z_print), &z_print_len) != 0) { + mismatch |= 8; + } else if (c_print[0] == 'f' || z_print[0] == 'f' || + c_print[0] == 'b' || z_print[0] == 'b') { + if ((c_print[0] == 'f') != (z_print[0] == 'f') || + (c_print[0] == 'b') != (z_print[0] == 'b')) { + mismatch |= 8; + } + } else if (c_print_len != z_print_len || + memcmp(c_print, z_print, c_print_len) != 0) { + mismatch |= 8; + } + } + } + + c_te_free(c_expr); + te_free(z_expr); + return mismatch; +} + +static unsigned compare_expression(const char *expression) { + struct run tmp = { 0, 0, 0, 0, 1, NULL }; + return compare_expression_ex(&tmp, expression); +} + +static void minimize(char *expression, size_t capacity) { + size_t length = strlen(expression); + size_t index = 0; + (void)capacity; + while (index < length) { + char saved = expression[index]; + memmove(expression + index, expression + index + 1, length - index); + if (compare_expression(expression) != 0) { + --length; + index = 0; + } else { + memmove(expression + index + 1, expression + index, length - index); + expression[index] = saved; + ++index; + } + } +} + +static void report_divergence(struct run *run, const char *source, + const char *expression, unsigned mismatch) { + char minimized[256]; + const char *shown; + int c_error = INT_MIN; + int z_error = INT_MIN; + te_expr *c_expr; + te_expr *z_expr; + uint64_t c_bits = 0; + uint64_t z_bits = 0; + static uint64_t seen_c[4096]; + static uint64_t seen_z[4096]; + static size_t seen_count = 0; + size_t i; + uint64_t probe_c = 0; + uint64_t probe_z = 0; + int probe_c_err = INT_MIN; + int probe_z_err = INT_MIN; + te_expr *probe_ce; + te_expr *probe_ze; + + if (strlen(expression) < sizeof(minimized)) { + strcpy(minimized, expression); + if (strlen(minimized) <= 24) { + minimize(minimized, sizeof(minimized)); + } + shown = minimized[0] == '\0' ? "" : minimized; + } else { + shown = expression; + } + + probe_ce = c_te_compile(shown, NULL, 0, &probe_c_err); + probe_ze = te_compile(shown, NULL, 0, &probe_z_err); + if (probe_ce) probe_c = f64_bits(c_te_eval(probe_ce)); + if (probe_ze) probe_z = f64_bits(te_eval(probe_ze)); + for (i = 0; i < seen_count; ++i) { + if (seen_c[i] == probe_c && seen_z[i] == probe_z && + /* also distinguish nullness via error sentinel */ + 1) { + c_te_free(probe_ce); + te_free(probe_ze); + return; + } + } + if (seen_count < sizeof(seen_c) / sizeof(seen_c[0])) { + seen_c[seen_count] = probe_c; + seen_z[seen_count] = probe_z; + ++seen_count; + } + c_te_free(probe_ce); + te_free(probe_ze); + + ++run->divergences; + /* Always capture te_print when recording a unique divergence. */ + run->compare_print = 1; + c_expr = c_te_compile(shown, NULL, 0, &c_error); + z_expr = te_compile(shown, NULL, 0, &z_error); + if (c_expr) c_bits = f64_bits(c_te_eval(c_expr)); + if (z_expr) z_bits = f64_bits(te_eval(z_expr)); + if (c_expr != NULL && z_expr != NULL) { + char c_print[4096]; + char z_print[4096]; + size_t c_print_len = 0; + size_t z_print_len = 0; + if (capture_print(c_te_print, c_expr, c_print, sizeof(c_print), &c_print_len) == 0 && + capture_print(te_print, z_expr, z_print, sizeof(z_print), &z_print_len) == 0) { + if (c_print_len != z_print_len || memcmp(c_print, z_print, c_print_len) != 0) { + mismatch |= 8; + } + } + } + fprintf(run->divergence_file, + "| %s | `%s` | c_null=%d z_null=%d c_err=%d z_err=%d " + "c_bits=0x%016" PRIx64 " z_bits=0x%016" PRIx64 " | %s%s%s%s |\n", + source, shown, + c_expr == NULL, z_expr == NULL, c_error, z_error, c_bits, z_bits, + (mismatch & 1) ? "compile-success " : "", + (mismatch & 2) ? "error-position " : "", + (mismatch & 4) ? "eval-bits " : "", + (mismatch & 8) ? "te_print-bytes" : ""); + fflush(run->divergence_file); + c_te_free(c_expr); + te_free(z_expr); +} + +static void check(struct run *run, const char *source, const char *expression) { + static size_t reports = 0; + unsigned mismatch = compare_expression_ex(run, expression); + if (mismatch == 0) return; + ++run->divergences; + if (reports < 32) { + ++reports; + report_divergence(run, source, expression, mismatch); + } +} + +static void enumerate_strings_rec(struct run *run, char *buffer, size_t length, + size_t index, const char *alphabet) { + const char *p; + if (index == length) { + buffer[length] = '\0'; + ++run->strings; + check(run, "strings", buffer); + return; + } + for (p = alphabet; *p != '\0'; ++p) { + buffer[index] = *p; + enumerate_strings_rec(run, buffer, length, index + 1, alphabet); + } +} + +static void enumerate_strings(struct run *run, unsigned max_length, + const char *alphabet) { + char buffer[64]; + unsigned length; + for (length = 0; length <= max_length; ++length) { + enumerate_strings_rec(run, buffer, length, 0, alphabet); + } +} + +static void free_trees(struct trees trees) { + size_t index; + for (index = 0; index < trees.len; ++index) free(trees.items[index]); + free(trees.items); +} + +static int push_tree(struct trees *trees, size_t *capacity, const char *text) { + if (trees->len == *capacity) { + size_t next = *capacity ? *capacity * 2 : 64; + char **items = realloc(trees->items, next * sizeof(*items)); + if (items == NULL) return -1; + trees->items = items; + *capacity = next; + } + trees->items[trees->len] = strdup(text); + if (trees->items[trees->len] == NULL) return -1; + ++trees->len; + return 0; +} + +/* + * AST enumerator over a fixed leaf set and operator/builtin vocabulary. + * Depth is the maximum operator/function nesting from root to leaf. + */ +static struct trees make_trees(unsigned depth) { + static const char *leaves[] = { + "0", "1", "-0", "1e-320", "1e308", + }; + static const char *unary_ops[] = { "-", "+" }; + static const char *binary_ops[] = { "+", "-", "*", "/", "^", "%" }; + static const char *unary_fns[] = { "sin", "cos", "abs", "sqrt", "fac", "log", "floor" }; + static const char *binary_fns[] = { "atan2", "pow", "ncr", "npr", "min", "max" }; + struct trees previous = { NULL, 0 }; + struct trees current = { NULL, 0 }; + size_t capacity = 0; + size_t i; + + if (depth == 0) { + for (i = 0; i < sizeof(leaves) / sizeof(leaves[0]); ++i) { + if (push_tree(¤t, &capacity, leaves[i]) != 0) goto oom; + } + return current; + } + + previous = make_trees(depth - 1); + if (previous.items == NULL && depth > 0) return current; + + for (i = 0; i < sizeof(leaves) / sizeof(leaves[0]); ++i) { + if (push_tree(¤t, &capacity, leaves[i]) != 0) goto oom; + } + for (i = 0; i < sizeof(unary_ops) / sizeof(unary_ops[0]); ++i) { + size_t child; + for (child = 0; child < previous.len; ++child) { + char buf[512]; + if (snprintf(buf, sizeof(buf), "(%s%s)", unary_ops[i], previous.items[child]) >= (int)sizeof(buf)) + continue; + if (push_tree(¤t, &capacity, buf) != 0) goto oom; + } + } + for (i = 0; i < sizeof(binary_ops) / sizeof(binary_ops[0]); ++i) { + size_t left; + for (left = 0; left < previous.len; ++left) { + size_t right; + for (right = 0; right < previous.len; ++right) { + char buf[1024]; + if (snprintf(buf, sizeof(buf), "(%s%s%s)", previous.items[left], + binary_ops[i], previous.items[right]) >= (int)sizeof(buf)) + continue; + if (push_tree(¤t, &capacity, buf) != 0) goto oom; + } + } + } + for (i = 0; i < sizeof(unary_fns) / sizeof(unary_fns[0]); ++i) { + size_t child; + for (child = 0; child < previous.len; ++child) { + char buf[512]; + if (snprintf(buf, sizeof(buf), "%s(%s)", unary_fns[i], previous.items[child]) >= (int)sizeof(buf)) + continue; + if (push_tree(¤t, &capacity, buf) != 0) goto oom; + } + } + for (i = 0; i < sizeof(binary_fns) / sizeof(binary_fns[0]); ++i) { + size_t left; + for (left = 0; left < previous.len; ++left) { + size_t right; + for (right = 0; right < previous.len; ++right) { + char buf[1024]; + if (snprintf(buf, sizeof(buf), "%s(%s,%s)", binary_fns[i], + previous.items[left], previous.items[right]) >= (int)sizeof(buf)) + continue; + if (push_tree(¤t, &capacity, buf) != 0) goto oom; + } + } + } + free_trees(previous); + return current; + +oom: + free_trees(previous); + free_trees(current); + return (struct trees){ NULL, 0 }; +} + +static int enumerate_asts(struct run *run, unsigned depth) { + int saved_print = run->compare_print; + /* AST space is large; compare null/error/bits exhaustively, then + * re-check unique divergences with te_print in report_divergence. */ + run->compare_print = 0; + static const char *leaves[] = { + "0", "1", "-0", "1e-320", "1e308", + }; + static const char *unary_ops[] = { "-", "+" }; + static const char *binary_ops[] = { "+", "-", "*", "/", "^", "%" }; + static const char *unary_fns[] = { "sin", "cos", "abs", "sqrt", "fac", "log", "floor" }; + static const char *binary_fns[] = { "atan2", "pow", "ncr", "npr", "min", "max" }; + struct trees previous; + size_t i; + + if (depth == 0) { + for (i = 0; i < sizeof(leaves) / sizeof(leaves[0]); ++i) { + ++run->asts; + check(run, "ast", leaves[i]); + } + return 0; + } + + previous = make_trees(depth - 1); + if (previous.items == NULL) return -1; + + for (i = 0; i < sizeof(leaves) / sizeof(leaves[0]); ++i) { + ++run->asts; + check(run, "ast", leaves[i]); + } + for (i = 0; i < sizeof(unary_ops) / sizeof(unary_ops[0]); ++i) { + size_t child; + for (child = 0; child < previous.len; ++child) { + char buf[512]; + if (snprintf(buf, sizeof(buf), "(%s%s)", unary_ops[i], previous.items[child]) >= (int)sizeof(buf)) + continue; + ++run->asts; + check(run, "ast", buf); + } + } + for (i = 0; i < sizeof(binary_ops) / sizeof(binary_ops[0]); ++i) { + size_t left; + for (left = 0; left < previous.len; ++left) { + size_t right; + for (right = 0; right < previous.len; ++right) { + char buf[1024]; + if (snprintf(buf, sizeof(buf), "(%s%s%s)", previous.items[left], + binary_ops[i], previous.items[right]) >= (int)sizeof(buf)) + continue; + ++run->asts; + check(run, "ast", buf); + } + } + } + for (i = 0; i < sizeof(unary_fns) / sizeof(unary_fns[0]); ++i) { + size_t child; + for (child = 0; child < previous.len; ++child) { + char buf[512]; + if (snprintf(buf, sizeof(buf), "%s(%s)", unary_fns[i], previous.items[child]) >= (int)sizeof(buf)) + continue; + ++run->asts; + check(run, "ast", buf); + } + } + for (i = 0; i < sizeof(binary_fns) / sizeof(binary_fns[0]); ++i) { + size_t left; + for (left = 0; left < previous.len; ++left) { + size_t right; + for (right = 0; right < previous.len; ++right) { + char buf[1024]; + if (snprintf(buf, sizeof(buf), "%s(%s,%s)", binary_fns[i], + previous.items[left], previous.items[right]) >= (int)sizeof(buf)) + continue; + ++run->asts; + check(run, "ast", buf); + } + } + } + free_trees(previous); + run->compare_print = saved_print; + return 0; +} + +int main(int argc, char **argv) { + /* + * Alphabet: digits, '.', operators + - * / ^ %, parentheses, comma, + * space, and identifier 'a'. Excludes letters beyond 'a' (combinatorial + * blow-up without new token classes) and quotes/escapes (not in grammar). + */ + const char *alphabet = "0123456789.+-*/^%(), a"; + const char *divergence_path = NULL; + unsigned string_max = 3; + unsigned ast_depth = 2; + struct run run = { 0, 0, 0, 0, 1, NULL }; + int index; + + for (index = 1; index < argc; ++index) { + if (strcmp(argv[index], "--string-max") == 0 && index + 1 < argc) { + string_max = (unsigned)strtoul(argv[++index], NULL, 10); + } else if (strcmp(argv[index], "--ast-depth") == 0 && index + 1 < argc) { + ast_depth = (unsigned)strtoul(argv[++index], NULL, 10); + } else if (strcmp(argv[index], "--alphabet") == 0 && index + 1 < argc) { + alphabet = argv[++index]; + } else if (strcmp(argv[index], "--divergences") == 0 && index + 1 < argc) { + divergence_path = argv[++index]; + } else { + fprintf(stderr, + "usage: %s [--string-max N] [--ast-depth N] [--alphabet S] " + "--divergences PATH\n", + argv[0]); + return 2; + } + } + if (divergence_path == NULL || string_max >= 64) return 2; + + run.divergence_file = fopen(divergence_path, "w"); + if (run.divergence_file == NULL) { + perror(divergence_path); + return 2; + } + fprintf(run.divergence_file, + "# Differential enumerator divergences\n\n" + "| Enumerator | Minimized input | Observed | Mismatch |\n" + "| --- | --- | --- | --- |\n"); + enumerate_strings(&run, string_max, alphabet); + if (enumerate_asts(&run, ast_depth) != 0) { + fclose(run.divergence_file); + fputs("AST generation ran out of memory\n", stderr); + return 2; + } + fclose(run.divergence_file); + + printf("{\"alphabet\":\"%s\",\"alphabet_size\":%zu," + "\"string_max_length\":%u," + "\"string_total\":%" PRIu64 ",\"ast_depth\":%u," + "\"ast_total\":%" PRIu64 ",\"total\":%" PRIu64 "," + "\"divergences\":%" PRIu64 "}\n", + alphabet, strlen(alphabet), string_max, run.strings, ast_depth, run.asts, + run.strings + run.asts, run.divergences); + return run.divergences == 0 ? 0 : 1; +} From fca672437d5929ecaaecd0ea21f8d6bd3c8942bf Mon Sep 17 00:00:00 2001 From: aadi-joshi Date: Sun, 2 Aug 2026 10:38:35 +0530 Subject: [PATCH 2/2] docs: clarify fac(NaN) divergence root cause --- docs/evidence/divergences.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/evidence/divergences.md b/docs/evidence/divergences.md index f62e440..0c14333 100644 --- a/docs/evidence/divergences.md +++ b/docs/evidence/divergences.md @@ -18,7 +18,7 @@ subexpressions such as `0/0` and `1%0`. Measured with the same in-process | Enumerator | Minimized input | Observed | Root cause | | --- | --- | --- | --- | -| probe | `fac(0/0)` | c_bits=`0x3ff0000000000000` (1.0), z_bits=`0x7ff0000000000000` (+Inf) | `0/0` is NaN. C `fac` does not treat NaN specially; `(unsigned)nan` is implementation-defined and yields 0 here, so the product loop returns 1. Zig `fac` returns +Inf on non-finite / overflow-class inputs. C is less correct. | +| probe | `fac(0/0)` | c_bits=`0x3ff0000000000000` (1.0), z_bits=`0x7ff0000000000000` (+Inf) | `0/0` is NaN. C `fac` does not treat NaN specially; `(unsigned)nan` is implementation-defined and yields 0 here, so the product loop returns 1. Zig `@intFromFloat(NaN)` takes a path that returns +Inf. Measured; C is the surprising 1.0. | | probe | `fac((0/0))` | same as above | Same after parentheses. | | probe | `fac(ncr(0,1))` | c=1.0, z=+Inf | `ncr(0,1)` is NaN (`n < r`); `fac(NaN)` hits the same C cast path. | | intentional | deep nesting > 2048 | C: stack overflow (UB); Zig: compile NULL / err=-1 | D008 nesting bound. Not run in-process against C. |