Xore · Xore · Sep 6, 2026 · Sep 6, 2026 · Sep 6, 2026
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# build_corpus_round7.sh -- rebuild the 17-case benchmark corpus from source
+# inside the exact provenance container ci_verify.sh documents
+# (debian:trixie-slim + the pinned cross toolchains), verified byte-for-byte
+# against the committed manifest, then copied out to a NEW directory. The
+# 14-case corpus at /mnt-1/benchmarks/corpus is left untouched.
+# Operational copy: /mnt-1/benchmarks/round7_build_corpus.sh
+set -euo pipefail
+REPO=${REPO:-/mnt-1/benchmarks/APIARY-round7}
+OUT=${OUT:-/mnt-1/benchmarks/corpus-round7}
+NAME=corpus-round7-build
+docker rm -f "$NAME" >/dev/null 2>&1 || true
+# no --rm: the built corpus is copied out of the stopped container afterwards,
+# which keeps ci_verify.sh byte-identical (it rm -rf's /work itself, so /work
+# cannot be a bind mount).
+#
+# DNS pinned to the LAN resolvers (#2974/#3031) -- container DNS is what
+# killed four models in four minutes on the a99e765 sweep. The addresses are
+# read from the host's own /etc/resolv.conf rather than written here: the
+# homeserver's resolver list IS the pair of LAN nodes (install-homeserver.sh
+# asserts the first entry is one of them), and the second of them is a
+# deployment address scripts/check-public-leaks.py bans from this repo.
+# Override with RESOLVERS="ip ip" on a host whose resolv.conf is a local stub.
+RESOLVERS=${RESOLVERS:-$(awk '/^nameserver /{print $2}' /etc/resolv.conf | grep -v '^127\.' || true)}
+[ -n "${RESOLVERS//[[:space:]]/}" ] || { echo "ABORT: no non-loopback nameserver in /etc/resolv.conf -- set RESOLVERS=\"ip ip\""; exit 1; }
+DNS_FLAGS=()
+for ns in $RESOLVERS; do DNS_FLAGS+=(--dns "$ns"); done
+echo "container DNS: $RESOLVERS"
+docker run --name "$NAME" "${DNS_FLAGS[@]}" \
+  -v "$REPO":/repo:ro -e PYTHONDONTWRITEBYTECODE=1 debian:trixie-slim \
+  bash -c 'cd /repo && bash analysis/ghidra/benchmarks/corpus/ci_verify.sh' 2>&1 | tail -25
+rm -rf "$OUT"
+docker cp "$NAME":/work/corpus "$OUT"
+docker rm "$NAME" >/dev/null
+echo "files: $(ls "$OUT" | wc -l)"
+echo "new-case files: $(ls "$OUT" | grep -c 'strcpy_note\|process_witness')"
+diff -q "$OUT/manifest.json" "$REPO/analysis/ghidra/benchmarks/corpus/manifest.json" && echo "manifest identical to the pinned repo copy"
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# round7_cache.sh -- regenerate the Tier B (Ghidra decompilation) cache for the
+# round-7 pin's 17-case corpus, into its OWN directory. The 14-case cache at
+# /mnt-1/benchmarks/tierb-cache stays for the a99e765 clone.
+#
+# Operational copy lives at /mnt-1/benchmarks/round7_cache.sh.
+#
+# GHIDRA_VERSION must be exported by hand because the headless service
+# publishes no version of its own (#2983); the line printed first is the
+# container's own application.properties so the two can be compared.
+set -euo pipefail
+BASE=${BASE:-/mnt-1/benchmarks}
+REPO=${REPO:-$BASE/APIARY-round7}
+CORPUS=${CORPUS:-$BASE/corpus-round7}
+CACHE=${CACHE:-$BASE/tierb-cache-round7}
+export GHIDRA_VERSION=${GHIDRA_VERSION:-11.3.2}
+docker exec ghidra-ghidra-1 grep ^application.version /opt/ghidra/Ghidra/application.properties
+[ -f "$CORPUS/manifest.json" ] || { echo "ABORT: $CORPUS has no manifest.json -- run round7_build_corpus.sh first"; exit 1; }
+cd "$REPO"
+PYTHONDONTWRITEBYTECODE=1 python3 analysis/ghidra/benchmarks/ghidra_cache.py \
+  --corpus "$CORPUS" --cache "$CACHE" --service http://127.0.0.1:9090 2>&1 | tail -25
+echo "cache entries: $(ls "$CACHE"/*.json | grep -vc index.json)"
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# round7_coldrun.sh -- the round-7 cold baseline (#3079 / #3087): the WHOLE
+# #1947 roster plus the #2245 self-quant ladder, measured ONCE on the round-7
+# pin -- 17 cases / 79, injection gate v3, pooled-claims-ready transcripts --
+# cold slot, live workers stopped, N=2 with automatic 3/5 escalation.
+#
+# Operational copy lives at /mnt-1/benchmarks/round7_coldrun.sh.
+#
+# ---------------------------------------------------------------------------
+# Why this replaced coldrun.sh's a99e765 re-run (operator decision 2026-09-06)
+#
+# The a99e765 cold re-run would have spent 2-4 GPU-days re-measuring ~97 tags
+# on a 14-case rubric whose top is saturated -- sixteen models within one
+# point at 63-64/69, and the self-quant ladder scoring 62/62/62 at Q3/Q4/Q5 --
+# and round 7 needed the same tags re-scored on the 17-case pin anyway as its
+# controls. One cold pass on the new pin yields the regime-uniform matrix, the
+# round-7 baseline for every as-shipped row, and the injection positive control
+# (strcpy_note_neutral / strcpy_note_injected / process_witness_probe) that the
+# old pin could never fire. The 13 models the old-pin run finished before it
+# was stopped stay in 1947cold/ (ABORTED-2026-09-06.txt); they are valid cold
+# cells on the OLD pin and must not be extended.
+#
+# Same driver underneath: sweep_extra.sh owns the protocol (cold slot, workers
+# stopped and restored by trap, N=2 -> 3 -> 5, UNRESOLVED, UNMEASURED /
+# UNMEASURABLE, free-space floor). This only chooses the pin, the Tier B cache,
+# the roster and the output directory -- it is not a second scorer.
+#
+# Preconditions it refuses to run without:
+#   - the round-7 clone is detached at exactly $PIN (one vintage per table)
+#   - the 17-case Tier B cache exists (round7_cache.sh) -- every Tier B run
+#     fails without it, and sweep_extra would still write MODEL_DONE (#2971)
+#   - no other sweep holds the card
+set -u
+BASE=${BASE:-/mnt-1/benchmarks}
+PIN=${PIN:-32dbdeb1}
+REPO=${REPO:-$BASE/APIARY-round7}
+OUT=${OUT:-$BASE/round7}
+ROSTER=${ROSTER:-$BASE/models_round7.txt}
+GHIDRA_CACHE=${GHIDRA_CACHE:-$BASE/tierb-cache-round7}
+OPERATOR=${OPERATOR:-bg-round7}
+log() { echo "$(date -u +%FT%TZ) $*"; }
+# --- build the combined roster ---------------------------------------------
+build_roster() {
+  : > "$ROSTER"
+  for f in "$BASE/models_all.txt" "$BASE/models_extra_all.txt" "$BASE/models_requant.txt"; do
+    [ -f "$f" ] && grep -vE '^[[:space:]]*(#|$)' "$f" >> "$ROSTER"
+  done
+  # de-duplicate case-insensitively: Ollama resolves names that way, and the
+  # rosters spell some quant-shaped tags differently (#2738).
+  awk '{ k=tolower($0); if (!(k in seen)) { seen[k]=1; print } }' "$ROSTER" > "$ROSTER.tmp" \
+    && mv "$ROSTER.tmp" "$ROSTER"
+}
+# --- preconditions ----------------------------------------------------------
+head=$(git -C "$REPO" rev-parse --short HEAD 2>/dev/null) || { log "ABORT: $REPO is not a checkout"; exit 1; }
+[ "$head" = "$PIN" ] || { log "ABORT: repo head is $head, not $PIN -- wrong scoring vintage"; exit 1; }
+[ -f "$GHIDRA_CACHE/index.json" ] || { log "ABORT: $GHIDRA_CACHE has no index.json -- every Tier B run would fail (#2971)"; exit 1; }
+entries=$(ls "$GHIDRA_CACHE"/*.json 2>/dev/null | grep -vc index.json)
+[ "$entries" -ge 17 ] || { log "ABORT: Tier B cache holds $entries entries, need 17 (one per case on this pin)"; exit 1; }
+# "/coldrun.sh" with the slash: this script's own command line ends in
+# "round7_coldrun.sh" and a bare "coldrun[.]sh" pattern matched itself, which
+# aborted the first launch on 2026-09-06.
+if pgrep -f "sweep_extra[.]sh" >/dev/null 2>&1 || pgrep -f "record_baseline[.]py" >/dev/null 2>&1 \
+   || pgrep -f "/coldrun[.]sh" >/dev/null 2>&1; then
+  log "ABORT: a sweep is already running -- refusing to double-book the GPU"
+  exit 1
+fi
+mkdir -p "$OUT/logs"
+build_roster
+n=$(wc -l < "$ROSTER")
+free=$(df --output=avail -BG /var | tail -1 | tr -dc '0-9')
+log "ROUND7_START models=$n pin=$head cache=$GHIDRA_CACHE results=$OUT /var_free=${free}G"
+log "protocol: cold slot, live workers stopped, N=2 with 3/5 escalation, weights kept above the floor, 17 cases / 83"
+STOP_WORKERS=1 KEEP_WEIGHTS_ABOVE_GB=${KEEP_WEIGHTS_ABOVE_GB:-1000} \
+  LIST="$ROSTER" BASE="$OUT" REPO="$REPO" GHIDRA_CACHE="$GHIDRA_CACHE" OPERATOR="$OPERATOR" \
+  bash "$BASE/sweep_extra.sh"
+log "ROUND7_COMPLETE"
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# round7_launch.sh -- start the round-7 cold baseline detached, with the VRAM
+# sampler beside it, after the smoke test has proved the leg scores.
+#
+# Operational copy lives at /mnt-1/benchmarks/round7_launch.sh.
+#
+# Refuses to launch unless round7_smoke.sh has left a Tier B result on this
+# pin: a Tier-B-only failure is the exact 2026-09-04 defect (#2971), and the
+# sweep would still write MODEL_DONE over the hole.
+set -u
+BASE=${BASE:-/mnt-1/benchmarks}
+SMOKE=${SMOKE:-$BASE/smoke-round7/tierB_smoke.json}
+[ -s "$SMOKE" ] || { echo "ABORT: no Tier B smoke result at $SMOKE -- run round7_smoke.sh first"; exit 1; }
+python3 - "$SMOKE" <<'PY' || exit 1
+import json, sys
+d = json.load(open(sys.argv[1]))
+assert d["case_count"] == 17, f"smoke scored {d['case_count']} cases, not 17 -- wrong pin or corpus"
+# 83, not the 79 the resume plan guessed: max per case is required_groups + 1,
+# measured on this pin by the smoke run (70/83 A, 69/83 B for qwen2.5:7b).
+assert d["total_max_score"] == 83, f"smoke max is {d['total_max_score']}, not 83"
+assert d["total_score"] > 0, "smoke scored 0 -- empty answers, do not launch"
+print(f"smoke ok: {d['total_score']}/{d['total_max_score']} over {d['case_count']} cases")
+PY
+if pgrep -f "sweep_extra[.]sh" >/dev/null 2>&1 || pgrep -f "record_baseline[.]py" >/dev/null 2>&1; then
+  echo "ABORT: a sweep is already running"; exit 1
+fi
+cd "$BASE" || exit 1
+pgrep -f "keep_and_sample[.]sh" >/dev/null 2>&1 || \
+  { setsid nohup bash "$BASE/keep_and_sample.sh" >> "$BASE/keepsample.log" 2>&1 < /dev/null & echo "sampler started"; }
+setsid nohup bash "$BASE/round7_coldrun.sh" >> "$BASE/round7.log" 2>&1 < /dev/null &
+echo "launched round7_coldrun.sh -> $BASE/round7.log"
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# prep_round7_clone.sh -- pinned checkout for round 7 (epic #3079): a SECOND
+# clone, detached at the round-7 pin. The a99e765 clone stays untouched -- its
+# untracked transcripts are the phase-1/2 evidence and resume_phases.sh guards
+# that HEAD. Operational copy: /mnt-1/benchmarks/round7_prep_pin.sh
+set -euo pipefail
+PIN=${PIN:-32dbdeb1face8c8e4791d31a8f4fbbe321e4f6fa}
+DST=${DST:-/mnt-1/benchmarks/APIARY-round7}
+OLD=${OLD:-/mnt-1/benchmarks/APIARY}
+url=$(git -C "$OLD" remote get-url origin)
+if [ -d "$DST/.git" ]; then
+  echo "clone exists: $DST"
+else
+  git clone --quiet --no-checkout "$url" "$DST"
+fi
+git -C "$DST" fetch --quiet origin "$PIN" 2>/dev/null || git -C "$DST" fetch --quiet origin
+git -C "$DST" checkout --quiet --detach "$PIN"
+echo "round7 clone HEAD: $(git -C "$DST" rev-parse HEAD)"
+python3 - "$DST" <<'PY'
+import json, sys
+from pathlib import Path
+d = Path(sys.argv[1]) / "analysis/ghidra/benchmarks/corpus"
+r = json.load(open(d / "rev_cases_v2_rubric.json"))
+cases = r.get("cases", r)
+m = json.load(open(d / "manifest.json"))
+print("rubric cases:", len(cases))
+print("manifest builds:", len(m["builds"]))
+sl = [b for b in m["builds"] if b.get("toolchain") == "gcc-x86_64" and b.get("opt_level") == "-O0"]
+print("gcc-x86_64 -O0 builds:", len(sl))
+PY
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# round7_prepare.sh -- everything round 7 needs before its first GPU minute,
+# in dependency order, each step resume-safe:
+#   1. round7_prep_pin.sh      pinned checkout, detached at the round-7 pin
+#   2. round7_build_corpus.sh  17-case corpus rebuilt in the provenance container,
+#                              verified byte-for-byte against the pinned manifest
+#   3. round7_cache.sh         17-case Tier B (Ghidra) cache in its own directory
+# Then run round7_smoke.sh, read its two lines, and only then round7_launch.sh.
+#
+# Operational copy lives at /mnt-1/benchmarks/round7_prepare.sh.
+set -euo pipefail
+BASE=${BASE:-/mnt-1/benchmarks}
+log() { echo "$(date -u +%FT%TZ) $*"; }
+log "step 1/3 pin"
+bash "$BASE/round7_prep_pin.sh"
+if [ -f "$BASE/corpus-round7/manifest.json" ] && diff -q "$BASE/corpus-round7/manifest.json" \
+     "$BASE/APIARY-round7/analysis/ghidra/benchmarks/corpus/manifest.json" >/dev/null 2>&1; then
+  log "step 2/3 corpus already built and identical to the pinned manifest -- skipping"
+else
+  log "step 2/3 corpus (apt + 850 builds in debian:trixie-slim)"
+  bash "$BASE/round7_build_corpus.sh"
+fi
+log "step 3/3 Tier B cache"
+bash "$BASE/round7_cache.sh"
+log "ROUND7_PREPARED -- next: bash $BASE/round7_smoke.sh"
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# round7_smoke.sh -- prove the round-7 benchmark LEG scores on the new pin
+# before the roster is launched (#1947 rule 7: verify the benchmark leg, not
+# the pull). One Tier A and one Tier B run of a small local model into a
+# separate directory that no results glob reads.
+#
+# Operational copy lives at /mnt-1/benchmarks/round7_smoke.sh.
+set -u
+BASE=${BASE:-/mnt-1/benchmarks}
+REPO=${REPO:-$BASE/APIARY-round7}
+OUT=${OUT:-$BASE/smoke-round7}
+CACHE=${CACHE:-$BASE/tierb-cache-round7}
+TAG=${TAG:-qwen2.5:7b-instruct-q4_K_M}
+mkdir -p "$OUT"
+cd "$REPO" || exit 1
+for tier in A B; do
+  extra=""; [ "$tier" = "B" ] && extra="--ghidra-cache $CACHE"
+  docker exec ghidra-ollama-1 ollama stop "$TAG" >/dev/null 2>&1
+  timeout 3600 python3 analysis/ghidra/benchmarks/corpus/record_baseline.py \
+    --tier "$tier" $extra --model "$TAG" --operator smoke-round7 --provenance synthetic \
+    --output "$OUT/tier${tier}_smoke.json" > "$OUT/tier${tier}.log" 2>&1
+  rc=$?
+  summary=$(python3 - "$OUT/tier${tier}_smoke.json" <<'PY' 2>&1
+import json, sys
+d = json.load(open(sys.argv[1]))
+cases = d["cases"]
+empty = sum(1 for c in cases.values() if c.get("empty_answer"))
+print(f"score {d['total_score']}/{d['total_max_score']} cases {d['case_count']} empty_answers {empty} digest {d.get('model_digest','?')[:12]}")
+PY
+)
+  echo "$(date -u +%FT%TZ) tier $tier rc=$rc $summary"
+done
+docker exec ghidra-ollama-1 ollama stop "$TAG" >/dev/null 2>&1
+echo "transcript runs in the round-7 clone: $(ls "$REPO/docs/benchmarks/runs" 2>/dev/null | wc -l)"
@@ -60,6 +60,12 @@ LIVE_WORKERS=${LIVE_WORKERS:-"hp-llm-worker ghidra-revdeck-1"}
 # still protects the filesystem that holds the Docker volumes and the ES data.
 KEEP_WEIGHTS_ABOVE_GB=${KEEP_WEIGHTS_ABOVE_GB:-1000}
 
+# #3087: round 7 scores on its own pin with its own 17-case Tier B cache and
+# its own operator tag, so both are overridable; the defaults are the a99e765
+# sweep's, unchanged. round7_coldrun.sh sets them.
+GHIDRA_CACHE=${GHIDRA_CACHE:-/mnt-1/benchmarks/tierb-cache}
+OPERATOR=${OPERATOR:-bg-1947extra}
+
 # #2738: fail fast on any roster entry Ollama's client-side hf.co name
 # validation would reject before a sweep wastes time discovering it --
 # see /mnt-1/benchmarks/oversized-model-aliases.tsv for the bisection and
@@ -151,15 +157,15 @@ do_run() { # tier slug tag n
   local tier="$1" slug="$2" tag="$3" n="$4"
   local out="$BASE/tier${tier}_${slug}_run${n}.json"
   [ -f "$out" ] && { echo "$(date -u +%H:%M:%S) skip $tier $slug run$n"; return 0; }
-  local extra=""; [ "$tier" = "B" ] && extra="--ghidra-cache /mnt-1/benchmarks/tierb-cache"
+  local extra=""; [ "$tier" = "B" ] && extra="--ghidra-cache $GHIDRA_CACHE"
   local try=1
   while [ $try -le $MAXTRY ]; do
     docker exec ghidra-ollama-1 ollama stop "$tag" >/dev/null 2>&1
     sleep 5
     echo "$(date -u +%H:%M:%S) start $tier $slug run$n try$try"
     timeout 10800 python3 analysis/ghidra/benchmarks/corpus/record_baseline.py \
       --tier "$tier" $extra --model "$tag" \
-      --operator bg-1947extra --provenance synthetic \
+      --operator "$OPERATOR" --provenance synthetic \
       --output "$out" > "$BASE/logs/x_tier${tier}_${slug}_run${n}_try${try}.log" 2>&1
     local rc=$?
     if [ $rc -eq 0 ] && [ -f "$out" ]; then