diff --git a/analysis/ghidra/benchmarks/corpus/chain_phase3.sh b/analysis/ghidra/benchmarks/corpus/chain_phase3.sh new file mode 100755 index 000000000..dd92c2104 --- /dev/null +++ b/analysis/ghidra/benchmarks/corpus/chain_phase3.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# chain_phase3.sh -- run phase 3's SCORING half once phase 2 has actually +# finished with the GPU. +# +# Operational copy lives at /mnt-1/benchmarks/chain_phase3.sh. Committed here +# because the previous generation of chain scripts (chain2b.sh, chain3.sh) were +# never committed, did not survive the rebuild, and took phases 2.5/3/5 with +# them (#2985). +# +# --------------------------------------------------------------------------- +# Why this does not wait on EXTRA_COMPLETE +# +# The old chain polled for the `EXTRA_COMPLETE` marker. That marker is written +# unconditionally at the end of the roster loop no matter how many entries +# failed, and it has now produced a false completion three separate times: +# once on the pull path, once on the Tier B path (#2971), and once when 12 +# models in a row died to container DNS in four minutes and the sweep declared +# itself finished anyway (#3031). +# +# So this waits on the thing that is actually true or false -- whether every +# roster entry has both tier files, or is explicitly marked UNMEASURED / +# UNMEASURABLE -- and it refuses to start if the sweep is still alive. +# +# It also declines to start on a suspicious roster: if a large share of the +# roster is unmeasured, that is an infrastructure failure to investigate, not a +# green light to spend GPU hours on the next phase. +set -u + +BASE=${BASE:-/mnt-1/benchmarks} +RESULTS=${RESULTS:-$BASE/1947full} +ROSTER=${ROSTER:-$BASE/models_extra_all.txt} +TAGLIST=${TAGLIST:-$BASE/models_requant.txt} +REPO=${REPO:-$BASE/APIARY} +MAX_WAIT_MIN=${MAX_WAIT_MIN:-2880} # 48h +MAX_UNMEASURED_PCT=${MAX_UNMEASURED_PCT:-25} + +log() { echo "$(date -u +%FT%TZ) $*"; } + +roster_state() { # -> "done total unmeasured" + python3 - "$ROSTER" "$RESULTS" <<'EOF' +import os, re, sys +roster = [l.strip() for l in open(sys.argv[1]) if l.strip() and not l.startswith("#")] +res = sys.argv[2] +slug = lambda t: re.sub(r"[^A-Za-z0-9._-]", "_", t) +done = unmeasured = 0 +for t in roster: + s = slug(t) + if os.path.exists(f"{res}/tierA_{s}_run1.json") and os.path.exists(f"{res}/tierB_{s}_run1.json"): + done += 1 + elif os.path.exists(f"{res}/UNMEASURED_{s}.status"): + unmeasured += 1 +print(done, len(roster), unmeasured) +EOF +} + +log "PHASE3_CHAIN_ARMED roster=$ROSTER taglist=$TAGLIST" + +for i in $(seq 1 "$MAX_WAIT_MIN"); do + if ! pgrep -f "sweep_extra.sh" >/dev/null 2>&1 && ! pgrep -f "record_baseline.py" >/dev/null 2>&1; then + read -r done total unmeasured <<<"$(roster_state)" + if [ $((done + unmeasured)) -ge "$total" ]; then + log "phase 2 settled: $done measured, $unmeasured unmeasured, $total total" + break + fi + log "sweep not running but roster incomplete ($done+$unmeasured/$total) -- waiting, it may be between models" + fi + sleep 60 +done + +read -r done total unmeasured <<<"$(roster_state)" +if [ $((done + unmeasured)) -lt "$total" ]; then + log "ABORT: phase 2 never settled ($done+$unmeasured/$total after ${MAX_WAIT_MIN}m)" + exit 1 +fi + +pct=$(( unmeasured * 100 / (total > 0 ? total : 1) )) +if [ "$pct" -gt "$MAX_UNMEASURED_PCT" ]; then + log "ABORT: $unmeasured/$total ($pct%) of the roster is UNMEASURED, over the ${MAX_UNMEASURED_PCT}% bar." + log " That is an infrastructure failure to investigate (see #3031), not a" + log " reason to start the next phase. Not spending GPU on it." + exit 1 +fi + +if [ ! -s "$TAGLIST" ]; then + log "ABORT: $TAGLIST is empty -- the ladder was never built. Run requant_sweep.sh first." + exit 1 +fi + +log "scoring $(wc -l < "$TAGLIST") self-quantized tags" +cd "$BASE" || exit 1 +LIST="$TAGLIST" BASE="$RESULTS" REPO="$REPO" bash "$BASE/sweep_extra.sh" +log "PHASE3_SCORING_COMPLETE" diff --git a/analysis/ghidra/benchmarks/corpus/f16_ladder_plan.txt b/analysis/ghidra/benchmarks/corpus/f16_ladder_plan.txt new file mode 100644 index 000000000..2ac0fcb37 --- /dev/null +++ b/analysis/ghidra/benchmarks/corpus/f16_ladder_plan.txt @@ -0,0 +1,37 @@ +# #1947 phase 3 / #2245 -- clean f16 quantization ladder. +# +# | | +# +# These are ORIGINAL safetensors repos, not GGUF repos. convert_hf_to_gguf.py +# needs the original weights; a GGUF repo has nothing to convert. Each row's +# base_model was read off the corresponding GGUF repo's card and verified to +# exist, be ungated, and carry config.json + safetensors shards (2026-09-05). +# +# Level choice is anchored on the measured envelope, not on habit: the RTX 4000 +# Ada holds 20475 MiB, and KV cache at the benchmark's served CONTEXT 32768 +# costs ~3-5 GB on a 26-35B model, so a fully-resident model has a WEIGHTS +# budget of roughly 16-17 GB. Every row therefore brackets that budget -- one +# level above it (spills, measured with offload), one at it, one or two below -- +# because the whole question #2245 asks is whether buying parameters with bits +# pays off, and that is only answerable if the ladder crosses the residency line. +# +# Each row's top level deliberately matches a quant already measured +# as-published in the phase-1/2 matrix, so every ladder has a same-base, +# same-scorer control rather than floating free. +# +# Two arms, per #2245 step 1: one dense, one MoE/A3B for the latency axis. + +# --- dense arm -------------------------------------------------------------- +# Ornith-1.0-35B: 70 GB of safetensors. Published Q4_K_M is 22 GB and spills +# (6.7 min/run against gemma-4-26B-A4B's 2.1 for identical work), which is +# exactly the "published quants stop where our card cannot use them" case. +# Scored 67/69 on ghidra Tier B in #1805-c -- the highest in that cohort -- so +# whether it can be made to fit is a real question and not an academic one. +llmfan46/Ornith-1.0-35B-uncensored-heretic | Q4_K_M Q3_K_M Q3_K_S IQ3_M | ornith-35b-selfquant + +# --- MoE arm ---------------------------------------------------------------- +# gemma-4-26B-A4B: 52 GB of safetensors. This is the promote candidate from +# PR #2641 -- fastest in the cohort at 2.1 min/run, clean on both gates. Its +# published ladder stops at Q4/Q5, so Q5_K_M (over budget, measured spilling) +# and Q3_K_M (under) are both points nobody has published for this card. +llmfan46/gemma-4-26B-A4B-it-ultra-uncensored-heretic | Q5_K_M Q4_K_M Q3_K_M | gemma4-26b-a4b-selfquant diff --git a/analysis/ghidra/benchmarks/corpus/requant_sweep.sh b/analysis/ghidra/benchmarks/corpus/requant_sweep.sh new file mode 100755 index 000000000..d19968847 --- /dev/null +++ b/analysis/ghidra/benchmarks/corpus/requant_sweep.sh @@ -0,0 +1,268 @@ +#!/usr/bin/env bash +# requant_sweep.sh -- #1947 phase 3 / #2245: build our own quantization ladder +# from an f16 master and measure every level with the same scorer, at the same +# pin, as the as-published rows it is being compared against. +# +# Operational copy runs from /mnt-1/benchmarks/requant_sweep.sh on the +# homeserver. Committed here because the original was never committed, did not +# survive the 2026-09-03/04 rebuild, and had to be rewritten from its issue +# (#2985). Keep the two in sync by hand. +# +# --------------------------------------------------------------------------- +# What this answers, and why it is not the cheap version +# +# #2245 asks whether a base one or two size classes above the current slots, +# pushed down a quantization ladder WE control, beats the smaller models we run +# today. The cheap version -- llama-quantize --allow-requantize on a GGUF we +# already hold -- is lossy-on-lossy and, per #2245's own comment, "a fit-and-cost +# probe, not a clean quality datapoint". The operator chose the full clean f16 +# ladder, so this converts from the original weights every time. +# +# Scoring is record_baseline.py at tiers A and B on the pinned harness, served +# through Ollama -- NOT corpus_eval.py via llama-server. #2245 step 4 requires a +# winner to beat the #1795/#1947 rows, and those were scored by record_baseline; +# numbers from two different scorers do not compare (rule 6 of #1947's six). +# This script therefore only BUILDS; it hands the finished tags to +# sweep_extra.sh, which already owns the cold-slot protocol, the N=2->3->5 +# escalation and the UNRESOLVED marker (#3036). +# +# --------------------------------------------------------------------------- +# Per base, resume-safe at every step: +# +# 1. download the HF snapshot (skipped if the f16 already exists) +# 2. convert_hf_to_gguf.py -> f16 GGUF (snapshot deleted straight after) +# 3. llama-quantize down each level (skipped per level if present) +# 4. ollama create a tag per level +# 5. hand the tags to sweep_extra.sh for scoring +# 6. delete the f16 master unless KEEP_F16=1 +# +# Re-running after an interruption picks up exactly where it stopped rather than +# re-downloading or re-converting anything already on disk. That matters: an f16 +# of a 27B base is ~55 GB and of a 123B base ~245 GB. +# +# --------------------------------------------------------------------------- +# Toolchain: one container, no host installs. +# +# ghcr.io/ggml-org/llama.cpp:full carries convert_hf_to_gguf.py, gguf-py, +# llama-quantize and llama-gguf-split. Verified on this host 2026-09-05. +# +# There is no llama.cpp checkout on the homeserver and no rex86-eval container, +# so the layout model-quant-benchmark/README.md documents does not exist here; +# do not try to use it. +# +# HF_TOKEN is read from ~/.cache/huggingface/token. Gated repos need it, and so +# does any snapshot download at a useful rate. +# +# Usage: +# bash requant_sweep.sh # work the whole plan +# PLAN=/path/to/plan.txt bash requant_sweep.sh +# BUILD_ONLY=1 bash requant_sweep.sh # build the ladder, don't score +set -u + +BASE=${BASE:-/mnt-1/benchmarks} +WORK=${WORK:-$BASE/f16work} +RESULTS=${RESULTS:-$BASE/1947full} +REPO=${REPO:-$BASE/APIARY} +PLAN=${PLAN:-$BASE/f16_ladder_plan.txt} +TAGLIST=${TAGLIST:-$BASE/models_requant.txt} +IMAGE=${IMAGE:-ghcr.io/ggml-org/llama.cpp:full} +OLLAMA=${OLLAMA:-ghidra-ollama-1} +KEEP_F16=${KEEP_F16:-0} +BUILD_ONLY=${BUILD_ONLY:-0} +# An f16 master plus one quant level is the peak; leave room for the largest +# base in the plan rather than discovering the ceiling mid-convert. +MIN_FREE_GB=${MIN_FREE_GB:-400} + +log() { echo "$(date -u +%FT%TZ) $*"; } +die() { log "ABORT: $*"; exit 1; } + +free_gb() { df --output=avail -BG "$1" | tail -1 | tr -dc '0-9'; } + +# --- preconditions, asserted not assumed ------------------------------------ +[ -f "$PLAN" ] || die "no plan file at $PLAN" +[ -d "$REPO" ] || die "no pinned repo at $REPO" +head=$(git -C "$REPO" rev-parse --short HEAD 2>/dev/null) || die "$REPO is not a git checkout" +[ "$head" = "a99e765" ] || die "repo head is $head, not a99e765 -- a different scoring vintage would split the matrix (#1947 rule 6)" +command -v docker >/dev/null || die "no docker" +docker ps --format '{{.Names}}' | grep -qx "$OLLAMA" || die "$OLLAMA is not running" +[ -r "$HOME/.cache/huggingface/token" ] || die "no HF token at ~/.cache/huggingface/token -- snapshot downloads need it" +docker image inspect "$IMAGE" >/dev/null 2>&1 || { log "pulling $IMAGE"; docker pull "$IMAGE" >/dev/null || die "cannot pull $IMAGE"; } + +mkdir -p "$WORK" "$RESULTS" +: > "$TAGLIST" + +log "REQUANT_START plan=$PLAN pin=$head free=$(free_gb "$WORK")G" + +# --- helpers ---------------------------------------------------------------- + +# Everything heavy runs in the toolchain container with $WORK bind-mounted. +# --user keeps the outputs owned by the invoking user: a root-owned file in a +# work tree is how #3024 bricked every CI runner, and the same trap applies here. +llama() { # + docker run --rm \ + -u "$(id -u):$(id -g)" \ + -e HOME=/work \ + -v "$WORK:/work" \ + -v "$HOME/.cache/huggingface:/work/.cache/huggingface" \ + --entrypoint "$1" "$IMAGE" "${@:2}" +} + +# Some published repos ship `extra_special_tokens` as a bare LIST where +# transformers requires a dict, and the conversion dies deep inside the +# tokenizer with a bare AttributeError: +# +# tokenization_utils_base.py: self.SPECIAL_TOKENS_ATTRIBUTES + list(special_tokens.keys()) +# AttributeError: 'list' object has no attribute 'keys' +# +# Seen on llmfan46/gemma-4-26B-A4B-it-ultra-uncensored-heretic, whose own +# sibling field `model_specific_special_tokens` IS a correctly-formed dict +# ({"audio_token": "<|audio|>", "boi_token": "<|image>", ...}), so the intended +# shape is unambiguous and the list is simply malformed upstream. +# +# The token is real -- <|video|> is in tokenizer.json's vocab -- so DELETING the +# field would silently drop a token the weights know about. Re-key it instead, +# following the sibling field's own <|x|> -> x_token convention, and log the +# rewrite: this is a deviation from the published artifact and #1947 rule 5 +# requires deviations to be recorded rather than quietly applied. +normalize_snapshot() { # snapshot_dir + python3 - "$1" <<'EOF' +import json, pathlib, re, sys +p = pathlib.Path(sys.argv[1]) / "tokenizer_config.json" +if not p.exists(): + sys.exit(0) +cfg = json.loads(p.read_text()) +v = cfg.get("extra_special_tokens") +if isinstance(v, list): + fixed = {} + for tok in v: + m = re.fullmatch(r"<\|(.+?)\|>", str(tok)) + key = (m.group(1) if m else re.sub(r"\W+", "_", str(tok)).strip("_")) + "_token" + fixed[key] = tok + cfg["extra_special_tokens"] = fixed + p.write_text(json.dumps(cfg, indent=2)) + print(f"NORMALIZED extra_special_tokens: list {v} -> dict {fixed}") +EOF +} + +snapshot_dl() { # repo dest + local repo="$1" dest="$2" + docker run --rm \ + -u "$(id -u):$(id -g)" \ + -e HOME=/work -e HF_HUB_ENABLE_HF_TRANSFER=0 \ + -v "$WORK:/work" \ + -v "$HOME/.cache/huggingface:/work/.cache/huggingface" \ + --entrypoint python3 "$IMAGE" -c " +import os, sys +from huggingface_hub import snapshot_download +tok = open('/work/.cache/huggingface/token').read().strip() +snapshot_download(repo_id='$repo', local_dir='$dest', token=tok, + allow_patterns=['*.json','*.safetensors','*.model','*.txt','*.py']) +print('snapshot ok') +" +} + +# --- the plan --------------------------------------------------------------- +# One line per base: +# | | +# e.g. +# Qwen/Qwen3-32B | Q5_K_M Q4_K_M Q3_K_M | qwen3-32b-selfquant + +# Trim with bash builtins, not `xargs`: xargs treats quotes as special, so an +# apostrophe anywhere in this file's prose comments makes it error out -- +# noisily and, worse, it would mangle any plan value that contained one. +trim() { local s="$1"; s="${s#"${s%%[![:space:]]*}"}"; printf '%s' "${s%"${s##*[![:space:]]}"}"; } + +while IFS='|' read -r REPO_ID LEVELS PREFIX; do + # comment/blank check happens BEFORE any processing of the line + case "$(trim "${REPO_ID:-}")" in \#*|'') continue;; esac + REPO_ID=$(trim "${REPO_ID:-}"); LEVELS=$(trim "${LEVELS:-}"); PREFIX=$(trim "${PREFIX:-}") + [ -n "$LEVELS" ] && [ -n "$PREFIX" ] || { log "SKIP malformed plan line for '$REPO_ID'"; continue; } + + name=$(echo "$REPO_ID" | tr '/' '_') + f16="$WORK/${name}-f16.gguf" + snap="$WORK/${name}-snapshot" + + log "=== $REPO_ID -> $PREFIX (levels: $LEVELS) ===" + + # --- 1+2. snapshot -> f16, both skipped once the f16 exists --------------- + if [ ! -f "$f16" ]; then + free=$(free_gb "$WORK") + [ "$free" -ge "$MIN_FREE_GB" ] || { log "SKIP $REPO_ID: only ${free}G free, need ${MIN_FREE_GB}G"; continue; } + if [ ! -d "$snap" ]; then + log "downloading snapshot $REPO_ID (${free}G free)" + snapshot_dl "$REPO_ID" "/work/${name}-snapshot" || { log "SNAPSHOT_FAILED $REPO_ID"; rm -rf "$snap"; continue; } + else + log "snapshot already present, reusing" + fi + normalize_snapshot "$snap" | while read -r l; do log "$l"; done + log "converting to f16" + if ! llama python3 /app/convert_hf_to_gguf.py "/work/${name}-snapshot" \ + --outfile "/work/${name}-f16.gguf" --outtype f16; then + log "CONVERT_FAILED $REPO_ID"; rm -f "$f16"; continue + fi + # The snapshot is quantize input only and is the bulk of the disk cost. + rm -rf "$snap" + log "f16 ready: $(du -h "$f16" | cut -f1)" + else + log "f16 already present: $(du -h "$f16" | cut -f1)" + fi + + # --- 3+4. ladder ---------------------------------------------------------- + for lvl in $LEVELS; do + out="$WORK/${name}-${lvl}.gguf" + tag="${PREFIX}:$(echo "$lvl" | tr '[:upper:]' '[:lower:]')" + + if [ ! -f "$out" ]; then + log "quantizing -> $lvl" + if ! llama /app/llama-quantize "/work/${name}-f16.gguf" "/work/${name}-${lvl}.gguf" "$lvl"; then + log "QUANT_FAILED $REPO_ID $lvl"; rm -f "$out"; continue + fi + fi + log "$lvl on disk: $(du -h "$out" | cut -f1)" + + if docker exec "$OLLAMA" ollama list 2>/dev/null | awk '{print $1}' | grep -qixF "$tag"; then + log "ollama tag already present: $tag" + else + # ollama create needs the GGUF inside its own volume; copy, import, drop. + vol=$(docker inspect "$OLLAMA" --format '{{range .Mounts}}{{if eq .Destination "/root/.ollama"}}{{.Source}}{{end}}{{end}}') + [ -n "$vol" ] || { log "cannot locate the ollama volume; skipping $tag"; continue; } + sudo -n cp "$out" "$vol/requant-import.gguf" || { log "IMPORT_COPY_FAILED $tag"; continue; } + printf 'FROM /root/.ollama/requant-import.gguf\n' | sudo -n tee "$vol/requant-import.Modelfile" >/dev/null + if docker exec "$OLLAMA" ollama create "$tag" -f /root/.ollama/requant-import.Modelfile >/dev/null 2>&1; then + log "created $tag" + else + log "OLLAMA_CREATE_FAILED $tag" + fi + sudo -n rm -f "$vol/requant-import.gguf" "$vol/requant-import.Modelfile" + fi + + docker exec "$OLLAMA" ollama list 2>/dev/null | awk '{print $1}' | grep -qixF "$tag" \ + && echo "$tag" >> "$TAGLIST" + done + + # --- 6. drop the f16 ------------------------------------------------------ + if [ "$KEEP_F16" != "1" ]; then + rm -f "$f16" + log "removed f16 master for $REPO_ID (free now $(free_gb "$WORK")G)" + fi +done < "$PLAN" + +log "LADDER_BUILT tags=$(wc -l < "$TAGLIST")" +[ -s "$TAGLIST" ] || { log "REQUANT_SWEEP_COMPLETE (nothing built)"; exit 0; } + +if [ "$BUILD_ONLY" = "1" ]; then + log "BUILD_ONLY set -- not scoring. Tags in $TAGLIST" + log "REQUANT_SWEEP_COMPLETE" + exit 0 +fi + +# --- 5. score, through the reviewed driver ---------------------------------- +# Not a reimplementation: sweep_extra.sh owns the cold-slot protocol, the +# N=2->3->5 escalation and the UNRESOLVED marker, and using it is what keeps +# these rows comparable to the as-published ones. Locally created tags are never +# `ollama pull`ed, so its own "already local -> will not delete" branch protects +# the ladder we just spent hours building. +log "scoring $(wc -l < "$TAGLIST") tags via sweep_extra.sh" +LIST="$TAGLIST" BASE="$RESULTS" REPO="$REPO" bash "$BASE/sweep_extra.sh" + +log "REQUANT_SWEEP_COMPLETE" diff --git a/analysis/ghidra/benchmarks/corpus/sweep_extra.sh b/analysis/ghidra/benchmarks/corpus/sweep_extra.sh index f0ff5c743..922970984 100755 --- a/analysis/ghidra/benchmarks/corpus/sweep_extra.sh +++ b/analysis/ghidra/benchmarks/corpus/sweep_extra.sh @@ -31,11 +31,16 @@ # used to go to /dev/null, which is what made #2728 slow to diagnose -- it # now lands in a per-model log under $BASE/logs. set -u -BASE=/mnt-1/benchmarks/1947full -REPO=/mnt-1/benchmarks/APIARY -LIST=/mnt-1/benchmarks/models_extra_all.txt -PRESEED=/mnt-1/benchmarks/preseed.sh -MAXTRY=3 +# Overridable so a second roster can reuse this driver instead of copying its +# run/escalation/UNRESOLVED logic. requant_sweep.sh (#2245) builds a ladder of +# self-quantized tags and then runs exactly this script over them, so the +# self-quant rows are scored by the same code, at the same pin, as every +# as-published row they are meant to be compared against. +BASE=${BASE:-/mnt-1/benchmarks/1947full} +REPO=${REPO:-/mnt-1/benchmarks/APIARY} +LIST=${LIST:-/mnt-1/benchmarks/models_extra_all.txt} +PRESEED=${PRESEED:-/mnt-1/benchmarks/preseed.sh} +MAXTRY=${MAXTRY:-3} # #2738: fail fast on any roster entry Ollama's client-side hf.co name # validation would reject before a sweep wastes time discovering it --