Skip to content

fix(benchmarks): snapshot sentinel + earlier lineage assert in T0 merge driver #3157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Xore merged 3 commits into from
Sep 9, 2026
Merged

fix(benchmarks): snapshot sentinel + earlier lineage assert in T0 merge driver #3157

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2ea54be
fix(benchmarks): add snapshot sentinel and move lineage assert earlier
Xore Sep 9, 2026
59809f6
fix(benchmarks): unprivileged snapshot sentinel, resume-preserving ba…
Xore Sep 9, 2026
409e6bf
sync main (post #3159 mnt-1 decommission)
Xore Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 49 additions & 18 deletions analysis/ghidra/benchmarks/corpus/round7_t0_merge_export.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,15 +267,30 @@ BASE16_C="$RUN_C/base_16bit"
# inside the same unsloth container used for the merge (it already has
# huggingface_hub and the /var/training bind).
have_base_snapshot() { [ -s "$BASE16/model.safetensors.index.json" ] || [ -s "$BASE16/model.safetensors" ]; }
if ! have_base_snapshot; then
# Completion sentinel, same shape as $MERGE_DONE above: snapshot_download has
# no atomic "done" marker of its own, so a partial download can leave
# index.json (or a lone shard) on disk and a re-run's have_base_snapshot()
# check reads that as finished, skips the download, and fails later at
# convert instead. Lives under $RUN, not $BASE16: the unsloth container
# writes $BASE16 as root, so this unprivileged host process could never
# create a marker file inside it.
SNAPSHOT_DONE="$RUN/.base_snapshot_done"
if [ ! -s "$SNAPSHOT_DONE" ]; then
# No rm -rf "$BASE16" here: $BASE16 is root-owned (written by the unsloth
# container), so an unprivileged rm would silently no-op, and
# snapshot_download resumes partial downloads on its own -- deleting the
# tree would only discard $BASE16/.cache/huggingface resume metadata and
# force a full ~15G re-download.
log "downloading untouched base $BASE_MODEL @ $BASE_REV (no adapter)"
exec_py "base snapshot ->" <<PY3 || die "base snapshot download failed"
from huggingface_hub import snapshot_download
p = snapshot_download("$BASE_MODEL", revision="$BASE_REV", local_dir="$BASE16_C")
print("base snapshot ->", p)
PY3
have_base_snapshot || die "base snapshot incomplete at $BASE16"
date -u +%FT%TZ > "$SNAPSHOT_DONE" || die "cannot write $SNAPSHOT_DONE"
fi
[ -s "$SNAPSHOT_DONE" ] || die "base snapshot incomplete at $BASE16"

BASE_GGUF_F16="$RUN/qwen2.5-coder-7b-base-f16.gguf"
if [ ! -s "$BASE_GGUF_F16" ]; then
Expand All @@ -284,6 +299,22 @@ if [ ! -s "$BASE_GGUF_F16" ]; then
/app/convert_hf_to_gguf.py "/var/training/runs/t0-rex86/base_16bit" --outfile "/var/training/runs/t0-rex86/qwen2.5-coder-7b-base-f16.gguf" --outtype f16 \
|| die "base convert_hf_to_gguf failed"
fi
[ -s "$BASE_GGUF_F16" ] || die "base convert produced no output at $BASE_GGUF_F16"

# ---------------------------------------------------------------------------
# 3c. Lineage assert (#3137), run before stage 4 registers anything in Ollama
# so a lineage collision cannot register. The base twin must be provably
# distinct weights from the merged model, not just a distinct file -- a
# re-quantised copy of the same tensors would still pass an [ -s ] guard.
# Both f16 GGUFs exist now; the manifest stage below reuses these shas
# instead of re-hashing two ~15GB files.
# ---------------------------------------------------------------------------
MERGED_F16_SHA=$(sha256sum "$GGUF_F16" | awk '{print $1}')
BASE_F16_SHA=$(sha256sum "$BASE_GGUF_F16" | awk '{print $1}')
[ -n "$MERGED_F16_SHA" ] || die "cannot hash $GGUF_F16"
[ -n "$BASE_F16_SHA" ] || die "cannot hash $BASE_GGUF_F16"
[ "$BASE_F16_SHA" != "$MERGED_F16_SHA" ] || die "base f16 GGUF matches merged f16 GGUF -- base twin is not untouched (#3137)"

BASE_Q4="$RUN/base-Q4_K_M.gguf"
BASE_Q8="$RUN/base-Q8_0.gguf"
[ -s "$BASE_Q4" ] || docker run --rm --entrypoint /app/llama-quantize -v /var/training:/var/training "$LLAMA_IMG" \
Expand Down Expand Up @@ -339,30 +370,30 @@ sudo -n rm -f "$STAGE"/*.gguf || log "warning: could not clean staged GGUFs from
# ---------------------------------------------------------------------------
# 5. Manifest: input SHAs, base revision, quant params, file sizes
# ---------------------------------------------------------------------------
python3 - "$RUN" "$ZIP_SIZE" "$ZIP_SHA" "$INNER_ACTUAL" "$INNER_ACTUAL_SIZE" <<'EOF' || die "manifest generation failed"
python3 - "$RUN" "$ZIP_SIZE" "$ZIP_SHA" "$INNER_ACTUAL" "$INNER_ACTUAL_SIZE" "$MERGED_F16_SHA" "$BASE_F16_SHA" <<'EOF' || die "manifest generation failed"
import hashlib, json, os, sys
run, zip_size, zip_sha, inner_sha, inner_size = (
sys.argv[1], int(sys.argv[2]), sys.argv[3], sys.argv[4], int(sys.argv[5]))
run, zip_size, zip_sha, inner_sha, inner_size, merged_f16_sha, base_f16_sha = (
sys.argv[1], int(sys.argv[2]), sys.argv[3], sys.argv[4], int(sys.argv[5]),
sys.argv[6], sys.argv[7])
def sha(p):
h = hashlib.sha256()
with open(p, "rb") as f:
for c in iter(lambda: f.read(1 << 20), b""):
h.update(c)
return h.hexdigest()
ggufs = {n: {"path": p, "bytes": os.path.getsize(p), "sha256": sha(p)}
for n, p in [("rex86-merged-f16", f"{run}/rex86-merged-f16.gguf"),
("rex86-merged-Q4_K_M", f"{run}/rex86-merged-Q4_K_M.gguf"),
("rex86-merged-Q8_0", f"{run}/rex86-merged-Q8_0.gguf"),
("base-f16", f"{run}/qwen2.5-coder-7b-base-f16.gguf"),
("base-Q4_K_M", f"{run}/base-Q4_K_M.gguf"),
("base-Q8_0", f"{run}/base-Q8_0.gguf")]}
# #3137: the base twins must be provably distinct weights from the merged
# model, not just distinct files -- a re-quantised copy of the same tensors
# would still pass an [ -s ] guard.
merged_f16_sha = ggufs["rex86-merged-f16"]["sha256"]
base_f16_sha = ggufs["base-f16"]["sha256"]
assert base_f16_sha != merged_f16_sha, \
"base f16 GGUF matches merged f16 GGUF -- base twin is not untouched (#3137)"
# rex86-merged-f16 and base-f16 shas were computed earlier in the shell (the
# #3137 lineage assert there runs before stage 4 registers anything in
# Ollama) -- reuse them here instead of re-hashing two ~15GB files.
precomputed_sha = {"rex86-merged-f16": merged_f16_sha, "base-f16": base_f16_sha}
ggufs = {}
for n, p in [("rex86-merged-f16", f"{run}/rex86-merged-f16.gguf"),
("rex86-merged-Q4_K_M", f"{run}/rex86-merged-Q4_K_M.gguf"),
("rex86-merged-Q8_0", f"{run}/rex86-merged-Q8_0.gguf"),
("base-f16", f"{run}/qwen2.5-coder-7b-base-f16.gguf"),
("base-Q4_K_M", f"{run}/base-Q4_K_M.gguf"),
("base-Q8_0", f"{run}/base-Q8_0.gguf")]:
ggufs[n] = {"path": p, "bytes": os.path.getsize(p),
"sha256": precomputed_sha[n] if n in precomputed_sha else sha(p)}
manifest = {
"experiment": "T0 pipeline proof (#3081)",
"base_model": "unsloth/Qwen2.5-Coder-7B",
Expand Down