diff --git a/benchmarks/atlas-hallucination-containment/README.md b/benchmarks/atlas-hallucination-containment/README.md new file mode 100644 index 0000000..a3a8a84 --- /dev/null +++ b/benchmarks/atlas-hallucination-containment/README.md @@ -0,0 +1,87 @@ +# Atlas Hallucination-Containment Benchmark (Protocol-C attested) + +A signed, tamper-evident record of an internal Aether Atlas experiment. The +numbers in [`manifest.json`](./manifest.json) were produced **on the Aether VPS5 +(Oracle) node** and signed in-process with that node's Ed25519 Protocol-C key. +You can verify, without trusting us, that these exact numbers were attested by +the holder of that key and have not been edited since. + +> **Honest scope.** This is **not** a public NLP hallucination benchmark +> (TruthfulQA, HaluEval, etc.). It measures one thing: how often the Atlas +> **surfacing gate** withholds a mislabelled / decayed ("hallucinated") cell +> before it can be served as a fact, on a controlled synthetic cell population. +> It is an engine-level containment measurement, not a model-level accuracy +> claim. We publish it because the *attestation* is the point: a number you can +> cryptographically tie to a run on our hardware. + +## Result + +| metric | value | +|---|---| +| containment mean (200 seeds) | **0.687** | +| sd | 0.066 | +| 5th-percentile floor (conservative) | **0.578** | +| median | 0.684 | +| range | 0.469 – 0.915 | +| Layer-1 admission reject-rate of hallucinated cells | **0.000** | + +**Read it as:** the surfacing gate contains ~**69% ± 7%** of hallucinated cells; +quote the **p05 floor (≈58%)** if you want a single conservative number. The +admission gate (Layer 1) rejects *none* of them — containment is entirely a +**Layer-3 staleness / re-validation** effect, not an admission filter. We state +that explicitly rather than imply a multi-layer filter that isn't doing the work. + +### Method (what each seed runs) + +100 cells/seed, 50/50 correct vs. hallucinated, through the real pipeline: + +``` +admit -> surface -> append real outcome -> re-surface with contra_r +containment = 1 - surfaced_hallucinated / hallucinated +``` + +200 deterministic seeds (0–199). The single-seed baseline is locked at +`containment >= 0.50` in the engine's own test suite; this is the multi-seed band +around it. The Atlas engine that ran it is proprietary, so this is **reproducible +by attestation, not from scratch** — the signature is the guarantee, not a +re-run on your machine. + +## Verify it yourself + +```bash +pip install cryptography +python verify_manifest.py manifest.json +``` + +Three independent checks, all must pass: + +1. `results_hash` — `sha256(canonical(results))` equals `results_sha256` + (change one digit of any number → fails). +2. `attestation_bind` — the signed attestation commits to that same hash. +3. `ed25519_signature` — the signature verifies under the embedded + `public_key_pem` (re-signing with a different key changes the fingerprint). + +## Trust model + +- **Signer:** VPS5 / Oracle node, Ed25519. The Oracle is the Atlas signing + + audit authority, so a benchmark it signs carries the same provenance as an + Atlas cell write. +- **Public-key fingerprint (anchor this):** + `sha256(raw pubkey) = 920dd4c8559005db10e80cb127bce512e382ef7afc5e4a9f4189fd282fab79e7` +- **Engine identity:** `aether-atlas-deployed@vps5:d0de38aea90de664` + (content hash of the deployed gate/core/store modules at run time). +- The private key never left the node; only the signature and public key are + published here. + +A valid signature proves the holder of the VPS5 Oracle key attested **these exact +numbers** at the recorded UTC. It does **not**, by itself, prove the key belongs +to Aether — anchor the fingerprint above against our published node key to close +that gap. + +## What is deliberately NOT here + +The Atlas LLM coding pool has a leakage-free SWE-bench *protocol*, but its current +input is a **synthetic** generator. We do **not** publish a SWE-bench +precision/F1 here, signed or otherwise, because a synthetic number is not a +benchmark result. That figure ships only after a real SWE-bench Verified run — +same attestation, real labels. diff --git a/benchmarks/atlas-hallucination-containment/manifest.json b/benchmarks/atlas-hallucination-containment/manifest.json new file mode 100644 index 0000000..56b5df9 --- /dev/null +++ b/benchmarks/atlas-hallucination-containment/manifest.json @@ -0,0 +1,230 @@ +{ + "attestation": { + "alg": "ed25519", + "engine": "aether-atlas-deployed@vps5:d0de38aea90de664", + "node": "VPS5-ORACLE", + "results_sha256": "068b1eb3af4134587542b73f6e484d2276e6b62a080490a487cfd0597be4c2b9", + "utc": "2026-06-13T15:30:31Z" + }, + "benchmark": "atlas-hallucination-containment", + "public_key_pem": "-----BEGIN PUBLIC KEY-----\nMCowBQYDK2VwAyEA9wIbGdvH+Gll5UHB8ATd/EkMHeOdfHRTOtnKCfqP2S0=\n-----END PUBLIC KEY-----\n", + "public_key_sha256": "920dd4c8559005db10e80cb127bce512e382ef7afc5e4a9f4189fd282fab79e7", + "results": { + "baseline_floor": 0.5, + "benchmark": "atlas-hallucination-containment", + "cells_per_seed": 100, + "layer1_admit_rate_hall_mean": 1.0, + "max": 0.914894, + "mean": 0.687343, + "median": 0.683569, + "method": "test_pipeline_statistics multi-seed (admit->surface->real->resurface contra_r)", + "min": 0.469388, + "n_seeds": 200, + "p05": 0.577735, + "per_seed_containment": [ + 0.734375, + 0.690909, + 0.659574, + 0.56, + 0.686275, + 0.794872, + 0.785714, + 0.65, + 0.740741, + 0.755102, + 0.791667, + 0.785714, + 0.65625, + 0.644444, + 0.914894, + 0.689655, + 0.62, + 0.688889, + 0.660377, + 0.769231, + 0.652174, + 0.823529, + 0.680851, + 0.618182, + 0.678571, + 0.733333, + 0.723404, + 0.76, + 0.603774, + 0.666667, + 0.810345, + 0.781818, + 0.723404, + 0.704545, + 0.701754, + 0.734694, + 0.673469, + 0.804348, + 0.784314, + 0.633333, + 0.717391, + 0.685185, + 0.540984, + 0.6, + 0.679245, + 0.682927, + 0.659574, + 0.650794, + 0.772727, + 0.679245, + 0.645833, + 0.674419, + 0.75, + 0.75, + 0.724138, + 0.807692, + 0.653061, + 0.644444, + 0.54, + 0.673469, + 0.708333, + 0.804348, + 0.666667, + 0.78, + 0.696429, + 0.767857, + 0.68, + 0.744681, + 0.702128, + 0.672727, + 0.693878, + 0.72, + 0.64, + 0.636364, + 0.745098, + 0.609756, + 0.702128, + 0.698113, + 0.698113, + 0.625, + 0.682927, + 0.673469, + 0.552632, + 0.68, + 0.645833, + 0.684211, + 0.654545, + 0.732143, + 0.577778, + 0.693878, + 0.714286, + 0.642857, + 0.686567, + 0.696429, + 0.711111, + 0.636364, + 0.608696, + 0.781818, + 0.576923, + 0.8125, + 0.648148, + 0.673469, + 0.709091, + 0.581818, + 0.774194, + 0.641509, + 0.732143, + 0.631579, + 0.632653, + 0.642857, + 0.545455, + 0.733333, + 0.622222, + 0.659574, + 0.591837, + 0.66, + 0.686275, + 0.705882, + 0.659091, + 0.660377, + 0.727273, + 0.666667, + 0.611111, + 0.617021, + 0.75, + 0.705882, + 0.659574, + 0.690476, + 0.685185, + 0.695652, + 0.636364, + 0.591837, + 0.666667, + 0.581818, + 0.745098, + 0.723404, + 0.666667, + 0.76087, + 0.754717, + 0.724138, + 0.658537, + 0.641509, + 0.55102, + 0.74, + 0.714286, + 0.673469, + 0.705882, + 0.529412, + 0.777778, + 0.707317, + 0.736842, + 0.709091, + 0.711538, + 0.689655, + 0.765957, + 0.666667, + 0.765957, + 0.755102, + 0.648148, + 0.733333, + 0.72093, + 0.72549, + 0.583333, + 0.58, + 0.651163, + 0.72549, + 0.731707, + 0.469388, + 0.716981, + 0.722222, + 0.653061, + 0.75, + 0.660377, + 0.666667, + 0.654545, + 0.818182, + 0.77551, + 0.616667, + 0.666667, + 0.682927, + 0.625, + 0.645833, + 0.681818, + 0.708333, + 0.625, + 0.707692, + 0.678571, + 0.857143, + 0.738462, + 0.8, + 0.622642, + 0.673077, + 0.66, + 0.54902, + 0.647059, + 0.64, + 0.730769, + 0.673077, + 0.708333, + 0.653061 + ], + "sd": 0.066159 + }, + "results_sha256": "068b1eb3af4134587542b73f6e484d2276e6b62a080490a487cfd0597be4c2b9", + "signature_ed25519": "a766dcd2286104411cc5f240404bd1f91a6d4d2820085f1b85001a54093b1c6404d13a65f7706ae97c97aee6568060e3b7174532e65fcaf21e4a4aef17fc1604" +} \ No newline at end of file diff --git a/benchmarks/atlas-hallucination-containment/verify_manifest.py b/benchmarks/atlas-hallucination-containment/verify_manifest.py new file mode 100644 index 0000000..85a79d0 --- /dev/null +++ b/benchmarks/atlas-hallucination-containment/verify_manifest.py @@ -0,0 +1,97 @@ +"""bench/verify_manifest.py +=========================== +Standalone verifier for a Protocol-C benchmark manifest. PUBLIC — ships with the +published benchmark so anyone can check the result was signed by the node key and +not edited after the fact. + +Three independent checks: + 1. results hash — sha256(canonical(manifest.results)) == manifest.results_sha256 + 2. attestation bind — manifest.attestation.results_sha256 == that same hash + 3. signature — Ed25519 verify(signature, canonical(attestation)) under public_key_pem + +Pass all three and the numbers in `results` are exactly what the holder of +`public_key_pem` signed. Change a single digit and check 1 fails; re-sign with a +different key and the published public-key fingerprint no longer matches the one +anchored in the node's infra. + + python verify_manifest.py manifest.json + +Only dependency outside the stdlib is `cryptography` (pip install cryptography). +""" +from __future__ import annotations + +import hashlib +import json +import sys + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + + +def _canonical(obj) -> bytes: + return json.dumps(obj, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def verify(manifest: dict) -> list: + """Return a list of (check_name, ok, detail).""" + checks = [] + + results_hash = hashlib.sha256(_canonical(manifest["results"])).hexdigest() + checks.append(( + "results_hash", + results_hash == manifest["results_sha256"], + f"{results_hash} vs {manifest['results_sha256']}", + )) + + att = manifest["attestation"] + checks.append(( + "attestation_bind", + att.get("results_sha256") == manifest["results_sha256"], + f"attestation binds {att.get('results_sha256')}", + )) + + ok_sig = False + detail = "" + try: + pub = serialization.load_pem_public_key(manifest["public_key_pem"].encode("ascii")) + if not isinstance(pub, Ed25519PublicKey): + detail = f"not an Ed25519 public key ({type(pub).__name__})" + else: + pub.verify(bytes.fromhex(manifest["signature_ed25519"]), _canonical(att)) + ok_sig = True + detail = f"signed by pubkey_sha256={manifest.get('public_key_sha256', '?')}" + except InvalidSignature: + detail = "INVALID signature for this public key" + except Exception as e: # noqa: BLE001 + detail = f"error: {e}" + checks.append(("ed25519_signature", ok_sig, detail)) + + return checks + + +def main(argv=None) -> int: + argv = argv if argv is not None else sys.argv[1:] + if not argv: + print("usage: python verify_manifest.py manifest.json", file=sys.stderr) + return 2 + with open(argv[0], "rb") as fh: + manifest = json.loads(fh.read()) + + checks = verify(manifest) + print(f"benchmark: {manifest.get('benchmark')}") + print(f"node: {manifest.get('attestation', {}).get('node')} " + f"utc: {manifest.get('attestation', {}).get('utc')}") + print(f"engine: {manifest.get('attestation', {}).get('engine')}") + print("-" * 56) + all_ok = True + for name, ok, detail in checks: + all_ok = all_ok and ok + print(f" [{'PASS' if ok else 'FAIL'}] {name}: {detail}") + print("-" * 56) + print("VERDICT:", "VERIFIED -- signed, unmodified." if all_ok else "FAILED -- do not trust.") + return 0 if all_ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main())