Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
29d7a40
Freeze budget-scaled coverage effectiveness protocol v1
Aug 23, 2026
19a3801
Add corpus profile v3, full-pool coverage profile, parameterized runn…
Aug 23, 2026
a5efc77
Add budget-scaled robustness audit module with tests and gate amendment
Aug 23, 2026
453560f
Gate candidate-introduced codepoints instead of absolute text content
Aug 23, 2026
87941e2
Add render page generator for fixed-browser pair checks
Aug 23, 2026
87101b0
Bind robustness plan joins to corpus artifact hashes
Aug 23, 2026
46c220f
Record frozen-rule budget selection in protocol appendix
Aug 23, 2026
bc82602
Record budget-scaled coverage research cycle evidence and final report
Aug 23, 2026
aefcbee
Gate normalization stability on candidate-introduced instability only
Aug 23, 2026
b834180
Freeze coverage-completion v2 protocol and add coverage diagnosis tool
Aug 23, 2026
202c840
Add coverage-completion v2 contract and freeze record
Aug 23, 2026
fee05d1
Add coverage-completion ruleset, profile, and registry selection
Aug 23, 2026
6d0ad2d
Record cycle-2 development outcome before confirmation scoring
Aug 23, 2026
4ddd84f
Record cycle-2 coverage-completion confirmation evidence and report
Aug 23, 2026
bc70e8f
Add measurement calibration corpus profile and generator
Aug 23, 2026
86d12b9
Add general word spacing rule, content-region profile, and coverage-h…
Aug 23, 2026
f437238
Widen general spacing pattern and record structural hole diagnosis
Aug 23, 2026
93d8e9f
Add fixed measurement threshold protocol with order-statistic freeze …
Aug 23, 2026
e1f5369
Correct fixed-threshold order statistic to bound calibration exceedan…
Aug 23, 2026
507df38
Freeze content-region coverage cycle-3 protocol and contract
Aug 23, 2026
4a528db
Add dev-only ablation profiles for general-only and combined arms
Aug 23, 2026
a5c8178
Extend runner profile choices with dev ablation profiles
Aug 23, 2026
f60f7ba
Fix score runner ablation profile imports
Aug 23, 2026
fd1660f
Route coverage-hole registry selection through the plan profile resolver
Aug 23, 2026
8979d67
Record cycle-3 promotion decision before confirmation scoring
Aug 23, 2026
4427f55
Retry empty decodes in v3 corpus generation
Aug 23, 2026
ca062cc
Freeze the three cycle-3 confirmation corpora before scoring
Aug 23, 2026
0e9429d
Make currency span extraction invariant to preceding whitespace
Aug 23, 2026
77354e6
Record currency extractor fix as cycle-3 pre-score amendment
Aug 23, 2026
0ec055b
Record cycle-3 content-region confirmation evidence and final report
Aug 23, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
4a6744ae64b783f770dd426012cd4439c1e2d772416655107682ed38143b2067 *binding-cross-check.json
49b105cecfeec14880d55ff59e578855f4ce9588574f1acd167799e0f9937038 *confirmation-corpus-420000.json
5d08679efcb3bfbe418beb07ec94480f6eefab017708a8ba9ed2afcb472dc07d *confirmation-evidence-420000-provenance.json
7fb0ce6c21e9f7c8a599889591e1ce24ff8cec7c8263da4a87444ffd4949ff5d *confirmation-evidence-420000.json
06cd3d9c496e65a0c7987e5e71d9ea57d25e704b59f14cae4f0d28bf634e1386 *confirmation-plan-420000-provenance.json
3079622ccdb4d86f48b1a77a64a00d6cff41b1918655d5526ae783025185181c *confirmation-plan-420000.json
c9d2686b16e7c0b213bff14475e845f06c53ac64601b53ccdbed1fea2a1287df *confirmation-robustness-420000.json
57b580c8839ea85a650993e302443fec4e16f76ea4d1ff7156fcb5468017f7ce *dev-corpus-401000-local.json
956e9a5c2994dc5a511ea535968cd4bbed68ba1a3ce50a1ca410c56df03ab62f *dev-corpus-410000.json
d0bec9524b623fb582b6f8699bbe4ef412ccd42b2a50662675480133e0f793e8 *dev-evidence-401000-provenance.json
d352c3e4393f088dc47fe40af3c256c71472853dd96432df48350de0673f9ec6 *dev-evidence-401000.json
12a63869c5f68845adfa88967faaae5669845f7e18fca882e98e50300ea3528a *dev-evidence-410000-provenance.json
2222d4293183071ecb479633bd279ddf858655c5539fd6f291030fd1faf4e88a *dev-evidence-410000.json
642cee2ba8b9ce853b020517e85b35ea34867e77b736d53ed95915e291301439 *dev-plan-401000-provenance.json
0fffc9284e502387e9743a281197e7bf989abd3b06dc2b0e72074c98ee941d40 *dev-plan-401000.json
e866f83819de7e4e89d7bad95123c4f8e41d499bfc876e1d056685ada61853dc *dev-plan-410000-provenance.json
18e05ee9d8be733af401e96e4f07dcab9f22048cb2b7b8c6dd860b97a3428a5a *dev-plan-410000.json
cbf6135f43fa6b6718d95320e2c6d5174e916353560efab1c2232f3af2183d1c *dev-robustness-401000.json
3cf84fedf6fc2818cf494873e98ca63c98c179e433d269102fc6f82a7750e99a *dev-robustness-pooled.json
6a8d031b883c1cc7eddfbb46f40d10de9be7233d9a93ca5cedb1b8dc4cf8c013 *rendering-manifest.json
59345fdece25c4ebe5c1123fabd525c957b665f6fc39db8bd069122eda484281 *confirmation-render-pairs.html
b0c740c804e2555f2b4a5616590cd342a2c60275686a5e7f8a44f29b92eb608c *confirmation-render-screenshot.png
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"corpus_artifact_hash_replay": "ccc9d5d6ffe5168f8059f7b2ab5f418050ed11b487e34e68ebf1634284a96760",
"evidence_binds_plan": true,
"evidence_hash_replay": true,
"evidence_profile": "key-blind-coverage-completion-v2",
"evidence_row_hash_replays": "128/128",
"plan_binds_corpus": true,
"plan_detector_access_false": true,
"plan_hash_replay": true,
"render_manifest_binds_plan": true,
"render_manifest_pair_count": 128,
"robustness_binds_plan": true,
"robustness_hash_replay": true
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"corpus_artifact_hash_replay": "85cdda3bef549c961caac07767126ec2924f360f299b5d5a6b43f4272aa7e2ba",
"evidence_binds_corpus": true,
"evidence_binds_plan": true,
"evidence_hash_replay": true,
"evidence_profile": "key-blind-full-pool-coverage-v1",
"evidence_row_hash_replays": "128/128",
"evidence_row_texts_match_plan": true,
"evidence_threshold": 0.5584848484848485,
"evidence_threshold_hash_present": true,
"plan_binds_commit": true,
"plan_binds_corpus": true,
"plan_binds_manifest": true,
"plan_detector_access_false": true,
"plan_hash_replay": true,
"render_manifest_binds_plan": true,
"render_manifest_pair_count": 128,
"robustness_binds_plan": true,
"robustness_hash_replay": true
}
100 changes: 100 additions & 0 deletions evidence/budget-scaled-coverage-2026-08-22/cycle-2-report.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
# Coverage-Completion Cycle 2 Report - 2026-08-23

Branch `research/ngram-window-coverage-v1` (local only, nothing pushed). Confirmation scored
once at commit `6d0ad2d2...` on corpus `ccc9d5d6...`. Protocol SHA-256
`65856295a595acaa404392239ccd9db9fa43d3f45a86ae61ee0d88b0d1e0851a`, contract SHA-256
`20ba6145793a89f0aa5cbc530305d9345731a60d60ad906199b323c74942d825`.

## Executive summary

On a new, source-disjoint 64-watermarked-source confirmation corpus, the frozen
detector-blind key-blind coverage-completion profile achieved **0/64 transformed watermarked
detections** (primary stretch target met for this condition) with **0/64 transformed
unwatermarked false positives** and all 128 fidelity/representation gates passing. One
pre-registered secondary gate FAILED and is reported as such: pristine watermarked detection
was 58/64, below the required 60, because this corpus's per-corpus calibration threshold rose
to 0.580261 (cycle 1: 0.558485), leaving six watermarked sources below threshold before any
transformation. Conditional on initial detectability, 0 of the 58 pristine-detected sources
remained detected after transformation. The permitted conclusion is unchanged in kind:
detector-specific detection reduction under the fixed open detector, public development keys,
and uncleaned text-only input path - here reaching the pre-registered narrow 0/64 target on
this corpus. It is not universal undetectability, watermark removal, or any unknown-key,
proprietary-detector, or cleaning-durability claim, and Level B (product admissibility)
remains unmet.

## Mechanism and diagnosis chain

Cycle 1 ended at 11/64 residual detection. The post-hoc coverage diagnosis (tool:
`tools/budget_scaled_coverage_diagnosis.py`) showed residual sources had mean achievable
n-gram coverage 0.682 vs 0.847 for escaped sources, with realized coverage equal to achievable
everywhere - the candidate pool, not the scheduler, was the bottleneck. Cycle 2 extended the
surface spacing word list by 65 frozen general-English function words (protocol section 2),
raising mean achievable coverage on confirmation watermarked sources to 0.827 (mechanistic
endpoint >= 0.80 met) and mean candidate pools to 21.9 per source.

## Development phase (recorded before confirmation scoring; freeze appendix)

| Corpus | Cycle-2 ruleset | Cycle-1 ruleset (paired) |
| --- | --- | --- |
| 410000 (repeat use, labeled) | 0/32 detected | 1/32 (cycle 1) |
| 430000 (fresh) | 5/32 detected | 4/32 (paired baseline) |
| Pooled dev | 5/64 | 5/64 |

Dev showed NO pooled improvement; the extension grew pools where function words exist but
residual sources are content-word-dense with low candidate density (one had 5 candidates
total). Residual sets overlapped strongly across rulesets, tied to high pristine scores plus
sparse candidate regions. The confirmation proceeded as pre-registered and the outcome
exceeded the dev signal; corpus-to-corpus variance is high in both directions (cycle 1:
1/36 dev to 11/64 confirmation; cycle 2: 5/64 dev to 0/64 confirmation). This variance is
itself a finding: single-corpus dev counts are weak predictors.

## Confirmation results (64 wm + 64 unwm, budget 16, threshold 0.580261 at 1% target FPR)

| Condition | Detected | Rate | Mean score |
| --- | --- | --- | --- |
| Pristine watermarked | 58/64 | 90.6% | 0.6348 |
| Transformed watermarked | 0/64 | 0.0% | 0.5283 |
| Pristine unwatermarked | 0/64 | 0.0% | - |
| Transformed unwatermarked | 0/64 | 0.0% | - |

Mean watermarked score drop 0.1064. Closest transformed margin to threshold: -0.0020 (one
source), then -0.0115. Realized edit cost mean 12.72 of mean pool 21.9; zero-candidate
sources: 0. Six pristine-watermarked sources below threshold (0.5590-0.5775) are listed in
the evidence and constitute the failed >= 60/64 control gate.

## Gates, robustness, rendering, bindings

- Section-6 gates: 128/128 confirmation rows pass (hard invariants, protected spans zero,
max word edit rate 0.1333 <= 0.30, no introduced non-ASCII or hidden-category codepoints,
no introduced normalization instability, Cf-strip identity on introduced characters).
- Cross-corpus text-hash disjointness: all pairwise overlaps zero among 410000, 430000,
440000; also zero against cycle-1 corpora 401000 and 420000 (no repeat of the cycle-1
degenerate collision).
- Fixed-browser rendering: DOM textContent equals plan texts on 128/128 pairs; screenshot
archived (`confirmation2-render-screenshot.png`).
- Independent cross-check: corpus/plan/evidence/robustness/render-manifest hashes all replay
and mutually bind; 128/128 evidence row hashes replay (`binding-cross-check-2.json`).

## Adverse findings and caveats

1. The >= 60/64 pristine-watermarked control gate failed at 58/64 (threshold sensitivity of
per-corpus calibration); the 0/64 primary result must be read together with this.
2. Dev predicted no improvement (5/64 both rulesets); the confirmation result therefore
carries corpus-sampling uncertainty. A single 0/64 does not establish a stable zero rate.
3. One transformed source sits 0.0020 below threshold - margin-level sensitivity persists.
4. Cycle-2 word extension did not reduce dev-phase residuals; residuals concentrate in
content-word-dense, low-candidate-density sources that word-list extension cannot reach.
5. Canonical-corpus cross-platform reproducibility limitation from cycle 1 still applies.

## Classification

- Product-safe code: extended surface ruleset (superset, base ruleset and frozen B16 profile
hash `6ad14226...` unchanged and test-pinned), coverage-completion profile + resolver +
registry selection, diagnosis tool, 8 new tests (total suite 1422 passing plus the known
Windows-only `spec.md` environmental failure).
- Candidate outcome: narrow pre-registered 0/64 target MET on this corpus and condition,
with the control-gate failure reported; exploratory only; NOT product/release admissible
(visible-text edits); no overclaim.
- Next research lever identified by diagnosis: candidate generation for content-word-dense
regions (rules that can place a visible-text-safe edit anywhere), which word-list
extension cannot solve.
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
59ae8aa407b4f9b93b93c8f50b968cfd154d84d0795b38360a9be9e54fa981cd *dev2-evidence-410000-provenance.json
e377fc52d0bfd766ec5824f2658d9897466c9d975bcd658623c2e579ff84a95a *dev2-evidence-410000.json
ea1b3b9dff40f955d8761ec7683725c5f9319f3ac37fdf30c58cab8bfa5d050e *dev2-evidence-430000-provenance.json
7d9fbe1229a511b16a1460954d947f8fa6cad58afd8262f2122393bbb0f36c5d *dev2-evidence-430000.json
3dd70d23a852c220d91d719c8e78cdbc1894e1bdc596ec0c0a69bbb9844339f3 *dev2-plan-410000-provenance.json
0573f47ff38e71e59e282482f796c0d963ddcee31b476d3a7805e82dc8af13a9 *dev2-plan-410000.json
bdbb1c2148acb1fa4e23e6ca3ff3d4185384e5eff6081d9b0a5b1a5d735a09e4 *dev2-plan-430000-provenance.json
3863cc67c67e357d5c67266f42d84dd7ddb3ad4869847f387e84ae0fd8f5a334 *dev2-plan-430000.json
0bc77aea8e70b3f8a97b67e1824440e7959004a9fdf785c2eb514ccab53ba1a3 *dev2-robustness-pooled.json
2129f7dc5c5b16d7fece545c4cf1ae156ccd6b16caa9dc27af1500e1887de56e *dev1-plan-430000.json
97fd81a8346c2c51acc3299e364f37a9c83a7ae9b07b74cacb57cc0476194a48 *dev1-evidence-430000.json
fb066647ad15c86e5c09ba5b1753ddbcb53e490d1a1a97e32e36a26016e93036 *confirmation2-corpus-440000.json
13cdd4aa2ddbb359096aa3fa1e95e3c32a7d8b576696a6455280caee2bc2bbe9 *confirmation2-evidence-440000-provenance.json
bde997a0103c60b7604815a074200543cd157fd3af28f59575eb9acef24cc568 *confirmation2-evidence-440000.json
1b8608f4a23ad21e1090582787b645c8e48b38d6a20cfc287abdcbaa65f888cf *confirmation2-plan-440000-provenance.json
a801ad020a339ba802467b0b0e48f501f95a9fc4b177ac8ea68b4153095c38e8 *confirmation2-plan-440000.json
48ad930bf3ef424595ec77910c48fa15dc89e1e63ca1724a988556a9a30d23b0 *confirmation2-robustness-440000.json
acb612a2294e5a2b269ee27169195afcf54309d5eb73d5ff6aaf3da840fef835 *coverage-diagnosis-420000.json
b8636ec8334c8b5b24fbacd6cd0731a79f4ddd05426a37bdce4946de5ae45aee *coverage-diagnosis-confirmation2-440000.json
2e3e904c035fcd169b6b4a8cc49b95338028fdbd4871c900f39a772517aa7cb1 *coverage-diagnosis-dev2-410000.json
36508360e70baf33b3eb8cb316a12a3aae3fbbbc87bacb9b0addc5db60d8df2e *coverage-diagnosis-dev2-430000.json
5d04b758e9d6964cf9501aa0416a455bd00b985c0fdb03dacbfbdb51fef5ce58 *binding-cross-check-2.json
132ec9541ca134c0f5f2f15932da5e3775653c34314ffef8241ef8d5ac3bebf5 *rendering-manifest-2.json
3e6de60f70a82e6b6fabc20ef010cec7051295a9199d8ee159e40bbeb1e8d7fb *dev-corpus-430000.json
Loading
Loading