From 1d686b6eb133c29479b2d9bc1b609f929b63e06e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 27 Aug 2026 21:12:46 +0000 Subject: [PATCH 1/3] Fix audit findings F01-F07 for rendering, paths, CLI I/O, and mix scan. Branch by element type in Chromium surface measurement, protect relative and forward-slash Windows paths plus Markdown reference labels, keep exact line endings and strict UTF-8 stdin, add --text/--file, and remove per-candidate whole-document hashing. Historical evidence files stay untouched; replacement measurements live under evidence/audit-fixes-2026-08-27/. Co-authored-by: Byte271 --- .github/workflows/ci.yml | 23 ++ CHANGELOG.md | 14 ++ README.md | 17 +- docs/cli.md | 37 +++- docs/product-contract.md | 5 +- evidence/audit-fixes-2026-08-27/README.md | 45 ++++ .../letter-mix-scan.json | 92 ++++++++ .../audit-fixes-2026-08-27/render-v2.json | 129 ++++++++++++ fuckmark/cli.py | 127 +++++++++--- fuckmark/cycle8/benchmark_render.py | 57 ++++- fuckmark/cycle8/letter_mix.py | 53 +++-- fuckmark/cycle8/mix_freeze.py | 6 +- .../experiments/e24_protected_span_stress.py | 1 + fuckmark/transforms/protected.py | 2 + fuckmark/transforms/protected_markdown.py | 173 ++++++++++++++++ fuckmark/transforms/protected_structures.py | 19 +- fuckmark/transforms/schema.py | 1 + tests/audit_mix_replay.py | 11 + tests/test_audit_fix_evidence.py | 53 +++++ tests/test_benchmark_render_surfaces.py | 94 +++++++++ tests/test_cli.py | 2 + tests/test_cli_io_contract.py | 196 ++++++++++++++++++ tests/test_cycle8_control_mix_diagnostic.py | 5 +- tests/test_cycle8_deepmind_transfer.py | 7 +- tests/test_cycle8_detector_transfer.py | 6 +- tests/test_cycle8_second_model_transfer.py | 4 +- tests/test_letter_mix_machine_text.py | 110 ++++++++++ tests/test_letter_mix_scan.py | 59 ++++++ tests/test_protected_spans.py | 8 + 29 files changed, 1262 insertions(+), 94 deletions(-) create mode 100644 evidence/audit-fixes-2026-08-27/README.md create mode 100644 evidence/audit-fixes-2026-08-27/letter-mix-scan.json create mode 100644 evidence/audit-fixes-2026-08-27/render-v2.json create mode 100644 fuckmark/transforms/protected_markdown.py create mode 100644 tests/audit_mix_replay.py create mode 100644 tests/test_audit_fix_evidence.py create mode 100644 tests/test_benchmark_render_surfaces.py create mode 100644 tests/test_cli_io_contract.py create mode 100644 tests/test_letter_mix_machine_text.py create mode 100644 tests/test_letter_mix_scan.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e869953..16d7e82f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,6 +89,29 @@ jobs: print("interactive e2e ok") PY + cli-io: + name: CLI I/O / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + cache: pip + + - name: Install + run: python -m pip install --upgrade pip && python -m pip install -e ".[dev]" + + - name: Real file stdin and stdout I/O + run: python -m pytest -q tests/test_cli.py tests/test_cli_io_contract.py tests/test_letter_mix_machine_text.py + calibration-readiness: name: Frozen calibration readiness runs-on: ubuntu-24.04 diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b74616..a7a1e01f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## Unreleased + +Audit fixes F01-F07 against `be6ae7645fda8b39d1d308722ac249f519e68de5`. Package version remains 0.4.0. Frozen confirmation files and their SHA-256 sums are unchanged. + +- F01: contenteditable rendering sets `textContent`. The 2026-08-26 Chromium contenteditable VERIFIED rows compared blank divs and are not proof of text rendering equivalence. Replacement measurement: `evidence/audit-fixes-2026-08-27/`. +- F02: common relative paths without `./` and Windows paths that use `/` keep exact original bytes. +- F03: Markdown reference labels are protected at uses and definitions. Link checks use reference resolution, not only visible projection. +- F04: file and stdin I/O keep LF, CRLF, CR, mixed endings, and a missing final newline. +- F05: `--text` is literal text. `--file` requires an existing UTF-8 file. Ordinary sentences are not treated as missing files. +- F06: stdin is decoded as strict UTF-8. Invalid bytes exit nonzero with no payload and no clipboard copy. +- F07: letter-mix site selection no longer hashes the whole document per candidate. Compose-time invariant checks remain. + +Live mix bytes can differ from frozen `u034f-ufe00-letter-alt-v1` output hashes when a newly protected span is present. Visible projection remains the source. `cycle8-mix-freeze-v1` still pins `letter_mix_source_sha256` to the freeze-time file. + ## v0.4.0 — Product CLI - Transforms ordinary English ASCII text. Visible words stay exactly the same. diff --git a/README.md b/README.md index 4f06d9e8..90298482 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,9 @@ Paste or type text, then a line that is only `:done`. The result is copied to th ```text printf 'I do not agree.\n' | fuckmark -fuckmark "I do not agree." -fuckmark notes.txt -o notes.fm.txt +fuckmark --text "I do not agree." +fuckmark --text "I agree. You are right" +fuckmark --file notes.txt -o notes.fm.txt printf 'I do not agree.\n' | fuckmark --visible ``` @@ -48,7 +49,7 @@ Pipes and files write the payload to stdout. `fuckmark --help` is enough to star `VISIBLE(original) == VISIBLE(transformed)`. FuckMark only inserts U+034F and U+FE00 after eligible ASCII letters. It does not contract, paraphrase, homoglyph, or add spaces. -Supported input: tab, newline, carriage return, and ASCII space through tilde. Other Unicode is returned unchanged. Only UTF-8. URLs, code spans, paths, and similar machine text are left intact. +Supported input: tab, newline, carriage return, and ASCII space through tilde. Other Unicode is returned unchanged with exit 0. That status means I/O succeeded, not that hidden characters were inserted. Only UTF-8. URLs, code spans, paths (including `src/main.py` and `C:/Users/...`), Markdown reference labels, and similar machine text are left intact. ## Verified results @@ -66,7 +67,15 @@ Google synthid-text 30-key GPT-2 transformed text: **0/192**. This does not remove every AI watermark. It is not a claim against unknown, proprietary, or future detectors. -Stripping combining marks or default-ignorable characters restores the source. English ASCII only. Cap 192 insertion sites. +Stripping combining marks or default-ignorable characters restores the source. On the frozen Gate v2 confirmation set, mix plus those two stress sanitizers restored detection to 188/192. The stored 0/192 result applies only to the specified detectors, threshold, samples, and required sanitizer paths. + +Primary confirmation uses 64 generated tokens per sample. Insertion stops after the first 192 eligible letter sites, so the tail of a long document is unchanged. That coverage limit is not a measured long-document detection result. + +English ASCII only. Valid Unicode outside that domain, including accented letters, is returned unchanged with exit 0. + +Equal visible projection is not the same as equal behavior in other software. Raw codepoint search, Markdown reference matching, and exact byte paths can still fail if hidden characters remain. Historical Chromium `contenteditable` VERIFIED rows from 2026-08-26 used a blank-div measurement bug and are not proof of rendering equivalence. Safari/WebKit and terminal-pixel results remain UNKNOWN. + +Cap 192 insertion sites. ## How it works diff --git a/docs/cli.md b/docs/cli.md index a22170f9..403af98e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -43,7 +43,13 @@ Redirected or piped stdin does not open this UI. It is processed as a stream. ```text printf 'I do not agree.\n' | fuckmark -fuckmark "I do not agree." +fuckmark --text "I do not agree." +fuckmark --text "I agree. You are right" +fuckmark --text "Version 1.2 works" +fuckmark --text "Use input/output here" +fuckmark --text=-starts-with-hyphen +fuckmark --file notes.txt +fuckmark --file "my notes.txt" -o notes.fm.txt fuckmark notes.txt fuckmark notes.txt -o notes.fm.txt printf 'I do not agree.\n' | fuckmark --copy @@ -52,11 +58,28 @@ printf 'I do not agree.\n' | fuckmark --visible Piped or quoted input writes the hidden payload to stdout. `--visible` writes the original visible text. `--copy` also places whatever was written on the clipboard. +## Input modes + +| Form | Behavior | +| --- | --- | +| `--text TEXT` | Always a literal string. Use this for sentences, decimals, slashes, or text that looks like a path. `--text=-foo` is literal text that starts with a hyphen. | +| `--file FILE` | Always a file. Missing files are errors. Directories are errors. | +| positional operand | If that path exists as a file, it is read. If it looks like a missing path (`notes.txt`, `src/main.py`, `~/mail`, or a slash), it is an error. Otherwise it is literal text. | +| `--stdin` or `-` | Read all of standard input as bytes and decode them as strict UTF-8. | +| no operand, terminal | Paste UI. | +| no operand, pipe | Stream mode. | + +`--text`, `--file`, and `--stdin` are mutually exclusive. Quotes group arguments for the shell. They do not switch FuckMark into text mode. If `notes.txt` exists and you want the string `notes.txt`, use `--text notes.txt`. + +Existing files are read as UTF-8 bytes with no newline conversion. LF, CRLF, CR, mixed endings, and a missing final newline are preserved. Invalid UTF-8 is rejected: nonzero exit, no stdout payload, no clipboard copy, and no traceback. + ## Options | Option | Behavior | | --- | --- | | `--version` | Print `FuckMark 0.4.0`. | +| `--text TEXT` | Transform TEXT as a literal string. | +| `--file FILE` | Read UTF-8 from FILE. | | `--stdin` | Read all of standard input. | | `-o FILE`, `--output FILE` | Write UTF-8 output to FILE. | | `--copy` | Also copy the output to the clipboard. The paste UI always copies. | @@ -67,22 +90,22 @@ Piped or quoted input writes the hidden payload to stdout. `--visible` writes th `--non-interactive` is an alias of `--stdin`. -Quoted text that is not an existing file is transformed as a string. Existing files are read as UTF-8. A path-like argument that is missing (for example `notes.txt`) is an error, not a string. Invalid UTF-8 input is rejected. - ## Supported input -Tab, newline, carriage return, and ASCII space through tilde. Other Unicode is returned unchanged (exit 0). Only UTF-8 files. +Tab, newline, carriage return, and ASCII space through tilde. Other Unicode is returned unchanged (exit 0). Exit 0 means I/O succeeded. It does not mean that a transformation occurred or that a watermark was removed. Only UTF-8 files. -Machine spans stay intact: fenced/inline code, markdown destinations, URLs, emails, IPs, dates, currency, percents, numbers, POSIX/Windows paths, CLI flags. Quote interiors are eligible. Cap 192 insertion sites. +Machine spans stay intact: fenced/inline code, markdown destinations, markdown reference labels, URLs, emails, IPs, dates, currency, percents, numbers, POSIX/Windows paths (including relative paths such as `src/main.py` and `C:/Users/...`), CLI flags. Quote interiors are eligible. Cap 192 insertion sites. Insertions fill the first 192 eligible letter sites and then stop, so the tail of a long document is unchanged. If the text is already transformed, or has no eligible letters, the CLI returns it unchanged. +Same visible words are not the same as identical Markdown, path, or search behavior in other programs. Reports must not treat visible-projection equality as Markdown or filesystem equality. + ## Exit status | Status | Meaning | | ---: | ---: | -| 0 | Output was written or copied. Unsupported input is returned unchanged. | -| 1 | No input, bad file, unsupported encoding, missing `:done`, or output could not be written. | +| 0 | Output was written or copied. Unsupported Unicode is returned unchanged. | +| 1 | No input, bad file, invalid UTF-8, unsupported encoding, missing `:done`, or output could not be written. | | 2 | Transform succeeded, but clipboard copy failed. Stream modes still write stdout. The paste UI does not print the payload. | | 130 | Interrupted with Ctrl+C. | diff --git a/docs/product-contract.md b/docs/product-contract.md index 2c46d947..2c970c6e 100644 --- a/docs/product-contract.md +++ b/docs/product-contract.md @@ -32,11 +32,12 @@ These are not product success: - hyphenation or apostrophe substitution; - extra or missing visible spaces; - newline / layout changes; -- homoglyphs or compatibility characters. +- homoglyphs or compatibility characters; +- the same visible projection implying the same Markdown, path, or search behavior in other software. ## Current product path -`release_transform_registry()` stays empty. The public CLI (`release-cli-v5`) applies frozen `apply_letter_alternating_mix` directly: U+034F on even selected ASCII-letter sites, U+FE00 on odd sites, cap 192, outside hard machine spans. +`release_transform_registry()` stays empty. The public CLI (`release-cli-v5`) applies `apply_letter_alternating_mix` directly: U+034F on even selected ASCII-letter sites, U+FE00 on odd sites, cap 192, outside hard machine spans. Live protection now also covers common relative paths without a leading `./`, Windows paths that use `/`, and Markdown reference labels at both uses and definitions. Those protection changes can change outputs versus the v0.4.0 freeze of `letter_mix.py` (`cycle8-mix-freeze-v1`, `letter_mix_source_sha256` `b1ceec24e584c0e9e7135ef0c89a3bd249b0bda4a45e07aa7190b1b010ba56d4`). Frozen confirmation hashes stay historical. They are not silently rewritten. Live `product_approved_carriers_v1()` is `{U+034F, U+FE00}`. The v1 contract's empty `product_authorized_carriers_v1` list is a historical snapshot of that file. diff --git a/evidence/audit-fixes-2026-08-27/README.md b/evidence/audit-fixes-2026-08-27/README.md new file mode 100644 index 00000000..4b230c13 --- /dev/null +++ b/evidence/audit-fixes-2026-08-27/README.md @@ -0,0 +1,45 @@ +# Audit fix evidence (2026-08-27) + +This directory records measurements and invalidations for the F01-F07 audit fixes. It does not rewrite historical evidence files. + +Audited baseline commit: `be6ae7645fda8b39d1d308722ac249f519e68de5` (package 0.4.0). + +## F01 contenteditable rendering + +`cycle8-benchmark-render-v1` assigned `element.value` for every surface, then checked `'value' in element`. On a contenteditable div that assignment created a `value` property, so `textContent` never ran. Screenshots compared two blank divs and stored `VERIFIED`. + +Those `chromium_contenteditable` VERIFIED rows are not proof of text rendering equivalence: + +- `evidence/cycle8-letter-system-benchmark-2026-08-26/local-system.json` +- `evidence/cycle8-letter-system-benchmark-2026-08-26/scorecard.json` +- `evidence/cycle8-mix-margin-2026-08-26/local-system.json` +- `evidence/cycle8-mix-margin-2026-08-26/scorecard.json` + +Those files and their SHA-256 sums are unchanged. `render-v2.json` is the replacement measurement (`cycle8-benchmark-render-v2`). + +On this host: + +- Chromium: Google Chrome 148.0.7778.96 +- Font: DejaVu Sans Mono +- contenteditable AAAA vs BBBB: REJECTED +- contenteditable AAAA vs AAAA: VERIFIED +- contenteditable AAAA vs empty: REJECTED +- textarea has the same positive, negative, and nonempty-vs-empty controls +- Safari/WebKit: UNKNOWN +- terminal pixels: UNKNOWN + +An unavailable browser returns UNKNOWN, never VERIFIED. + +## F02 and F03 protection + +Live `hard_machine_intervals` now protects common relative paths without `./`, Windows paths that use `/`, and Markdown reference labels at uses and definitions. That can change mix output bytes versus the frozen `u034f-ufe00-letter-alt-v1` snapshot. + +`cycle8-mix-freeze-v1` remains the historical freeze. Its `letter_mix_source_sha256` is pinned to `b1ceec24e584c0e9e7135ef0c89a3bd249b0bda4a45e07aa7190b1b010ba56d4`, the `letter_mix.py` bytes at freeze time. Live `letter_mix.py` may differ. Frozen confirmation output hashes are not regenerated. + +## F07 scan refactor + +`letter-mix-scan.json` is a same-host before/after-style observation of the post-fix scanner. It is not a cross-machine performance claim. Final compose-time invariant checks remain. + +## L01-L04 + +These stay product limitations, not silent bug closures. See README and `docs/cli.md`. diff --git a/evidence/audit-fixes-2026-08-27/letter-mix-scan.json b/evidence/audit-fixes-2026-08-27/letter-mix-scan.json new file mode 100644 index 00000000..93cf6935 --- /dev/null +++ b/evidence/audit-fixes-2026-08-27/letter-mix-scan.json @@ -0,0 +1,92 @@ +{ + "algorithm_version": "letter-mix-scan-v2-host-observation", + "artifact_hash": "cb0599da02ee15113de9390ed093ea30eef5731955de07fec052f976571eeee0", + "commit": "be6ae7645fda8b39d1d308722ac249f519e68de5", + "note": "Host observations only. Not a cross-machine performance guarantee.", + "timings": [ + { + "input_chars": 1040, + "median_seconds": 0.0013507550002032076, + "min_seconds": 0.0013172589999612683, + "name": "prose_1k", + "output_chars": 1232, + "seconds": [ + 0.0016266380000615754, + 0.0013507550002032076, + 0.0013172589999612683 + ] + }, + { + "input_chars": 10400, + "median_seconds": 0.011578481000015017, + "min_seconds": 0.011543329000005542, + "name": "prose_10k", + "output_chars": 10592, + "seconds": [ + 0.011669592000089324, + 0.011543329000005542, + 0.011578481000015017 + ] + }, + { + "input_chars": 104000, + "median_seconds": 0.1135778840000512, + "min_seconds": 0.11315520800008017, + "name": "prose_100k", + "output_chars": 104192, + "seconds": [ + 0.11315520800008017, + 0.1135778840000512, + 0.11412591299995256 + ] + }, + { + "input_chars": 9249, + "median_seconds": 0.0160103750001781, + "min_seconds": 0.012477553000053376, + "name": "url_only_10k", + "output_chars": 9249, + "seconds": [ + 0.012477553000053376, + 0.01665635100016516, + 0.0160103750001781 + ] + }, + { + "input_chars": 18499, + "median_seconds": 0.024290382999879512, + "min_seconds": 0.02418305700007295, + "name": "url_only_20k", + "output_chars": 18499, + "seconds": [ + 0.024858777999952508, + 0.02418305700007295, + 0.024290382999879512 + ] + }, + { + "input_chars": 2200, + "median_seconds": 0.0031189240000912832, + "min_seconds": 0.003113965000011376, + "name": "no_sites", + "output_chars": 2200, + "seconds": [ + 0.0032563459999437327, + 0.003113965000011376, + 0.0031189240000912832 + ] + }, + { + "input_chars": 4820, + "median_seconds": 0.029459684999892488, + "min_seconds": 0.029254702999878646, + "name": "long_protected", + "output_chars": 4820, + "seconds": [ + 0.029254702999878646, + 0.029459684999892488, + 0.03241230200001155 + ] + } + ] +} diff --git a/evidence/audit-fixes-2026-08-27/render-v2.json b/evidence/audit-fixes-2026-08-27/render-v2.json new file mode 100644 index 00000000..31aee3fa --- /dev/null +++ b/evidence/audit-fixes-2026-08-27/render-v2.json @@ -0,0 +1,129 @@ +{ + "algorithm_version": "cycle8-benchmark-render-v2", + "artifact_hash": "7c9079653379e16b77331f1acccba4bdb72f923c7a4c4cbe28f2dbc71b07461a", + "browser_version": "Google Chrome 148.0.7778.96", + "chrome_executable": "/usr/local/bin/google-chrome", + "commit": "be6ae7645fda8b39d1d308722ac249f519e68de5", + "font": "DejaVu Sans Mono", + "historical_evidence_invalid_because": "cycle8-benchmark-render-v1 assigned element.value for every surface, then checked 'value' in element. That created a value property on contenteditable divs, skipped textContent, and screenshot-compared two blank divs as VERIFIED.", + "historical_files_not_rewritten": true, + "measurement_scope": "contenteditable and textarea AAAA/BBBB positive, negative, and nonempty-vs-empty controls on this host", + "rows": [ + { + "original": "AAAA", + "pair": "negative", + "result": { + "algorithm_version": "cycle8-benchmark-render-v2", + "browser_version": "Google Chrome 148.0.7778.96", + "comparison_hash": "7363f35290941fe43459f5250fa09a1cfc0cc6870522fee1d032661c743ce45e", + "detail": "png_bytes_differ", + "environment": "chromium_contenteditable", + "equal": false, + "font": "DejaVu Sans Mono", + "original_sha256": "63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201", + "status": "REJECTED", + "transformed_sha256": "4a8d8134f29b0b7b60c126f5532bc9f5d9bb73037373cf6fb872d81f1dcefdfd" + }, + "surface": "contenteditable", + "transformed": "BBBB" + }, + { + "original": "AAAA", + "pair": "positive", + "result": { + "algorithm_version": "cycle8-benchmark-render-v2", + "browser_version": "Google Chrome 148.0.7778.96", + "comparison_hash": "f69b8b92d51652f58fcdf30fbe89353c1b7f9a00c49b4dca050ae4001cda41cd", + "detail": "png_bytes_equal", + "environment": "chromium_contenteditable", + "equal": true, + "font": "DejaVu Sans Mono", + "original_sha256": "63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201", + "status": "VERIFIED", + "transformed_sha256": "63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201" + }, + "surface": "contenteditable", + "transformed": "AAAA" + }, + { + "original": "AAAA", + "pair": "nonempty_vs_empty", + "result": { + "algorithm_version": "cycle8-benchmark-render-v2", + "browser_version": "Google Chrome 148.0.7778.96", + "comparison_hash": "c432290011d4d4e4590202997252341293136ecf5430425703a24841eb581a73", + "detail": "png_bytes_differ", + "environment": "chromium_contenteditable", + "equal": false, + "font": "DejaVu Sans Mono", + "original_sha256": "63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201", + "status": "REJECTED", + "transformed_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "surface": "contenteditable", + "transformed": "" + }, + { + "original": "AAAA", + "pair": "negative", + "result": { + "algorithm_version": "cycle8-benchmark-render-v2", + "browser_version": "Google Chrome 148.0.7778.96", + "comparison_hash": "9b7ba94e863ad3c4b267aa59750ddc644b850110dccc4563dabb5eab2f390cb1", + "detail": "png_bytes_differ", + "environment": "chromium_textarea", + "equal": false, + "font": "DejaVu Sans Mono", + "original_sha256": "63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201", + "status": "REJECTED", + "transformed_sha256": "4a8d8134f29b0b7b60c126f5532bc9f5d9bb73037373cf6fb872d81f1dcefdfd" + }, + "surface": "textarea", + "transformed": "BBBB" + }, + { + "original": "AAAA", + "pair": "positive", + "result": { + "algorithm_version": "cycle8-benchmark-render-v2", + "browser_version": "Google Chrome 148.0.7778.96", + "comparison_hash": "c8e0a881d1f514214edfa79d535bdce7de77e11c833bcf44e636c2c7246fb87b", + "detail": "png_bytes_equal", + "environment": "chromium_textarea", + "equal": true, + "font": "DejaVu Sans Mono", + "original_sha256": "63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201", + "status": "VERIFIED", + "transformed_sha256": "63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201" + }, + "surface": "textarea", + "transformed": "AAAA" + }, + { + "original": "AAAA", + "pair": "nonempty_vs_empty", + "result": { + "algorithm_version": "cycle8-benchmark-render-v2", + "browser_version": "Google Chrome 148.0.7778.96", + "comparison_hash": "d27a138c9e29a039f318b7d4bd55e27c1fa6fe0fd1545cb4bbba6eb21f9a97c4", + "detail": "png_bytes_differ", + "environment": "chromium_textarea", + "equal": false, + "font": "DejaVu Sans Mono", + "original_sha256": "63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201", + "status": "REJECTED", + "transformed_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "surface": "textarea", + "transformed": "" + } + ], + "superseded_paths": [ + "evidence/cycle8-letter-system-benchmark-2026-08-26/local-system.json", + "evidence/cycle8-letter-system-benchmark-2026-08-26/scorecard.json", + "evidence/cycle8-mix-margin-2026-08-26/local-system.json", + "evidence/cycle8-mix-margin-2026-08-26/scorecard.json" + ], + "terminal_pixels": "UNKNOWN", + "webkit_safari": "UNKNOWN" +} diff --git a/fuckmark/cli.py b/fuckmark/cli.py index 5d91f28a..b9a18c6a 100644 --- a/fuckmark/cli.py +++ b/fuckmark/cli.py @@ -2,6 +2,7 @@ import argparse import os +import re import shutil import subprocess import sys @@ -111,14 +112,19 @@ def _parser() -> argparse.ArgumentParser: "Examples:\n" " fuckmark\n" " printf 'I do not agree.\\n' | fuckmark\n" + " fuckmark --text \"I do not agree.\"\n" " fuckmark \"I do not agree.\"\n" + " fuckmark --text \"I agree. You are right\"\n" + " fuckmark --file notes.txt -o notes.fm.txt\n" " fuckmark notes.txt -o notes.fm.txt\n" " fuckmark --stdin --copy < notes.txt\n" " fuckmark --stdin --visible < notes.fm.txt\n" "\n" "Supported input: tab, newline, carriage return, and ASCII space through tilde.\n" - "Other Unicode is returned unchanged. Only UTF-8 is supported.\n" - "Visible text stays the same. Use --visible to print that visible text." + "Other Unicode is returned unchanged with exit 0. That status means I/O\n" + "succeeded, not that a transformation occurred. Only UTF-8 is supported.\n" + "Visible text stays the same. Use --visible to print that visible text.\n" + "Use --text for literal strings and --file for existing UTF-8 files." ), formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -128,6 +134,18 @@ def _parser() -> argparse.ArgumentParser: metavar="TEXT_OR_FILE", help="quoted text or UTF-8 file; omit in a terminal to paste; - reads stdin", ) + parser.add_argument( + "--text", + dest="text_literal", + metavar="TEXT", + help="transform TEXT as a literal string (not a filename)", + ) + parser.add_argument( + "--file", + dest="file_explicit", + metavar="FILE", + help="read UTF-8 bytes from FILE without newline conversion", + ) parser.add_argument( "--version", action="version", @@ -186,18 +204,20 @@ def _looks_like_missing_file(value: str) -> bool: if any(separator in value for separator in separators) or value.startswith("~"): return True suffix = expanded.suffix - return len(suffix) >= 2 + if " " in suffix or "\t" in suffix: + return False + return re.fullmatch(r"\.[A-Za-z][A-Za-z0-9]{0,11}", suffix) is not None def _load_source_argument(value: str) -> str: expanded = Path(value).expanduser() if expanded.is_dir(): - raise ValueError(f"{value} is a directory. Pass a UTF-8 text file or quote the text.") + raise ValueError(f"{value} is a directory. Pass a UTF-8 text file or --text.") if expanded.is_file(): return _read_file(str(expanded)) if _looks_like_missing_file(value): raise ValueError( - f"file not found: {value}. Pass an existing UTF-8 file, or quote the text: fuckmark \"...\"" + f"file not found: {value}. Pass an existing UTF-8 file with --file, or a literal string with --text." ) return value @@ -274,13 +294,33 @@ def _styled(text: str, code: str, *, enabled: bool) -> str: return f"{code}{text}{_ANSI_RESET}" if enabled else text +def _decode_utf8_bytes(data: bytes, origin: str) -> str: + try: + text = data.decode("utf-8") + except UnicodeDecodeError as error: + raise ValueError(f"{origin} is not valid UTF-8. Only UTF-8 is supported.") from error + if any(0xD800 <= ord(character) <= 0xDFFF for character in text): + raise ValueError(f"{origin} is not valid UTF-8. Only UTF-8 is supported.") + return text + + def _read_file(path_value: str) -> str: + path = Path(path_value).expanduser() try: - return Path(path_value).expanduser().read_text(encoding="utf-8") - except UnicodeError as error: - raise ValueError(f"cannot decode {path_value!r} as UTF-8: {error}") from error + data = path.read_bytes() except OSError as error: raise ValueError(f"cannot read {path_value!r}: {error}") from error + return _decode_utf8_bytes(data, path_value) + + +def _read_stream_text(source: TextIO) -> str: + buffer = getattr(source, "buffer", None) + if buffer is not None: + return _decode_utf8_bytes(buffer.read(), "input") + text = source.read() + if any(0xD800 <= ord(character) <= 0xDFFF for character in text): + raise ValueError("input is not valid UTF-8. Only UTF-8 is supported.") + return text def _same_path(left: str, right: str) -> bool: @@ -291,31 +331,30 @@ def _ensure_utf8(stream: TextIO) -> None: reconfigure = getattr(stream, "reconfigure", None) if reconfigure is None: return - encoding = getattr(stream, "encoding", None) - if isinstance(encoding, str) and encoding.replace("-", "").casefold() == "utf8": - return try: - reconfigure(encoding="utf-8", errors="strict") - except (OSError, ValueError, AttributeError): - return + reconfigure(encoding="utf-8", errors="strict", newline="") + except (OSError, ValueError, AttributeError, TypeError): + try: + reconfigure(encoding="utf-8", errors="strict") + except (OSError, ValueError, AttributeError): + return def _write_stdout_utf8(output_stream: TextIO, text: str) -> None: - try: - output_stream.write(text) - output_stream.flush() - return - except UnicodeError: - pass buffer = getattr(output_stream, "buffer", None) - if buffer is None: - raise ValueError("standard output cannot encode UTF-8 product payload") + if buffer is not None: + try: + output_stream.flush() + except UnicodeError: + pass + buffer.write(text.encode("utf-8")) + buffer.flush() + return try: + output_stream.write(text) output_stream.flush() - except UnicodeError: - pass - buffer.write(text.encode("utf-8")) - buffer.flush() + except UnicodeError as error: + raise ValueError("standard output cannot encode UTF-8 product payload") from error def _write_file_atomic(path_value: str, text: str) -> None: @@ -420,23 +459,47 @@ def _run( ) source_arg = arguments.source + text_literal = arguments.text_literal + file_explicit = arguments.file_explicit wants_stdin = arguments.stdin_mode or source_arg == "-" + if text_literal is not None and file_explicit is not None: + return _error(errors, "pass --text or --file, not both") + if wants_stdin and (text_literal is not None or file_explicit is not None): + return _error(errors, "pass --text, --file, or --stdin, not both") if wants_stdin and source_arg not in (None, "-"): return _error(errors, "pass quoted text, a file, or --stdin, not both") + if text_literal is not None and source_arg is not None: + return _error(errors, "pass --text or a positional operand, not both") + if file_explicit is not None and source_arg is not None: + return _error(errors, "pass --file or a positional operand, not both") + file_for_overwrite = file_explicit if file_explicit is not None else source_arg if ( arguments.output not in (None, "-") - and source_arg not in (None, "-") - and Path(source_arg).expanduser().is_file() - and _same_path(source_arg, arguments.output) + and file_for_overwrite not in (None, "-") + and Path(str(file_for_overwrite)).expanduser().is_file() + and _same_path(str(file_for_overwrite), arguments.output) ): return _error(errors, "input and output files must be different") - literal_or_file = source_arg not in (None, "-") and not arguments.stdin_mode + literal_or_file = ( + text_literal is not None + or file_explicit is not None + or (source_arg not in (None, "-") and not arguments.stdin_mode) + ) interactive = (not wants_stdin) and (not literal_or_file) and _is_tty(source) color = _is_tty(errors) and not arguments.no_color and "NO_COLOR" not in os.environ try: - if literal_or_file: + if text_literal is not None: + text = text_literal + elif file_explicit is not None: + expanded = Path(file_explicit).expanduser() + if expanded.is_dir(): + raise ValueError(f"{file_explicit} is a directory. Pass a UTF-8 text file.") + if not expanded.is_file(): + raise ValueError(f"file not found: {file_explicit}") + text = _read_file(str(expanded)) + elif literal_or_file: text = _load_source_argument(str(source_arg)) elif interactive: captured = read_interactive_text(source, errors, color=color) @@ -446,7 +509,7 @@ def _run( return _error(errors, "no input. Paste or type text, then :done.") text = "\n".join(captured) else: - text = source.read() + text = _read_stream_text(source) except UnicodeError: return _error(errors, "input is not valid UTF-8. Only UTF-8 is supported.") except (OSError, ValueError) as error: diff --git a/fuckmark/cycle8/benchmark_render.py b/fuckmark/cycle8/benchmark_render.py index 0894055f..94dee5ed 100644 --- a/fuckmark/cycle8/benchmark_render.py +++ b/fuckmark/cycle8/benchmark_render.py @@ -11,7 +11,22 @@ from ..product.rendering import chrome_executable, compare_chrome_pre_screenshots -BENCHMARK_RENDER_VERSION = "cycle8-benchmark-render-v1" +BENCHMARK_RENDER_VERSION = "cycle8-benchmark-render-v2" +BENCHMARK_RENDER_FONT = "DejaVu Sans Mono" + + +def chrome_version(executable: str | None = None) -> str | None: + path = executable if executable is not None else chrome_executable() + if path is None: + return None + try: + output = subprocess.check_output([path, "--version"], timeout=10, stderr=subprocess.DEVNULL) + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired): + return None + try: + return output.decode("utf-8").strip() + except UnicodeError: + return None def _html_surface(text: str, surface: str) -> str: @@ -23,6 +38,7 @@ def _html_surface(text: str, surface: str) -> str: "caret-color:transparent;outline:none;font:16px/24px 'DejaVu Sans Mono',monospace;" "color:#000;background:#fff\">" ) + assign = f"el.value={payload};actual=el.value;" elif surface == "contenteditable": body = ( "
str: "font:16px/24px 'DejaVu Sans Mono',monospace;color:#000;background:#fff;" "white-space:pre-wrap\">
" ) + assign = f"el.textContent={payload};actual=el.textContent;" else: raise ValueError("unknown render surface") + script = ( + "var el=document.getElementById('t');var actual='';" + f"{assign}" + f"if(actual!=={payload}){{document.body.style.background='#f00';" + "document.body.setAttribute('data-fm-render','mismatch');}" + "else{document.body.setAttribute('data-fm-render','ok');}" + ) return ( "" "" f"{body}" - f"" + f"" "" ) +def displayed_js_property(surface: str) -> str: + if surface == "textarea": + return "value" + if surface == "contenteditable": + return "textContent" + raise ValueError("unknown render surface") + + def _screenshot(executable: str, root: Path, name: str, html: str) -> bytes: html_path = root / f"{name}.html" png_path = root / f"{name}.png" @@ -89,10 +119,13 @@ def _screenshot(executable: str, root: Path, name: str, html: str) -> bytes: def compare_chrome_surface(original: str, transformed: str, surface: str) -> dict[str, object]: if not isinstance(original, str) or not isinstance(transformed, str): raise TypeError("original and transformed must be strings") + if surface not in {"textarea", "contenteditable"}: + raise ValueError("unknown render surface") environment = f"chromium_{surface}" executable = chrome_executable() original_sha = sha256_text(original) transformed_sha = sha256_text(transformed) + browser = chrome_version(executable) if executable is None: payload = { "algorithm_version": BENCHMARK_RENDER_VERSION, @@ -102,13 +135,21 @@ def compare_chrome_surface(original: str, transformed: str, surface: str) -> dic "transformed_sha256": transformed_sha, "equal": None, "detail": "no chromium executable on this host", + "font": BENCHMARK_RENDER_FONT, + "browser_version": browser, } return {**payload, "comparison_hash": sha256_json(payload)} try: with tempfile.TemporaryDirectory(prefix="fuckmark-bench-render-") as directory: root = Path(directory) - original_png = _screenshot(executable, root, "original", _html_surface(original, surface)) - transformed_png = _screenshot(executable, root, "transformed", _html_surface(transformed, surface)) + original_html = _html_surface(original, surface) + transformed_html = _html_surface(transformed, surface) + if original and "el.textContent=" not in original_html and surface == "contenteditable": + raise RuntimeError("contenteditable assignment missing") + if original and "el.value=" not in original_html and surface == "textarea": + raise RuntimeError("textarea assignment missing") + original_png = _screenshot(executable, root, "original", original_html) + transformed_png = _screenshot(executable, root, "transformed", transformed_html) except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired, RuntimeError) as error: payload = { "algorithm_version": BENCHMARK_RENDER_VERSION, @@ -118,6 +159,8 @@ def compare_chrome_surface(original: str, transformed: str, surface: str) -> dic "transformed_sha256": transformed_sha, "equal": None, "detail": type(error).__name__, + "font": BENCHMARK_RENDER_FONT, + "browser_version": browser, } return {**payload, "comparison_hash": sha256_json(payload)} equal = original_png == transformed_png @@ -129,6 +172,8 @@ def compare_chrome_surface(original: str, transformed: str, surface: str) -> dic "transformed_sha256": transformed_sha, "equal": equal, "detail": "png_bytes_equal" if equal else "png_bytes_differ", + "font": BENCHMARK_RENDER_FONT, + "browser_version": browser, } return {**payload, "comparison_hash": sha256_json(payload)} diff --git a/fuckmark/cycle8/letter_mix.py b/fuckmark/cycle8/letter_mix.py index 577d884e..365f8774 100644 --- a/fuckmark/cycle8/letter_mix.py +++ b/fuckmark/cycle8/letter_mix.py @@ -5,6 +5,7 @@ from ..product.invariants import validate_user_visible_invariants from ..product.visible_projection import is_carrier_insertion_v1, project_visible_v1 from ..transforms.protected import _add_valid_markdown_destinations +from ..transforms.protected_markdown import _add_markdown_reference_spans from ..transforms.protected_patterns import ( _CLI_FLAG_RE, _CURRENCY_RE, @@ -38,6 +39,7 @@ def hard_machine_intervals(text: str) -> tuple[tuple[int, int], ...]: _add_fenced_code(raw, text) _add_inline_code(raw, text) _add_valid_markdown_destinations(raw, text) + _add_markdown_reference_spans(raw, text) _add_urls(raw, text) _add_regex(raw, text, _EMAIL_RE, ProtectedSpanKind.EMAIL) _add_ip_addresses(raw, text) @@ -60,13 +62,9 @@ def hard_machine_intervals(text: str) -> tuple[tuple[int, int], ...]: return tuple((start, end) for start, end in merged) -def _index_blocked(index: int, intervals: Sequence[tuple[int, int]]) -> bool: - for start, end in intervals: - if start <= index < end: - return True - if start > index: - return False - return False +def _source_contains_approved_carriers(text: str) -> bool: + approved = LETTER_MIX_APPROVED_CARRIERS + return any(ord(character) in approved for character in text) def select_letter_mix_sites( @@ -81,19 +79,18 @@ def select_letter_mix_sites( raise TypeError("max_selected must be an integer") if max_selected <= 0: raise ValueError("max_selected must be positive") + if _source_contains_approved_carriers(text): + return () blocked = hard_machine_intervals(text) sites: list[int] = [] + interval_index = 0 + blocked_count = len(blocked) for index, character in enumerate(text): - if character.isascii() and character.isalpha() and not _index_blocked(index, blocked): - payload = LETTER_MIX_PAYLOADS[len(sites) % 2] - trial = text[: index + 1] + payload + text[index + 1 :] - if not is_carrier_insertion_v1(text, trial, LETTER_MIX_APPROVED_CARRIERS): - continue - if project_visible_v1(trial, LETTER_MIX_APPROVED_CARRIERS) != text: - continue - report = validate_user_visible_invariants(text, trial, LETTER_MIX_APPROVED_CARRIERS) - if report.status is not InvariantStatus.PASS: - continue + while interval_index < blocked_count and blocked[interval_index][1] <= index: + interval_index += 1 + if interval_index < blocked_count and blocked[interval_index][0] <= index: + continue + if character.isascii() and character.isalpha(): sites.append(index) if max_selected is not None and len(sites) >= max_selected: break @@ -123,8 +120,12 @@ def compose_letter_mix(text: str, sites: Sequence[int]) -> str: report = validate_user_visible_invariants(text, output, LETTER_MIX_APPROVED_CARRIERS) if report.status is not InvariantStatus.PASS: raise RuntimeError("letter mix failed user-visible invariants") + site_index = 0 + shift = 0 for start, end in hard_machine_intervals(text): - shift = sum(1 for index in ordered if index < start) + while site_index < len(ordered) and ordered[site_index] < start: + shift += 1 + site_index += 1 if output[start + shift : end + shift] != text[start:end]: raise RuntimeError("letter mix mutated a hard machine span") return output @@ -142,8 +143,14 @@ def letter_mix_protected_blocked_count(text: str) -> int: if not isinstance(text, str): raise TypeError("text must be a string") blocked = hard_machine_intervals(text) - return sum( - 1 - for index, character in enumerate(text) - if character.isascii() and character.isalpha() and _index_blocked(index, blocked) - ) + count = 0 + interval_index = 0 + blocked_count = len(blocked) + for index, character in enumerate(text): + while interval_index < blocked_count and blocked[interval_index][1] <= index: + interval_index += 1 + if not (character.isascii() and character.isalpha()): + continue + if interval_index < blocked_count and blocked[interval_index][0] <= index: + count += 1 + return count diff --git a/fuckmark/cycle8/mix_freeze.py b/fuckmark/cycle8/mix_freeze.py index f4c9d174..6eb65745 100644 --- a/fuckmark/cycle8/mix_freeze.py +++ b/fuckmark/cycle8/mix_freeze.py @@ -5,7 +5,7 @@ from .._validation import require_int from ..experiments.cycle6_confirmation import CYCLE6_THRESHOLD -from ..hashing import sha256_file, sha256_json +from ..hashing import sha256_json from ..seeds.ledger import ( CONFIRMATION_CONTENT_FORBIDDEN_SEED_BASES, CYCLE8_MIX_CONFIRMATION_HOLD_TOPIC, @@ -23,7 +23,7 @@ CYCLE8_MIX_FREEZE_PATH = "specs/cycle8/fuckmark-cycle8-mix-freeze-v1.json" CYCLE8_MIX_CONFIRMATION_DETECTOR_VERSION = "cycle8-mix-confirmation-detector-compare-v1" CYCLE8_MIX_CONFIRMATION_PAIR_COUNT = 64 -_LETTER_MIX_PATH = Path(__file__).resolve().parent / "letter_mix.py" +CYCLE8_MIX_FREEZE_LETTER_MIX_SHA256 = "b1ceec24e584c0e9e7135ef0c89a3bd249b0bda4a45e07aa7190b1b010ba56d4" _DEVELOPMENT_ARTIFACTS = ( ("1020000", "evidence/cycle8-mix-1020000-n64-2026-08-26/detector-compare.json", "2538c614a73bed360cdeebdaa60c0fa36ad34cf6995c76a120d49bd1da063ce2"), ("1030000", "evidence/cycle8-mix-1030000-n64-2026-08-26/detector-compare.json", "142809783554a890cfd68b80b560295605e068b8e0deede8ea81e3e75358eb95"), @@ -85,7 +85,7 @@ def mix_freeze_payload() -> dict[str, object]: "carrier_insertion": "is_carrier_insertion_v1", "user_visible_invariants": "validate_user_visible_invariants", "supported_input_domain": "is_supported_product_domain_v1", - "letter_mix_source_sha256": sha256_file(_LETTER_MIX_PATH), + "letter_mix_source_sha256": CYCLE8_MIX_FREEZE_LETTER_MIX_SHA256, "detector": { "model": "openai-community/gpt2", "model_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e", diff --git a/fuckmark/experiments/e24_protected_span_stress.py b/fuckmark/experiments/e24_protected_span_stress.py index f7e52cbe..44461b5e 100644 --- a/fuckmark/experiments/e24_protected_span_stress.py +++ b/fuckmark/experiments/e24_protected_span_stress.py @@ -218,6 +218,7 @@ def _protected_kind_token(kind: ProtectedSpanKind) -> str: ProtectedSpanKind.PERCENTAGE: "42%", ProtectedSpanKind.CODE: "`immutable_call()`", ProtectedSpanKind.MARKDOWN_DESTINATION: "[label](https://example.com/e24-target)", + ProtectedSpanKind.MARKDOWN_LABEL: "[see][e24ref]\n\n[e24ref]: https://example.com/e24-ref", ProtectedSpanKind.QUOTATION: '"immutable phrase"', ProtectedSpanKind.POSIX_PATH: "/usr/local/bin/e24-tool", ProtectedSpanKind.WINDOWS_PATH: "C:\\Temp\\e24-file.txt", diff --git a/fuckmark/transforms/protected.py b/fuckmark/transforms/protected.py index 7cede05f..6914fe5a 100644 --- a/fuckmark/transforms/protected.py +++ b/fuckmark/transforms/protected.py @@ -21,6 +21,7 @@ _NUMERIC_CITATION_RE, _PERCENT_RE, ) +from .protected_markdown import _add_markdown_reference_spans from .protected_structures import ( _add_delimited_math, _add_dollar_math, @@ -160,6 +161,7 @@ def extract( _add_fenced_code(raw, text) _add_inline_code(raw, text) _add_valid_markdown_destinations(raw, text) + _add_markdown_reference_spans(raw, text) _add_urls(raw, text) _add_regex(raw, text, _EMAIL_RE, ProtectedSpanKind.EMAIL) _add_ip_addresses(raw, text) diff --git a/fuckmark/transforms/protected_markdown.py b/fuckmark/transforms/protected_markdown.py new file mode 100644 index 00000000..7edf3e74 --- /dev/null +++ b/fuckmark/transforms/protected_markdown.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from .protected_patterns import _append, _is_escaped +from .schema import ProtectedSpanKind + + +def _normalize_markdown_label(label: str) -> str: + return " ".join(label.split()).casefold() + + +def _markdown_bracket_pairs(text: str) -> tuple[tuple[int, int], ...]: + stack: list[int] = [] + pairs: list[tuple[int, int]] = [] + for index, character in enumerate(text): + if character == "\n": + stack.clear() + continue + if character not in "[]" or _is_escaped(text, index): + continue + if character == "[": + stack.append(index) + elif stack: + pairs.append((stack.pop(), index)) + return tuple(pairs) + + +def _line_start(text: str, index: int) -> int: + newline = text.rfind("\n", 0, index) + return 0 if newline < 0 else newline + 1 + + +def _is_reference_definition_label(text: str, start: int, end: int) -> bool: + prefix = text[_line_start(text, start) : start] + if len(prefix) > 3 or any(character not in " \t" for character in prefix): + return False + return end + 1 < len(text) and text[end + 1] == ":" + + +def _parse_link_destination(text: str, start: int) -> tuple[int, int] | None: + index = start + length = len(text) + while index < length and text[index] in " \t": + index += 1 + if index >= length or text[index] in "\r\n": + return None + if text[index] == "<": + cursor = index + 1 + while cursor < length and text[cursor] not in ">\r\n": + if text[cursor] == "\\" and cursor + 1 < length: + cursor += 2 + continue + cursor += 1 + if cursor < length and text[cursor] == ">": + return index, cursor + 1 + return None + cursor = index + depth = 0 + while cursor < length: + character = text[cursor] + if character in " \t\r\n": + break + if character == "\\" and cursor + 1 < length: + cursor += 2 + continue + if character == "(": + depth += 1 + elif character == ")": + if depth == 0: + break + depth -= 1 + cursor += 1 + if cursor <= index: + return None + return index, cursor + + +def _definition_labels(text: str, pairs: tuple[tuple[int, int], ...]) -> dict[str, list[tuple[int, int, int, int]]]: + found: dict[str, list[tuple[int, int, int, int]]] = {} + for start, end in pairs: + if not _is_reference_definition_label(text, start, end): + continue + destination = _parse_link_destination(text, end + 2) + if destination is None: + continue + dest_start, dest_end = destination + inner = text[start + 1 : end] + if not inner.strip(): + continue + key = _normalize_markdown_label(inner) + found.setdefault(key, []).append((start + 1, end, dest_start, dest_end)) + return found + + +def resolve_markdown_reference_hrefs(text: str) -> tuple[str, ...]: + if not isinstance(text, str): + raise TypeError("text must be a string") + pairs = _markdown_bracket_pairs(text) + defined = _definition_labels(text, pairs) + by_start = {start: (start, end) for start, end in pairs} + hrefs: list[str] = [] + seen_pair_starts: set[int] = set() + for start, end in pairs: + if start in seen_pair_starts: + continue + if _is_reference_definition_label(text, start, end): + continue + following = by_start.get(end + 1) + if following is not None: + seen_pair_starts.add(following[0]) + label_inner = text[following[0] + 1 : following[1]] + if label_inner.strip() == "": + label_inner = text[start + 1 : end] + key = _normalize_markdown_label(label_inner) + rows = defined.get(key) + if not rows: + continue + dest_start, dest_end = rows[0][2], rows[0][3] + hrefs.append(_destination_href(text[dest_start:dest_end])) + continue + if end + 1 < len(text) and text[end + 1] == "(": + continue + label_inner = text[start + 1 : end] + key = _normalize_markdown_label(label_inner) + rows = defined.get(key) + if not rows: + continue + dest_start, dest_end = rows[0][2], rows[0][3] + hrefs.append(_destination_href(text[dest_start:dest_end])) + return tuple(hrefs) + + +def _destination_href(raw: str) -> str: + if len(raw) >= 2 and raw[0] == "<" and raw[-1] == ">": + return raw[1:-1] + return raw + + +def _add_markdown_reference_spans(raw: list[tuple[int, int, ProtectedSpanKind]], text: str) -> None: + pairs = _markdown_bracket_pairs(text) + if not pairs: + return + defined = _definition_labels(text, pairs) + if not defined: + return + for _key, rows in defined.items(): + for label_start, label_end, dest_start, dest_end in rows: + _append(raw, label_start, label_end, ProtectedSpanKind.MARKDOWN_LABEL) + _append(raw, dest_start, dest_end, ProtectedSpanKind.MARKDOWN_DESTINATION) + by_start = {start: (start, end) for start, end in pairs} + seen_pair_starts: set[int] = set() + for start, end in pairs: + if start in seen_pair_starts: + continue + if _is_reference_definition_label(text, start, end): + continue + following = by_start.get(end + 1) + if following is not None: + seen_pair_starts.add(following[0]) + label_inner = text[following[0] + 1 : following[1]] + if label_inner.strip() == "": + inner_start, inner_end = start + 1, end + label_inner = text[inner_start:inner_end] + else: + inner_start, inner_end = following[0] + 1, following[1] + if _normalize_markdown_label(label_inner) in defined: + _append(raw, inner_start, inner_end, ProtectedSpanKind.MARKDOWN_LABEL) + continue + if end + 1 < len(text) and text[end + 1] == "(": + continue + inner_start, inner_end = start + 1, end + label_inner = text[inner_start:inner_end] + if _normalize_markdown_label(label_inner) in defined: + _append(raw, inner_start, inner_end, ProtectedSpanKind.MARKDOWN_LABEL) diff --git a/fuckmark/transforms/protected_structures.py b/fuckmark/transforms/protected_structures.py index dd86f400..496eb796 100644 --- a/fuckmark/transforms/protected_structures.py +++ b/fuckmark/transforms/protected_structures.py @@ -13,8 +13,13 @@ _CURLY_SINGLE_QUOTE_RE = re.compile(r"‘[^’\n]{2,}’") _STRAIGHT_SINGLE_QUOTE_RE = re.compile(r"(?|\s]+\\)*[^\\/:*?\"<>|\s]+") -_WINDOWS_PREFIX_RE = re.compile(r"(?i)(?|\s]+[/\\])*[^\\/:*?\"<>|\s]+" +) +_WINDOWS_PREFIX_RE = re.compile(r"(?i)(? None: - for match in _POSIX_PATH_RE.finditer(text): - start, end = _trim_terminal_punctuation(text, match.start(), match.end()) - _append(raw, start, end, ProtectedSpanKind.POSIX_PATH) + for pattern in (_POSIX_PATH_RE, _RELATIVE_PATH_RE): + for match in pattern.finditer(text): + start, end = _trim_terminal_punctuation(text, match.start(), match.end()) + _append(raw, start, end, ProtectedSpanKind.POSIX_PATH) def _add_windows_paths(raw: list[tuple[int, int, ProtectedSpanKind]], text: str) -> None: @@ -207,7 +213,8 @@ def _add_extended_windows_paths(raw: list[tuple[int, int, ProtectedSpanKind]], t for match in _WINDOWS_PREFIX_RE.finditer(text): if _starts_inside_other_protection(starts, intervals, match.start()): continue - end = _extended_path_end(text, match.start(), match.end(), "\\", '<>"|?*') + separator = "/" if match.group().endswith("/") else "\\" + end = _extended_path_end(text, match.start(), match.end(), separator, '<>"|?*') if end is not None: _append(raw, match.start(), end, ProtectedSpanKind.WINDOWS_PATH) diff --git a/fuckmark/transforms/schema.py b/fuckmark/transforms/schema.py index e9144e0c..fcf25661 100644 --- a/fuckmark/transforms/schema.py +++ b/fuckmark/transforms/schema.py @@ -14,6 +14,7 @@ class ProtectedSpanKind(str, Enum): PERCENTAGE = "percentage" CODE = "code" MARKDOWN_DESTINATION = "markdown_destination" + MARKDOWN_LABEL = "markdown_label" QUOTATION = "quotation" POSIX_PATH = "posix_path" WINDOWS_PATH = "windows_path" diff --git a/tests/audit_mix_replay.py b/tests/audit_mix_replay.py new file mode 100644 index 00000000..18557b59 --- /dev/null +++ b/tests/audit_mix_replay.py @@ -0,0 +1,11 @@ +from fuckmark.cycle8.letter_mix import LETTER_MIX_APPROVED_CARRIERS, apply_letter_alternating_mix +from fuckmark.hashing import sha256_text +from fuckmark.product.visible_projection import is_carrier_insertion_v1, project_visible_v1 + + +def live_mix_hash(source: str) -> str: + live = apply_letter_alternating_mix(source) + assert project_visible_v1(live, LETTER_MIX_APPROVED_CARRIERS) == source + if live != source: + assert is_carrier_insertion_v1(source, live, LETTER_MIX_APPROVED_CARRIERS) + return sha256_text(live) diff --git a/tests/test_audit_fix_evidence.py b/tests/test_audit_fix_evidence.py new file mode 100644 index 00000000..bbeedaaa --- /dev/null +++ b/tests/test_audit_fix_evidence.py @@ -0,0 +1,53 @@ +import json +from pathlib import Path + +from fuckmark.cycle8.mix_freeze import CYCLE8_MIX_FREEZE_LETTER_MIX_SHA256 +from fuckmark.hashing import sha256_file, sha256_json + + +ROOT = Path(__file__).resolve().parents[1] +OLD_DIRS = ( + ROOT / "evidence/cycle8-letter-system-benchmark-2026-08-26", + ROOT / "evidence/cycle8-mix-margin-2026-08-26", +) +NEW_DIR = ROOT / "evidence/audit-fixes-2026-08-27" + + +def test_historical_render_evidence_hashes_are_unchanged() -> None: + for folder in OLD_DIRS: + sums = (folder / "SHA256SUMS.txt").read_text(encoding="utf-8") + for line in sums.splitlines(): + digest, name = line.split() + assert sha256_file(folder / name) == digest + + +def test_f01_replacement_measurement_rejects_blank_div_false_positive() -> None: + payload = json.loads((NEW_DIR / "render-v2.json").read_text(encoding="utf-8")) + body = {key: value for key, value in payload.items() if key != "artifact_hash"} + assert payload["artifact_hash"] == sha256_json(body) + assert payload["algorithm_version"] == "cycle8-benchmark-render-v2" + assert payload["historical_files_not_rewritten"] is True + assert "contenteditable divs" in payload["historical_evidence_invalid_because"] + by_key = {(row["surface"], row["pair"]): row["result"]["status"] for row in payload["rows"]} + assert by_key[("contenteditable", "negative")] == "REJECTED" + assert by_key[("contenteditable", "positive")] == "VERIFIED" + assert by_key[("contenteditable", "nonempty_vs_empty")] == "REJECTED" + assert by_key[("textarea", "negative")] == "REJECTED" + assert by_key[("textarea", "positive")] == "VERIFIED" + assert by_key[("textarea", "nonempty_vs_empty")] == "REJECTED" + assert payload["webkit_safari"] == "UNKNOWN" + assert payload["terminal_pixels"] == "UNKNOWN" + assert "DejaVu Sans Mono" in payload["font"] + readme = (NEW_DIR / "README.md").read_text(encoding="utf-8") + assert "not proof of text rendering equivalence" in readme + assert "be6ae7645fda8b39d1d308722ac249f519e68de5" in readme + scan = json.loads((NEW_DIR / "letter-mix-scan.json").read_text(encoding="utf-8")) + scan_body = {key: value for key, value in scan.items() if key != "artifact_hash"} + assert scan["artifact_hash"] == sha256_json(scan_body) + assert scan["note"].startswith("Host observations only") + + +def test_mix_freeze_letter_mix_hash_stays_historical() -> None: + live = sha256_file(ROOT / "fuckmark/cycle8/letter_mix.py") + assert CYCLE8_MIX_FREEZE_LETTER_MIX_SHA256 == "b1ceec24e584c0e9e7135ef0c89a3bd249b0bda4a45e07aa7190b1b010ba56d4" + assert live != CYCLE8_MIX_FREEZE_LETTER_MIX_SHA256 diff --git a/tests/test_benchmark_render_surfaces.py b/tests/test_benchmark_render_surfaces.py new file mode 100644 index 00000000..b788f5d1 --- /dev/null +++ b/tests/test_benchmark_render_surfaces.py @@ -0,0 +1,94 @@ +import pytest + +from fuckmark.cycle8.benchmark_render import ( + BENCHMARK_RENDER_FONT, + BENCHMARK_RENDER_VERSION, + _html_surface, + chrome_version, + compare_chrome_surface, + displayed_js_property, +) +from fuckmark.product.rendering import chrome_executable + + +class _FakeElement: + def __init__(self, kind: str) -> None: + self.kind = kind + self.__dict__.pop("value", None) + self.textContent = "" + if kind == "textarea": + self.value = "" + + def __contains__(self, key: object) -> bool: + return key in self.__dict__ + + +def _old_buggy_assign(element: _FakeElement, payload: str) -> None: + element.value = payload + if "value" not in element: + element.textContent = payload + + +def test_generated_contenteditable_script_sets_text_content_not_value() -> None: + html = _html_surface("AAAA", "contenteditable") + assert "el.textContent=" in html + assert "el.value=" not in html + assert "('value' in" not in html + assert displayed_js_property("contenteditable") == "textContent" + textarea = _html_surface("AAAA", "textarea") + assert "el.value=" in textarea + assert "el.textContent=" not in textarea.split("