From 59c4f3b0d402d07a78059527c1f8ec1fdd725271 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 28 Aug 2026 20:05:59 +0000 Subject: [PATCH 01/19] =?UTF-8?q?Fix=20audit=20findings=20D01=E2=80=93G01?= =?UTF-8?q?=20for=20the=200.4.1=20product=20CLI.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Protect multiline Markdown, HTML, indented code, and remaining path/URI forms; report transformation outcomes and distinct exit statuses; align installers and docs with carrier insertion; and record live-versus-frozen mix hashes plus Chromium payload render evidence without rewriting frozen confirmation files. Co-authored-by: Byte271 --- CHANGELOG.md | 26 +- README.md | 81 ++++-- docs/cli.md | 19 +- docs/cycle8/gate-v2.md | 7 + docs/cycle8/mix-second-model-transfer.md | 7 + docs/install.md | 53 ++-- docs/limits.md | 74 +++++ docs/product-contract.md | 2 +- docs/release.md | 23 +- docs/research.md | 34 ++- docs/website.md | 37 +++ evidence/audit-fixes-2026-08-28/README.md | 23 ++ .../audit-fixes-2026-08-28/SHA256SUMS.txt | 2 + .../render-payload-v2.json | 126 +++++++++ .../CHECKSUM-PROVENANCE.md | 15 ++ fuckmark/__init__.py | 2 +- fuckmark/cli.py | 251 ++++++++++++++--- fuckmark/cycle8/benchmark_render.py | 38 ++- fuckmark/cycle8/letter_mix.py | 39 +-- fuckmark/product/domain.py | 1 + fuckmark/product/rendering.py | 51 +++- fuckmark/transforms/protected.py | 71 ++--- fuckmark/transforms/protected_markdown.py | 119 +++++++-- fuckmark/transforms/protected_patterns.py | 60 ++++- fuckmark/transforms/protected_structures.py | 162 ++++++++++- pyproject.toml | 6 +- tests/audit_mix_replay.py | 42 +++ tests/test_audit_d03_d11.py | 252 ++++++++++++++++++ tests/test_cli.py | 9 +- tests/test_cycle8_control_mix_diagnostic.py | 9 +- tests/test_cycle8_deepmind_transfer.py | 16 +- tests/test_cycle8_detector_transfer.py | 9 +- tests/test_cycle8_gate_v2.py | 15 ++ tests/test_cycle8_second_model_transfer.py | 9 +- tests/test_evidence_manifests.py | 43 +++ tests/test_install_release_hardening.py | 25 +- tests/test_project_identity.py | 4 +- tests/test_render_coverage.py | 56 ++++ tools/install/unix.sh | 13 +- tools/install/windows.ps1 | 8 +- tools/verify_release_install.py | 30 +++ 41 files changed, 1608 insertions(+), 261 deletions(-) create mode 100644 docs/cycle8/gate-v2.md create mode 100644 docs/cycle8/mix-second-model-transfer.md create mode 100644 docs/limits.md create mode 100644 docs/website.md create mode 100644 evidence/audit-fixes-2026-08-28/README.md create mode 100644 evidence/audit-fixes-2026-08-28/SHA256SUMS.txt create mode 100644 evidence/audit-fixes-2026-08-28/render-payload-v2.json create mode 100644 evidence/cycle7-stage-a-2026-08-25/CHECKSUM-PROVENANCE.md create mode 100644 tests/test_audit_d03_d11.py create mode 100644 tests/test_evidence_manifests.py create mode 100644 tests/test_render_coverage.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a7a1e01f..5abe8de1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,18 @@ # Changelog -## Unreleased - -Audit fixes F01-F07 against `be6ae7645fda8b39d1d308722ac249f519e68de5`. Package version remains 0.4.0. Frozen confirmation files and their SHA-256 sums are unchanged. - -- F01: contenteditable rendering sets `textContent`. The 2026-08-26 Chromium contenteditable VERIFIED rows compared blank divs and are not proof of text rendering equivalence. Replacement measurement: `evidence/audit-fixes-2026-08-27/`. -- F02: common relative paths without `./` and Windows paths that use `/` keep exact original bytes. -- F03: Markdown reference labels are protected at uses and definitions. Link checks use reference resolution, not only visible projection. -- F04: file and stdin I/O keep LF, CRLF, CR, mixed endings, and a missing final newline. -- F05: `--text` is literal text. `--file` requires an existing UTF-8 file. Ordinary sentences are not treated as missing files. -- F06: stdin is decoded as strict UTF-8. Invalid bytes exit nonzero with no payload and no clipboard copy. -- F07: letter-mix site selection no longer hashes the whole document per candidate. Compose-time invariant checks remain. - -Live mix bytes can differ from frozen `u034f-ufe00-letter-alt-v1` output hashes when a newly protected span is present. Visible projection remains the source. `cycle8-mix-freeze-v1` still pins `letter_mix_source_sha256` to the freeze-time file. +## v0.4.1 + +Audit findings D01-D16, E01-E04, L01-L06, P01, and G01 against `d7dc98b7ee99fc46b767c93444a9f051fdbea2ba`. Do not retag v0.4.0. Frozen confirmation files and their SHA-256 sums are unchanged. + +- Public copy, installer messages, and this source tree describe carrier insertion. They no longer say the CLI returns text unchanged. `docs/website.md` is the controlled website/installer copy. Do not pipe `https://d.q1z.org/mark` into a shell. +- Package version is 0.4.1. The published v0.4.0 wheel remains the last GitHub Release artifact and does not implement `--text` / `--file`. +- Markdown reference labels may span lines and line endings (LF, CRLF, CR). Definitions may put the destination on the next line and may appear in blockquotes or lists. Multiline inline destinations are protected. HTML tags, HTML entities, and indented code are protected. +- Extensionless relative paths such as `scripts/build`, last-component spaces such as `C:/Users/Alice/My final notes.txt`, and `ftp://` URIs are protected. `and/or` remains eligible. +- CLI outcomes: stderr reasons and `--status` distinguish transformed, unsupported Unicode, already-transformed, no eligible sites, site-cap, and internal failure. Clipboard partial success is exit 3. Usage errors remain exit 2. Internal transform failure is exit 4. Output is written before clipboard copy. Stdout write failures no longer escape as a traceback. +- Unix installer PATH uses `FUCKMARK_BIN`. Windows launcher is written as UTF-16 LE with BOM. +- Transfer replay tests compare live output hashes to stored hashes. Documented unwatermarked DeepMind drift is listed; unexplained drift fails. +- Historical Cycle 7 Stage A README checksum mismatch is a CRLF-vs-LF provenance note, not a rewritten README. +- Limits, sanitizer, length, tokenizer, and compatibility matrices: `docs/limits.md`. ## v0.4.0 — Product CLI diff --git a/README.md b/README.md index 90298482..f69472f4 100644 --- a/README.md +++ b/README.md @@ -9,47 +9,80 @@ # FuckMark -FuckMark is a UTF-8 command that inserts hidden Unicode into ordinary English ASCII text **without changing what you see**. +FuckMark is a UTF-8 command for **ordinary English ASCII text**. It inserts hidden Unicode (U+034F and U+FE00) after eligible letters **without changing the words you see**. -On confirmation-scale GPT-2 / SynthID tests, transformed text was not flagged, visible words stayed identical, and output was deterministic. +It is a constrained research/product CLI with measured SynthID / GPT-2 results. It is **not** a general watermark remover, not a paraphraser, and not a claim against unknown or proprietary detectors. Website: [mark.q1z.org](https://mark.q1z.org) · License: MIT -## Install +## Who it is for -Python 3.11 or newer. From this repository: +People who paste or pipe **English ASCII** (tab, CR, LF, space through tilde) and want a local, deterministic, detector-blind insertion that keeps visible text identical. + +If the input contains an emoji, a curly apostrophe, an accent, or any other Unicode, FuckMark copies it unchanged and says so. + +## Install from this repository + +Python 3.11 or newer. Get the code, then keep using the venv path (or activate the venv). A venv is not on PATH by itself. + +Linux / macOS: ```text +git clone https://github.com/byte271/FuckMark.git +cd FuckMark python3 -m venv .venv .venv/bin/python -m pip install . .venv/bin/fuckmark --version +printf 'I do not agree.\n' | .venv/bin/fuckmark --visible +printf 'I do not agree.\n' | .venv/bin/fuckmark --status >/tmp/fm.out +``` + +Windows (PowerShell): + +```text +git clone https://github.com/byte271/FuckMark.git +cd FuckMark +py -3.12 -m venv .venv +.venv\Scripts\python.exe -m pip install . +.venv\Scripts\fuckmark.exe --version ``` -That prints `FuckMark 0.4.0`. v0.4.0 wheel SHA-256: `5a6ac62c8bb8d7ddd9e5bc9cb6cee6e3eb181ac5f397b4a6645ef86468ee932f`. See [`docs/install.md`](docs/install.md). Do not pipe `https://d.q1z.org/mark` into a shell. +That prints `FuckMark 0.4.1` for this source tree. + +Do not pipe `https://d.q1z.org/mark` into a shell. The supported install is a clone or a checksummed GitHub Release wheel. See [`docs/install.md`](docs/install.md) and [`docs/website.md`](docs/website.md). + +Last **published** wheel is still **v0.4.0** (SHA-256 `5a6ac62c8bb8d7ddd9e5bc9cb6cee6e3eb181ac5f397b4a6645ef86468ee932f`). That wheel does not implement `--text` / `--file`. Use this 0.4.1 tree or wait for the v0.4.1 GitHub Release. Do not retag v0.4.0. + +If clipboard copy fails, pipe instead: `printf 'I do not agree.\n' | .venv/bin/fuckmark`. ## Use +Keep using the venv command from the install step. Examples below assume `.venv/bin/fuckmark` on Unix. + ```text -fuckmark +.venv/bin/fuckmark ``` -Paste or type text, then a line that is only `:done`. The result is copied to the clipboard. You still read the same words; the hidden payload is not printed. +Paste or type text, then a line that is only `:done`. The result is copied to the clipboard. Status on stderr says whether hidden characters were inserted. ```text -printf 'I do not agree.\n' | fuckmark -fuckmark --text "I do not agree." -fuckmark --text "I agree. You are right" -fuckmark --file notes.txt -o notes.fm.txt -printf 'I do not agree.\n' | fuckmark --visible +printf 'I do not agree.\n' | .venv/bin/fuckmark +.venv/bin/fuckmark --text "I do not agree." +.venv/bin/fuckmark --text "I agree. You are right" +.venv/bin/fuckmark --file notes.txt -o notes.fm.txt +printf 'I do not agree.\n' | .venv/bin/fuckmark --visible +printf 'I do not agree.\n' | .venv/bin/fuckmark --status >/tmp/fm.out ``` -Pipes and files write the payload to stdout. `fuckmark --help` is enough to start. +Pipes and files write the payload to stdout. `--visible` prints the original visible text. `--status` writes a machine-readable outcome to stderr. `fuckmark --help` is enough to start. ## What it guarantees -`VISIBLE(original) == VISIBLE(transformed)`. FuckMark only inserts U+034F and U+FE00 after eligible ASCII letters. It does not contract, paraphrase, homoglyph, or add spaces. +`VISIBLE(original) == VISIBLE(transformed)`. FuckMark only inserts U+034F and U+FE00 after eligible ASCII letters. It does not contract, paraphrase, homoglyph, or add spaces. Transformation selection does not use detectors or watermark keys. -Supported input: tab, newline, carriage return, and ASCII space through tilde. Other Unicode is returned unchanged with exit 0. That status means I/O succeeded, not that hidden characters were inserted. Only UTF-8. URLs, code spans, paths (including `src/main.py` and `C:/Users/...`), Markdown reference labels, and similar machine text are left intact. +Supported input: tab, newline, carriage return, and ASCII space through tilde. Other Unicode is returned unchanged with exit 0 and a stderr reason. That status means I/O succeeded, not that hidden characters were inserted or that a watermark was removed. Only UTF-8. + +Machine text is left intact when recognized: fenced/inline/indented code, HTML tags and entities, markdown destinations and reference labels (including multiline and container forms), URLs including `ftp://`, emails, and paths such as `src/main.py`, `scripts/build`, and `C:/Users/Alice/My final notes.txt`. Ambiguous prose such as `and/or` stays eligible. ## Verified results @@ -63,19 +96,23 @@ Supported input: tab, newline, carriage return, and ASCII space through tilde. O Google synthid-text 30-key GPT-2 transformed text: **0/192**. +These numbers are frozen Gate v2 confirmation on GPT-2 / SynthID, 64-token samples, threshold and sanitizer paths as recorded. They are not a universal zero-rate guarantee. See [`docs/limits.md`](docs/limits.md). + ## Limitations -This does not remove every AI watermark. It is not a claim against unknown, proprietary, or future detectors. +Stripping combining marks or default-ignorable characters restores the source and the detectable watermark. On the frozen Gate v2 set, those two stress sanitizers restored detection to 188/192. NFC, NFKC, Cf stripping, and whitespace collapse are not that removal step. + +English ASCII only. One unsupported character disables transformation of the whole input. -Stripping combining marks or default-ignorable characters restores the source. On the frozen Gate v2 confirmation set, mix plus those two stress sanitizers restored detection to 188/192. The stored 0/192 result applies only to the specified detectors, threshold, samples, and required sanitizer paths. +Insertion stops after the first 192 eligible letter sites. A long document is mostly unchanged after that point. -Primary confirmation uses 64 generated tokens per sample. Insertion stops after the first 192 eligible letter sites, so the tail of a long document is unchanged. That coverage limit is not a measured long-document detection result. +Primary evidence uses GPT-2-generated 64-token samples and GPT-2 BPE. DistilGPT2 still uses that tokenizer family. This is not evidence against generic AI-authorship classifiers, unknown proprietary detectors, every SynthID deployment, or C2PA. -English ASCII only. Valid Unicode outside that domain, including accented letters, is returned unchanged with exit 0. +Hidden characters change raw substring search, some editors, and GPT-2 token counts (about 8.9x on the stored Gate v2 watermarked set). Downstream software does not automatically call `visible_contains()`. -Equal visible projection is not the same as equal behavior in other software. Raw codepoint search, Markdown reference matching, and exact byte paths can still fail if hidden characters remain. Historical Chromium `contenteditable` VERIFIED rows from 2026-08-26 used a blank-div measurement bug and are not proof of rendering equivalence. Safari/WebKit and terminal-pixel results remain UNKNOWN. +Cap 192 insertion sites. See [`docs/limits.md`](docs/limits.md). -Cap 192 insertion sites. +Historical Chromium `contenteditable` VERIFIED rows from 2026-08-26 used a blank-div measurement bug and are not proof of rendering equivalence. Replacement controls exist; actual product-payload pixel requalification and Safari/WebKit/terminal pixels remain UNKNOWN. Viewport screenshots cannot prove whole-document equality. ## How it works @@ -83,7 +120,7 @@ Eligible ASCII letters receive an alternating pair of invisible characters (U+03 ## Research -Frozen evidence, hashes, and protocols: [`docs/research.md`](docs/research.md). Product contract: [`docs/product-contract.md`](docs/product-contract.md). CLI: [`docs/cli.md`](docs/cli.md). +Frozen evidence, hashes, and protocols: [`docs/research.md`](docs/research.md). Product contract: [`docs/product-contract.md`](docs/product-contract.md). CLI: [`docs/cli.md`](docs/cli.md). Limits matrix: [`docs/limits.md`](docs/limits.md). ## Development diff --git a/docs/cli.md b/docs/cli.md index 403af98e..6f64f310 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -31,6 +31,7 @@ Enter :done on a new line when finished. Processing... ✓ Copied to clipboard +FuckMark: inserted 54 hidden characters. ``` Blank lines are kept. `:done` ends the session only when it is the entire line. The transformed payload is copied and not printed. @@ -54,6 +55,7 @@ fuckmark notes.txt fuckmark notes.txt -o notes.fm.txt printf 'I do not agree.\n' | fuckmark --copy printf 'I do not agree.\n' | fuckmark --visible +printf 'I do not agree.\n' | fuckmark --status >/tmp/fm.out ``` Piped or quoted input writes the hidden payload to stdout. `--visible` writes the original visible text. `--copy` also places whatever was written on the clipboard. @@ -77,15 +79,16 @@ Existing files are read as UTF-8 bytes with no newline conversion. LF, CRLF, CR, | Option | Behavior | | --- | --- | -| `--version` | Print `FuckMark 0.4.0`. | +| `--version` | Print `FuckMark 0.4.1`. | | `--text TEXT` | Transform TEXT as a literal string. | | `--file FILE` | Read UTF-8 from FILE. | | `--stdin` | Read all of standard input. | -| `-o FILE`, `--output FILE` | Write UTF-8 output to FILE. | +| `-o FILE`, `--output FILE` | Write UTF-8 output to FILE. The file is written before any clipboard copy. | | `--copy` | Also copy the output to the clipboard. The paste UI always copies. | | `--visible` | Print the visible text (no hidden characters). | | `--encoding NAME` | Only `utf-8`. `latin-1`, `ascii`, and `cp1252` are rejected. | | `-q`, `--quiet` | Hide non-essential status messages. | +| `--status` | Write one `fuckmark-status` line to stderr (result, insertions, sites, last_index, capped). | | `--no-color` | Disable color on stderr. `NO_COLOR` does the same. | `--non-interactive` is an alias of `--stdin`. @@ -94,9 +97,9 @@ Existing files are read as UTF-8 bytes with no newline conversion. LF, CRLF, CR, Tab, newline, carriage return, and ASCII space through tilde. Other Unicode is returned unchanged (exit 0). Exit 0 means I/O succeeded. It does not mean that a transformation occurred or that a watermark was removed. Only UTF-8 files. -Machine spans stay intact: fenced/inline code, markdown destinations, markdown reference labels, URLs, emails, IPs, dates, currency, percents, numbers, POSIX/Windows paths (including relative paths such as `src/main.py` and `C:/Users/...`), CLI flags. Quote interiors are eligible. Cap 192 insertion sites. Insertions fill the first 192 eligible letter sites and then stop, so the tail of a long document is unchanged. +Machine spans stay intact: fenced/inline/indented code, HTML tags and entities, markdown destinations (including multiline), markdown reference labels (including multiline, container, and CR line endings), URLs (including `ftp://`), emails, IPs, dates, currency, percents, numbers, POSIX/Windows paths (including `src/main.py`, `scripts/build`, `C:/Users/Alice/My final notes.txt`), CLI flags. Quote interiors are eligible. Cap 192 insertion sites. Insertions fill the first 192 eligible letter sites and then stop, so the tail of a long document is unchanged. -If the text is already transformed, or has no eligible letters, the CLI returns it unchanged. +If the text is already transformed, has no eligible letters, or is outside the ASCII domain, the CLI returns it unchanged and reports that on stderr unless `-q`. `--status` always reports the reason. Exit 0 still means I/O succeeded, not watermark removal. Same visible words are not the same as identical Markdown, path, or search behavior in other programs. Reports must not treat visible-projection equality as Markdown or filesystem equality. @@ -104,9 +107,11 @@ Same visible words are not the same as identical Markdown, path, or search behav | Status | Meaning | | ---: | ---: | -| 0 | Output was written or copied. Unsupported Unicode is returned unchanged. | -| 1 | No input, bad file, invalid UTF-8, unsupported encoding, missing `:done`, or output could not be written. | -| 2 | Transform succeeded, but clipboard copy failed. Stream modes still write stdout. The paste UI does not print the payload. | +| 0 | Output was written or copied. Unsupported Unicode, already-transformed text, no eligible sites, and the site cap are reported on stderr. This is not watermark removal. | +| 1 | No input, bad file, invalid UTF-8, unsupported encoding, missing `:done`, input too large, or output could not be written. | +| 2 | Usage error (`argparse`: unknown option or bad flags). Transformation did not run. | +| 3 | Transform I/O succeeded, but clipboard copy failed. Stream modes still write stdout. The paste UI does not print the payload. | +| 4 | Internal transform failure. Source is returned unchanged. Do not treat this as a completed transformation. | | 130 | Interrupted with Ctrl+C. | ## Clipboard diff --git a/docs/cycle8/gate-v2.md b/docs/cycle8/gate-v2.md new file mode 100644 index 00000000..26ecb897 --- /dev/null +++ b/docs/cycle8/gate-v2.md @@ -0,0 +1,7 @@ +# Gate v2 (archive pointer) + +Historical research write-up for publishability Gate v2. The machine-readable gate is `specs/cycle8/fuckmark-cycle8-publishability-gate-v2.json`. Confirmation scorecard: `evidence/cycle8-gate-v2-confirmation-2026-08-27/`. + +At confirmation time some READMEs said the public CLI remained empty. That sentence is historical. Live authorization is `cycle8-product-authorization-v1` / `release-cli-v5` letter-mix. Do not treat archive "CLI remains empty" lines as current product behavior. + +H16 local audit: `evidence/h16-local/README.md`. diff --git a/docs/cycle8/mix-second-model-transfer.md b/docs/cycle8/mix-second-model-transfer.md new file mode 100644 index 00000000..5fa48469 --- /dev/null +++ b/docs/cycle8/mix-second-model-transfer.md @@ -0,0 +1,7 @@ +# Mix second-model transfer (archive pointer) + +Historical pointer for the DistilGPT2 n=16 mix transfer (`evidence/cycle8-mix-distilgpt2-1090000-n16-2026-08-27/`). + +Status: `HYPOTHESIS`. Not mix-freeze confirmation. Not confirmation-scale. The tokenizer is still GPT-2 BPE, so this is not an independent tokenizer-family validation. + +See `docs/research.md` and `docs/limits.md`. diff --git a/docs/install.md b/docs/install.md index 2218b2c3..3c3a1cfa 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2,30 +2,50 @@ Python 3.11 or newer. The public CLI is `fuckmark` / `FuckMark` / `Fuckmark`. -Do not pipe `https://d.q1z.org/mark` into a shell. +Do not pipe `https://d.q1z.org/mark` into a shell. The live website must match [`website.md`](website.md). + +This source tree is **0.4.1**. The last published GitHub Release wheel is **v0.4.0**. That wheel does not implement `--text` or `--file`. Do not retag v0.4.0. The v0.4.1 wheel SHA-256 is recorded after the GitHub Release exists. ## From this repository (works now) +Linux / macOS, from a clean shell with no FuckMark on PATH: + ```text +git clone https://github.com/byte271/FuckMark.git +cd FuckMark python3 -m venv .venv .venv/bin/python -m pip install . .venv/bin/fuckmark --version +printf 'I do not agree.\n' | .venv/bin/fuckmark --visible +``` + +Keep using `.venv/bin/fuckmark` unless you activate the venv: + +```text +. .venv/bin/activate +fuckmark --version ``` -On Windows, create the venv with `py -3.12 -m venv .venv` and use `.venv\Scripts\python.exe`. +Windows PowerShell: -`python3 -m pip install git+https://github.com/byte271/FuckMark.git` installs the same product from `main`. +```text +git clone https://github.com/byte271/FuckMark.git +cd FuckMark +py -3.12 -m venv .venv +.venv\Scripts\python.exe -m pip install . +.venv\Scripts\fuckmark.exe --version +``` + +`python3 -m pip install git+https://github.com/byte271/FuckMark.git` installs the same product from `main` after this branch merges. ## Tagged wheel -v0.4.0 wheel SHA-256: +v0.4.0 wheel SHA-256 (last published GitHub Release; missing `--text` / `--file`): ```text 5a6ac62c8bb8d7ddd9e5bc9cb6cee6e3eb181ac5f397b4a6645ef86468ee932f fuckmark-0.4.0-py3-none-any.whl ``` -The GitHub Release wheel is the checksummed install. - ```text python3 -m venv .venv .venv/bin/python -m pip install https://github.com/byte271/FuckMark/releases/download/v0.4.0/fuckmark-0.4.0-py3-none-any.whl @@ -47,7 +67,7 @@ That tag is not retagged. ## In-repo installer (tagged wheel + checksum) -These scripts download the GitHub Release wheel, verify `SHA256SUMS.txt`, install into a user virtualenv, and print `fuckmark --help`. They do not start the CLI, do not use sudo, and do not install Python. They default to `v0.4.0`. +These scripts download the GitHub Release wheel, verify `SHA256SUMS.txt`, install into a user virtualenv, and print `fuckmark --help`. They do not start the CLI, do not use sudo, and do not install Python. They default to `v0.4.1` once that GitHub Release exists. Until then, install from this clone as above. Override: `FUCKMARK_RELEASE_TAG=v0.4.0` for the last published wheel. Linux / macOS, from a clone: @@ -61,18 +81,21 @@ Windows PowerShell, from a clone: powershell -ExecutionPolicy Bypass -File tools/install/windows.ps1 ``` -Optional override: `FUCKMARK_RELEASE_TAG=v0.4.0`. +`FUCKMARK_BIN` is the launcher directory and the directory added to PATH. `FUCKMARK_HOME` is the venv root. Windows launchers are UTF-16 LE so Unicode profile paths are preserved. ## Verify ```text -fuckmark --version -printf 'I do not agree.\n' | fuckmark --visible +.venv/bin/fuckmark --version +printf 'I do not agree.\n' | .venv/bin/fuckmark --visible +printf 'I do not agree.\n' | .venv/bin/fuckmark --status >/tmp/fm.out ``` -`--version` must print `FuckMark 0.4.0`. `--visible` must print `I do not agree.` +This tree prints `FuckMark 0.4.1`. `--visible` prints `I do not agree.` `--status` writes a `fuckmark-status` line to stderr. Installation success is not watermark removal. + +In a terminal, `fuckmark` with no arguments opens the paste UI. Finish with `:done`. The result is copied, not printed. Stderr says whether hidden characters were inserted. -In a terminal, `fuckmark` with no arguments opens the paste UI. Finish with `:done`. The result is copied, not printed. +If clipboard copy fails, pipe text: `printf 'I do not agree.\n' | .venv/bin/fuckmark`. ## Development @@ -83,8 +106,8 @@ python -m pytest ## Troubleshooting -If the command is not found, open a new terminal so PATH updates load. +If the command is not found, use the venv path from the install step, or open a new terminal so PATH updates load. -On Linux, clipboard copy needs `wl-copy`, `xclip`, `xsel`, or `clip.exe`. Stream `--copy` still prints the text if copy fails (exit 2). The paste UI does not print the payload; pipe text if you need stdout. +On Linux, clipboard copy needs `wl-copy`, `xclip`, `xsel`, or `clip.exe`. Stream `--copy` still prints the text if copy fails (exit 3). The paste UI does not print the payload; pipe text if you need stdout. -Website: [mark.q1z.org](https://mark.q1z.org). +Website: [mark.q1z.org](https://mark.q1z.org). Controlled copy: [`website.md`](website.md). diff --git a/docs/limits.md b/docs/limits.md new file mode 100644 index 00000000..f8bd818b --- /dev/null +++ b/docs/limits.md @@ -0,0 +1,74 @@ +# Supported scope and failure matrix + +This page is the public limit list for FuckMark 0.4.1. Frozen detector numbers are not regenerated here. Do not retune on spent confirmation corpora. + +## L01 — Sanitizers + +| Arm | Restores source? | Gate v2 mix WM (frozen) | +| --- | --- | --- | +| raw | no | 0/192 | +| nfc | no | 0/192 | +| nfkc | no | 0/192 | +| cf_strip | no | 0/192 | +| nfkc_cf_strip | no | 0/192 | +| ws_collapse | no | 0/192 | +| ws_collapse_nfkc_cf_strip | no | 0/192 | +| lm-watermarking UnicodeSanitizer | no (mangles; does not reconstruct) | 0/192 | +| Mn-strip / combining-mark strip | **yes** | 188/192 (diagnostic restore) | +| default-ignorable strip | **yes** | 188/192 (diagnostic restore) | + +If `R(T(x)) = x` and a detector sees `R(T(x))`, it sees the original watermarked source. The insertion-only mechanism cannot claim immunity to that removal. Demonstrations must show both the successful required-sanitizer paths and this source-restoring failure path. + +## L02 — Input domain + +Supported code points: U+0009, U+000A, U+000D, U+0020..U+007E. + +| Input | Behavior | +| --- | --- | +| `I do not agree.` | transformed (exit 0, reason `transformed`) | +| `I don't agree.` (U+2019) | unchanged (exit 0, reason `unsupported-domain`) | +| `Hello 😀` | unchanged | +| `café` | unchanged | +| BOM, NBSP, CJK | unchanged | +| already contains U+034F or U+FE00 | unchanged (`already-transformed`) | + +The product does not strip a BOM, normalize accents, or transliterate to force eligibility. + +## L03 — Length and the 192-site cap + +Only the first 192 eligible ASCII-letter sites receive insertions. For a long document, report `--status` fields `sites`, `last_index`, and `capped=yes`. A 9,000-character paragraph sequence can stop near source index 245, leaving thousands of trailing characters unchanged. That is a coverage limit, not a long-document detection result. + +## L04 — Detector / model / tokenizer evidence + +| Family | Model | Tokenizer | n | Label | +| --- | --- | --- | --- | --- | +| Gate v2 confirmation | GPT-2 | GPT-2 BPE | 192 pairs | VERIFIED on that protocol | +| DeepMind 30-key transfer | GPT-2 | GPT-2 BPE | 192 pairs | HYPOTHESIS / configuration check | +| DistilGPT2 n=16 | DistilGPT2 | GPT-2 BPE | 16 pairs | HYPOTHESIS; not a second tokenizer family | + +Zero detections in a finite set is not a universal zero-rate. Not claimed: generic AI-authorship classifiers, unknown proprietary detectors, every SynthID deployment, C2PA. + +## L05 — Downstream behavior + +| Path | Status | +| --- | --- | +| Raw substring search for `do not` after mix | fails (proven) | +| `visible_contains()` | matches visible text | +| Other editors, screen readers, clipboard apps | not all measured; do not assert they all fail | + +`--visible` is the reversible plain-text export. `--status` reports the outcome without inspecting invisible characters. + +## L06 — Token overhead (GPT-2, stored Gate v2 watermarked geometry) + +| Quantity | Stored value | +| --- | --- | +| Original tokens | 58–64, mean 63.90 | +| Transformed tokens | 139–605, mean 566.61 | +| Mean ratio | about 8.87x | +| Bytes added at 192 insertions | 480 UTF-8 bytes | + +These GPT-2 ratios are not measurements for unrelated commercial tokenizers. + +## P01 — Size budget + +Maximum product input: 2,000,000 characters. Larger input is a documented error (exit 1), not a silent no-op. Bracket-heavy scans precompute line starts so they do not grow as repeated prefix copies. diff --git a/docs/product-contract.md b/docs/product-contract.md index 2c970c6e..541bff40 100644 --- a/docs/product-contract.md +++ b/docs/product-contract.md @@ -37,7 +37,7 @@ These are not product success: ## Current product path -`release_transform_registry()` stays empty. The public CLI (`release-cli-v5`) applies `apply_letter_alternating_mix` directly: U+034F on even selected ASCII-letter sites, U+FE00 on odd sites, cap 192, outside hard machine spans. Live protection now also covers common relative paths without a leading `./`, Windows paths that use `/`, and Markdown reference labels at both uses and definitions. Those protection changes can change outputs versus the v0.4.0 freeze of `letter_mix.py` (`cycle8-mix-freeze-v1`, `letter_mix_source_sha256` `b1ceec24e584c0e9e7135ef0c89a3bd249b0bda4a45e07aa7190b1b010ba56d4`). Frozen confirmation hashes stay historical. They are not silently rewritten. +`release_transform_registry()` stays empty. The public CLI (`release-cli-v5`) applies `apply_letter_alternating_mix` directly: U+034F on even selected ASCII-letter sites, U+FE00 on odd sites, cap 192, outside hard machine spans. Live protection covers fenced/inline/indented code, HTML tags and entities, Markdown destinations and reference labels (including multiline and container forms), URLs including `ftp://`, and paths including `src/main.py`, `scripts/build`, and last-component spaces. Those protection changes can change outputs versus the v0.4.0 freeze of `letter_mix.py` (`cycle8-mix-freeze-v1`, `letter_mix_source_sha256` `b1ceec24e584c0e9e7135ef0c89a3bd249b0bda4a45e07aa7190b1b010ba56d4`). Frozen confirmation hashes stay historical. They are not silently rewritten. Live `product_approved_carriers_v1()` is `{U+034F, U+FE00}`. The v1 contract's empty `product_authorized_carriers_v1` list is a historical snapshot of that file. diff --git a/docs/release.md b/docs/release.md index 5452f73f..fa5c2edc 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,10 +1,10 @@ -# v0.4.0 Release Process +# v0.4.1 Release Process -FuckMark v0.4.0 is the Gate v2 product-authorization release. The public CLI algorithm identity is `release-cli-v5`. It applies frozen `u034f-ufe00-letter-alt-v1` while keeping `release_transform_registry()` empty. The historical `v0.3.0` tag is the identity CLI (`release-cli-v4`). The historical `v0.2.0` tag still applies contractions. Do not retag `v0.3.0`. +FuckMark v0.4.1 is the audit-fix release on top of the Gate v2 product-authorization CLI. The public CLI algorithm identity remains `release-cli-v5`. Do not retag `v0.4.0`. The historical `v0.3.0` tag is the identity CLI (`release-cli-v4`). The historical `v0.2.0` tag still applies contractions. ## Release invariants -1. `pyproject.toml`, `fuckmark.__version__`, `uv.lock`, installed CLI output, and the Git tag must agree on `0.4.0` / `v0.4.0`. +1. `pyproject.toml`, `fuckmark.__version__`, installed CLI output, and the Git tag must agree on `0.4.1` / `v0.4.1`. 2. The release registry is not broadened merely because a development experiment succeeds. Product authorization additionally requires exact visible-projection equality and the frozen mix apply path. 3. U+200C remains diagnostic-only and is never promoted into automatic CLI behavior. 4. Visible-edit historical catalogs (contractions, Cycle 6 spacing, Cycle 7 durable families) remain replayable and must not silently become release defaults. @@ -12,13 +12,16 @@ FuckMark v0.4.0 is the Gate v2 product-authorization release. The public CLI alg 6. The release must build and clean-install on Linux, macOS, and Windows. 7. Every published GitHub Release must be backed by an immutable `v*` tag. 8. The v1 mix sanitizer gate stays FAIL. `required_sanitizers_keep` is not weakened. Do not generate `950000`. +9. CI for the published artifact must exercise `--text`, `--file`, `--status`, and the documented exit statuses, not only a piped sentence. ## Required release sequence -1. Merge this v0.4.0 branch to green `main`. Clone install (`python -m pip install .`) already runs the product CLI from that tree. The GitHub Release wheel URL is valid only after the next two steps. +1. Merge this v0.4.1 branch to green `main`. Clone install (`python -m pip install .`) already runs the product CLI from that tree. The GitHub Release wheel URL is valid only after the next two steps. 2. On the resulting `main` push, rerun the cross-platform package matrix. That push must **not** create tags, publish a GitHub Release, or delete branches. -3. Create and push the immutable `v0.4.0` tag on that merge commit yourself. The workflow never runs `git tag`. Do not tag a pull-request SHA if `main` will be a squash merge. -4. Publish with `workflow_dispatch` and `publish_github_release=true` on that same commit. The job refuses to run unless `v0.4.0` already exists and points at the dispatch SHA. If the GitHub Release is missing, it uploads the verified wheel and sdist. It does not delete merged branches. +3. Create and push the immutable `v0.4.1` tag on that merge commit yourself. The workflow never runs `git tag`. Do not tag a pull-request SHA if `main` will be a squash merge. Do not retag `v0.4.0`. +4. Publish with `workflow_dispatch` and `publish_github_release=true` on that same commit. The job refuses to run unless `v0.4.1` already exists and points at the dispatch SHA. If the GitHub Release is missing, it uploads the verified wheel and sdist. It does not delete merged branches. +5. Record the published v0.4.1 wheel SHA-256 in README and `docs/install.md` in a follow-up commit, as was done for v0.4.0. +6. Deploy [mark.q1z.org](https://mark.q1z.org) and `https://d.q1z.org/mark` from `docs/website.md`, including the PowerShell User-Agent response. ## Publication workflow @@ -30,14 +33,14 @@ Direct tag pushes do not trigger the workflow. There is no hard-coded release ta ## Claim boundary -Gate v2 confirmation is GPT-2 / Hugging Face SynthID Weighted Mean plus the real UnicodeSanitizer, with existing DeepMind 30-key GPT-2 transfer. It is not a universal watermark-removal guarantee. Mn-strip and default-ignorable-strip still restore mix. DistilGPT2 n=16 and mean-versus-weighted-mean remain `HYPOTHESIS`. +Gate v2 confirmation is GPT-2 / Hugging Face SynthID Weighted Mean plus the real UnicodeSanitizer, with existing DeepMind 30-key GPT-2 transfer. It is not a universal watermark-removal guarantee. Mn-strip and default-ignorable-strip still restore mix. DistilGPT2 n=16 and mean-versus-weighted-mean remain `HYPOTHESIS`. See `docs/limits.md`. ## Historical boundary -`specs/fuckmark-v0.1.0-release-readiness-baseline.json` remains an immutable historical artifact. v0.4.0 does not rewrite the old baseline's gate statuses or reinterpret the repository state it captured. +`specs/fuckmark-v0.1.0-release-readiness-baseline.json` remains an immutable historical artifact. v0.4.1 does not rewrite the old baseline's gate statuses or reinterpret the repository state it captured. -The `v0.3.0` GitHub Release remains the historical identity CLI. The `v0.2.0` GitHub Release remains the historical contraction CLI. Current product behavior is v0.4.0 letter-mix. +The `v0.4.0` GitHub Release remains the first product-CLI publication (without later `--text` / `--file` and protection fixes). The `v0.3.0` GitHub Release remains the historical identity CLI. The `v0.2.0` GitHub Release remains the historical contraction CLI. ## Distribution scope -The GitHub Release is the authoritative v0.4.0 publication produced by this repository workflow. Publishing to another package index is a separate action and must not be implied unless that publication actually occurs and its credentials/workflow are configured. +The GitHub Release is the authoritative v0.4.1 publication produced by this repository workflow. Publishing to another package index is a separate action and must not be implied unless that publication actually occurs and its credentials/workflow are configured. diff --git a/docs/research.md b/docs/research.md index 70f5c8a5..7fad64fb 100644 --- a/docs/research.md +++ b/docs/research.md @@ -1,11 +1,35 @@ # Research archive -This directory's user manuals are [`install.md`](install.md) and [`cli.md`](cli.md). +This directory's user manuals are [`install.md`](install.md), [`cli.md`](cli.md), [`limits.md`](limits.md), and [`website.md`](website.md). -Frozen scientific records are not the user manual. They stay in the repository so the product claims can be audited: +Frozen scientific records are not the user manual. They stay in the repository so the product claims can be audited. -- [`evidence/`](../evidence/) — confirmation corpora, scorecards, and hashes -- [`specs/`](../specs/) — frozen contracts and protocol JSON -- [`evidence/frozen-spec-revision-2/spec.md`](../evidence/frozen-spec-revision-2/spec.md) — historical research specification (old project name; not the product identity) +## Live versus historical authorization + +| Record | What it is | CLI at that time | +| --- | --- | --- | +| Gate v2 confirmation READMEs | Frozen 188/192 identity and 0/192 mix after required sanitizers | Historical sentence "CLI remains empty" / `process_text` identity **at confirmation time**. Live product is authorized letter-mix. | +| `cycle8-product-authorization-v1` | Live authorization | Public CLI applies mix (`release-cli-v5`) | +| mix-freeze / mix-confirmation 830000+ | Historical 0/192 on earlier seeds | Not rewritten | + +## Evidence index + +- Gate v2 confirmation: [`evidence/cycle8-gate-v2-confirmation-2026-08-27/README.md`](../evidence/cycle8-gate-v2-confirmation-2026-08-27/README.md) +- DeepMind 30-key transfer: [`evidence/cycle8-mix-deepmind-transfer-2026-08-27/README.md`](../evidence/cycle8-mix-deepmind-transfer-2026-08-27/README.md) +- DistilGPT2 n=16: [`evidence/cycle8-mix-distilgpt2-1090000-n16-2026-08-27/README.md`](../evidence/cycle8-mix-distilgpt2-1090000-n16-2026-08-27/README.md) +- H16 sanitizer audit: [`evidence/h16-local/README.md`](../evidence/h16-local/README.md) +- F01–F07 render/path notes: [`evidence/audit-fixes-2026-08-27/README.md`](../evidence/audit-fixes-2026-08-27/README.md) +- 0.4.1 audit follow-up: [`evidence/audit-fixes-2026-08-28/README.md`](../evidence/audit-fixes-2026-08-28/README.md) + +Cycle 8 archive pointers: + +- [`cycle8/gate-v2.md`](cycle8/gate-v2.md) +- [`cycle8/mix-second-model-transfer.md`](cycle8/mix-second-model-transfer.md) + +Also: + +- [`../evidence/`](../evidence/) — confirmation corpora, scorecards, and hashes +- [`../specs/`](../specs/) — frozen contracts and protocol JSON +- [`../evidence/frozen-spec-revision-2/spec.md`](../evidence/frozen-spec-revision-2/spec.md) — historical research specification (old project name; not the product identity) Do not treat those files as installation or usage docs. diff --git a/docs/website.md b/docs/website.md new file mode 100644 index 00000000..003c9609 --- /dev/null +++ b/docs/website.md @@ -0,0 +1,37 @@ +# Website and public installer copy + +This file is the controlled source for [mark.q1z.org](https://mark.q1z.org) and the `https://d.q1z.org/mark` endpoints. Repository CI cannot update those hosts. The live HTML and User-Agent-specific PowerShell response must match this page after each release. + +Do not pipe `https://d.q1z.org/mark` into a shell. Do not invoke that URL with `iex`. Those commands are disallowed. + +## What the product does + +FuckMark inserts U+034F and U+FE00 into ordinary English ASCII text without changing the visible words. v0.4.1 (this tree) already inserts those characters. It does not return the input unchanged until a future release. + +It is not a general watermark remover. + +## Install (Linux / macOS) + +```text +git clone https://github.com/byte271/FuckMark.git +cd FuckMark +python3 -m venv .venv +.venv/bin/python -m pip install . +.venv/bin/fuckmark --version +``` + +## Install (Windows) + +```text +git clone https://github.com/byte271/FuckMark.git +cd FuckMark +py -3.12 -m venv .venv +.venv\Scripts\python.exe -m pip install . +.venv\Scripts\fuckmark.exe --version +``` + +Tagged wheel: GitHub Release `SHA256SUMS.txt` only. Last published wheel: v0.4.0. Source tree: 0.4.1. + +## Results to show, with boundaries + +Gate v2 confirmation (GPT-2 / SynthID, 64-token samples): unmodified watermarked **188/192**; transformed after required sanitizers **0/192**; visible text **192/192**. Stripping combining marks or default-ignorable characters restores the source and **188/192** detections. English ASCII only. Cap 192 letter sites. See [`limits.md`](limits.md). diff --git a/evidence/audit-fixes-2026-08-28/README.md b/evidence/audit-fixes-2026-08-28/README.md new file mode 100644 index 00000000..df5448af --- /dev/null +++ b/evidence/audit-fixes-2026-08-28/README.md @@ -0,0 +1,23 @@ +# Audit follow-up (2026-08-28) + +This directory records 0.4.1 audit follow-up that does not rewrite frozen confirmation files. + +Audited main: `d7dc98b7ee99fc46b767c93444a9f051fdbea2ba`. + +## D16 + +`evidence/cycle7-stage-a-2026-08-25/README.md` is LF in Git. The historical `SHA256SUMS.txt` digest is the CRLF encoding of the same text. See `evidence/cycle7-stage-a-2026-08-25/CHECKSUM-PROVENANCE.md`. The original manifest is unchanged. + +## E01 + +Transfer tests now compare `live_mix_hash(source)` to the stored `mix.text_sha256`. Permitted drift is listed in `tests/audit_mix_replay.py` with old and new hashes. Gate v2 confirmation live mix hashes still match all 384 stored transformed hashes. Unexplained drift fails the test. Stored detector scores are not inherited for drifted samples. + +## E02 / E03 + +`render-v2.json` in `evidence/audit-fixes-2026-08-27/` remains the frozen AAAA/BBBB control harness. It is not rewritten. + +`render-payload-v2.json` in this directory is a dated this-host Chromium measurement of the actual U+034F/U+FE00 mix sentence `I do not agree.` on `pre`, `textarea`, and `contenteditable`, plus AAAA negative/positive controls and a below-fold pair that must be `INCOMPLETE` rather than VERIFIED. Safari/WebKit and terminal pixels remain UNKNOWN. Viewport equality is only claimed for the captured window. No UNKNOWN-to-PASS promotion. + +## E04 + +Restored archive pointers: `docs/cycle8/gate-v2.md` and `docs/cycle8/mix-second-model-transfer.md`. Current map: `docs/research.md`. diff --git a/evidence/audit-fixes-2026-08-28/SHA256SUMS.txt b/evidence/audit-fixes-2026-08-28/SHA256SUMS.txt new file mode 100644 index 00000000..f0818743 --- /dev/null +++ b/evidence/audit-fixes-2026-08-28/SHA256SUMS.txt @@ -0,0 +1,2 @@ +ccebd27402ba8d73da2f622772d80dd05ecade51026694bdbc1a4fc13a0a61bb README.md +c71121e5d8b21cb625d4484ef32fc312f737afc4c47337c77fb62cd448306970 render-payload-v2.json diff --git a/evidence/audit-fixes-2026-08-28/render-payload-v2.json b/evidence/audit-fixes-2026-08-28/render-payload-v2.json new file mode 100644 index 00000000..d1949d89 --- /dev/null +++ b/evidence/audit-fixes-2026-08-28/render-payload-v2.json @@ -0,0 +1,126 @@ +{ + "algorithm_version": "product-reference-render-v2", + "browser_version": "Google Chrome 148.0.7778.96", + "chrome_executable": "/usr/local/bin/google-chrome", + "font": "DejaVu Sans Mono", + "harness": "product-reference-render-v2", + "historical_render_v2_controls_not_rewritten": true, + "historical_evidence_invalid_because": "cycle8-benchmark-render-v1 compared blank contenteditable divs. Frozen AAAA/BBBB controls remain in evidence/audit-fixes-2026-08-27/render-v2.json.", + "measurement_scope": "this-host Chromium headless pre/textarea/contenteditable for one mix sentence plus AAAA controls and a below-fold INCOMPLETE check. Not Safari/WebKit. Not terminal pixels. Not whole-document proof beyond the captured window.", + "platform": "linux", + "rows": [ + { + "pair": "mix_payload", + "surface": "pre", + "original": "I do not agree.", + "transformed": "I͏ d︀o͏ n︀o͏t︀ a͏g︀r͏e︀e͏.", + "result": { + "algorithm_version": "product-reference-render-v2", + "environment": "chromium_headless", + "status": "VERIFIED", + "original_sha256": "3067cc798b2263d2f6c31afb17b24bb0f421aa505d406145b423e681d6a037fe", + "transformed_sha256": "5c0409ead4bf62c9ba0e3a5f82b98cba18bc36ae6c85edad380f1e0d3f018ef5", + "equal": true, + "detail": "png_bytes_equal", + "comparison_hash": "3a64fa1f51738d9415b3a3fffa4b1de12d221467846d9fc47df00468e1737ff8", + "font": "DejaVu Sans Mono", + "browser_version": "Google Chrome 148.0.7778.96" + } + }, + { + "pair": "mix_payload", + "surface": "textarea", + "original": "I do not agree.", + "transformed": "I͏ d︀o͏ n︀o͏t︀ a͏g︀r͏e︀e͏.", + "result": { + "algorithm_version": "cycle8-benchmark-render-v2", + "environment": "chromium_textarea", + "status": "VERIFIED", + "original_sha256": "3067cc798b2263d2f6c31afb17b24bb0f421aa505d406145b423e681d6a037fe", + "transformed_sha256": "5c0409ead4bf62c9ba0e3a5f82b98cba18bc36ae6c85edad380f1e0d3f018ef5", + "equal": true, + "detail": "png_bytes_equal", + "font": "DejaVu Sans Mono", + "browser_version": "Google Chrome 148.0.7778.96", + "comparison_hash": "4f6f681d0b49c3f4fb0cd81e88c3992ec2bdff9e5d26c830ee757aab0e8a656f" + } + }, + { + "pair": "mix_payload", + "surface": "contenteditable", + "original": "I do not agree.", + "transformed": "I͏ d︀o͏ n︀o͏t︀ a͏g︀r͏e︀e͏.", + "result": { + "algorithm_version": "cycle8-benchmark-render-v2", + "environment": "chromium_contenteditable", + "status": "VERIFIED", + "original_sha256": "3067cc798b2263d2f6c31afb17b24bb0f421aa505d406145b423e681d6a037fe", + "transformed_sha256": "5c0409ead4bf62c9ba0e3a5f82b98cba18bc36ae6c85edad380f1e0d3f018ef5", + "equal": true, + "detail": "png_bytes_equal", + "font": "DejaVu Sans Mono", + "browser_version": "Google Chrome 148.0.7778.96", + "comparison_hash": "e0c0d730a01ecaaed5d9b7f3129246a72bfe568bfb344d7ff425575d9ae7bc0d" + } + }, + { + "pair": "negative", + "surface": "contenteditable", + "original": "AAAA", + "transformed": "BBBB", + "result": { + "algorithm_version": "cycle8-benchmark-render-v2", + "environment": "chromium_contenteditable", + "status": "REJECTED", + "original_sha256": "63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201", + "transformed_sha256": "4a8d8134f29b0b7b60c126f5532bc9f5d9bb73037373cf6fb872d81f1dcefdfd", + "equal": false, + "detail": "png_bytes_differ", + "font": "DejaVu Sans Mono", + "browser_version": "Google Chrome 148.0.7778.96", + "comparison_hash": "7363f35290941fe43459f5250fa09a1cfc0cc6870522fee1d032661c743ce45e" + } + }, + { + "pair": "positive", + "surface": "textarea", + "original": "AAAA", + "transformed": "AAAA", + "result": { + "algorithm_version": "cycle8-benchmark-render-v2", + "environment": "chromium_textarea", + "status": "VERIFIED", + "original_sha256": "63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201", + "transformed_sha256": "63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201", + "equal": true, + "detail": "png_bytes_equal", + "font": "DejaVu Sans Mono", + "browser_version": "Google Chrome 148.0.7778.96", + "comparison_hash": "c8e0a881d1f514214edfa79d535bdce7de77e11c833bcf44e636c2c7246fb87b" + } + }, + { + "pair": "below_fold", + "surface": "pre", + "original_sha256": "4677942dfa3e74b5dea7484661a2485bb73ba422eb72d311fdb39372c019c615", + "transformed_sha256": "307dffe4cb27b00adc53d540a7a2aa91d234515e49955354a85eabd14bc13b2c", + "original_chars": 5, + "transformed_chars": 2005, + "result": { + "algorithm_version": "product-reference-render-v2", + "environment": "chromium_headless", + "status": "INCOMPLETE", + "original_sha256": "4677942dfa3e74b5dea7484661a2485bb73ba422eb72d311fdb39372c019c615", + "transformed_sha256": "307dffe4cb27b00adc53d540a7a2aa91d234515e49955354a85eabd14bc13b2c", + "equal": null, + "detail": "content exceeds captured window", + "comparison_hash": "b8f1912f139aa2ed85d2271304b5a5e2c10ebf2e61cc3c2705833ca4fc23c925", + "font": "DejaVu Sans Mono", + "browser_version": "Google Chrome 148.0.7778.96" + } + } + ], + "terminal_pixels": "UNKNOWN", + "webkit_safari": "UNKNOWN", + "artifact_hash": "0266d1097518040e660f4cb851fa3e44d7baaa0b4e24f9a49f21474e01547179" +} diff --git a/evidence/cycle7-stage-a-2026-08-25/CHECKSUM-PROVENANCE.md b/evidence/cycle7-stage-a-2026-08-25/CHECKSUM-PROVENANCE.md new file mode 100644 index 00000000..4a6ec264 --- /dev/null +++ b/evidence/cycle7-stage-a-2026-08-25/CHECKSUM-PROVENANCE.md @@ -0,0 +1,15 @@ +# Cycle 7 Stage A README checksum provenance + +The frozen `SHA256SUMS.txt` in this directory lists: + +```text +833d1d94fcea01bc09855f0af3a4bcc18df1010065fb3e2825f8e2e31733c2eb README.md +``` + +The Git object and working tree store the same README with LF line endings. SHA-256 of those LF bytes is: + +```text +d7b7de433f5ca231ef2d9de586774ffdfe255b1625f902fa0bf9780019d62709 +``` + +SHA-256 of the same text encoded with CRLF is the digest in `SHA256SUMS.txt`. The scientific text did not change. The original manifest is not rewritten. CI treats this README as a documented CRLF-vs-LF exception: the listed digest must match the CRLF reconstruction, and the Git bytes must stay LF. diff --git a/fuckmark/__init__.py b/fuckmark/__init__.py index e82f808d..3f71d2d8 100644 --- a/fuckmark/__init__.py +++ b/fuckmark/__init__.py @@ -1,5 +1,5 @@ __project_name__ = "FuckMark" -__version__ = "0.4.0" +__version__ = "0.4.1" from .adapters import ( DEEPMIND_REFERENCE_ADAPTER_ID, diff --git a/fuckmark/cli.py b/fuckmark/cli.py index b9a18c6a..1dad1aff 100644 --- a/fuckmark/cli.py +++ b/fuckmark/cli.py @@ -13,8 +13,13 @@ from typing import TextIO from . import __project_name__, __version__ -from .cycle8.letter_mix import LETTER_MIX_APPROVED_CARRIERS, apply_letter_alternating_mix -from .product.domain import is_supported_product_domain_v1 +from .cycle8.letter_mix import ( + LETTER_MIX_APPROVED_CARRIERS, + LETTER_MIX_MAX_SELECTED, + compose_letter_mix, + select_letter_mix_sites, +) +from .product.domain import PRODUCT_MAX_INPUT_CHARS, is_supported_product_domain_v1 from .product.encodings import require_supported_product_encoding from .product.visible_projection import ( is_carrier_insertion_v1, @@ -37,10 +42,29 @@ class ClipboardUnavailableError(RuntimeError): pass +EXIT_OK = 0 +EXIT_ERROR = 1 +EXIT_USAGE = 2 +EXIT_CLIPBOARD = 3 +EXIT_INTERNAL = 4 +EXIT_INTERRUPT = 130 +REASON_TRANSFORMED = "transformed" +REASON_SITE_CAP = "site-cap" +REASON_UNSUPPORTED_DOMAIN = "unsupported-domain" +REASON_ALREADY_TRANSFORMED = "already-transformed" +REASON_NO_ELIGIBLE_SITES = "no-eligible-sites" +REASON_INTERNAL_ERROR = "internal-error" +REASON_TOO_LARGE = "too-large" + + @dataclass(frozen=True, slots=True) class ProcessResult: output_text: str change_count: int + reason: str = REASON_TRANSFORMED + last_source_index: int | None = None + site_count: int = 0 + capped: bool = False @property def changed(self) -> bool: @@ -51,23 +75,38 @@ def transform_text(text: str) -> ProcessResult: if not isinstance(text, str): raise TypeError("text must be a string") try: + if len(text) > PRODUCT_MAX_INPUT_CHARS: + return ProcessResult(text, 0, reason=REASON_TOO_LARGE) approved = product_approved_carriers_v1() if approved != frozenset(LETTER_MIX_APPROVED_CARRIERS): - return ProcessResult(text, 0) - if not is_supported_product_domain_v1(text): - return ProcessResult(text, 0) + return ProcessResult(text, 0, reason=REASON_INTERNAL_ERROR) if any(ord(character) in approved for character in text): - return ProcessResult(text, 0) - applied = apply_letter_alternating_mix(text) + return ProcessResult(text, 0, reason=REASON_ALREADY_TRANSFORMED) + if not is_supported_product_domain_v1(text): + return ProcessResult(text, 0, reason=REASON_UNSUPPORTED_DOMAIN) + probe = select_letter_mix_sites(text, max_selected=LETTER_MIX_MAX_SELECTED + 1) + capped = len(probe) > LETTER_MIX_MAX_SELECTED + sites = probe[:LETTER_MIX_MAX_SELECTED] + if not sites: + return ProcessResult(text, 0, reason=REASON_NO_ELIGIBLE_SITES) + applied = compose_letter_mix(text, sites) if applied == text: - return ProcessResult(text, 0) + return ProcessResult(text, 0, reason=REASON_NO_ELIGIBLE_SITES) if not is_carrier_insertion_v1(text, applied, approved): - return ProcessResult(text, 0) + return ProcessResult(text, 0, reason=REASON_INTERNAL_ERROR) if project_visible_v1(applied, approved) != text: - return ProcessResult(text, 0) - return ProcessResult(applied, len(applied) - len(text)) + return ProcessResult(text, 0, reason=REASON_INTERNAL_ERROR) + reason = REASON_SITE_CAP if capped else REASON_TRANSFORMED + return ProcessResult( + applied, + len(applied) - len(text), + reason=reason, + last_source_index=sites[-1], + site_count=len(sites), + capped=capped, + ) except (KeyError, TypeError, ValueError, RuntimeError): - return ProcessResult(text, 0) + return ProcessResult(text, 0, reason=REASON_INTERNAL_ERROR) def process_text(text: str) -> str: @@ -122,9 +161,10 @@ def _parser() -> argparse.ArgumentParser: "\n" "Supported input: tab, newline, carriage return, and ASCII space through tilde.\n" "Other Unicode is returned unchanged with exit 0. That status means I/O\n" - "succeeded, not that a transformation occurred. Only UTF-8 is supported.\n" + "succeeded, not that hidden characters were inserted. Only UTF-8 is supported.\n" "Visible text stays the same. Use --visible to print that visible text.\n" - "Use --text for literal strings and --file for existing UTF-8 files." + "Use --text for literal strings and --file for existing UTF-8 files.\n" + "Use --status for a machine-readable outcome on stderr." ), formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -186,6 +226,12 @@ def _parser() -> argparse.ArgumentParser: action="store_true", help="hide non-essential status messages", ) + parser.add_argument( + "--status", + dest="status_report", + action="store_true", + help="write a machine-readable outcome line to stderr", + ) parser.add_argument( "--no-color", action="store_true", @@ -340,19 +386,49 @@ def _ensure_utf8(stream: TextIO) -> None: return +def _abandon_stdio(stream: TextIO) -> None: + try: + handle = stream.fileno() + except (OSError, ValueError, AttributeError): + return + try: + stream.flush() + except (OSError, ValueError, UnicodeError, BrokenPipeError): + pass + try: + null_fd = os.open(os.devnull, os.O_WRONLY) + try: + os.dup2(null_fd, handle) + finally: + os.close(null_fd) + except OSError: + pass + try: + stream.close() + except (OSError, ValueError): + pass + + def _write_stdout_utf8(output_stream: TextIO, text: str) -> None: buffer = getattr(output_stream, "buffer", None) - if buffer is not None: - try: - output_stream.flush() - except UnicodeError: - pass - buffer.write(text.encode("utf-8")) - buffer.flush() - return try: + if buffer is not None: + try: + output_stream.flush() + except UnicodeError: + pass + buffer.write(text.encode("utf-8")) + buffer.flush() + return output_stream.write(text) output_stream.flush() + except BrokenPipeError as error: + _abandon_stdio(output_stream) + raise ValueError("standard output pipe closed") from error + except OSError as error: + _abandon_stdio(output_stream) + detail = getattr(error, "strerror", None) or str(error) + raise ValueError(f"cannot write standard output: {detail}") from error except UnicodeError as error: raise ValueError("standard output cannot encode UTF-8 product payload") from error @@ -402,6 +478,56 @@ def _status(errors: TextIO, message: str, *, enabled: bool, code: str) -> None: errors.flush() +def _human_reason(result: ProcessResult) -> str: + if result.reason == REASON_TRANSFORMED: + return f"inserted {result.change_count} hidden characters" + if result.reason == REASON_SITE_CAP: + return ( + f"inserted {result.change_count} hidden characters, then stopped at the " + f"{LETTER_MIX_MAX_SELECTED}-site cap; trailing text is unchanged" + ) + if result.reason == REASON_UNSUPPORTED_DOMAIN: + return "no hidden characters inserted (unsupported Unicode). I/O succeeded; this is not watermark removal" + if result.reason == REASON_ALREADY_TRANSFORMED: + return "no hidden characters inserted (input already contains the payload)" + if result.reason == REASON_NO_ELIGIBLE_SITES: + return "no hidden characters inserted (no eligible ASCII letter sites)" + if result.reason == REASON_TOO_LARGE: + return f"input is too large (max {PRODUCT_MAX_INPUT_CHARS} characters)" + return "transformation failed internally; source returned unchanged" + + +def _machine_status_line(result: ProcessResult) -> str: + last = "" if result.last_source_index is None else str(result.last_source_index) + capped = "yes" if result.capped else "no" + return ( + "fuckmark-status " + f"result={result.reason} insertions={result.change_count} " + f"sites={result.site_count} last_index={last} capped={capped}" + ) + + +def _emit_outcome( + errors: TextIO, + result: ProcessResult, + *, + interactive: bool, + quiet: bool, + status_report: bool, + color: bool, +) -> None: + if status_report: + errors.write(_machine_status_line(result) + "\n") + errors.flush() + if result.reason == REASON_INTERNAL_ERROR or result.reason == REASON_TOO_LARGE: + return + if quiet: + return + if result.reason == REASON_TRANSFORMED and not interactive: + return + _status(errors, f"FuckMark: {_human_reason(result)}.", enabled=color, code=_ANSI_YELLOW if result.reason != REASON_TRANSFORMED else _ANSI_GREEN) + + def main( input_stream: TextIO | None = None, output_stream: TextIO | None = None, @@ -424,13 +550,30 @@ def main( for value in (input_stream, output_stream, clipboard_writer, error_stream) ), ) + except BrokenPipeError: + try: + errors.write("FuckMark: standard output pipe closed\n") + errors.flush() + except (OSError, UnicodeError, ValueError, BrokenPipeError): + pass + _abandon_stdio(output) + return EXIT_ERROR except KeyboardInterrupt: try: errors.write("\n") errors.flush() - except (OSError, UnicodeError, ValueError): + except (OSError, UnicodeError, ValueError, BrokenPipeError): + pass + return EXIT_INTERRUPT + except OSError as error: + try: + detail = getattr(error, "strerror", None) or str(error) + errors.write(f"FuckMark: cannot write standard output: {detail}\n") + errors.flush() + except (OSError, UnicodeError, ValueError, BrokenPipeError): pass - return 130 + _abandon_stdio(output) + return EXIT_ERROR def _run( @@ -528,10 +671,24 @@ def _run( except (KeyError, TypeError, ValueError) as error: return _error(errors, f"could not transform the text: {error}") + if result.reason == REASON_TOO_LARGE: + return _error(errors, _human_reason(result)) + if result.reason == REASON_INTERNAL_ERROR: + _error(errors, _human_reason(result)) + return EXIT_INTERNAL + output_text = result.output_text if arguments.visible: output_text = project_visible_v1(output_text, product_approved_carriers_v1()) + wrote_payload = False + if (not interactive) or arguments.output not in (None, "-"): + try: + _write_result(output_text, arguments.output, output) + wrote_payload = True + except (ValueError, OSError) as error: + return _error(errors, str(error)) + should_copy = arguments.copy or interactive copy_failed: Exception | None = None if should_copy: @@ -541,17 +698,18 @@ def _run( except Exception as error: copy_failed = error - wrote_payload = False - if (not interactive) or arguments.output not in (None, "-"): - try: - _write_result(output_text, arguments.output, output) - wrote_payload = True - except ValueError as error: - return _error(errors, str(error)) - if interactive and copy_failed is None: _status(errors, _COPIED, enabled=color, code=_ANSI_GREEN) + _emit_outcome( + errors, + result, + interactive=interactive, + quiet=arguments.quiet, + status_report=arguments.status_report, + color=color, + ) + if copy_failed is not None: tool_failed = "no supported clipboard command found" not in str(copy_failed) if wrote_payload: @@ -568,9 +726,32 @@ def _run( enabled=color, code=_ANSI_YELLOW, ) - return 2 - return 0 + return EXIT_CLIPBOARD + return EXIT_OK if __name__ == "__main__": - raise SystemExit(main()) + status = EXIT_ERROR + try: + status = main() + except SystemExit as error: + code = error.code + if code is None: + status = EXIT_OK + elif isinstance(code, int): + status = code + else: + status = EXIT_ERROR + except BrokenPipeError: + status = EXIT_ERROR + except KeyboardInterrupt: + status = EXIT_INTERRUPT + except OSError: + status = EXIT_ERROR + for stream in (sys.stdout, sys.stderr): + try: + stream.flush() + except (OSError, ValueError, BrokenPipeError): + pass + _abandon_stdio(sys.stdout) + os._exit(status) diff --git a/fuckmark/cycle8/benchmark_render.py b/fuckmark/cycle8/benchmark_render.py index 94dee5ed..5b5ce7d4 100644 --- a/fuckmark/cycle8/benchmark_render.py +++ b/fuckmark/cycle8/benchmark_render.py @@ -8,7 +8,7 @@ from pathlib import Path from ..hashing import sha256_json, sha256_text -from ..product.rendering import chrome_executable, compare_chrome_pre_screenshots +from ..product.rendering import chrome_executable, compare_chrome_pre_screenshots, render_window_size BENCHMARK_RENDER_VERSION = "cycle8-benchmark-render-v2" @@ -29,12 +29,12 @@ def chrome_version(executable: str | None = None) -> str | None: return None -def _html_surface(text: str, surface: str) -> str: +def _html_surface(text: str, surface: str, *, box_height: int = 160) -> str: payload = json.dumps(text, ensure_ascii=False).replace("<", "\\u003c").replace(">", "\\u003e") if surface == "textarea": body = ( "" ) @@ -42,7 +42,7 @@ def _html_surface(text: str, surface: str) -> str: elif surface == "contenteditable": body = ( "
" ) @@ -73,7 +73,7 @@ def displayed_js_property(surface: str) -> str: raise ValueError("unknown render surface") -def _screenshot(executable: str, root: Path, name: str, html: str) -> bytes: +def _screenshot(executable: str, root: Path, name: str, html: str, *, width: int, height: int) -> bytes: html_path = root / f"{name}.html" png_path = root / f"{name}.png" html_path.write_text(html, encoding="utf-8") @@ -88,7 +88,7 @@ def _screenshot(executable: str, root: Path, name: str, html: str) -> bytes: "--no-first-run", "--disable-background-networking", "--virtual-time-budget=5000", - "--window-size=800,240", + f"--window-size={width},{height}", f"--screenshot={png_path}", html_path.as_uri(), ], @@ -140,16 +140,34 @@ def compare_chrome_surface(original: str, transformed: str, surface: str) -> dic } return {**payload, "comparison_hash": sha256_json(payload)} try: + longest = original if len(original) >= len(transformed) else transformed + width, height, complete, content_height = render_window_size(longest, min_height=240) + if not complete: + payload = { + "algorithm_version": BENCHMARK_RENDER_VERSION, + "environment": environment, + "status": "INCOMPLETE", + "original_sha256": original_sha, + "transformed_sha256": transformed_sha, + "equal": None, + "detail": "content exceeds captured window", + "font": BENCHMARK_RENDER_FONT, + "browser_version": browser, + "window_width": width, + "window_height": height, + "content_height": content_height, + } + return {**payload, "comparison_hash": sha256_json(payload)} with tempfile.TemporaryDirectory(prefix="fuckmark-bench-render-") as directory: root = Path(directory) - original_html = _html_surface(original, surface) - transformed_html = _html_surface(transformed, surface) + original_html = _html_surface(original, surface, box_height=max(160, height - 80)) + transformed_html = _html_surface(transformed, surface, box_height=max(160, height - 80)) if original and "el.textContent=" not in original_html and surface == "contenteditable": raise RuntimeError("contenteditable assignment missing") if original and "el.value=" not in original_html and surface == "textarea": raise RuntimeError("textarea assignment missing") - original_png = _screenshot(executable, root, "original", original_html) - transformed_png = _screenshot(executable, root, "transformed", transformed_html) + original_png = _screenshot(executable, root, "original", original_html, width=width, height=height) + transformed_png = _screenshot(executable, root, "transformed", transformed_html, width=width, height=height) except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired, RuntimeError) as error: payload = { "algorithm_version": BENCHMARK_RENDER_VERSION, diff --git a/fuckmark/cycle8/letter_mix.py b/fuckmark/cycle8/letter_mix.py index 365f8774..04f7afa3 100644 --- a/fuckmark/cycle8/letter_mix.py +++ b/fuckmark/cycle8/letter_mix.py @@ -4,27 +4,7 @@ from ..product.invariants import validate_user_visible_invariants from ..product.visible_projection import is_carrier_insertion_v1, project_visible_v1 -from ..transforms.protected import _add_valid_markdown_destinations -from ..transforms.protected_markdown import _add_markdown_reference_spans -from ..transforms.protected_patterns import ( - _CLI_FLAG_RE, - _CURRENCY_RE, - _EMAIL_RE, - _NUMBER_RE, - _PERCENT_RE, - _add_dates, - _add_ip_addresses, - _add_regex, - _add_urls, -) -from ..transforms.protected_structures import ( - _add_extended_posix_paths, - _add_extended_windows_paths, - _add_fenced_code, - _add_inline_code, - _add_posix_paths, - _add_windows_paths, -) +from ..transforms.protected import add_hard_machine_spans from ..transforms.schema import InvariantStatus, ProtectedSpanKind LETTER_MIX_APPROVED_CARRIERS = (0x034F, 0xFE00) @@ -36,22 +16,7 @@ def hard_machine_intervals(text: str) -> tuple[tuple[int, int], ...]: if not isinstance(text, str): raise TypeError("text must be a string") raw: list[tuple[int, int, ProtectedSpanKind]] = [] - _add_fenced_code(raw, text) - _add_inline_code(raw, text) - _add_valid_markdown_destinations(raw, text) - _add_markdown_reference_spans(raw, text) - _add_urls(raw, text) - _add_regex(raw, text, _EMAIL_RE, ProtectedSpanKind.EMAIL) - _add_ip_addresses(raw, text) - _add_dates(raw, text) - _add_regex(raw, text, _CURRENCY_RE, ProtectedSpanKind.CURRENCY) - _add_regex(raw, text, _PERCENT_RE, ProtectedSpanKind.PERCENTAGE) - _add_regex(raw, text, _NUMBER_RE, ProtectedSpanKind.NUMBER) - _add_posix_paths(raw, text) - _add_extended_posix_paths(raw, text) - _add_windows_paths(raw, text) - _add_extended_windows_paths(raw, text) - _add_regex(raw, text, _CLI_FLAG_RE, ProtectedSpanKind.CLI_FLAG) + add_hard_machine_spans(raw, text) ordered = sorted((start, end) for start, end, _kind in raw) merged: list[list[int]] = [] for start, end in ordered: diff --git a/fuckmark/product/domain.py b/fuckmark/product/domain.py index 42e13aaf..6ae7afa6 100644 --- a/fuckmark/product/domain.py +++ b/fuckmark/product/domain.py @@ -3,6 +3,7 @@ PRODUCT_DOMAIN_ID = "ordinary-english-ascii-v1" _ALLOWED = frozenset({9, 10, 13, *range(0x20, 0x7F)}) +PRODUCT_MAX_INPUT_CHARS = 2_000_000 def is_supported_product_domain_v1(text: str) -> bool: diff --git a/fuckmark/product/rendering.py b/fuckmark/product/rendering.py index c30c7487..63c560cc 100644 --- a/fuckmark/product/rendering.py +++ b/fuckmark/product/rendering.py @@ -11,8 +11,26 @@ from ..hashing import sha256_json, sha256_text -RENDERING_HARNESS_VERSION = "product-reference-render-v1" +RENDERING_HARNESS_VERSION = "product-reference-render-v2" _CHROME_NAMES = ("google-chrome", "google-chrome-stable", "chromium", "chromium-browser", "chrome") +_RENDER_MAX_HEIGHT = 8000 +_RENDER_WIDTH = 800 + + +def render_window_size(text: str, *, min_height: int) -> tuple[int, int, bool, int]: + if not isinstance(text, str): + raise TypeError("text must be a string") + columns = max(1, (_RENDER_WIDTH - 40) // 10) + wrapped = 0 + lines = text.splitlines() or [""] + if text.endswith(("\n", "\r")): + lines = [*lines, ""] + for line in lines: + wrapped += max(1, (len(line) + columns - 1) // columns) + content_height = 16 + wrapped * 24 + 32 + complete = content_height <= _RENDER_MAX_HEIGHT + height = min(max(content_height, min_height), _RENDER_MAX_HEIGHT) + return _RENDER_WIDTH, height, complete, content_height @dataclass(frozen=True, slots=True) @@ -71,8 +89,31 @@ def compare_chrome_pre_screenshots(original: str, transformed: str) -> Reference try: with tempfile.TemporaryDirectory(prefix="fuckmark-render-") as directory: root = Path(directory) - original_png = _render_pre(executable, root, "original", original) - transformed_png = _render_pre(executable, root, "transformed", transformed) + width, height, complete, content_height = render_window_size(original if len(original) >= len(transformed) else transformed, min_height=200) + if not complete: + payload = { + "algorithm_version": RENDERING_HARNESS_VERSION, + "environment": "chromium_headless", + "status": "INCOMPLETE", + "original_sha256": sha256_text(original), + "transformed_sha256": sha256_text(transformed), + "equal": None, + "detail": "content exceeds captured window", + "window_width": width, + "window_height": height, + "content_height": content_height, + } + return ReferenceRenderComparison( + "chromium_headless", + "INCOMPLETE", + sha256_text(original), + sha256_text(transformed), + None, + "content exceeds captured window", + sha256_json(payload), + ) + original_png = _render_pre(executable, root, "original", original, width=width, height=height) + transformed_png = _render_pre(executable, root, "transformed", transformed, width=width, height=height) except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: payload = { "algorithm_version": RENDERING_HARNESS_VERSION, @@ -127,7 +168,7 @@ def _html_page(text: str) -> str: ) -def _render_pre(executable: str, root: Path, name: str, text: str) -> bytes: +def _render_pre(executable: str, root: Path, name: str, text: str, *, width: int = 800, height: int = 200) -> bytes: html_path = root / f"{name}.html" png_path = root / f"{name}.png" html_path.write_text(_html_page(text), encoding="utf-8") @@ -142,7 +183,7 @@ def _render_pre(executable: str, root: Path, name: str, text: str) -> bytes: "--no-first-run", "--disable-background-networking", "--virtual-time-budget=5000", - "--window-size=800,200", + f"--window-size={width},{height}", f"--screenshot={png_path}", html_path.as_uri(), ], diff --git a/fuckmark/transforms/protected.py b/fuckmark/transforms/protected.py index 6914fe5a..0e9feffe 100644 --- a/fuckmark/transforms/protected.py +++ b/fuckmark/transforms/protected.py @@ -21,7 +21,7 @@ _NUMERIC_CITATION_RE, _PERCENT_RE, ) -from .protected_markdown import _add_markdown_reference_spans +from .protected_markdown import _add_markdown_reference_spans, markdown_label_closers from .protected_structures import ( _add_delimited_math, _add_dollar_math, @@ -29,6 +29,8 @@ _add_extended_posix_paths, _add_extended_windows_paths, _add_fenced_code, + _add_html_markup, + _add_indented_code, _add_inline_code, _add_markdown_destinations, _add_posix_paths, @@ -38,7 +40,6 @@ ) from .schema import ProtectedSpanKind - PROTECTED_SPAN_ALGORITHM_VERSION = "protected-span-extractor-v4" _MAX_IDENTIFIER_SCAN_WORK = 50_000_000 @@ -65,34 +66,8 @@ def _merge_spans(text: str, raw: Sequence[tuple[int, int, ProtectedSpanKind]]) - return tuple(output) -def _is_escaped(text: str, index: int) -> bool: - count = 0 - cursor = index - 1 - while cursor >= 0 and text[cursor] == "\\": - count += 1 - cursor -= 1 - return count % 2 == 1 - - -def _markdown_label_closers(text: str) -> frozenset[int]: - stack: list[int] = [] - closers: set[int] = set() - for index, character in enumerate(text): - if character == "\n": - stack.clear() - continue - if character not in "[]" or _is_escaped(text, index): - continue - if character == "[": - stack.append(index) - elif stack: - stack.pop() - closers.add(index) - return frozenset(closers) - - def _add_valid_markdown_destinations(raw: list[tuple[int, int, ProtectedSpanKind]], text: str) -> None: - closers = _markdown_label_closers(text) + closers = markdown_label_closers(text) if not closers: return temporary: list[tuple[int, int, ProtectedSpanKind]] = [] @@ -102,6 +77,27 @@ def _add_valid_markdown_destinations(raw: list[tuple[int, int, ProtectedSpanKind _append(raw, start, end, kind) +def add_hard_machine_spans(raw: list[tuple[int, int, ProtectedSpanKind]], text: str) -> None: + _add_fenced_code(raw, text) + _add_inline_code(raw, text) + _add_indented_code(raw, text) + _add_html_markup(raw, text) + _add_valid_markdown_destinations(raw, text) + _add_markdown_reference_spans(raw, text) + _add_urls(raw, text) + _add_regex(raw, text, _EMAIL_RE, ProtectedSpanKind.EMAIL) + _add_ip_addresses(raw, text) + _add_dates(raw, text) + _add_regex(raw, text, _CURRENCY_RE, ProtectedSpanKind.CURRENCY) + _add_regex(raw, text, _PERCENT_RE, ProtectedSpanKind.PERCENTAGE) + _add_regex(raw, text, _NUMBER_RE, ProtectedSpanKind.NUMBER) + _add_posix_paths(raw, text) + _add_extended_posix_paths(raw, text) + _add_windows_paths(raw, text) + _add_extended_windows_paths(raw, text) + _add_regex(raw, text, _CLI_FLAG_RE, ProtectedSpanKind.CLI_FLAG) + + def _identifier_matches(text: str, identifier: str) -> Iterator[tuple[int, int]]: escaped = re.escape(identifier) if identifier[0].isalnum() or identifier[0] == "_": @@ -158,26 +154,11 @@ def extract( if value.end > len(text): raise ValueError("user protected range extends beyond text") raw: list[tuple[int, int, ProtectedSpanKind]] = [] - _add_fenced_code(raw, text) - _add_inline_code(raw, text) - _add_valid_markdown_destinations(raw, text) - _add_markdown_reference_spans(raw, text) - _add_urls(raw, text) - _add_regex(raw, text, _EMAIL_RE, ProtectedSpanKind.EMAIL) - _add_ip_addresses(raw, text) - _add_dates(raw, text) - _add_regex(raw, text, _CURRENCY_RE, ProtectedSpanKind.CURRENCY) - _add_regex(raw, text, _PERCENT_RE, ProtectedSpanKind.PERCENTAGE) - _add_regex(raw, text, _NUMBER_RE, ProtectedSpanKind.NUMBER) + add_hard_machine_spans(raw, text) if include_quotations: _add_double_quotations(raw, text) _add_regex(raw, text, _CURLY_SINGLE_QUOTE_RE, ProtectedSpanKind.QUOTATION) _add_regex(raw, text, _STRAIGHT_SINGLE_QUOTE_RE, ProtectedSpanKind.QUOTATION) - _add_posix_paths(raw, text) - _add_extended_posix_paths(raw, text) - _add_windows_paths(raw, text) - _add_extended_windows_paths(raw, text) - _add_regex(raw, text, _CLI_FLAG_RE, ProtectedSpanKind.CLI_FLAG) _add_regex(raw, text, _NUMERIC_CITATION_RE, ProtectedSpanKind.CITATION) _add_regex(raw, text, _AUTHOR_YEAR_CITATION_RE, ProtectedSpanKind.CITATION) _add_dollar_math(raw, text) diff --git a/fuckmark/transforms/protected_markdown.py b/fuckmark/transforms/protected_markdown.py index 7edf3e74..ef54300e 100644 --- a/fuckmark/transforms/protected_markdown.py +++ b/fuckmark/transforms/protected_markdown.py @@ -1,8 +1,18 @@ from __future__ import annotations -from .protected_patterns import _append, _is_escaped +from bisect import bisect_right + +from .protected_patterns import ( + _append, + _is_escaped, + _line_content_end, + _line_starts, + _skip_reference_whitespace, +) from .schema import ProtectedSpanKind +_MAX_MARKDOWN_LABEL = 999 + def _normalize_markdown_label(label: str) -> str: return " ".join(label.split()).casefold() @@ -12,36 +22,90 @@ def _markdown_bracket_pairs(text: str) -> tuple[tuple[int, int], ...]: stack: list[int] = [] pairs: list[tuple[int, int]] = [] for index, character in enumerate(text): - if character == "\n": - stack.clear() - continue if character not in "[]" or _is_escaped(text, index): continue if character == "[": stack.append(index) - elif stack: - pairs.append((stack.pop(), index)) + continue + if not stack: + continue + start = stack.pop() + inner = index - start - 1 + if inner > _MAX_MARKDOWN_LABEL: + continue + pairs.append((start, index)) return tuple(pairs) -def _line_start(text: str, index: int) -> int: - newline = text.rfind("\n", 0, index) - return 0 if newline < 0 else newline + 1 +def _line_index(line_starts: tuple[int, ...], index: int) -> int: + return bisect_right(line_starts, index) - 1 + + +def _skip_blockquotes(text: str, index: int, limit: int) -> int: + while True: + spaces = 0 + cursor = index + while cursor < limit and spaces < 4 and text[cursor] == " ": + spaces += 1 + cursor += 1 + if cursor < limit and text[cursor] == ">": + index = cursor + 1 + if index < limit and text[index] in " \t": + index += 1 + continue + return index -def _is_reference_definition_label(text: str, start: int, end: int) -> bool: - prefix = text[_line_start(text, start) : start] - if len(prefix) > 3 or any(character not in " \t" for character in prefix): +def _skip_list_marker(text: str, index: int, limit: int) -> int: + spaces = 0 + cursor = index + while cursor < limit and spaces < 4 and text[cursor] == " ": + spaces += 1 + cursor += 1 + if spaces >= 4: + return index + if cursor < limit and text[cursor] in "-+*": + nxt = cursor + 1 + if nxt < limit and text[nxt] in " \t": + return nxt + 1 + return index + digits = 0 + while cursor < limit and text[cursor].isdigit() and digits < 9: + cursor += 1 + digits += 1 + if digits and cursor < limit and text[cursor] in ".)": + cursor += 1 + if cursor < limit and text[cursor] in " \t": + return cursor + 1 + return index + + +def _is_reference_definition_label( + text: str, + start: int, + end: int, + line_starts: tuple[int, ...], +) -> bool: + line_i = _line_index(line_starts, start) + line_start = line_starts[line_i] + next_start = line_starts[line_i + 1] if line_i + 1 < len(line_starts) else len(text) + limit = _line_content_end(text, line_start, next_start) + cursor = _skip_blockquotes(text, line_start, limit) + cursor = _skip_list_marker(text, cursor, limit) + indent = 0 + while cursor < start and indent < 4 and text[cursor] == " ": + indent += 1 + cursor += 1 + if cursor != start or indent >= 4: return False - return end + 1 < len(text) and text[end + 1] == ":" + after = _skip_reference_whitespace(text, end + 1) + return after < len(text) and text[after] == ":" def _parse_link_destination(text: str, start: int) -> tuple[int, int] | None: index = start length = len(text) - while index < length and text[index] in " \t": - index += 1 - if index >= length or text[index] in "\r\n": + if index >= length: return None if text[index] == "<": cursor = index + 1 @@ -57,7 +121,7 @@ def _parse_link_destination(text: str, start: int) -> tuple[int, int] | None: depth = 0 while cursor < length: character = text[cursor] - if character in " \t\r\n": + if character in " \t\r\n" or ord(character) < 32: break if character == "\\" and cursor + 1 < length: cursor += 2 @@ -76,16 +140,23 @@ def _parse_link_destination(text: str, start: int) -> tuple[int, int] | None: def _definition_labels(text: str, pairs: tuple[tuple[int, int], ...]) -> dict[str, list[tuple[int, int, int, int]]]: found: dict[str, list[tuple[int, int, int, int]]] = {} + line_starts = _line_starts(text) for start, end in pairs: - if not _is_reference_definition_label(text, start, end): + if not _is_reference_definition_label(text, start, end, line_starts): continue - destination = _parse_link_destination(text, end + 2) + after = _skip_reference_whitespace(text, end + 1) + if after >= len(text) or text[after] != ":": + continue + dest_from = _skip_reference_whitespace(text, after + 1) + destination = _parse_link_destination(text, dest_from) if destination is None: continue dest_start, dest_end = destination inner = text[start + 1 : end] if not inner.strip(): continue + if end - start - 1 > _MAX_MARKDOWN_LABEL: + continue key = _normalize_markdown_label(inner) found.setdefault(key, []).append((start + 1, end, dest_start, dest_end)) return found @@ -99,10 +170,11 @@ def resolve_markdown_reference_hrefs(text: str) -> tuple[str, ...]: by_start = {start: (start, end) for start, end in pairs} hrefs: list[str] = [] seen_pair_starts: set[int] = set() + line_starts = _line_starts(text) for start, end in pairs: if start in seen_pair_starts: continue - if _is_reference_definition_label(text, start, end): + if _is_reference_definition_label(text, start, end, line_starts): continue following = by_start.get(end + 1) if following is not None: @@ -142,6 +214,7 @@ def _add_markdown_reference_spans(raw: list[tuple[int, int, ProtectedSpanKind]], defined = _definition_labels(text, pairs) if not defined: return + line_starts = _line_starts(text) for _key, rows in defined.items(): for label_start, label_end, dest_start, dest_end in rows: _append(raw, label_start, label_end, ProtectedSpanKind.MARKDOWN_LABEL) @@ -151,7 +224,7 @@ def _add_markdown_reference_spans(raw: list[tuple[int, int, ProtectedSpanKind]], for start, end in pairs: if start in seen_pair_starts: continue - if _is_reference_definition_label(text, start, end): + if _is_reference_definition_label(text, start, end, line_starts): continue following = by_start.get(end + 1) if following is not None: @@ -171,3 +244,7 @@ def _add_markdown_reference_spans(raw: list[tuple[int, int, ProtectedSpanKind]], label_inner = text[inner_start:inner_end] if _normalize_markdown_label(label_inner) in defined: _append(raw, inner_start, inner_end, ProtectedSpanKind.MARKDOWN_LABEL) + + +def markdown_label_closers(text: str) -> frozenset[int]: + return frozenset(end for _start, end in _markdown_bracket_pairs(text)) diff --git a/fuckmark/transforms/protected_patterns.py b/fuckmark/transforms/protected_patterns.py index b3a2fd2f..c92b3180 100644 --- a/fuckmark/transforms/protected_patterns.py +++ b/fuckmark/transforms/protected_patterns.py @@ -8,7 +8,7 @@ _MAX_PROTECTED_ITEMS = 100_000 _HWS = r"[^\S\r\n]" -_URL_RE = re.compile(r"\b(?ai:https?://|www\.)[^\s<>\"']+") +_URL_RE = re.compile(r"\b(?ai:(?:https?|ftps?|sftp|file|ws|wss|git|ssh|s3)://|www\.)[^\s<>\"']+") _BARE_DOMAIN_RE = re.compile(r"(?\"']*)?") _EMAIL_RE = re.compile(r"(? int: return len(text) if end < 0 else end +def _line_starts(text: str) -> tuple[int, ...]: + starts = [0] + index = 0 + length = len(text) + while index < length: + character = text[index] + if character == "\r": + index += 1 + if index < length and text[index] == "\n": + index += 1 + starts.append(index) + continue + if character == "\n": + starts.append(index + 1) + index += 1 + continue + index += 1 + return tuple(starts) + + +def _line_content_end(text: str, line_start: int, next_line_start: int) -> int: + end = next_line_start + if end > line_start and text[end - 1] == "\n": + end -= 1 + if end > line_start and text[end - 1] == "\r": + end -= 1 + return end + if end > line_start and text[end - 1] == "\r": + return end - 1 + return end + + +def _skip_spaces_tabs(text: str, index: int, limit: int | None = None) -> int: + end = len(text) if limit is None else limit + while index < end and text[index] in " \t": + index += 1 + return index + + +def _skip_one_line_ending(text: str, index: int) -> int: + if index < len(text) and text[index] == "\r": + index += 1 + if index < len(text) and text[index] == "\n": + index += 1 + return index + if index < len(text) and text[index] == "\n": + return index + 1 + return index + + +def _skip_reference_whitespace(text: str, index: int) -> int: + index = _skip_spaces_tabs(text, index) + advanced = _skip_one_line_ending(text, index) + if advanced != index: + index = _skip_spaces_tabs(text, advanced) + return index + + def _trim_terminal_punctuation(text: str, start: int, end: int) -> tuple[int, int]: while end > start and text[end - 1] in ".,;:!?": end -= 1 diff --git a/fuckmark/transforms/protected_structures.py b/fuckmark/transforms/protected_structures.py index 496eb796..58a6218f 100644 --- a/fuckmark/transforms/protected_structures.py +++ b/fuckmark/transforms/protected_structures.py @@ -4,7 +4,7 @@ from bisect import bisect_right from collections.abc import Sequence -from .protected_patterns import _append, _is_escaped, _line_end, _trim_terminal_punctuation +from .protected_patterns import _append, _is_escaped, _line_content_end, _line_end, _line_starts, _trim_terminal_punctuation from .schema import ProtectedSpanKind _INLINE_CODE_RUN_RE = re.compile(r"`+") @@ -16,6 +16,82 @@ _RELATIVE_PATH_RE = re.compile( r"(?\r\n]{0,1024})?/?>") +_HTML_ENTITY_RE = re.compile(r"&(?:[A-Za-z][A-Za-z0-9]{0,31}|#[0-9]{1,7}|#x[0-9A-Fa-f]{1,6});") +_PROSE_SLASH_PAIRS = frozenset( + { + "and/or", + "or/and", + "either/or", + "he/she", + "she/he", + "his/her", + "her/his", + "yes/no", + "no/yes", + "on/off", + "off/on", + "true/false", + "false/true", + "n/a", + "w/o", + "c/o", + "a/k/a", + "i/o", + "input/output", + "output/input", + "read/write", + "write/read", + "high/low", + "low/high", + "left/right", + "right/left", + "up/down", + "down/up", + "plus/minus", + "minus/plus", + } +) +_PATH_ROOTS = frozenset( + { + "src", + "lib", + "bin", + "sbin", + "scripts", + "tests", + "docs", + "dist", + "tmp", + "temp", + "usr", + "var", + "opt", + "etc", + "include", + "vendor", + "pkg", + "pkgs", + "tools", + "assets", + "static", + "cmake", + "modules", + "third_party", + "node_modules", + } +) _WINDOWS_PATH_RE = re.compile( r"(?i)(?|\s]+[/\\])*[^\\/:*?\"<>|\s]+" ) @@ -49,6 +125,39 @@ def _paragraph_end(text: str, start: int) -> int: return len(text) if match is None else match.start() +def _add_indented_code(raw: list[tuple[int, int, ProtectedSpanKind]], text: str) -> None: + starts = _line_starts(text) + count = len(starts) + for index, start in enumerate(starts): + next_start = starts[index + 1] if index + 1 < count else len(text) + limit = _line_content_end(text, start, next_start) + cursor = start + while True: + spaces = 0 + look = cursor + while look < limit and spaces < 4 and text[look] == " ": + spaces += 1 + look += 1 + if look < limit and text[look] == ">": + cursor = look + 1 + if cursor < limit and text[cursor] in " \t": + cursor += 1 + continue + break + if cursor < limit and text[cursor] == "\t": + _append(raw, cursor, limit, ProtectedSpanKind.CODE) + continue + if cursor + 4 <= limit and text[cursor : cursor + 4] == " ": + _append(raw, cursor, limit, ProtectedSpanKind.CODE) + + +def _add_html_markup(raw: list[tuple[int, int, ProtectedSpanKind]], text: str) -> None: + for match in _HTML_TAG_RE.finditer(text): + _append(raw, match.start(), match.end(), ProtectedSpanKind.CODE) + for match in _HTML_ENTITY_RE.finditer(text): + _append(raw, match.start(), match.end(), ProtectedSpanKind.CODE) + + def _add_inline_code(raw: list[tuple[int, int, ProtectedSpanKind]], text: str) -> None: runs = tuple(match for match in _INLINE_CODE_RUN_RE.finditer(text) if not _is_escaped(text, match.start())) if not runs: @@ -119,7 +228,7 @@ def _add_markdown_destinations(raw: list[tuple[int, int, ProtectedSpanKind]], te depth = 1 index = start escaped = False - while index < len(text) and text[index] != "\n": + while index < len(text): character = text[index] if escaped: escaped = False @@ -143,10 +252,49 @@ def _add_markdown_destinations(raw: list[tuple[int, int, ProtectedSpanKind]], te cursor = max(index, start + 1) +def _accept_extensionless_relative(token: str) -> bool: + compact = token.strip("/").casefold() + if compact in _PROSE_SLASH_PAIRS: + return False + parts = [part for part in token.strip("/").split("/") if part] + if len(parts) < 2: + return False + if len(parts) >= 3: + return True + if parts[0].casefold() in _PATH_ROOTS: + return True + return any( + any(character in part for character in "._-") or any(character.isdigit() for character in part) + for part in parts + ) + + +def _last_component_has_space(text: str, start: int, end: int, separators: str) -> bool: + chunk = text[start:end] + last = -1 + for separator in separators: + last = max(last, chunk.rfind(separator)) + if last < 0: + return False + return " " in chunk[last + 1 :] + + def _add_posix_paths(raw: list[tuple[int, int, ProtectedSpanKind]], text: str) -> None: - for pattern in (_POSIX_PATH_RE, _RELATIVE_PATH_RE): - for match in pattern.finditer(text): - start, end = _trim_terminal_punctuation(text, match.start(), match.end()) + for match in _POSIX_PATH_RE.finditer(text): + start, end = _trim_terminal_punctuation(text, match.start(), match.end()) + _append(raw, start, end, ProtectedSpanKind.POSIX_PATH) + for match in _RELATIVE_PATH_RE.finditer(text): + start, end = _trim_terminal_punctuation(text, match.start(), match.end()) + _append(raw, start, end, ProtectedSpanKind.POSIX_PATH) + for match in _EXTENSIONLESS_RELATIVE_PATH_RE.finditer(text): + start, end = _trim_terminal_punctuation(text, match.start(), match.end()) + token = text[start:end] + if not _accept_extensionless_relative(token): + continue + _append(raw, start, end, ProtectedSpanKind.POSIX_PATH) + for match in _POSIX_SPACED_FILE_RE.finditer(text): + start, end = _trim_terminal_punctuation(text, match.start(), match.end()) + if _last_component_has_space(text, start, end, "/"): _append(raw, start, end, ProtectedSpanKind.POSIX_PATH) @@ -154,6 +302,10 @@ def _add_windows_paths(raw: list[tuple[int, int, ProtectedSpanKind]], text: str) for match in _WINDOWS_PATH_RE.finditer(text): start, end = _trim_terminal_punctuation(text, match.start(), match.end()) _append(raw, start, end, ProtectedSpanKind.WINDOWS_PATH) + for match in _WINDOWS_SPACED_FILE_RE.finditer(text): + start, end = _trim_terminal_punctuation(text, match.start(), match.end()) + if _last_component_has_space(text, start, end, "/\\"): + _append(raw, start, end, ProtectedSpanKind.WINDOWS_PATH) def _path_scan_limit(text: str, start: int, forbidden: str) -> int: diff --git a/pyproject.toml b/pyproject.toml index 60b301a3..1ca2dad4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "fuckmark" -version = "0.4.0" -description = "CLI that inserts a frozen invisible Unicode payload into English ASCII text without changing what you see" +version = "0.4.1" +description = "CLI that inserts frozen invisible Unicode into English ASCII text without changing what you see. Constrained SynthID results; not a general watermark remover." readme = "README.md" requires-python = ">=3.11" dependencies = [] @@ -37,7 +37,7 @@ Fuckmark = "fuckmark.cli:main" fuckmark = "fuckmark.cli:main" [project.optional-dependencies] -dev = ["pytest>=8.3"] +dev = ["pytest>=8.3", "markdown-it-py==4.2.0"] research = ["fonttools>=4.50", "uharfbuzz>=0.39", "tokenizers>=0.20"] [tool.pytest.ini_options] diff --git a/tests/audit_mix_replay.py b/tests/audit_mix_replay.py index 18557b59..eaa1af8e 100644 --- a/tests/audit_mix_replay.py +++ b/tests/audit_mix_replay.py @@ -2,6 +2,30 @@ from fuckmark.hashing import sha256_text from fuckmark.product.visible_projection import is_carrier_insertion_v1, project_visible_v1 +PERMITTED_LIVE_MIX_DRIFT = { + "cycle8-1080000-51-structured_instructional-unwatermarked": { + "family": "deepmind-transfer", + "label": "unwatermarked", + "scope": "live path protection of sys/types.h after freeze; stored detector scores do not transfer", + "stored_mix_sha256": "3da2b2707ff2307585661008922d3f25ddd91ccfa60040321a5277b08f64193d", + "live_mix_sha256": "f6d4464ebbbf3adeae5b118e80151d38ad8e7511f47dc55aa309fd938a6fb860", + }, + "cycle8-1080000-55-structured_instructional-unwatermarked": { + "family": "deepmind-transfer", + "label": "unwatermarked", + "scope": "live HTML include and sys/types.h protection after freeze; stored detector scores do not transfer", + "stored_mix_sha256": "14ed554609655ce2aaf22ca7d36d0d946babdc368d7276c15c443894dbc27c72", + "live_mix_sha256": "e5c6da6a45f00a3b614752bef46c852e2a9ae2384d06fa83d4d679cddd1ab660", + }, + "cycle8-840000-13-technical_explanation-watermarked": { + "family": "mix-confirmation-mean-transfer", + "label": "watermarked", + "scope": "tests/test_validation is a live extensionless path; Gate v2 confirmation hashes are unchanged; stored mean-transfer scores do not transfer", + "stored_mix_sha256": "096dadebdf8d309decaa841fb58d7bc0573d7c4d95eab306becfcfe3d21486b2", + "live_mix_sha256": "845ec79d12040a9a77505d7da8b78b6de3e5c8ab94f9dc3cfc5ae8f6222642b9", + }, +} + def live_mix_hash(source: str) -> str: live = apply_letter_alternating_mix(source) @@ -9,3 +33,21 @@ def live_mix_hash(source: str) -> str: if live != source: assert is_carrier_insertion_v1(source, live, LETTER_MIX_APPROVED_CARRIERS) return sha256_text(live) + + +def assert_live_mix_matches_stored(sample_id: str, source: str, stored_hash: str, *, label: str) -> str: + live = live_mix_hash(source) + if live == stored_hash: + return live + allowed = PERMITTED_LIVE_MIX_DRIFT.get(sample_id) + if allowed is None: + raise AssertionError(f"unexplained live mix drift for {sample_id}") + if str(label) == "watermarked" and allowed.get("label") != "watermarked": + raise AssertionError(f"watermarked sample drifted: {sample_id}") + recorded_stored = allowed.get("stored_mix_sha256") + if recorded_stored is not None and stored_hash != recorded_stored: + raise AssertionError(f"permitted stored mix hash changed for {sample_id}") + expected = allowed.get("live_mix_sha256") + if expected is not None and live != expected: + raise AssertionError(f"permitted drift hash changed for {sample_id}") + return live diff --git a/tests/test_audit_d03_d11.py b/tests/test_audit_d03_d11.py new file mode 100644 index 00000000..c0914d1e --- /dev/null +++ b/tests/test_audit_d03_d11.py @@ -0,0 +1,252 @@ +from io import StringIO +import os +from pathlib import Path +import subprocess +import sys +import time + +import pytest + +from fuckmark.cli import ( + EXIT_CLIPBOARD, + EXIT_ERROR, + EXIT_INTERNAL, + EXIT_OK, + REASON_INTERNAL_ERROR, + REASON_NO_ELIGIBLE_SITES, + REASON_SITE_CAP, + REASON_TOO_LARGE, + REASON_TRANSFORMED, + REASON_UNSUPPORTED_DOMAIN, + main, + transform_text, +) +from fuckmark.cycle8.letter_mix import apply_letter_alternating_mix, hard_machine_intervals +from fuckmark.product.domain import PRODUCT_MAX_INPUT_CHARS +from fuckmark.product.visible_projection import project_visible_v1 +from fuckmark.transforms.protected_markdown import resolve_markdown_reference_hrefs + + +CARRIERS = ("\u034f", "\ufe00") +ROOT = Path(__file__).resolve().parents[1] + + +def _strip(text: str) -> str: + for carrier in CARRIERS: + text = text.replace(carrier, "") + return text + + +def _commonmark_hrefs(text: str) -> tuple[str, ...]: + markdown_it = pytest.importorskip("markdown_it") + tokens = markdown_it.MarkdownIt("commonmark").parse(text) + hrefs: list[str] = [] + for token in tokens: + children = token.children or () + for child in children: + if child.type == "link_open": + hrefs.append(child.attrGet("href") or "") + return tuple(hrefs) + + +def _commonmark_html(text: str) -> str: + markdown_it = pytest.importorskip("markdown_it") + return markdown_it.MarkdownIt("commonmark").render(text) + + +def _cli_bytes(*args: str, stdin: bytes | None = None) -> subprocess.CompletedProcess[bytes]: + env = os.environ.copy() + env["PYTHONPATH"] = str(ROOT) + env["PYTHONUTF8"] = "1" + return subprocess.run( + [sys.executable, "-m", "fuckmark.cli", *args], + input=stdin, + cwd=ROOT, + env=env, + capture_output=True, + ) + + +@pytest.mark.parametrize( + "source", + ( + "[foo\nbar][]\n\n[foo bar]: https://example.com\n", + "[ref]\n\n[ref]:\n https://example.com\n", + "> [ref]\n>\n> [ref]: https://example.com\n", + "- [ref]\n\n- [ref]: https://example.com\n", + "[ref]\r\r[ref]: https://example.com\r", + "[click][ref]\n\n[ref]: https://example.com\n", + ), +) +def test_markdown_reference_hrefs_match_commonmark_before_and_after(source: str) -> None: + before = _commonmark_hrefs(source) + assert before + applied = apply_letter_alternating_mix(source) + assert _strip(applied) == source + assert _commonmark_hrefs(applied) == before + assert resolve_markdown_reference_hrefs(source) == resolve_markdown_reference_hrefs(applied) + + +def test_multiline_inline_destination_keeps_href() -> None: + source = "[hello](\nrelative-file\n)\n" + before = _commonmark_hrefs(source) + assert before == ("relative-file",) + applied = apply_letter_alternating_mix(source) + assert _commonmark_hrefs(applied) == before + assert "relative-file" in applied + assert _strip(applied) == source + + +def test_html_entity_and_indented_code_are_stable() -> None: + html = "Hello\n" + applied_html = apply_letter_alternating_mix(html) + assert "" in applied_html and "" in applied_html + assert _strip(_commonmark_html(applied_html)) == _strip(_commonmark_html(html)) + entity = "Fish & chips.\n" + applied_entity = apply_letter_alternating_mix(entity) + assert "&" in applied_entity + assert _strip(_commonmark_html(applied_entity)) == _strip(_commonmark_html(entity)) + indented = " print(\"hello\")\n" + applied_code = apply_letter_alternating_mix(indented) + assert 'print("hello")' in applied_code + assert _strip(applied_code) == indented + + +def test_paths_and_ftp_uri_are_protected_without_and_or() -> None: + cases = ( + ("See scripts/build now.", "scripts/build"), + ("Open C:/Users/Alice/My final notes.txt now.", "C:/Users/Alice/My final notes.txt"), + ("Read /tmp/My final notes.txt now.", "/tmp/My final notes.txt"), + ("Get ftp://example.com/file now.", "ftp://example.com/file"), + ) + for source, token in cases: + applied = apply_letter_alternating_mix(source) + assert token in applied + assert _strip(applied) == source + covered = "".join(source[start:end] for start, end in hard_machine_intervals(source)) + assert token in covered + prose = "Use and/or input/output here." + covered = "".join(prose[start:end] for start, end in hard_machine_intervals(prose)) + assert "and/or" not in covered + assert "input/output" not in covered + assert apply_letter_alternating_mix(prose) != prose + math = "B2 = (B1 / B2) 2.2, B4.B2" + math_covered = "".join(math[start:end] for start, end in hard_machine_intervals(math)) + assert "/ B2)" not in math_covered + assert "B4.B2" not in math_covered + messages = "Use a series of test/error messages." + message_covered = "".join(messages[start:end] for start, end in hard_machine_intervals(messages)) + assert "test/error" not in message_covered + assert apply_letter_alternating_mix(messages) != messages + + +def test_transform_reasons_are_distinct() -> None: + changed = transform_text("I do not agree.") + assert changed.reason == REASON_TRANSFORMED + assert changed.change_count > 0 + unsupported = transform_text("I do not agree " + chr(0x00E9) + ".") + assert unsupported.reason == REASON_UNSUPPORTED_DOMAIN + assert unsupported.change_count == 0 + mixed = transform_text(changed.output_text) + assert mixed.reason == "already-transformed" + none = transform_text("123.") + assert none.reason == REASON_NO_ELIGIBLE_SITES + long = "abcdefghijklmnopqrstuvwxyz" * 12 + capped = transform_text(long) + assert capped.reason == REASON_SITE_CAP + assert capped.capped is True + huge = transform_text("a" * (PRODUCT_MAX_INPUT_CHARS + 1)) + assert huge.reason == REASON_TOO_LARGE + + +def test_internal_transform_failure_is_not_silent_success(monkeypatch) -> None: + def boom(text: str, sites): + raise RuntimeError("invariant") + + monkeypatch.setattr("fuckmark.cli.compose_letter_mix", boom) + result = transform_text("I do not agree.") + assert result.reason == REASON_INTERNAL_ERROR + assert result.output_text == "I do not agree." + output = StringIO() + errors = StringIO() + status = main(StringIO(""), output, error_stream=errors, argv=("--text", "I do not agree.")) + assert status == EXIT_INTERNAL + assert output.getvalue() == "" + assert "internally" in errors.getvalue() + + +def test_clipboard_runs_after_successful_file_write(tmp_path: Path) -> None: + copied: list[str] = [] + missing = tmp_path / "no-such-dir" / "out.txt" + status = main( + StringIO(""), + StringIO(), + copied.append, + error_stream=StringIO(), + argv=("--text", "hello", "--copy", "-o", str(missing)), + ) + assert status == EXIT_ERROR + assert copied == [] + target = tmp_path / "out.txt" + status = main( + StringIO(""), + StringIO(), + copied.append, + error_stream=StringIO(), + argv=("--text", "hello", "--copy", "-o", str(target)), + ) + assert status == EXIT_OK + assert copied == [apply_letter_alternating_mix("hello")] + + +def test_usage_error_keeps_exit_two_and_clipboard_uses_three() -> None: + usage = _cli_bytes("--not-a-real-flag") + assert usage.returncode == 2 + assert usage.stdout == b"" + output = StringIO() + errors = StringIO() + + def fail(_: str) -> None: + raise RuntimeError("clipboard unavailable") + + status = main(StringIO("I do not agree.\n"), output, fail, error_stream=errors, argv=("--stdin", "--copy")) + assert status == EXIT_CLIPBOARD + assert apply_letter_alternating_mix("I do not agree.\n") in output.getvalue() + + +@pytest.mark.skipif(not Path("/dev/full").exists(), reason="no /dev/full") +def test_stdout_full_device_has_no_traceback() -> None: + env = os.environ.copy() + env["PYTHONPATH"] = str(ROOT) + env["PYTHONUTF8"] = "1" + with Path("/dev/full").open("wb") as handle: + result = subprocess.run( + [sys.executable, "-m", "fuckmark.cli", "--text", "hello"], + cwd=ROOT, + env=env, + stdout=handle, + stderr=subprocess.PIPE, + ) + stderr = result.stderr.decode("utf-8", errors="replace") + assert result.returncode == EXIT_ERROR + assert "Traceback" not in stderr + assert "standard output" in stderr or "No space" in stderr or "cannot write" in stderr + + +def test_status_flag_reports_noop_without_touching_payload() -> None: + source = "I don" + chr(0x2019) + "t agree." + result = _cli_bytes("--text", source, "--status") + assert result.returncode == EXIT_OK + assert result.stdout == source.encode("utf-8") + stderr = result.stderr.decode("utf-8") + assert "fuckmark-status" in stderr + assert "unsupported-domain" in stderr + + +def test_bracket_heavy_scan_stays_bounded() -> None: + payload = "[abc]" * 32000 + start = time.perf_counter() + apply_letter_alternating_mix(payload) + elapsed = time.perf_counter() - start + assert elapsed < 8.0 + assert project_visible_v1(apply_letter_alternating_mix(payload), (0x034F, 0xFE00)) == payload diff --git a/tests/test_cli.py b/tests/test_cli.py index cfcccc5d..bcd3d115 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -174,6 +174,7 @@ def test_cli_main_interactive_copies_fail_closed_unicode_without_printing() -> N assert copied == [source_text] assert output.getvalue() == "" assert "Copied to clipboard" in errors.getvalue() + assert "unsupported Unicode" in errors.getvalue() or "hidden characters" in errors.getvalue() assert "\u034f" not in copied[0] @@ -186,7 +187,7 @@ def fail(_: str) -> None: raise RuntimeError("clipboard unavailable") status = main(source, output, fail, error_stream=errors) - assert status == 2 + assert status == 3 assert output.getvalue() == "" ui = errors.getvalue() assert "clipboard copy failed" in ui @@ -302,7 +303,7 @@ def fail(_: str) -> None: raise RuntimeError("clipboard unavailable") status = main(source, output, fail, error_stream=errors, argv=("--stdin", "--copy")) - assert status == 2 + assert status == 3 rendered = output.getvalue() expected = apply_letter_alternating_mix("I do not agree.\n") assert expected in rendered @@ -468,7 +469,7 @@ def test_cli_stdin_returns_unsupported_unicode_unchanged() -> None: status = main(StringIO(source_text), output, error_stream=errors, argv=("--stdin",)) assert status == 0 assert output.getvalue() == source_text - assert errors.getvalue() == "" + assert "unsupported Unicode" in errors.getvalue() or "hidden characters" in errors.getvalue() def test_cli_stdin_keeps_multiline_visible_text() -> None: @@ -492,7 +493,7 @@ def test_cli_stdin_respects_selected_site_cap() -> None: applied = output.getvalue() assert applied.count("\u034f") + applied.count("\ufe00") == 192 assert project_visible_v1(applied, APPROVED) == source_text - assert errors.getvalue() == "" + assert "site cap" in errors.getvalue() or "192" in errors.getvalue() def test_cli_rejects_invalid_utf8_stdin() -> None: diff --git a/tests/test_cycle8_control_mix_diagnostic.py b/tests/test_cycle8_control_mix_diagnostic.py index fceb6c32..dff4cdab 100644 --- a/tests/test_cycle8_control_mix_diagnostic.py +++ b/tests/test_cycle8_control_mix_diagnostic.py @@ -6,7 +6,7 @@ from fuckmark.hashing import sha256_file, sha256_json, sha256_text from fuckmark.product.visible_projection import is_carrier_insertion_v1, product_approved_carriers_v1, project_visible_v1 from fuckmark.transforms.registry import release_transform_registry -from tests.audit_mix_replay import live_mix_hash +from tests.audit_mix_replay import assert_live_mix_matches_stored _DIAGNOSTIC = "evidence/cycle8-control-mix-diagnostic-920000-n16-2026-08-27" @@ -48,7 +48,12 @@ def test_control_mix_diagnostic_920000_is_seen_hypothesis_zero() -> None: source = by_id[row["sample_id"]]["text"] assert sha256_text(source) == row["source_sha256"] control = apply_control_alternating_mix(source) - live_mix_hash(source) + assert_live_mix_matches_stored( + str(row["sample_id"]), + source, + row["mix"]["text_sha256"], + label=str(row.get("label", "")), + ) assert sha256_text(control) == row["control_mix"]["text_sha256"] assert is_carrier_insertion_v1(source, control, CONTROL_MIX_APPROVED_CARRIERS) assert project_visible_v1(control, CONTROL_MIX_APPROVED_CARRIERS) == source diff --git a/tests/test_cycle8_deepmind_transfer.py b/tests/test_cycle8_deepmind_transfer.py index 5af86c29..c623358f 100644 --- a/tests/test_cycle8_deepmind_transfer.py +++ b/tests/test_cycle8_deepmind_transfer.py @@ -12,7 +12,7 @@ from fuckmark.hashing import sha256_file, sha256_json, sha256_text from fuckmark.product.visible_projection import product_approved_carriers_v1 from fuckmark.transforms.registry import release_transform_registry -from tests.audit_mix_replay import live_mix_hash +from tests.audit_mix_replay import assert_live_mix_matches_stored _920000 = "evidence/cycle8-mix-deepmind-30key-920000-n16-2026-08-27" @@ -53,7 +53,12 @@ def test_deepmind_920000_n16_is_independent_configuration_hypothesis() -> None: assert "text" not in row assert "text" not in row["mix"] assert row["source_sha256"] == sample["text_sha256"] == sha256_text(source) - live_mix_hash(source) + assert_live_mix_matches_stored( + str(sample["sample_id"]), + source, + row["mix"]["text_sha256"], + label=str(row.get("label", sample.get("label", ""))), + ) assert row["visible_ok"] is True sums = (root / "SHA256SUMS.txt").read_text(encoding="utf-8") for line in sums.splitlines(): @@ -108,7 +113,12 @@ def test_deepmind_n192_transfer_is_independent_configuration_hypothesis() -> Non assert "text" not in row assert "text" not in row["mix"] assert row["source_sha256"] == sample["text_sha256"] == sha256_text(source) - live_mix_hash(source) + assert_live_mix_matches_stored( + str(sample["sample_id"]), + source, + row["mix"]["text_sha256"], + label=str(row.get("label", sample.get("label", ""))), + ) assert row["visible_ok"] is True sums_root = root / Path(relative).parent for line in (sums_root / "SHA256SUMS.txt").read_text(encoding="utf-8").splitlines(): diff --git a/tests/test_cycle8_detector_transfer.py b/tests/test_cycle8_detector_transfer.py index 9f0df7e7..4b1619aa 100644 --- a/tests/test_cycle8_detector_transfer.py +++ b/tests/test_cycle8_detector_transfer.py @@ -13,7 +13,7 @@ from fuckmark.hashing import sha256_file, sha256_json, sha256_text from fuckmark.product.visible_projection import product_approved_carriers_v1 from fuckmark.transforms.registry import release_transform_registry -from tests.audit_mix_replay import live_mix_hash +from tests.audit_mix_replay import assert_live_mix_matches_stored def test_mix_mean_transfer_scorecard_is_hypothesis_on_the_same_adapter() -> None: @@ -55,7 +55,12 @@ def test_mix_mean_transfer_rows_replay_live_mix_without_rewriting_hashes() -> No assert "text" not in row assert "text" not in row["mix"] assert row["source_sha256"] == sample["text_sha256"] == sha256_text(source) - live_mix_hash(source) + assert_live_mix_matches_stored( + str(sample["sample_id"]), + source, + row["mix"]["text_sha256"], + label=str(row.get("label", sample.get("label", ""))), + ) root = Path(__file__).resolve().parents[1] / Path(CYCLE8_MIX_DETECTOR_TRANSFER_PATH).parent sums = (root / "SHA256SUMS.txt").read_text(encoding="utf-8") for line in sums.splitlines(): diff --git a/tests/test_cycle8_gate_v2.py b/tests/test_cycle8_gate_v2.py index 7c1f6e45..914313fc 100644 --- a/tests/test_cycle8_gate_v2.py +++ b/tests/test_cycle8_gate_v2.py @@ -1,4 +1,5 @@ from pathlib import Path +import json import pytest @@ -19,6 +20,7 @@ assert_gate_v2_confirmation_generation_seed, build_gate_v2_confirmation_scorecard, gate_v2_confirmation_artifact_dir, + gate_v2_confirmation_artifact_path, gate_v2_confirmation_artifacts_present, gate_v2_payload, load_gate_v2, @@ -41,6 +43,7 @@ row_for_seed_base, ) from fuckmark.transforms.registry import release_transform_registry +from tests.audit_mix_replay import assert_live_mix_matches_stored def test_gate_v2_spec_is_committed_and_confirmed(): @@ -221,6 +224,18 @@ def test_twelve_context_rescan_is_recorded_and_does_not_rewrite_ab(): assert payload["shaping_invisible_per_context"]["arabic"] == 390 +def test_gate_v2_live_mix_matches_stored_transformed_hashes() -> None: + arm = "u034f-ufe00-letter-alt-v1" + for seed_base in GATE_V2_CONFIRMATION_SEED_BASES: + artifact = json.loads(Path(gate_v2_confirmation_artifact_path(seed_base)).read_text(encoding="utf-8")) + by_text = {sample["sample_id"]: sample["text"] for sample in artifact["samples"]} + assert len(artifact["geometry_rows"]) == 128 + for row in artifact["geometry_rows"]: + source = by_text[row["sample_id"]] + stored = row["arms"][arm]["transformed_text_hash"] + assert_live_mix_matches_stored(str(row["sample_id"]), source, stored, label=str(row["label"])) + + def test_gate_v2_hash_is_canonical(): payload = gate_v2_payload() body = {key: value for key, value in payload.items() if key != "gate_hash"} diff --git a/tests/test_cycle8_second_model_transfer.py b/tests/test_cycle8_second_model_transfer.py index 34851a9d..63615678 100644 --- a/tests/test_cycle8_second_model_transfer.py +++ b/tests/test_cycle8_second_model_transfer.py @@ -12,7 +12,7 @@ from fuckmark.hashing import sha256_file, sha256_json, sha256_text from fuckmark.product.visible_projection import product_approved_carriers_v1 from fuckmark.transforms.registry import release_transform_registry -from tests.audit_mix_replay import live_mix_hash +from tests.audit_mix_replay import assert_live_mix_matches_stored _ROOT = "evidence/cycle8-mix-distilgpt2-1090000-n16-2026-08-27" @@ -57,7 +57,12 @@ def test_distilgpt2_n16_is_second_model_hypothesis_not_confirmation() -> None: assert "text" not in row assert "text" not in row["mix"] assert row["source_sha256"] == sample["text_sha256"] == sha256_text(source) - live_mix_hash(source) + assert_live_mix_matches_stored( + str(sample["sample_id"]), + source, + row["mix"]["text_sha256"], + label=str(row.get("label", sample.get("label", ""))), + ) assert row["visible_ok"] is True sums = (root / "SHA256SUMS.txt").read_text(encoding="utf-8") for line in sums.splitlines(): diff --git a/tests/test_evidence_manifests.py b/tests/test_evidence_manifests.py new file mode 100644 index 00000000..090858de --- /dev/null +++ b/tests/test_evidence_manifests.py @@ -0,0 +1,43 @@ +from pathlib import Path + +from fuckmark.hashing import sha256_bytes, sha256_file + + +ROOT = Path(__file__).resolve().parents[1] +CRLF_README = ROOT / "evidence/cycle7-stage-a-2026-08-25/README.md" +CRLF_DIGEST = "833d1d94fcea01bc09855f0af3a4bcc18df1010065fb3e2825f8e2e31733c2eb" +LF_DIGEST = "d7b7de433f5ca231ef2d9de586774ffdfe255b1625f902fa0bf9780019d62709" + + +def test_every_sha256sums_entry_matches_or_has_provenance() -> None: + manifests = tuple(sorted(ROOT.glob("evidence/**/SHA256SUMS.txt"))) + assert manifests + for manifest in manifests: + folder = manifest.parent + for line in manifest.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + digest, name = line.split() + path = folder / name + assert path.is_file(), path + actual = sha256_file(path) + if path.resolve() == CRLF_README.resolve() and digest == CRLF_DIGEST: + assert actual == LF_DIGEST + assert sha256_bytes(path.read_bytes().replace(b"\n", b"\r\n")) == CRLF_DIGEST + continue + assert actual == digest, path + + +def test_cycle8_archive_pointers_exist() -> None: + assert (ROOT / "docs/cycle8/gate-v2.md").is_file() + assert (ROOT / "docs/cycle8/mix-second-model-transfer.md").is_file() + distil = (ROOT / "evidence/cycle8-mix-distilgpt2-1090000-n16-2026-08-27/README.md").read_text(encoding="utf-8") + h16 = (ROOT / "evidence/h16-local/README.md").read_text(encoding="utf-8") + assert "docs/cycle8/mix-second-model-transfer.md" in distil + assert "docs/cycle8/gate-v2.md" in h16 + assert (ROOT / "docs/limits.md").is_file() + assert (ROOT / "docs/website.md").is_file() + website = (ROOT / "docs/website.md").read_text(encoding="utf-8") + assert "| sh" not in website + assert "| iex" not in website + assert "curl -fsSL https://d.q1z.org/mark | sh" not in website diff --git a/tests/test_install_release_hardening.py b/tests/test_install_release_hardening.py index 9cccacb6..c8615ee5 100644 --- a/tests/test_install_release_hardening.py +++ b/tests/test_install_release_hardening.py @@ -72,13 +72,16 @@ def test_unix_installer_uses_tagged_release_checksum_and_does_not_start_cli() -> assert UNIX_INSTALLER.stat().st_mode & 0o111 assert "releases/download" in text assert "SHA256SUMS" in text - assert "v0.4.0" in text + assert "v0.4.1" in text + assert "returns input text unchanged" not in text + assert 'export PATH="$HOME/.local/bin:$PATH"' not in text assert "main.zip" not in text assert "force-reinstall" not in text assert not any(line.strip().startswith("sudo") for line in text.splitlines()) assert "apt-get" not in text assert text.count("-m fuckmark.cli") == 1 assert 'Command: fuckmark --help' in text + assert "hidden Unicode" in text assert not text.rstrip().endswith('-m fuckmark.cli') @@ -86,13 +89,17 @@ def test_windows_installer_uses_tagged_release_checksum_and_does_not_start_cli() text = WINDOWS_INSTALLER.read_text(encoding="utf-8") assert "releases/download" in text assert "SHA256SUMS" in text - assert "v0.4.0" in text + assert "v0.4.1" in text + assert "returns input text unchanged" not in text + assert "[Text.Encoding]::ASCII" not in text + assert "UnicodeEncoding" in text assert "main.zip" not in text assert "force-reinstall" not in text assert "winget install" not in text assert "Starting FuckMark" not in text assert text.count("-m fuckmark.cli") == 1 assert "Command: fuckmark --help" in text + assert "hidden Unicode" in text def test_verify_release_install_loads_mix_from_repository_root() -> None: @@ -136,3 +143,17 @@ def test_install_docs_do_not_recommend_live_main_or_pipe_installers() -> None: assert "5a6ac62c8bb8d7ddd9e5bc9cb6cee6e3eb181ac5f397b4a6645ef86468ee932f" in readme assert "5a6ac62c8bb8d7ddd9e5bc9cb6cee6e3eb181ac5f397b4a6645ef86468ee932f" in install assert "cb4ee7b6c06d1dde8c612c237df78f68f8364bc74bf469086288e55a2d5c9325" in install + assert ".venv/bin/fuckmark --version" in readme + assert ".venv/bin/fuckmark --visible" in readme or ".venv/bin/fuckmark --status" in readme + website = (ROOT / "docs/website.md").read_text(encoding="utf-8") + assert "| sh" not in website + assert "| iex" not in website + + +def test_unix_path_config_quotes_custom_bin_directory() -> None: + text = UNIX_INSTALLER.read_text(encoding="utf-8") + assert 'BIN="${FUCKMARK_BIN:-$HOME/.local/bin}"' in text + assert 'quoted="\'${escaped}\'"' in text or "quoted=\"'${escaped}'\"" in text + assert "export PATH=${quoted}" in text + assert "fish_add_path ${quoted}" in text + assert "$HOME/.local/bin:$PATH" not in text diff --git a/tests/test_project_identity.py b/tests/test_project_identity.py index f14e8334..c814bfb1 100644 --- a/tests/test_project_identity.py +++ b/tests/test_project_identity.py @@ -8,9 +8,9 @@ def test_project_identity_is_fuckmark_v040() -> None: root = Path(__file__).resolve().parents[1] project = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8"))["project"] assert project["name"] == "fuckmark" - assert project["version"] == "0.4.0" + assert project["version"] == "0.4.1" assert fuckmark.__project_name__ == "FuckMark" - assert fuckmark.__version__ == "0.4.0" + assert fuckmark.__version__ == "0.4.1" def test_legacy_project_identity_is_absent_outside_immutable_spec() -> None: diff --git a/tests/test_render_coverage.py b/tests/test_render_coverage.py new file mode 100644 index 00000000..aa602823 --- /dev/null +++ b/tests/test_render_coverage.py @@ -0,0 +1,56 @@ +from fuckmark.cycle8.letter_mix import apply_letter_alternating_mix +from fuckmark.cycle8.benchmark_render import _html_surface +from fuckmark.product.rendering import RENDERING_HARNESS_VERSION, _html_page, compare_chrome_pre_screenshots, render_window_size + + +def test_render_window_covers_below_fold_text() -> None: + short = "AAAA\n" + long = ("AAAA\n" * 40) + "BBBB\n" + width, height, complete, content_height = render_window_size(long, min_height=200) + assert complete is True + assert content_height > 240 + assert height >= content_height + below = render_window_size(short, min_height=200) + assert below[1] == 200 or below[3] <= 200 + + +def test_html_page_contains_actual_mix_payload() -> None: + source = "I do not agree." + mixed = apply_letter_alternating_mix(source) + assert "\u034f" in mixed or "\ufe00" in mixed + page = _html_page(mixed) + assert mixed in page + surface = _html_surface(mixed, "contenteditable", box_height=400) + assert "textContent=" in surface + assert "el.value=" not in surface + unknown = compare_chrome_pre_screenshots(source, mixed) + assert unknown.status in {"UNKNOWN", "VERIFIED", "REJECTED", "INCOMPLETE"} + if unknown.status == "VERIFIED": + assert unknown.equal is True + assert RENDERING_HARNESS_VERSION == "product-reference-render-v2" + + +def test_payload_render_artifact_keeps_controls_and_does_not_promote_unknown() -> None: + from json import loads + from pathlib import Path + + from fuckmark.hashing import sha256_json + + folder = Path(__file__).resolve().parents[1] / "evidence/audit-fixes-2026-08-28" + payload = loads((folder / "render-payload-v2.json").read_text(encoding="utf-8")) + body = {key: value for key, value in payload.items() if key != "artifact_hash"} + assert payload["artifact_hash"] == sha256_json(body) + assert payload["webkit_safari"] == "UNKNOWN" + assert payload["terminal_pixels"] == "UNKNOWN" + by_key = {(row["surface"], row["pair"]): row["result"]["status"] for row in payload["rows"]} + assert by_key[("pre", "mix_payload")] == "VERIFIED" + assert by_key[("textarea", "mix_payload")] == "VERIFIED" + assert by_key[("contenteditable", "mix_payload")] == "VERIFIED" + assert by_key[("contenteditable", "negative")] == "REJECTED" + assert by_key[("textarea", "positive")] == "VERIFIED" + assert by_key[("pre", "below_fold")] == "INCOMPLETE" + frozen = Path(__file__).resolve().parents[1] / "evidence/audit-fixes-2026-08-27/render-v2.json" + frozen_payload = loads(frozen.read_text(encoding="utf-8")) + frozen_body = {key: value for key, value in frozen_payload.items() if key != "artifact_hash"} + assert frozen_payload["artifact_hash"] == sha256_json(frozen_body) + assert frozen_payload["artifact_hash"] == "79f45bfa0914d673c919beba70b15e9c00191630c61ef82fb9703b7ff9da4ff2" diff --git a/tools/install/unix.sh b/tools/install/unix.sh index 6a4b19ab..38d4089b 100755 --- a/tools/install/unix.sh +++ b/tools/install/unix.sh @@ -1,7 +1,7 @@ #!/bin/sh set -eu -RELEASE_TAG="${FUCKMARK_RELEASE_TAG:-v0.4.0}" +RELEASE_TAG="${FUCKMARK_RELEASE_TAG:-v0.4.1}" PACKAGE_VERSION="${RELEASE_TAG#v}" WHEEL_NAME="fuckmark-${PACKAGE_VERSION}-py3-none-any.whl" RELEASE_BASE="https://github.com/byte271/FuckMark/releases/download/${RELEASE_TAG}" @@ -24,7 +24,9 @@ find_python() { ensure_path() { shell_name="$(basename "${SHELL:-sh}")" - line='export PATH="$HOME/.local/bin:$PATH"' + escaped=$(printf "%s" "$BIN" | sed "s/'/'\\\\''/g") + quoted="'${escaped}'" + line="export PATH=${quoted}:\"\$PATH\"" case "$shell_name" in zsh) file="$HOME/.zshrc" @@ -36,7 +38,7 @@ ensure_path() { file="$HOME/.config/fish/config.fish" mkdir -p "$(dirname "$file")" touch "$file" - fish_line='fish_add_path "$HOME/.local/bin"' + fish_line="fish_add_path ${quoted}" grep -F "$fish_line" "$file" >/dev/null 2>&1 || printf '\n%s\n' "$fish_line" >> "$file" return 0 ;; @@ -121,8 +123,9 @@ chmod +x "$BIN/fuckmark" ensure_path -printf '\n%s\n%s\n%s\n%s\n\n' \ +printf '\n%s\n%s\n%s\n%s\n%s\n\n' \ "FuckMark ${PACKAGE_VERSION} installed." \ - "The public CLI currently returns input text unchanged." \ + "The public CLI inserts hidden Unicode into ordinary English ASCII text." \ + "Installation success is not watermark removal. Check --status for the outcome." \ "Command: fuckmark --help" \ "Open a new terminal if the command is not on PATH yet." diff --git a/tools/install/windows.ps1 b/tools/install/windows.ps1 index 52709bbe..674519da 100644 --- a/tools/install/windows.ps1 +++ b/tools/install/windows.ps1 @@ -2,7 +2,7 @@ $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -$ReleaseTag = if ($env:FUCKMARK_RELEASE_TAG) { $env:FUCKMARK_RELEASE_TAG } else { "v0.4.0" } +$ReleaseTag = if ($env:FUCKMARK_RELEASE_TAG) { $env:FUCKMARK_RELEASE_TAG } else { "v0.4.1" } $PackageVersion = $ReleaseTag.TrimStart("v") $WheelName = "fuckmark-$PackageVersion-py3-none-any.whl" $ReleaseBase = "https://github.com/byte271/FuckMark/releases/download/$ReleaseTag" @@ -98,7 +98,8 @@ if ($LASTEXITCODE -ne 0) { } $LauncherBody = "@echo off`r`n`"$Python`" -m fuckmark.cli %*`r`n" -[IO.File]::WriteAllText($Launcher, $LauncherBody, [Text.Encoding]::ASCII) +$Utf16 = New-Object System.Text.UnicodeEncoding $false, $true +[IO.File]::WriteAllText($Launcher, $LauncherBody, $Utf16) $UserPath = [Environment]::GetEnvironmentVariable("Path", "User") $Entries = @() @@ -114,7 +115,8 @@ if (($env:Path -split ";") -notcontains $Bin) { Write-Host "" Write-Host "FuckMark $PackageVersion installed." -Write-Host "The public CLI currently returns input text unchanged." +Write-Host "The public CLI inserts hidden Unicode into ordinary English ASCII text." +Write-Host "Installation success is not watermark removal. Check --status for the outcome." Write-Host "Command: fuckmark --help" Write-Host "Open a new terminal if the command is not on PATH yet." Write-Host "" diff --git a/tools/verify_release_install.py b/tools/verify_release_install.py index 27da6b88..de4b117f 100644 --- a/tools/verify_release_install.py +++ b/tools/verify_release_install.py @@ -107,6 +107,36 @@ def _verify_artifact(artifact: Path) -> None: raise RuntimeError( f"installed CLI quoted argument failed for {command.name}: stdout={quoted.stdout!r} stderr={quoted.stderr!r}" ) + text_flag = _run( + [str(command), "--text", "I do not agree."], + capture_output=True, + env=environment, + ) + if text_flag.stdout != expected_arg or text_flag.stderr: + raise RuntimeError( + f"installed CLI --text failed for {command.name}: stdout={text_flag.stdout!r} stderr={text_flag.stderr!r}" + ) + source_file = root / "notes.txt" + source_file.write_text("I do not agree.\n", encoding="utf-8") + file_flag = _run( + [str(command), "--file", str(source_file)], + capture_output=True, + env=environment, + ) + expected_file = apply_letter_alternating_mix("I do not agree.\n") + if file_flag.stdout != expected_file or file_flag.stderr: + raise RuntimeError( + f"installed CLI --file failed for {command.name}: stdout={file_flag.stdout!r} stderr={file_flag.stderr!r}" + ) + status_line = _run( + [str(command), "--text", "I do not agree.", "--status"], + capture_output=True, + env=environment, + ) + if status_line.stdout != expected_arg or "fuckmark-status" not in status_line.stderr: + raise RuntimeError( + f"installed CLI --status failed for {command.name}: stdout={status_line.stdout!r} stderr={status_line.stderr!r}" + ) help_text = _run([str(command), "--help"], capture_output=True, env=environment).stdout if "fuckmark" not in help_text.casefold() or "--visible" not in help_text or ":done" not in help_text: raise RuntimeError(f"installed CLI --help failed for {command.name}: {help_text!r}") From 75796b7993ce8e8967657ef3b299912770503c12 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 28 Aug 2026 20:12:31 +0000 Subject: [PATCH 02/19] Update uv.lock for 0.4.1 and the markdown-it-py dev extra. Co-authored-by: Byte271 --- uv.lock | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index ef33c724..495ca758 100644 --- a/uv.lock +++ b/uv.lock @@ -111,11 +111,12 @@ wheels = [ [[package]] name = "fuckmark" -version = "0.4.0" +version = "0.4.1" source = { editable = "." } [package.optional-dependencies] dev = [ + { name = "markdown-it-py" }, { name = "pytest" }, ] research = [ @@ -127,6 +128,7 @@ research = [ [package.metadata] requires-dist = [ { name = "fonttools", marker = "extra == 'research'", specifier = ">=4.50" }, + { name = "markdown-it-py", marker = "extra == 'dev'", specifier = "==4.2.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3" }, { name = "tokenizers", marker = "extra == 'research'", specifier = ">=0.20" }, { name = "uharfbuzz", marker = "extra == 'research'", specifier = ">=0.39" }, @@ -232,6 +234,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "packaging" version = "26.3" From 83c30c7e4aa3e492ce2f8463771d4a81b1b10e4f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 28 Aug 2026 21:25:16 +0000 Subject: [PATCH 03/19] Make sanitizer reversal and processing status first-class UX. Add a no-install demo with baked CLI samples, louder unsupported-domain and coverage reporting, --inspect maps, --status on hard failures, ASCII Windows cmd launchers, and root-level spaced path protection. Co-authored-by: Byte271 --- CHANGELOG.md | 7 +- README.md | 51 +- docs/cli.md | 21 +- docs/demo.html | 560 ++++++++++++++++++++ docs/install.md | 4 +- docs/limits.md | 2 + docs/release.md | 2 +- docs/research.md | 2 +- docs/website.md | 22 +- fuckmark/cli.py | 189 ++++++- fuckmark/product/domain.py | 10 + fuckmark/transforms/protected_structures.py | 4 +- tests/test_audit_d03_d11.py | 45 ++ tests/test_cli.py | 7 + tests/test_demo_page.py | 74 +++ tests/test_evidence_manifests.py | 2 + tests/test_install_release_hardening.py | 10 +- tools/install/windows.ps1 | 44 +- 18 files changed, 1002 insertions(+), 54 deletions(-) create mode 100644 docs/demo.html create mode 100644 tests/test_demo_page.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5abe8de1..828c6c7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,10 @@ Audit findings D01-D16, E01-E04, L01-L06, P01, and G01 against `d7dc98b7ee99fc46 - Public copy, installer messages, and this source tree describe carrier insertion. They no longer say the CLI returns text unchanged. `docs/website.md` is the controlled website/installer copy. Do not pipe `https://d.q1z.org/mark` into a shell. - Package version is 0.4.1. The published v0.4.0 wheel remains the last GitHub Release artifact and does not implement `--text` / `--file`. - Markdown reference labels may span lines and line endings (LF, CRLF, CR). Definitions may put the destination on the next line and may appear in blockquotes or lists. Multiline inline destinations are protected. HTML tags, HTML entities, and indented code are protected. -- Extensionless relative paths such as `scripts/build`, last-component spaces such as `C:/Users/Alice/My final notes.txt`, and `ftp://` URIs are protected. `and/or` remains eligible. -- CLI outcomes: stderr reasons and `--status` distinguish transformed, unsupported Unicode, already-transformed, no eligible sites, site-cap, and internal failure. Clipboard partial success is exit 3. Usage errors remain exit 2. Internal transform failure is exit 4. Output is written before clipboard copy. Stdout write failures no longer escape as a traceback. -- Unix installer PATH uses `FUCKMARK_BIN`. Windows launcher is written as UTF-16 LE with BOM. +- Extensionless relative paths such as `scripts/build`, last-component spaces such as `C:/My final notes.txt` and `C:/Users/Alice/My final notes.txt`, and `ftp://` URIs are protected. `and/or` remains eligible. +- CLI outcomes: stderr reasons and `--status` distinguish transformed, unsupported Unicode, already-transformed, no eligible sites, site-cap, and internal failure. `--status` includes `processed`, `source_length`, and `first_unsupported`, including for `too-large` and internal failure. `--inspect` prints a character-level map. Interactive runs state coverage and sanitizer reversal. Clipboard partial success is exit 3. Usage errors remain exit 2. Internal transform failure is exit 4. Output is written before clipboard copy. Stdout write failures no longer escape as a traceback. +- No-install demo: `docs/demo.html` (baked CLI samples, character diffs, reversal, frozen GPT-2 scores). Deploy as `mark.q1z.org/demo.html`. Demo numbers are not a guarantee for user text. +- Unix installer PATH uses `FUCKMARK_BIN`. Windows `fuckmark.cmd` is ASCII for `cmd.exe`; non-ASCII Python paths use an ASCII `.cmd` trampoline to UTF-8 `fuckmark.ps1`. - Transfer replay tests compare live output hashes to stored hashes. Documented unwatermarked DeepMind drift is listed; unexplained drift fails. - Historical Cycle 7 Stage A README checksum mismatch is a CRLF-vs-LF provenance note, not a rewritten README. - Limits, sanitizer, length, tokenizer, and compatibility matrices: `docs/limits.md`. diff --git a/README.md b/README.md index f69472f4..60ef09f7 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,42 @@ It is a constrained research/product CLI with measured SynthID / GPT-2 results. Website: [mark.q1z.org](https://mark.q1z.org) · License: MIT -## Who it is for +## No-install demo (start here) + +Open [`docs/demo.html`](docs/demo.html) in a browser. `file://` works; no Python install required. After website deploy: [mark.q1z.org/demo.html](https://mark.q1z.org/demo.html). + +The demo shows: + +- character-level differences for fixed CLI samples (U+034F / U+FE00) +- processed vs not processed, reason, insertions, sites, `last_index`, cap +- Mn-strip / default-ignorable strip restoring the source +- frozen Gate v2 detection numbers on the fixed GPT-2 corpus + +It does **not** score your paste against a detector and does **not** promise the same detection outcome for arbitrary user text. + +## Honest limits (read these) + +### 1. Sanitizer reversal + +Insertion-only means clearing combining marks or default-ignorable characters restores the original text. On the frozen Gate v2 set: **0/192** after required sanitizers, then **188/192** again after Mn-strip or default-ignorable strip. That is a designed limitation, not a tuning gap. -People who paste or pipe **English ASCII** (tab, CR, LF, space through tilde) and want a local, deterministic, detector-blind insertion that keeps visible text identical. +### 2. Unsupported input must look like failure to process + +Supported code points: tab, CR, LF, and ASCII space through tilde. Chinese characters, emoji, accents, and curly apostrophes (U+2019) cause the **whole** input to be returned unchanged with exit 0. Exit 0 means I/O succeeded, not that hidden characters were inserted. + +Example: `I don't agree.` with a curly apostrophe is **not processed**. Use `--status` or `--inspect`. The paste UI and stderr say `not processed` with the first unsupported code point. + +### 3. Frozen scores are not “AI detector rate reduction” + +Primary confirmation used GPT-2 / SynthID with 64-token samples (192 pairs). DistilGPT2 n=16 still used the GPT-2 tokenizer. The product only fills the first 192 eligible letter sites; long documents stay mostly unchanged after that. See [`docs/limits.md`](docs/limits.md). + +### 4. Install is still a terminal tool + +Python 3.11+, venv, and a CLI remain the supported install. The no-install demo above is the path for people who need visible feedback first. + +## Who it is for -If the input contains an emoji, a curly apostrophe, an accent, or any other Unicode, FuckMark copies it unchanged and says so. +People who paste or pipe **English ASCII** and want a local, deterministic, detector-blind insertion that keeps visible text identical. ## Install from this repository @@ -63,7 +94,7 @@ Keep using the venv command from the install step. Examples below assume `.venv/ .venv/bin/fuckmark ``` -Paste or type text, then a line that is only `:done`. The result is copied to the clipboard. Status on stderr says whether hidden characters were inserted. +Paste or type text, then a line that is only `:done`. The result is copied to the clipboard. Status on stderr says whether hidden characters were inserted, with insertions, sites, `last_index`, and cap. Successful interactive runs also remind you that stripping combining marks restores the source. ```text printf 'I do not agree.\n' | .venv/bin/fuckmark @@ -72,9 +103,10 @@ printf 'I do not agree.\n' | .venv/bin/fuckmark .venv/bin/fuckmark --file notes.txt -o notes.fm.txt printf 'I do not agree.\n' | .venv/bin/fuckmark --visible printf 'I do not agree.\n' | .venv/bin/fuckmark --status >/tmp/fm.out +printf 'I do not agree.\n' | .venv/bin/fuckmark --inspect >/tmp/fm.out ``` -Pipes and files write the payload to stdout. `--visible` prints the original visible text. `--status` writes a machine-readable outcome to stderr. `fuckmark --help` is enough to start. +Pipes and files write the payload to stdout. `--visible` prints the original visible text. `--status` writes a machine-readable outcome to stderr. `--inspect` writes a character-level map to stderr. `fuckmark --help` is enough to start. ## What it guarantees @@ -82,21 +114,22 @@ Pipes and files write the payload to stdout. `--visible` prints the original vis Supported input: tab, newline, carriage return, and ASCII space through tilde. Other Unicode is returned unchanged with exit 0 and a stderr reason. That status means I/O succeeded, not that hidden characters were inserted or that a watermark was removed. Only UTF-8. -Machine text is left intact when recognized: fenced/inline/indented code, HTML tags and entities, markdown destinations and reference labels (including multiline and container forms), URLs including `ftp://`, emails, and paths such as `src/main.py`, `scripts/build`, and `C:/Users/Alice/My final notes.txt`. Ambiguous prose such as `and/or` stays eligible. +Machine text is left intact when recognized: fenced/inline/indented code, HTML tags and entities, markdown destinations and reference labels (including multiline and container forms), URLs including `ftp://`, emails, and paths such as `src/main.py`, `scripts/build`, `C:/My final notes.txt`, and `C:/Users/Alice/My final notes.txt`. Ambiguous prose such as `and/or` stays eligible. -## Verified results +## Frozen GPT-2 / SynthID confirmation (not a product guarantee) | Check | Result | | --- | --- | | Unmodified watermarked text still detected | **188/192** | | Transformed text after required sanitizers | **0/192** | +| Transformed text after Mn-strip / DI-strip | **188/192** (source restored) | | Transformed unwatermarked text | **0/192** | | Exact visible text | **192/192** | | Transformed text after UnicodeSanitizer | **0/192** | Google synthid-text 30-key GPT-2 transformed text: **0/192**. -These numbers are frozen Gate v2 confirmation on GPT-2 / SynthID, 64-token samples, threshold and sanitizer paths as recorded. They are not a universal zero-rate guarantee. See [`docs/limits.md`](docs/limits.md). +These numbers are frozen Gate v2 confirmation on GPT-2 / SynthID, 64-token samples, threshold and sanitizer paths as recorded. They are not a universal zero-rate guarantee and are not “AI detector rate reduction” for arbitrary platforms. See [`docs/limits.md`](docs/limits.md) and the [no-install demo](docs/demo.html). ## Limitations @@ -120,7 +153,7 @@ Eligible ASCII letters receive an alternating pair of invisible characters (U+03 ## Research -Frozen evidence, hashes, and protocols: [`docs/research.md`](docs/research.md). Product contract: [`docs/product-contract.md`](docs/product-contract.md). CLI: [`docs/cli.md`](docs/cli.md). Limits matrix: [`docs/limits.md`](docs/limits.md). +Frozen evidence, hashes, and protocols: [`docs/research.md`](docs/research.md). Product contract: [`docs/product-contract.md`](docs/product-contract.md). CLI: [`docs/cli.md`](docs/cli.md). Limits matrix: [`docs/limits.md`](docs/limits.md). Demo: [`docs/demo.html`](docs/demo.html). ## Development diff --git a/docs/cli.md b/docs/cli.md index 6f64f310..840b5917 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -20,6 +20,7 @@ fuckmark FuckMark Paste or type your text below. +English ASCII only. Curly apostrophes, accents, and emoji are not processed. Enter :done on a new line when finished. > I do not agree. @@ -31,10 +32,12 @@ Enter :done on a new line when finished. Processing... ✓ Copied to clipboard -FuckMark: inserted 54 hidden characters. +FuckMark: processed: inserted 54 hidden characters. +FuckMark: processed=yes reason=transformed insertions=54 sites=54 last_index=... source_length=... capped=no. +FuckMark: stripping combining marks or default-ignorable characters restores the source. ``` -Blank lines are kept. `:done` ends the session only when it is the entire line. The transformed payload is copied and not printed. +Blank lines are kept. `:done` ends the session only when it is the entire line. The transformed payload is copied and not printed. Interactive stderr always states processed vs not processed, insertions, sites, `last_index`, `source_length`, and cap. If clipboard copy fails, nothing is printed. Pipe text instead: `printf 'I do not agree.\n' | fuckmark`. Ctrl+C exits cleanly. EOF without `:done` is an error and copies nothing. @@ -56,9 +59,12 @@ fuckmark notes.txt -o notes.fm.txt printf 'I do not agree.\n' | fuckmark --copy printf 'I do not agree.\n' | fuckmark --visible printf 'I do not agree.\n' | fuckmark --status >/tmp/fm.out +printf 'I do not agree.\n' | fuckmark --inspect >/tmp/fm.out +fuckmark --text "I don’t agree." --status ``` -Piped or quoted input writes the hidden payload to stdout. `--visible` writes the original visible text. `--copy` also places whatever was written on the clipboard. + +Piped or quoted input writes the hidden payload to stdout. `--visible` writes the original visible text. `--copy` also places whatever was written on the clipboard. Stream mode stays silent on full success unless `--status` or `--inspect`. No-ops and the site cap always report on stderr unless `-q`. ## Input modes @@ -88,7 +94,8 @@ Existing files are read as UTF-8 bytes with no newline conversion. LF, CRLF, CR, | `--visible` | Print the visible text (no hidden characters). | | `--encoding NAME` | Only `utf-8`. `latin-1`, `ascii`, and `cp1252` are rejected. | | `-q`, `--quiet` | Hide non-essential status messages. | -| `--status` | Write one `fuckmark-status` line to stderr (result, insertions, sites, last_index, capped). | +| `--status` | Write one `fuckmark-status` line to stderr (`result`, `processed`, `insertions`, `sites`, `last_index`, `source_length`, `capped`, `first_unsupported`). | +| `--inspect` | Write a character-level coverage map to stderr. Stdout stays the payload. | | `--no-color` | Disable color on stderr. `NO_COLOR` does the same. | `--non-interactive` is an alias of `--stdin`. @@ -97,9 +104,11 @@ Existing files are read as UTF-8 bytes with no newline conversion. LF, CRLF, CR, Tab, newline, carriage return, and ASCII space through tilde. Other Unicode is returned unchanged (exit 0). Exit 0 means I/O succeeded. It does not mean that a transformation occurred or that a watermark was removed. Only UTF-8 files. -Machine spans stay intact: fenced/inline/indented code, HTML tags and entities, markdown destinations (including multiline), markdown reference labels (including multiline, container, and CR line endings), URLs (including `ftp://`), emails, IPs, dates, currency, percents, numbers, POSIX/Windows paths (including `src/main.py`, `scripts/build`, `C:/Users/Alice/My final notes.txt`), CLI flags. Quote interiors are eligible. Cap 192 insertion sites. Insertions fill the first 192 eligible letter sites and then stop, so the tail of a long document is unchanged. +Example: `I don't agree.` with U+2019 is reason `unsupported-domain`, `processed=no`, and `first_unsupported=U+2019@5`. The straight ASCII apostrophe in `I don't agree.` is processed. + +Machine spans stay intact: fenced/inline/indented code, HTML tags and entities, markdown destinations (including multiline), markdown reference labels (including multiline, container, and CR line endings), URLs (including `ftp://`), emails, IPs, dates, currency, percents, numbers, POSIX/Windows paths (including `src/main.py`, `scripts/build`, `C:/My final notes.txt`, `C:/Users/Alice/My final notes.txt`), CLI flags. Quote interiors are eligible. Cap 192 insertion sites. Insertions fill the first 192 eligible letter sites and then stop, so the tail of a long document is unchanged. -If the text is already transformed, has no eligible letters, or is outside the ASCII domain, the CLI returns it unchanged and reports that on stderr unless `-q`. `--status` always reports the reason. Exit 0 still means I/O succeeded, not watermark removal. +If the text is already transformed, has no eligible letters, or is outside the ASCII domain, the CLI returns it unchanged and reports that on stderr unless `-q`. `--status` always reports the reason, including for `too-large` and internal failure. Exit 0 still means I/O succeeded, not watermark removal. Same visible words are not the same as identical Markdown, path, or search behavior in other programs. Reports must not treat visible-projection equality as Markdown or filesystem equality. diff --git a/docs/demo.html b/docs/demo.html new file mode 100644 index 00000000..a7e837d6 --- /dev/null +++ b/docs/demo.html @@ -0,0 +1,560 @@ + + + + + +FuckMark no-install demo + + + +
+ FuckMark +
+

FuckMark no-install demo

+

Fixed CLI samples only. Shows character differences, processing status, sanitizer reversal, and frozen GPT-2 / SynthID numbers. This page does not score your text against a detector and does not guarantee results on any platform.

+
+
+
+ + +
+

Frozen detection comparison (fixed corpus, not your paste)

+

Gate v2 confirmation: GPT-2 / SynthID, 64-token samples, 192 pairs. DistilGPT2 n=16 still used the GPT-2 tokenizer. The product stops after 192 letter sites. These numbers are not a general AI-detector rate reduction.

+ + + + + + + + + + + +
ArmDetectionsWhat it means
Unmodified watermarked188/192Baseline still detected
Transformed + required sanitizers0/192Required-path success on this corpus
Transformed + Mn-strip188/192Source restored; watermark returns
Transformed + default-ignorable strip188/192Source restored; watermark returns
Exact visible text192/192Words on screen unchanged
+

Do not treat this table as a guarantee for arbitrary user text, commercial platforms, unknown detectors, or long documents past the 192-site cap.

+
+ +
+

Fixed CLI samples

+

Outputs were produced by the repository CLI and baked into this file so file:// works with no network.

+
+
+ +
+
+ U+034F insertion + U+FE00 insertion + Unsupported / restored mismatch +
+
+ + + + + +
+

Use the strip buttons on a transformed sample. Restoring the source is expected and brings frozen detections back to 188/192 on that corpus.

+
+ +
+

Local paste inspector (no detector, no insertion)

+

Checks whether text is ordinary English ASCII the CLI accepts. It does not run the mix transform, does not protect Markdown/paths, and does not query any detector. The CLI remains the source of truth.

+ +
+ + +
+
+
+

Unsupported Unicode returns exit 0 from the CLI with reason unsupported-domain. That is I/O success, not a completed transformation.

+
+ + +
+ + + + diff --git a/docs/install.md b/docs/install.md index 3c3a1cfa..ba70af6e 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2,6 +2,8 @@ Python 3.11 or newer. The public CLI is `fuckmark` / `FuckMark` / `Fuckmark`. +Before installing, open the no-install demo: [`demo.html`](demo.html) (`file://` works). After website deploy: [mark.q1z.org/demo.html](https://mark.q1z.org/demo.html). The demo shows character differences, processing status, sanitizer reversal, and frozen GPT-2 scores on fixed samples only. + Do not pipe `https://d.q1z.org/mark` into a shell. The live website must match [`website.md`](website.md). This source tree is **0.4.1**. The last published GitHub Release wheel is **v0.4.0**. That wheel does not implement `--text` or `--file`. Do not retag v0.4.0. The v0.4.1 wheel SHA-256 is recorded after the GitHub Release exists. @@ -81,7 +83,7 @@ Windows PowerShell, from a clone: powershell -ExecutionPolicy Bypass -File tools/install/windows.ps1 ``` -`FUCKMARK_BIN` is the launcher directory and the directory added to PATH. `FUCKMARK_HOME` is the venv root. Windows launchers are UTF-16 LE so Unicode profile paths are preserved. +`FUCKMARK_BIN` is the launcher directory and the directory added to PATH. `FUCKMARK_HOME` is the venv root. The Windows `fuckmark.cmd` launcher is ASCII so `cmd.exe` can run it. When the Python path is non-ASCII, an ASCII `.cmd` trampoline calls a UTF-8 `fuckmark.ps1`. ## Verify diff --git a/docs/limits.md b/docs/limits.md index f8bd818b..23f6306e 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -2,6 +2,8 @@ This page is the public limit list for FuckMark 0.4.1. Frozen detector numbers are not regenerated here. Do not retune on spent confirmation corpora. +No-install walkthrough of these limits: [`demo.html`](demo.html). + ## L01 — Sanitizers | Arm | Restores source? | Gate v2 mix WM (frozen) | diff --git a/docs/release.md b/docs/release.md index fa5c2edc..18d90c2d 100644 --- a/docs/release.md +++ b/docs/release.md @@ -21,7 +21,7 @@ FuckMark v0.4.1 is the audit-fix release on top of the Gate v2 product-authoriza 3. Create and push the immutable `v0.4.1` tag on that merge commit yourself. The workflow never runs `git tag`. Do not tag a pull-request SHA if `main` will be a squash merge. Do not retag `v0.4.0`. 4. Publish with `workflow_dispatch` and `publish_github_release=true` on that same commit. The job refuses to run unless `v0.4.1` already exists and points at the dispatch SHA. If the GitHub Release is missing, it uploads the verified wheel and sdist. It does not delete merged branches. 5. Record the published v0.4.1 wheel SHA-256 in README and `docs/install.md` in a follow-up commit, as was done for v0.4.0. -6. Deploy [mark.q1z.org](https://mark.q1z.org) and `https://d.q1z.org/mark` from `docs/website.md`, including the PowerShell User-Agent response. +6. Deploy [mark.q1z.org](https://mark.q1z.org), `https://mark.q1z.org/demo.html` from `docs/demo.html`, and `https://d.q1z.org/mark` from `docs/website.md`, including the PowerShell User-Agent response. The demo must stay static, work from `file://`, and must not run detectors on visitor paste. ## Publication workflow diff --git a/docs/research.md b/docs/research.md index 7fad64fb..2afe9879 100644 --- a/docs/research.md +++ b/docs/research.md @@ -1,6 +1,6 @@ # Research archive -This directory's user manuals are [`install.md`](install.md), [`cli.md`](cli.md), [`limits.md`](limits.md), and [`website.md`](website.md). +This directory's user manuals are [`install.md`](install.md), [`cli.md`](cli.md), [`limits.md`](limits.md), [`website.md`](website.md), and the no-install [`demo.html`](demo.html). The demo uses fixed samples and frozen scores; it is not a live detector. Frozen scientific records are not the user manual. They stay in the repository so the product claims can be audited. diff --git a/docs/website.md b/docs/website.md index 003c9609..5f01e053 100644 --- a/docs/website.md +++ b/docs/website.md @@ -4,12 +4,32 @@ This file is the controlled source for [mark.q1z.org](https://mark.q1z.org) and Do not pipe `https://d.q1z.org/mark` into a shell. Do not invoke that URL with `iex`. Those commands are disallowed. +## Lead with the no-install demo + +Deploy [`demo.html`](demo.html) as `https://mark.q1z.org/demo.html`. Keep it a static file that works from `file://` with baked CLI samples. The live page must show: + +- character differences for fixed samples (U+034F / U+FE00) +- processed vs not processed, reason, insertions, sites, `last_index`, capped +- Mn-strip / default-ignorable strip restoring the source (0/192 required-sanitizer success and 188/192 restore failure) +- frozen Gate v2 numbers with GPT-2 / 64-token / 192-site-cap scope + +Do not run detectors on visitor paste. Do not present demo numbers as a guarantee for arbitrary user text. + +Homepage copy should link the demo before install instructions. + ## What the product does FuckMark inserts U+034F and U+FE00 into ordinary English ASCII text without changing the visible words. v0.4.1 (this tree) already inserts those characters. It does not return the input unchanged until a future release. It is not a general watermark remover. +## Honest limits on the homepage + +1. Clearing combining marks or default-ignorable characters restores the source and restores frozen detections to 188/192. +2. Curly apostrophes, accents, emoji, and other non-ASCII leave the whole input unchanged with exit 0. That is not a completed transformation. +3. Frozen 0/192 results are GPT-2 / SynthID confirmation, not a general AI-detector claim. +4. Install remains Python 3.11+ and a CLI; the static demo is the no-install path. + ## Install (Linux / macOS) ```text @@ -34,4 +54,4 @@ Tagged wheel: GitHub Release `SHA256SUMS.txt` only. Last published wheel: v0.4.0 ## Results to show, with boundaries -Gate v2 confirmation (GPT-2 / SynthID, 64-token samples): unmodified watermarked **188/192**; transformed after required sanitizers **0/192**; visible text **192/192**. Stripping combining marks or default-ignorable characters restores the source and **188/192** detections. English ASCII only. Cap 192 letter sites. See [`limits.md`](limits.md). +Gate v2 confirmation (GPT-2 / SynthID, 64-token samples): unmodified watermarked **188/192**; transformed after required sanitizers **0/192**; visible text **192/192**. Stripping combining marks or default-ignorable characters restores the source and **188/192** detections. English ASCII only. Cap 192 letter sites. See [`limits.md`](limits.md) and [`demo.html`](demo.html). diff --git a/fuckmark/cli.py b/fuckmark/cli.py index 1dad1aff..2cb6148d 100644 --- a/fuckmark/cli.py +++ b/fuckmark/cli.py @@ -19,7 +19,11 @@ compose_letter_mix, select_letter_mix_sites, ) -from .product.domain import PRODUCT_MAX_INPUT_CHARS, is_supported_product_domain_v1 +from .product.domain import ( + PRODUCT_MAX_INPUT_CHARS, + first_unsupported_product_domain_v1, + is_supported_product_domain_v1, +) from .product.encodings import require_supported_product_encoding from .product.visible_projection import ( is_carrier_insertion_v1, @@ -65,37 +69,61 @@ class ProcessResult: last_source_index: int | None = None site_count: int = 0 capped: bool = False + source_length: int = 0 + first_unsupported: str = "" @property def changed(self) -> bool: return self.change_count > 0 + @property + def processed(self) -> bool: + return self.change_count > 0 + + +def _unsupported_token(text: str) -> str: + found = first_unsupported_product_domain_v1(text) + if found is None: + return "" + return f"U+{found[1]:04X}@{found[0]}" + + +def _unchanged(text: str, reason: str) -> ProcessResult: + token = _unsupported_token(text) if reason == REASON_UNSUPPORTED_DOMAIN else "" + return ProcessResult( + text, + 0, + reason=reason, + source_length=len(text), + first_unsupported=token, + ) + def transform_text(text: str) -> ProcessResult: if not isinstance(text, str): raise TypeError("text must be a string") try: if len(text) > PRODUCT_MAX_INPUT_CHARS: - return ProcessResult(text, 0, reason=REASON_TOO_LARGE) + return _unchanged(text, REASON_TOO_LARGE) approved = product_approved_carriers_v1() if approved != frozenset(LETTER_MIX_APPROVED_CARRIERS): - return ProcessResult(text, 0, reason=REASON_INTERNAL_ERROR) + return _unchanged(text, REASON_INTERNAL_ERROR) if any(ord(character) in approved for character in text): - return ProcessResult(text, 0, reason=REASON_ALREADY_TRANSFORMED) + return _unchanged(text, REASON_ALREADY_TRANSFORMED) if not is_supported_product_domain_v1(text): - return ProcessResult(text, 0, reason=REASON_UNSUPPORTED_DOMAIN) + return _unchanged(text, REASON_UNSUPPORTED_DOMAIN) probe = select_letter_mix_sites(text, max_selected=LETTER_MIX_MAX_SELECTED + 1) capped = len(probe) > LETTER_MIX_MAX_SELECTED sites = probe[:LETTER_MIX_MAX_SELECTED] if not sites: - return ProcessResult(text, 0, reason=REASON_NO_ELIGIBLE_SITES) + return _unchanged(text, REASON_NO_ELIGIBLE_SITES) applied = compose_letter_mix(text, sites) if applied == text: - return ProcessResult(text, 0, reason=REASON_NO_ELIGIBLE_SITES) + return _unchanged(text, REASON_NO_ELIGIBLE_SITES) if not is_carrier_insertion_v1(text, applied, approved): - return ProcessResult(text, 0, reason=REASON_INTERNAL_ERROR) + return _unchanged(text, REASON_INTERNAL_ERROR) if project_visible_v1(applied, approved) != text: - return ProcessResult(text, 0, reason=REASON_INTERNAL_ERROR) + return _unchanged(text, REASON_INTERNAL_ERROR) reason = REASON_SITE_CAP if capped else REASON_TRANSFORMED return ProcessResult( applied, @@ -104,9 +132,10 @@ def transform_text(text: str) -> ProcessResult: last_source_index=sites[-1], site_count=len(sites), capped=capped, + source_length=len(text), ) except (KeyError, TypeError, ValueError, RuntimeError): - return ProcessResult(text, 0, reason=REASON_INTERNAL_ERROR) + return _unchanged(text, REASON_INTERNAL_ERROR) def process_text(text: str) -> str: @@ -118,6 +147,7 @@ def read_interactive_text(input_stream: TextIO, ui_stream: TextIO, *, color: boo ui_stream.write( "\n" "Paste or type your text below.\n" + "English ASCII only. Curly apostrophes, accents, and emoji are not processed.\n" "Enter :done on a new line when finished.\n" "\n" ) @@ -160,11 +190,13 @@ def _parser() -> argparse.ArgumentParser: " fuckmark --stdin --visible < notes.fm.txt\n" "\n" "Supported input: tab, newline, carriage return, and ASCII space through tilde.\n" - "Other Unicode is returned unchanged with exit 0. That status means I/O\n" - "succeeded, not that hidden characters were inserted. Only UTF-8 is supported.\n" - "Visible text stays the same. Use --visible to print that visible text.\n" + "Other Unicode (including curly apostrophes) is returned unchanged with exit 0.\n" + "Exit 0 means I/O succeeded, not that hidden characters were inserted.\n" + "Only UTF-8 is supported. Visible text stays the same.\n" + "Use --visible to print that visible text.\n" "Use --text for literal strings and --file for existing UTF-8 files.\n" - "Use --status for a machine-readable outcome on stderr." + "Use --status for processed/reason/insertions/coverage on stderr.\n" + "Use --inspect for a character-level insertion map on stderr." ), formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -232,6 +264,12 @@ def _parser() -> argparse.ArgumentParser: action="store_true", help="write a machine-readable outcome line to stderr", ) + parser.add_argument( + "--inspect", + dest="inspect_report", + action="store_true", + help="write a character-level coverage map to stderr; stdout stays the payload", + ) parser.add_argument( "--no-color", action="store_true", @@ -480,33 +518,101 @@ def _status(errors: TextIO, message: str, *, enabled: bool, code: str) -> None: def _human_reason(result: ProcessResult) -> str: if result.reason == REASON_TRANSFORMED: - return f"inserted {result.change_count} hidden characters" + return f"processed: inserted {result.change_count} hidden characters" if result.reason == REASON_SITE_CAP: return ( - f"inserted {result.change_count} hidden characters, then stopped at the " - f"{LETTER_MIX_MAX_SELECTED}-site cap; trailing text is unchanged" + f"processed with coverage limit: inserted {result.change_count} hidden characters, " + f"then stopped at the {LETTER_MIX_MAX_SELECTED}-site cap; trailing text is unchanged" ) if result.reason == REASON_UNSUPPORTED_DOMAIN: - return "no hidden characters inserted (unsupported Unicode). I/O succeeded; this is not watermark removal" + loc = result.first_unsupported or "non-ASCII" + note = ( + f"not processed: unsupported Unicode ({loc}). " + "Accents, emoji, CJK, and curly quotes disable the whole input" + ) + if result.first_unsupported.startswith("U+2019@"): + note += ( + ". English curly apostrophes are not ASCII, so a sentence such as " + "I don\u2019t agree. is returned unchanged" + ) + note += ( + ". Exit 0 means I/O succeeded, not that hidden characters were inserted" + ) + return note if result.reason == REASON_ALREADY_TRANSFORMED: - return "no hidden characters inserted (input already contains the payload)" + return "not processed: input already contains the payload" if result.reason == REASON_NO_ELIGIBLE_SITES: - return "no hidden characters inserted (no eligible ASCII letter sites)" + return "not processed: no eligible ASCII letter sites" if result.reason == REASON_TOO_LARGE: - return f"input is too large (max {PRODUCT_MAX_INPUT_CHARS} characters)" - return "transformation failed internally; source returned unchanged" + return f"not processed: input is too large (max {PRODUCT_MAX_INPUT_CHARS} characters)" + return "not processed: transformation failed internally; source returned unchanged" + + +def _coverage_line(result: ProcessResult) -> str: + last = "" if result.last_source_index is None else str(result.last_source_index) + capped = "yes" if result.capped else "no" + processed = "yes" if result.processed else "no" + return ( + f"processed={processed} reason={result.reason} " + f"insertions={result.change_count} sites={result.site_count} " + f"last_index={last} source_length={result.source_length} capped={capped}" + ) def _machine_status_line(result: ProcessResult) -> str: last = "" if result.last_source_index is None else str(result.last_source_index) capped = "yes" if result.capped else "no" + processed = "yes" if result.processed else "no" + unsupported = result.first_unsupported return ( "fuckmark-status " - f"result={result.reason} insertions={result.change_count} " - f"sites={result.site_count} last_index={last} capped={capped}" + f"result={result.reason} processed={processed} insertions={result.change_count} " + f"sites={result.site_count} last_index={last} " + f"source_length={result.source_length} capped={capped} " + f"first_unsupported={unsupported}" ) +def _inspect_map(result: ProcessResult) -> str: + pieces: list[str] = [] + for character in result.output_text: + codepoint = ord(character) + if codepoint == 0x034F: + pieces.append("[U+034F]") + elif codepoint == 0xFE00: + pieces.append("[U+FE00]") + elif character == "\n": + pieces.append("\\n") + elif character == "\r": + pieces.append("\\r") + elif character == "\t": + pieces.append("\\t") + else: + pieces.append(character) + blob = "".join(pieces) + if len(blob) > 8000: + return blob[:8000] + "..." + return blob + + +def _emit_inspect(errors: TextIO, result: ProcessResult) -> None: + errors.write(f"fuckmark-inspect {_coverage_line(result)}\n") + if result.first_unsupported: + errors.write(f"fuckmark-inspect-unsupported {result.first_unsupported}\n") + if result.change_count > 0: + errors.write(f"fuckmark-inspect-map {_inspect_map(result)}\n") + errors.write( + "fuckmark-inspect-note stripping combining marks or " + "default-ignorable characters restores the source\n" + ) + elif result.reason == REASON_UNSUPPORTED_DOMAIN: + errors.write( + "fuckmark-inspect-note the whole input is returned unchanged; " + "exit 0 is not proof of insertion\n" + ) + errors.flush() + + def _emit_outcome( errors: TextIO, result: ProcessResult, @@ -514,18 +620,32 @@ def _emit_outcome( interactive: bool, quiet: bool, status_report: bool, + inspect_report: bool, color: bool, ) -> None: if status_report: errors.write(_machine_status_line(result) + "\n") errors.flush() + if inspect_report: + _emit_inspect(errors, result) if result.reason == REASON_INTERNAL_ERROR or result.reason == REASON_TOO_LARGE: return if quiet: return if result.reason == REASON_TRANSFORMED and not interactive: return - _status(errors, f"FuckMark: {_human_reason(result)}.", enabled=color, code=_ANSI_YELLOW if result.reason != REASON_TRANSFORMED else _ANSI_GREEN) + warn = result.reason != REASON_TRANSFORMED and result.reason != REASON_SITE_CAP + code = _ANSI_YELLOW if warn else _ANSI_GREEN + _status(errors, f"FuckMark: {_human_reason(result)}.", enabled=color, code=code) + if interactive or result.reason != REASON_TRANSFORMED: + _status(errors, f"FuckMark: {_coverage_line(result)}.", enabled=color, code=code) + if interactive and result.change_count > 0: + _status( + errors, + "FuckMark: stripping combining marks or default-ignorable characters restores the source.", + enabled=color, + code=_ANSI_YELLOW, + ) def main( @@ -672,8 +792,26 @@ def _run( return _error(errors, f"could not transform the text: {error}") if result.reason == REASON_TOO_LARGE: + _emit_outcome( + errors, + result, + interactive=interactive, + quiet=arguments.quiet, + status_report=arguments.status_report, + inspect_report=arguments.inspect_report, + color=color, + ) return _error(errors, _human_reason(result)) if result.reason == REASON_INTERNAL_ERROR: + _emit_outcome( + errors, + result, + interactive=interactive, + quiet=arguments.quiet, + status_report=arguments.status_report, + inspect_report=arguments.inspect_report, + color=color, + ) _error(errors, _human_reason(result)) return EXIT_INTERNAL @@ -707,6 +845,7 @@ def _run( interactive=interactive, quiet=arguments.quiet, status_report=arguments.status_report, + inspect_report=arguments.inspect_report, color=color, ) diff --git a/fuckmark/product/domain.py b/fuckmark/product/domain.py index 6ae7afa6..e57ad05a 100644 --- a/fuckmark/product/domain.py +++ b/fuckmark/product/domain.py @@ -10,3 +10,13 @@ def is_supported_product_domain_v1(text: str) -> bool: if not isinstance(text, str): raise TypeError("text must be a string") return all(ord(character) in _ALLOWED for character in text) + + +def first_unsupported_product_domain_v1(text: str) -> tuple[int, int] | None: + if not isinstance(text, str): + raise TypeError("text must be a string") + for index, character in enumerate(text): + codepoint = ord(character) + if codepoint not in _ALLOWED: + return index, codepoint + return None diff --git a/fuckmark/transforms/protected_structures.py b/fuckmark/transforms/protected_structures.py index 58a6218f..3ef01fbd 100644 --- a/fuckmark/transforms/protected_structures.py +++ b/fuckmark/transforms/protected_structures.py @@ -22,10 +22,10 @@ _FILENAME_TOKEN = r"[A-Za-z0-9._~+@%'-]+" _SPACED_BASENAME = rf"(?:{_FILENAME_TOKEN} ){{1,3}}{_FILENAME_TOKEN}\.[A-Za-z][A-Za-z0-9]{{0,11}}" _WINDOWS_SPACED_FILE_RE = re.compile( - rf"(?i)(?\r\n]{0,1024})?/?>") _HTML_ENTITY_RE = re.compile(r"&(?:[A-Za-z][A-Za-z0-9]{0,31}|#[0-9]{1,7}|#x[0-9A-Fa-f]{1,6});") diff --git a/tests/test_audit_d03_d11.py b/tests/test_audit_d03_d11.py index c0914d1e..f81e14c7 100644 --- a/tests/test_audit_d03_d11.py +++ b/tests/test_audit_d03_d11.py @@ -116,7 +116,10 @@ def test_paths_and_ftp_uri_are_protected_without_and_or() -> None: cases = ( ("See scripts/build now.", "scripts/build"), ("Open C:/Users/Alice/My final notes.txt now.", "C:/Users/Alice/My final notes.txt"), + ("Open C:/My final notes.txt now.", "C:/My final notes.txt"), ("Read /tmp/My final notes.txt now.", "/tmp/My final notes.txt"), + ("Read /My final notes.txt now.", "/My final notes.txt"), + ("Read ~/My final notes.txt now.", "~/My final notes.txt"), ("Get ftp://example.com/file now.", "ftp://example.com/file"), ) for source, token in cases: @@ -241,6 +244,48 @@ def test_status_flag_reports_noop_without_touching_payload() -> None: stderr = result.stderr.decode("utf-8") assert "fuckmark-status" in stderr assert "unsupported-domain" in stderr + assert "processed=no" in stderr + assert "source_length=14" in stderr + assert "first_unsupported=U+2019@5" in stderr + assert "not processed" in stderr + assert "curly" in stderr.casefold() + + +def test_status_flag_reports_too_large_and_inspect_map() -> None: + huge = ("a" * (PRODUCT_MAX_INPUT_CHARS + 1)).encode("utf-8") + too_large = _cli_bytes("--stdin", "--status", stdin=huge) + assert too_large.returncode == EXIT_ERROR + stderr = too_large.stderr.decode("utf-8") + assert "fuckmark-status" in stderr + assert "too-large" in stderr + assert "processed=no" in stderr + source = "I do not agree." + inspected = _cli_bytes("--text", source, "--inspect") + assert inspected.returncode == EXIT_OK + assert inspected.stdout == apply_letter_alternating_mix(source).encode("utf-8") + inspect_err = inspected.stderr.decode("utf-8") + assert "fuckmark-inspect" in inspect_err + assert "[U+034F]" in inspect_err + assert "restores the source" in inspect_err + + +def test_internal_failure_emits_status_when_requested(monkeypatch) -> None: + def boom(text: str, sites): + raise RuntimeError("invariant") + + monkeypatch.setattr("fuckmark.cli.compose_letter_mix", boom) + output = StringIO() + errors = StringIO() + status = main( + StringIO(""), + output, + error_stream=errors, + argv=("--text", "I do not agree.", "--status"), + ) + assert status == EXIT_INTERNAL + assert output.getvalue() == "" + assert "fuckmark-status" in errors.getvalue() + assert "internal-error" in errors.getvalue() def test_bracket_heavy_scan_stays_bounded() -> None: diff --git a/tests/test_cli.py b/tests/test_cli.py index bcd3d115..e63c1ea8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -121,6 +121,7 @@ def test_cli_reads_multiline_paste_until_done_and_keeps_blank_lines() -> None: ui = prompt.getvalue() assert ui.startswith("FuckMark\n") assert "Paste or type your text below." in ui + assert "English ASCII only" in ui assert ":done" in ui assert ui.count("> ") == 5 @@ -145,6 +146,9 @@ def test_cli_main_interactive_copies_without_printing_payload() -> None: assert "FuckMark" in ui assert "Processing..." in ui assert "Copied to clipboard" in ui + assert "processed=yes" in ui + assert "source_length=" in ui + assert "restores the source" in ui assert expected not in ui assert "I don't agree." not in ui assert project_visible_v1(expected, APPROVED) == "I do not agree." @@ -420,9 +424,12 @@ def test_cli_help_documents_file_pipe_clipboard_and_visible_contract(capsys) -> assert "fuckmark \"I do not agree.\"" in rendered assert "--text" in rendered assert "--file" in rendered + assert "--status" in rendered + assert "--inspect" in rendered assert "standard input" in rendered.casefold() or "--stdin" in rendered assert ":done" in rendered assert "clipboard" in rendered.casefold() + assert "curly" in rendered.casefold() def test_cli_quoted_text_argument_transforms_without_a_file() -> None: diff --git a/tests/test_demo_page.py b/tests/test_demo_page.py new file mode 100644 index 00000000..36d81965 --- /dev/null +++ b/tests/test_demo_page.py @@ -0,0 +1,74 @@ +import json +import re +import unicodedata +from pathlib import Path + +from fuckmark.cli import transform_text + + +ROOT = Path(__file__).resolve().parents[1] +DEMO = ROOT / "docs" / "demo.html" + + +def _samples() -> list[dict]: + html = DEMO.read_text(encoding="utf-8") + match = re.search( + r'', + html, + flags=re.S, + ) + assert match is not None + payload = json.loads(match.group(1)) + assert isinstance(payload, list) and payload + return payload + + +def test_demo_page_is_self_contained_and_honest() -> None: + html = DEMO.read_text(encoding="utf-8") + assert "fetch(" not in html + assert "XMLHttpRequest" not in html + assert "no-install demo" in html.casefold() + assert "188/192" in html + assert "0/192" in html + assert "not a general" in html.casefold() or "not a guarantee" in html.casefold() + assert "Mn" in html + assert "Default_Ignorable_Code_Point" in html + assert "unsupported-domain" in html + assert "detector score" in html.casefold() + assert "file://" in html or "file://" in html.casefold() + assert "GPT-2" in html + assert "192" in html + assert ' + + diff --git a/docs/rec.html b/docs/rec.html new file mode 100644 index 00000000..67d2a37e --- /dev/null +++ b/docs/rec.html @@ -0,0 +1,26 @@ + + + + + +Mark + + + + + + diff --git a/docs/website.md b/docs/website.md index 64e96cc9..e40ede50 100644 --- a/docs/website.md +++ b/docs/website.md @@ -4,9 +4,28 @@ This file is the controlled source for [mark.q1z.org](https://mark.q1z.org) and Do not pipe `https://d.q1z.org/mark` into a shell. Do not invoke that URL with `iex`. Those commands are disallowed. -## Lead with the no-install demo +## Lead with the browser tool, then the research demo -Deploy [`demo.html`](demo.html) as `https://mark.q1z.org/demo.html`. Keep it a static file that works from `file://` with baked CLI samples. The live page must show: +Deploy these static files (repository CI cannot push them; copy after each release): + +| Repo file | Public URL | +| --- | --- | +| [`mark.html`](mark.html) | `https://mark.q1z.org/mark.html` (browser tool) | +| [`rec.html`](rec.html) | gate that waits for `go.txt`, then opens `mark.html?demo=1` | +| [`demo.html`](demo.html) | `https://mark.q1z.org/demo.html` (research / CLI samples) | + +### `mark.html` (product paste UI) + +Runs in the browser. On **Remove marks** it first runs the same closed-set FuckMark insertion scan as `fuckmark --detect` (U+034F, U+FE00, C0/C1, U+20DD, U+13430-U+13438, U+FFF9-U+FFFB). + +- Hit: strip those insertions, copy the cleaned text, keep visible words. +- Miss: show the no-watermark card (Chinese copy in `mark.html`) and the contact form that mails `Fhelp@q1z.org`. + +Do not strip emoji variation selectors, ZWJ, or other non-approved characters. Do not query GPT-2, SynthID, or any remote detector. Icon scripts may load from a CDN; the scan itself is local. + +### `demo.html` (research walkthrough) + +Keep it a static file that works from `file://` with baked CLI samples. The live page must show: - character differences for fixed samples (U+034F / U+FE00 plus C0/C1 / Me / Cf / annotation residuals) - processed vs not processed, reason, insertions, sites, `last_index`, capped @@ -14,9 +33,9 @@ Deploy [`demo.html`](demo.html) as `https://mark.q1z.org/demo.html`. Keep it a s - Mn-strip / default-ignorable strip leaving control residuals (source not restored; four-layer restore census 0/192) - frozen Gate v2 numbers with GPT-2 / 64-token / historical mark-only scope -Do not query GPT-2, SynthID, or any remote detector on visitor paste. The local scan only looks for FuckMark insertion characters. If none are found, the demo offers contact at `Fhelp@q1z.org`. Do not present demo numbers as a guarantee for arbitrary user text. +If none are found, the demo offers contact at `Fhelp@q1z.org`. Do not present demo numbers as a guarantee for arbitrary user text. -Homepage copy should link the demo before install instructions. +Homepage copy should link `mark.html` (tool) and `demo.html` (evidence) before install instructions. ## What the product does diff --git a/tests/test_mark_page.py b/tests/test_mark_page.py new file mode 100644 index 00000000..5745eafc --- /dev/null +++ b/tests/test_mark_page.py @@ -0,0 +1,51 @@ +import re +from pathlib import Path + +from fuckmark.cli import process_text +from fuckmark.product.detect import DETECT_CONTACT_EMAIL, detect_fuckmark_insertions + + +ROOT = Path(__file__).resolve().parents[1] +MARK = ROOT / "docs" / "mark.html" +REC = ROOT / "docs" / "rec.html" + + +def test_mark_page_is_closed_set_detector_with_contact() -> None: + html = MARK.read_text(encoding="utf-8") + assert "FM_CARRIER_SET" in html + assert "detectFuckMark" in html + assert "0x13430" in html + assert "length: 9" in html + assert "\\uFFF9" in html + assert "\\u20DD" in html + assert "FE00-\\uFE0F" not in html + assert "\\u200B-\\u200D" not in html + assert DETECT_CONTACT_EMAIL in html + assert f"mailto:{DETECT_CONTACT_EMAIL}" in html + assert "Fmark@q1z.org" not in html + assert "\u6211\u4eec\u6ca1\u6709\u4ece\u8fd9\u91cc\u9762\u68c0\u6d4b\u5230\u6c34\u5370" in html + assert "\u8054\u7cfb\u6211\u4eec" in html + assert "\u4ec0\u4e48\uff1f\u60a8\u8ba4\u4e3a\u8fd9\u91cc\u9762\u6709\u6c34\u5370\uff1f" in html + assert "closed-set" in html.casefold() + assert "not a general" in html.casefold() + assert "markedSample" in html + assert "FM_CCS" in html + assert "FM_CFS" in html + + +def test_mark_page_demo_sample_is_detectable_by_python_detector() -> None: + source = "Hello from Q1z. Visible text stays." + mixed = process_text(source) + hit = detect_fuckmark_insertions(mixed) + assert hit.detected is True + assert hit.found > 0 + miss = detect_fuckmark_insertions(source) + assert miss.detected is False + + +def test_rec_page_redirects_to_mark_demo() -> None: + html = REC.read_text(encoding="utf-8") + assert "mark.html" in html + assert "demo=1" in html + assert "go.txt" in html + assert re.search(r"location\.replace\(\s*[\"']mark\.html", html) From 0d273d2fb737e5b7eedee30fdf9c5985ad12c05a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 29 Aug 2026 18:40:08 +0000 Subject: [PATCH 17/19] Add fuckmark web and switch detector UI copy to English. Serve the packaged mark.html browser tool with `fuckmark web` for beginners who prefer a page over the CLI. Miss/contact copy is English only. CLI identity is release-cli-v12. Co-authored-by: Byte271 --- CHANGELOG.md | 5 +- README.md | 12 +- docs/cli.md | 13 + docs/cycle8/gate-v2.md | 2 +- docs/demo.html | 2 +- docs/install.md | 6 +- docs/mark.html | 6 +- docs/product-contract.md | 2 +- docs/release.md | 2 +- docs/research.md | 2 +- docs/website.md | 4 +- fuckmark/cli.py | 8 +- fuckmark/cycle8/gate_v2.py | 4 +- fuckmark/product/authorization.py | 6 +- fuckmark/web.py | 151 ++ fuckmark/webui/__init__.py | 0 fuckmark/webui/mark.html | 1576 +++++++++++++++++ pyproject.toml | 3 + ...kmark-cycle8-product-authorization-v2.json | 2 +- tests/test_cli.py | 5 +- tests/test_cycle8_mix_publishability.py | 2 +- tests/test_demo_page.py | 6 +- tests/test_install_release_hardening.py | 7 +- tests/test_mark_page.py | 8 +- tests/test_product_authorization.py | 2 +- tests/test_web.py | 71 + 26 files changed, 1875 insertions(+), 32 deletions(-) create mode 100644 fuckmark/web.py create mode 100644 fuckmark/webui/__init__.py create mode 100644 fuckmark/webui/mark.html create mode 100644 tests/test_web.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e048f533..4f917338 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,14 @@ Audit findings D01-D16, E01-E04, L01-L06, P01, and G01 against `d7dc98b7ee99fc46b767c93444a9f051fdbea2ba`. Do not retag v0.4.0. Frozen confirmation files and their SHA-256 sums are unchanged. -- Live mix is five-layer (`u034f-ufe00-cc-me-cf-ia-letter-alt-v1`, `release-cli-v11`): each eligible grapheme cluster gets U+034F or U+FE00, a C0/C1 control, enclosing Me (U+20DD), a cycling Egyptian hieroglyph format control (U+13430-U+13438), and a cycling interlinear annotation control (U+FFF9-U+FFFB). UnicodeSanitizer turns those annotation controls into spaces, so Mn then Me then UnicodeSanitizer then frozen Cf-strip no longer restores the source. Sites are Latin, Greek, Cyrillic, Han, Kana, Hangul syllables, and emoji. NFD Latin is marked after the combining sequence so normalization does not split the cluster. Visible projection stays identical. Mn-strip, default-ignorable strip, UnicodeSanitizer orderings, Mn then Me then UnicodeSanitizer, and the required sanitizer bundle no longer restore the source. Chromium pre-pixel probes reject the live Me-layer mix; historical mark-only remains pixel-equal where measured. The Egyptian Cf cycle stays the Unicode 14 assigned range U+13430-U+13438. +- Live mix is five-layer (`u034f-ufe00-cc-me-cf-ia-letter-alt-v1`, `release-cli-v12`): each eligible grapheme cluster gets U+034F or U+FE00, a C0/C1 control, enclosing Me (U+20DD), a cycling Egyptian hieroglyph format control (U+13430-U+13438), and a cycling interlinear annotation control (U+FFF9-U+FFFB). UnicodeSanitizer turns those annotation controls into spaces, so Mn then Me then UnicodeSanitizer then frozen Cf-strip no longer restores the source. Sites are Latin, Greek, Cyrillic, Han, Kana, Hangul syllables, and emoji. NFD Latin is marked after the combining sequence so normalization does not split the cluster. Visible projection stays identical. Mn-strip, default-ignorable strip, UnicodeSanitizer orderings, Mn then Me then UnicodeSanitizer, and the required sanitizer bundle no longer restore the source. Chromium pre-pixel probes reject the live Me-layer mix; historical mark-only remains pixel-equal where measured. The Egyptian Cf cycle stays the Unicode 14 assigned range U+13430-U+13438. - Exploratory restore census of frozen Gate v2 watermarked sources on seeds 1200000, 1210000, and 1220000 (n=192, detector not run): four-layer **0/192** restore under Mn then Me then UnicodeSanitizer, and **0/192** match to `UnicodeSanitizer(source)`, with Cf residual on every row. Historical triple-layer under that same path matches `UnicodeSanitizer(source)` **192/192**. Confirmation artifacts were not rewritten. - Exploratory GPT-2 / SynthID rescore of those same frozen sources remains historical triple-layer evidence (`u034f-ufe00-cc-me-letter-alt-v1`): **0/192** on raw, Mn-strip, DI-strip, UnicodeSanitizer, Mn then US, DI then US, US then Mn, required-bundle, and bundle then US. Historical dual-layer / mark-only under Mn then US returns to **182/192** detections. - Exploratory DistilGPT2 rescore of frozen seed 1090000 watermarked sources (n=16, `HYPOTHESIS`) remains historical triple-layer evidence: identity **16/16**. Historical triple-layer **0/16** after Mn then UnicodeSanitizer and after required-bundle then UnicodeSanitizer. Frozen DistilGPT2 scorecard was not rewritten. - Latin letters outside ASCII, Han syllables, and emoji-only input are processed. Curly apostrophes stay in the visible text and are reported as `first_unsupported`. Processed letters and emoji are not leftovers. Joining scripts such as Arabic stay unchanged. Site cap is 4096 (five insertions per site). - `--detect` and the no-install demo scan for FuckMark insertion characters only. A miss prints that no watermark was detected and points to `Fhelp@q1z.org`. That scan is not a general AI-watermark detector. -- Website tool source is now in-repo: `docs/mark.html` (plus `docs/rec.html` gate). It uses the same closed-set scan, strips approved insertions on a hit, and on a miss shows the Chinese no-watermark card with mailto `Fhelp@q1z.org`. It no longer strips emoji VS / ZWJ ranges. +- Website tool source is now in-repo: `docs/mark.html` (plus `docs/rec.html` gate), packaged as `fuckmark/webui/mark.html`. It uses the same closed-set scan, strips approved insertions on a hit, and on a miss shows an English no-watermark card with mailto `Fhelp@q1z.org`. It no longer strips emoji VS / ZWJ ranges. +- `fuckmark web` serves that local browser tool for beginners who prefer a page over the CLI (`http://127.0.0.1:8765/mark.html`). - Product authorization v2 records the five-layer path. Gate v2 confirmation hashes stay historical. The v1 mix publishability report records the historical triple-layer fixture measurements and remains `product_authorized: false`. - Public copy, installer messages, and this source tree describe carrier insertion. They no longer say the CLI returns text unchanged. `docs/website.md` is the controlled website/installer copy. Do not pipe `https://d.q1z.org/mark` into a shell. - Package version is 0.4.1. The published v0.4.0 wheel remains the last GitHub Release artifact and does not implement `--text` / `--file`. diff --git a/README.md b/README.md index 55983b74..bf6c2996 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,18 @@ It is a constrained research/product CLI with measured SynthID / GPT-2 results. Website: [mark.q1z.org](https://mark.q1z.org) · License: MIT -## Browser tool and no-install demo (start here) +## Browser tool (start here if you dislike the CLI) -- Product paste UI: [`docs/mark.html`](docs/mark.html) — after deploy: [mark.q1z.org/mark.html](https://mark.q1z.org/mark.html) +```text +fuckmark web +``` + +Opens a local page at `http://127.0.0.1:8765/mark.html`. Paste text in the browser. No other CLI flags required. Use `--no-open` to print the URL only, or `--port 0` to pick a free port. + +- Product paste UI source: [`docs/mark.html`](docs/mark.html) (packaged as `fuckmark/webui/mark.html`) — after deploy: [mark.q1z.org/mark.html](https://mark.q1z.org/mark.html) - Research demo: [`docs/demo.html`](docs/demo.html) — `file://` works; after deploy: [mark.q1z.org/demo.html](https://mark.q1z.org/demo.html) -`mark.html` runs a local closed-set FuckMark scan, strips approved insertions when found, and on a miss shows the no-watermark card with contact to `Fhelp@q1z.org`. +`mark.html` runs a local closed-set FuckMark scan, strips approved insertions when found, and on a miss shows an English no-watermark card with contact to `Fhelp@q1z.org`. The research demo also includes: diff --git a/docs/cli.md b/docs/cli.md index 240399e5..c4ac3a04 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -63,6 +63,7 @@ printf 'I do not agree.\n' | fuckmark --status >/tmp/fm.out printf 'I do not agree.\n' | fuckmark --inspect >/tmp/fm.out fuckmark --detect --text "I do not agree." printf 'paste\n' | fuckmark --detect +fuckmark web fuckmark --text "I don’t agree." --status ``` @@ -102,6 +103,18 @@ Existing files are read as UTF-8 bytes with no newline conversion. LF, CRLF, CR, | `--detect` | Scan for FuckMark insertions without transforming. Stdout is the detect report. If none are found, the report includes `Fhelp@q1z.org`. | | `--no-color` | Disable color on stderr. `NO_COLOR` does the same. | +### `fuckmark web` + +Open the local browser tool (same UI as `docs/mark.html`). Aimed at beginners who prefer a page over pipes and flags. + +```text +fuckmark web +fuckmark web --no-open +fuckmark web --port 9000 +``` + +Default URL: `http://127.0.0.1:8765/mark.html`. Press Ctrl+C to stop the server. + `--non-interactive` is an alias of `--stdin`. ## Supported input diff --git a/docs/cycle8/gate-v2.md b/docs/cycle8/gate-v2.md index 252a437f..b8fcd601 100644 --- a/docs/cycle8/gate-v2.md +++ b/docs/cycle8/gate-v2.md @@ -2,6 +2,6 @@ Historical research write-up for publishability Gate v2. The machine-readable gate is `specs/cycle8/fuckmark-cycle8-publishability-gate-v2.json`. Confirmation scorecard: `evidence/cycle8-gate-v2-confirmation-2026-08-27/`. -At confirmation time some READMEs said the public CLI remained empty. That sentence is historical. Live authorization is `cycle8-product-authorization-v2` / `release-cli-v11` five-layer mix. Frozen Gate v2 confirmation remains the historical mark-only corpus. Do not treat archive "CLI remains empty" lines as current product behavior. +At confirmation time some READMEs said the public CLI remained empty. That sentence is historical. Live authorization is `cycle8-product-authorization-v2` / `release-cli-v12` five-layer mix. Frozen Gate v2 confirmation remains the historical mark-only corpus. Do not treat archive "CLI remains empty" lines as current product behavior. H16 local audit: `evidence/h16-local/README.md`. diff --git a/docs/demo.html b/docs/demo.html index ff759ea7..46e9c9bf 100644 --- a/docs/demo.html +++ b/docs/demo.html @@ -832,7 +832,7 @@

Local paste inspector (eligibility, no network)

setStat("first", first) ); } else { - resultBox.innerHTML = '

我们没有从这里面检测到水印

什么？您认为这里面有水印？

联系我们

'; + resultBox.innerHTML = '

We did not detect a watermark in this text.

What? You think there is a watermark in this?

Contact us

'; statusBox.append( setStat("local scan", "not detected", "warn"), setStat("found", "0"), diff --git a/docs/install.md b/docs/install.md index 332bf50a..38ebcfcb 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2,7 +2,7 @@ Python 3.11 or newer. The public CLI is `fuckmark` / `FuckMark` / `Fuckmark`. -Before installing, open the no-install demo: [`demo.html`](demo.html) (`file://` works). After website deploy: [mark.q1z.org/demo.html](https://mark.q1z.org/demo.html). The demo shows character differences, processing status, Mn/DI residual checks, and frozen GPT-2 scores on fixed samples only. +Prefer a browser over the CLI? After install, run `fuckmark web` to open the local paste tool. Before installing, you can also open [`mark.html`](mark.html) or the research demo [`demo.html`](demo.html) with `file://`. After website deploy: [mark.q1z.org/mark.html](https://mark.q1z.org/mark.html) and [mark.q1z.org/demo.html](https://mark.q1z.org/demo.html). Do not pipe `https://d.q1z.org/mark` into a shell. The live website must match [`website.md`](website.md). @@ -18,14 +18,18 @@ cd FuckMark python3 -m venv .venv .venv/bin/python -m pip install . .venv/bin/fuckmark --version +.venv/bin/fuckmark web printf 'I do not agree.\n' | .venv/bin/fuckmark --visible ``` +`fuckmark web` opens `http://127.0.0.1:8765/mark.html` for beginners who do not want pipes or flags. + Keep using `.venv/bin/fuckmark` unless you activate the venv: ```text . .venv/bin/activate fuckmark --version +fuckmark web ``` Windows PowerShell: diff --git a/docs/mark.html b/docs/mark.html index 5512269b..a34fd0e1 100644 --- a/docs/mark.html +++ b/docs/mark.html @@ -644,7 +644,7 @@

FuckMark

-

Paste text. FuckMark checks for its
insertion characters, then strips them.

+

No CLI needed. Paste text here.
FuckMark checks for its insertions, then strips them.

@@ -692,9 +692,9 @@

Success

-

我们没有从这里面检测到水印

+

We did not detect a watermark in this text.

Closed-set scan of FuckMark insertion characters only. Not a general AI-watermark detector.

-

什么？您认为这里面有水印？

+

What? You think there is a watermark in this?

diff --git a/docs/product-contract.md b/docs/product-contract.md index 683d3ee6..8c8a448b 100644 --- a/docs/product-contract.md +++ b/docs/product-contract.md @@ -37,7 +37,7 @@ These are not product success: ## Current product path -`release_transform_registry()` stays empty. The public CLI (`release-cli-v11`) applies `apply_letter_alternating_mix` directly: U+034F on even selected letter/emoji grapheme clusters, U+FE00 on odd sites, plus a cycling C0/C1 control, enclosing Me (U+20DD), a cycling Egyptian hieroglyph format control (U+13430-U+13438), and a cycling interlinear annotation control (U+FFF9-U+FFFB), cap 4096, outside hard machine spans. Latin (including U+00E9), Greek, Cyrillic, Han, Kana, Hangul syllables, and emoji are processed. NFD clusters receive insertions after the combining sequence. Live protection covers fenced/inline/indented code, HTML tags and entities, Markdown destinations and reference labels (including multiline and container forms), URLs including `ftp://`, and paths including `src/main.py`, `scripts/build`, and last-component spaces. Those protection changes can change outputs versus the v0.4.0 freeze of `letter_mix.py` (`cycle8-mix-freeze-v1`, `letter_mix_source_sha256` `b1ceec24e584c0e9e7135ef0c89a3bd249b0bda4a45e07aa7190b1b010ba56d4`). Frozen confirmation hashes stay historical. They are not silently rewritten. `--detect` is a closed-set scan of those approved insertions and is not a general AI-watermark detector. +`release_transform_registry()` stays empty. The public CLI (`release-cli-v12`) applies `apply_letter_alternating_mix` directly: U+034F on even selected letter/emoji grapheme clusters, U+FE00 on odd sites, plus a cycling C0/C1 control, enclosing Me (U+20DD), a cycling Egyptian hieroglyph format control (U+13430-U+13438), and a cycling interlinear annotation control (U+FFF9-U+FFFB), cap 4096, outside hard machine spans. Latin (including U+00E9), Greek, Cyrillic, Han, Kana, Hangul syllables, and emoji are processed. NFD clusters receive insertions after the combining sequence. Live protection covers fenced/inline/indented code, HTML tags and entities, Markdown destinations and reference labels (including multiline and container forms), URLs including `ftp://`, and paths including `src/main.py`, `scripts/build`, and last-component spaces. Those protection changes can change outputs versus the v0.4.0 freeze of `letter_mix.py` (`cycle8-mix-freeze-v1`, `letter_mix_source_sha256` `b1ceec24e584c0e9e7135ef0c89a3bd249b0bda4a45e07aa7190b1b010ba56d4`). Frozen confirmation hashes stay historical. They are not silently rewritten. `--detect` is a closed-set scan of those approved insertions and is not a general AI-watermark detector. Live `product_approved_carriers_v1()` is the five-layer set `{U+034F, U+FE00}` plus C0/C1 controls (DEL and C1 except NEL) plus U+20DD plus U+13430-U+13438 plus U+FFF9-U+FFFB. Visible projection strips those carriers; Me may decorate glyphs in some renderers. The v1 contract's empty `product_authorized_carriers_v1` list is a historical snapshot of that file. diff --git a/docs/release.md b/docs/release.md index 9d9392e1..c3dc42bc 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,6 +1,6 @@ # v0.4.1 Release Process -FuckMark v0.4.1 is the audit-fix and five-layer mix release on top of the Gate v2 product-authorization CLI. The public CLI algorithm identity is `release-cli-v11`. Do not retag `v0.4.0`. The historical `v0.3.0` tag is the identity CLI (`release-cli-v4`). The historical `v0.2.0` tag still applies contractions. +FuckMark v0.4.1 is the audit-fix and five-layer mix release on top of the Gate v2 product-authorization CLI. The public CLI algorithm identity is `release-cli-v12`. Do not retag `v0.4.0`. The historical `v0.3.0` tag is the identity CLI (`release-cli-v4`). The historical `v0.2.0` tag still applies contractions. ## Release invariants diff --git a/docs/research.md b/docs/research.md index f30cd05c..742635f4 100644 --- a/docs/research.md +++ b/docs/research.md @@ -10,7 +10,7 @@ Frozen scientific records are not the user manual. They stay in the repository s | --- | --- | --- | | Gate v2 confirmation READMEs | Frozen 188/192 identity and 0/192 mix after required sanitizers | Historical sentence "CLI remains empty" / `process_text` identity **at confirmation time**. Live product is authorized letter-mix. | | `cycle8-product-authorization-v1` | Historical mark-only authorization | Public CLI applied U+034F / U+FE00 (`release-cli-v5`) | -| `cycle8-product-authorization-v2` | Live five-layer authorization | Public CLI applies mark plus C0/C1 plus Me plus Cf plus interlinear annotation (`release-cli-v11`) | +| `cycle8-product-authorization-v2` | Live five-layer authorization | Public CLI applies mark plus C0/C1 plus Me plus Cf plus interlinear annotation (`release-cli-v12`) | | mix-freeze / mix-confirmation 830000+ | Historical 0/192 on earlier seeds | Not rewritten | ## Evidence index diff --git a/docs/website.md b/docs/website.md index e40ede50..d1e9cf6f 100644 --- a/docs/website.md +++ b/docs/website.md @@ -19,10 +19,12 @@ Deploy these static files (repository CI cannot push them; copy after each relea Runs in the browser. On **Remove marks** it first runs the same closed-set FuckMark insertion scan as `fuckmark --detect` (U+034F, U+FE00, C0/C1, U+20DD, U+13430-U+13438, U+FFF9-U+FFFB). - Hit: strip those insertions, copy the cleaned text, keep visible words. -- Miss: show the no-watermark card (Chinese copy in `mark.html`) and the contact form that mails `Fhelp@q1z.org`. +- Miss: show the English no-watermark card and the contact form that mails `Fhelp@q1z.org`. Do not strip emoji variation selectors, ZWJ, or other non-approved characters. Do not query GPT-2, SynthID, or any remote detector. Icon scripts may load from a CDN; the scan itself is local. +Local beginners can run `fuckmark web` after install. That serves the same `mark.html` tool from the package (`fuckmark/webui/mark.html`, kept identical to `docs/mark.html`) on `http://127.0.0.1:8765/mark.html`. + ### `demo.html` (research walkthrough) Keep it a static file that works from `file://` with baked CLI samples. The live page must show: diff --git a/fuckmark/cli.py b/fuckmark/cli.py index f28bea6e..66eb70d2 100644 --- a/fuckmark/cli.py +++ b/fuckmark/cli.py @@ -26,6 +26,7 @@ detect_human_report, detect_machine_line, ) +from .web import run_web_argv from .product.domain import ( PRODUCT_MAX_INPUT_CHARS, is_supported_product_domain_v1, @@ -39,7 +40,7 @@ INTERACTIVE_DONE = ":done" -RELEASE_CLI_ALGORITHM_VERSION = "release-cli-v11" +RELEASE_CLI_ALGORITHM_VERSION = "release-cli-v12" _ANSI_BLUE = "\033[38;5;39m" _ANSI_GREEN = "\033[38;5;40m" _ANSI_YELLOW = "\033[38;5;214m" @@ -216,6 +217,7 @@ def _parser() -> argparse.ArgumentParser: " fuckmark --stdin --visible < notes.fm.txt\n" " fuckmark --detect --text \"I do not agree.\"\n" " printf 'paste\\n' | fuckmark --detect\n" + " fuckmark web\n" "\n" "Latin, Greek, Cyrillic, Han, Kana, Hangul syllable, and emoji sites are processed\n" "even when surrounding text has other Unicode, including curly apostrophes.\n" @@ -231,6 +233,7 @@ def _parser() -> argparse.ArgumentParser: "Use --detect to scan for FuckMark insertions without transforming.\n" "If --detect finds none, the report tells you to contact " f"{DETECT_CONTACT_EMAIL}.\n" + "Use fuckmark web to open the local browser tool (beginner-friendly).\n" "Mn-strip, default-ignorable strip, UnicodeSanitizer combinations, and Cf-strip after UnicodeSanitizer leave Me/Cc/Cf residuals and spaces." ), formatter_class=argparse.RawDescriptionHelpFormatter, @@ -803,6 +806,9 @@ def _run( parser_argv = argv if parser_argv is None and injected: parser_argv = () + raw_argv = list(sys.argv[1:] if parser_argv is None else parser_argv) + if raw_argv and raw_argv[0] == "web": + return run_web_argv(raw_argv[1:], errors) arguments = _parser().parse_args(parser_argv) _ensure_utf8(source) _ensure_utf8(output) diff --git a/fuckmark/cycle8/gate_v2.py b/fuckmark/cycle8/gate_v2.py index c75dc791..e66df188 100644 --- a/fuckmark/cycle8/gate_v2.py +++ b/fuckmark/cycle8/gate_v2.py @@ -316,8 +316,8 @@ def assert_gate_v2_committed() -> None: raise ValueError("product_approved_carriers_v1 must be the frozen mix carriers") if process_text("I do not agree.") == "I do not agree.": raise ValueError("authorized CLI must apply the frozen letter mix") - if RELEASE_CLI_ALGORITHM_VERSION != "release-cli-v11": - raise ValueError("authorized CLI must report release-cli-v11") + if RELEASE_CLI_ALGORITHM_VERSION != "release-cli-v12": + raise ValueError("authorized CLI must report release-cli-v12") scorecard = build_gate_v2_confirmation_scorecard() if scorecard["confirmation"] is not True: raise ValueError("Gate v2 confirmation scorecard must pass") diff --git a/fuckmark/product/authorization.py b/fuckmark/product/authorization.py index bffb041e..8a71542a 100644 --- a/fuckmark/product/authorization.py +++ b/fuckmark/product/authorization.py @@ -39,7 +39,7 @@ PRODUCT_AUTHORIZATION_VERSION = "cycle8-product-authorization-v2" PRODUCT_AUTHORIZATION_PATH = "specs/cycle8/fuckmark-cycle8-product-authorization-v2.json" -PRODUCT_AUTHORIZATION_HASH = "07499a27e8abcfca5d6d524bc577c59b7631a052959d8e2b46970d27fe35cecf" +PRODUCT_AUTHORIZATION_HASH = "b919483473fb98be0242db86c5eee827874f06508e2e5042b97102d50476967f" _MIX_FREEZE_HASH = "2286aa201bd9cb70136f2895740489136aa1ba7cfd9471c6e233fe201af41986" _MIX_CONFIRMATION_SCORECARD_HASH = "a4911189af7f38d34252452821d90df1188bfe05025fe33c028c4b670eecbcce" _AUDIT_SOURCE = "I do not agree." @@ -175,8 +175,8 @@ def assert_product_authorization_committed() -> None: raise ValueError("product authorization spec does not match the live payload") if disk["product_authorized"] is not True: raise ValueError("product authorization spec must authorize") - if disk["cli_algorithm_version"] != "release-cli-v11": - raise ValueError("product authorization must use release-cli-v11") + if disk["cli_algorithm_version"] != "release-cli-v12": + raise ValueError("product authorization must use release-cli-v12") if disk["mix_sanitizer_gate_v1"] != "PASS": raise ValueError("product authorization must record the durable sanitizer gate PASS") if disk["identities"]["mix_freeze_hash"] != _MIX_FREEZE_HASH: diff --git a/fuckmark/web.py b/fuckmark/web.py new file mode 100644 index 00000000..021a68fb --- /dev/null +++ b/fuckmark/web.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import argparse +import functools +import http.server +import socketserver +import threading +import time +import webbrowser +from collections.abc import Callable +from pathlib import Path +from typing import TextIO + + +DEFAULT_WEB_HOST = "127.0.0.1" +DEFAULT_WEB_PORT = 8765 + + +def mark_html_path() -> Path: + packaged = Path(__file__).resolve().parent / "webui" / "mark.html" + if packaged.is_file(): + return packaged + repo = Path(__file__).resolve().parents[1] / "docs" / "mark.html" + if repo.is_file(): + return repo + raise FileNotFoundError("mark.html is not installed with this FuckMark build") + + +def web_root() -> Path: + return mark_html_path().resolve().parent + + +def _web_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="fuckmark web", + description=( + "Open the FuckMark browser tool locally. " + "For beginners who prefer a page over the CLI." + ), + ) + parser.add_argument( + "--host", + default=DEFAULT_WEB_HOST, + help=f"bind address (default {DEFAULT_WEB_HOST})", + ) + parser.add_argument( + "--port", + type=int, + default=DEFAULT_WEB_PORT, + help=f"port (default {DEFAULT_WEB_PORT}; 0 picks a free port)", + ) + parser.add_argument( + "--no-open", + action="store_true", + help="print the URL without opening a browser", + ) + return parser + + +class _MarkHandler(http.server.SimpleHTTPRequestHandler): + def log_message(self, format: str, *args) -> None: + return + + def do_GET(self) -> None: + if self.path in {"/", "/index.html"}: + self.path = "/mark.html" + return super().do_GET() + + +def serve_mark_web( + *, + host: str = DEFAULT_WEB_HOST, + port: int = DEFAULT_WEB_PORT, + open_browser: bool = True, + errors: TextIO | None = None, + serve_seconds: float | None = None, + on_ready: Callable[[str, int], None] | None = None, +) -> tuple[str, int]: + root = web_root() + page = mark_html_path() + if page.name != "mark.html" or not page.is_file(): + raise FileNotFoundError("mark.html is missing") + handler = functools.partial(_MarkHandler, directory=str(root)) + + class _Server(socketserver.TCPServer): + allow_reuse_address = True + + server = _Server((host, port), handler) + bound_host, bound_port = server.server_address[:2] + bound_port = int(bound_port) + url = f"http://{bound_host}:{bound_port}/mark.html" + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + if errors is not None: + errors.write(f"FuckMark web: {url}\n") + errors.write("FuckMark web: press Ctrl+C to stop.\n") + errors.flush() + if on_ready is not None: + on_ready(url, bound_port) + if open_browser: + webbrowser.open(url) + if serve_seconds is None: + stop = threading.Event() + while not stop.wait(0.5): + if not thread.is_alive(): + break + else: + time.sleep(max(0.0, serve_seconds)) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2.0) + return url, bound_port + + +def run_web_argv(argv: list[str], errors: TextIO) -> int: + try: + arguments = _web_parser().parse_args(argv) + except SystemExit as error: + code = error.code + if code is None: + return 0 + return int(code) if isinstance(code, int) else 1 + if arguments.port < 0 or arguments.port > 65535: + errors.write("FuckMark: --port must be between 0 and 65535\n") + errors.flush() + return 1 + try: + mark_html_path() + except FileNotFoundError as error: + errors.write(f"FuckMark: {error}\n") + errors.flush() + return 1 + try: + serve_mark_web( + host=arguments.host, + port=arguments.port, + open_browser=not arguments.no_open, + errors=errors, + serve_seconds=None, + ) + except KeyboardInterrupt: + errors.write("\n") + errors.flush() + return 130 + except OSError as error: + errors.write(f"FuckMark: could not start web server: {error}\n") + errors.flush() + return 1 + return 0 diff --git a/fuckmark/webui/__init__.py b/fuckmark/webui/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fuckmark/webui/mark.html b/fuckmark/webui/mark.html new file mode 100644 index 00000000..a34fd0e1 --- /dev/null +++ b/fuckmark/webui/mark.html @@ -0,0 +1,1576 @@ + + + + + +FuckMark — Remove statistical marks + + + + + + + + + + + + +
+ + +
+
+

FuckMark

+

No CLI needed. Paste text here.
FuckMark checks for its insertions, then strips them.

+ +
+
+
Input0 chars
+
+ +
+
+ + + +
+
+
+ +
+

Success

+

Text copied to your clipboard.

+
+
+ +
+
History0
+
    +
    +
    +
    + +

    Runs entirely in your browser. Local FuckMark scan only — nothing is uploaded.

    +
    + +
    + + +
    +

    We did not detect a watermark in this text.

    +

    Closed-set scan of FuckMark insertion characters only. Not a general AI-watermark detector.

    +

    What? You think there is a watermark in this?

    + +
    + + +
    +
    +
    + +
    + + + + diff --git a/pyproject.toml b/pyproject.toml index 1ca2dad4..21bc71b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,3 +48,6 @@ addopts = "-q" [tool.setuptools.packages.find] where = ["."] include = ["fuckmark*"] + +[tool.setuptools.package-data] +fuckmark = ["webui/*.html"] diff --git a/specs/cycle8/fuckmark-cycle8-product-authorization-v2.json b/specs/cycle8/fuckmark-cycle8-product-authorization-v2.json index 5ed406dd..f911c818 100644 --- a/specs/cycle8/fuckmark-cycle8-product-authorization-v2.json +++ b/specs/cycle8/fuckmark-cycle8-product-authorization-v2.json @@ -1 +1 @@ -{"algorithm_version":"cycle8-product-authorization-v2","apply_path":"apply_letter_alternating_mix","authorization_hash":"07499a27e8abcfca5d6d524bc577c59b7631a052959d8e2b46970d27fe35cecf","carriers":[847,65024,127,128,129,130,131,132,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,8413,78896,78897,78898,78899,78900,78901,78902,78903,78904,65529,65530,65531],"cli_algorithm_version":"release-cli-v11","detect_mechanism_id":"fuckmark-carrier-scan-v1","do_not_generate_950000":true,"do_not_retag_v030":true,"fail_closed":["source_already_contains_approved_carriers","visible_projection_mismatch","carrier_insertion_mismatch","apply_error","no_eligible_letter_sites"],"identities":{"gate_v2_confirmation_scorecard_hash":"3df98598fa1f9fb3951029f105b43dfd5f3e83a9ec69fd5c160b31686b0ad6c9","gate_v2_hash":"66da1101bc6621023a0bb2b98d40f37ada0384f19a750d4c8b0c48de1c2cae68","mix_confirmation_scorecard_hash":"a4911189af7f38d34252452821d90df1188bfe05025fe33c028c4b670eecbcce","mix_confirmation_scorecard_version":"cycle8-mix-confirmation-scorecard-v1","mix_freeze_hash":"2286aa201bd9cb70136f2895740489136aa1ba7cfd9471c6e233fe201af41986","mix_freeze_version":"cycle8-mix-freeze-v1","mix_publishability_hash":"52e25a1e4078f7bb55de729b4ab85ba4d4aa26f544fe4fa56bf3bb79d3a4bdc8","product_contract_hash":"5afd79586f82e31d0d673acbebebf0ac00804cff74b9f644f000bddfd3dc07d1"},"live":{"already_mixed_identity":true,"approved_carriers":[127,128,129,130,131,132,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,847,8413,65024,65529,65530,65531,78896,78897,78898,78899,78900,78901,78902,78903,78904],"curly_apostrophe_leftover_reported":true,"detector_finds_live_mix":true,"detector_rejects_plain_source":true,"di_me_us_cf_does_not_restore_source":true,"di_me_us_does_not_restore_source":true,"di_strip_does_not_restore_source":true,"di_then_us_does_not_restore_source":true,"emoji_only_first_unsupported_empty":true,"emoji_only_processed":true,"han_only_first_unsupported_empty":true,"han_syllable_processed":true,"latin_letter_only_processed":true,"latin_only_first_unsupported_empty":true,"mixed_unicode_processed":true,"mn_me_cc_does_not_restore_source":true,"mn_me_us_cf_does_not_restore_source":true,"mn_me_us_does_not_restore_source":true,"mn_strip_does_not_restore_source":true,"mn_then_us_does_not_restore_source":true,"nfd_cluster_inserts_after_combining_mark":true,"nfd_cluster_keeps_base_then_mark":true,"no_letter_identity":true,"process_text_equals_mix":true,"required_bundle_then_us_does_not_restore_source":true,"supported_domain":true,"visible_projection_equals_source":true},"max_selected":4096,"mechanism_id":"u034f-ufe00-cc-me-cf-ia-letter-alt-v1","mix_sanitizer_gate_v1":"PASS","notes":"Product authorization of five-layer u034f-ufe00-cc-me-cf-ia-letter-alt-v1. Mark plus Cc plus enclosing Me (U+20DD) plus cycling Egyptian hieroglyph format controls (U+13430-U+13438) plus cycling interlinear annotation controls (U+FFF9-U+FFFB) keep Mn-strip, default-ignorable-strip, UnicodeSanitizer orderings, Mn then Me then UnicodeSanitizer, Mn then Me then UnicodeSanitizer then frozen cf_strip, and the required sanitizer bundle from restoring the source. UnicodeSanitizer turns the annotation controls into spaces, so cf_strip after it cannot rebuild the original spacing. Visible projection strips approved carriers; Me may decorate glyphs in some renderers. Live sites are grapheme clusters of Latin, Greek, Cyrillic, Han, Kana, Hangul syllables, and emoji, including NFD Latin where insertions follow the combining sequence. Historical Gate v2 confirmation remains the frozen GPT-2 evidence for the prior mark-only arm. first_unsupported reports leftover non-ASCII that is not inside an eligible letter or emoji cluster. --detect is a closed-set scan of approved insertions, not a general AI-watermark detector.","product_authorized":true,"release_registry_empty":true,"required_sanitizer_bundle_not_weakened":true} +{"algorithm_version":"cycle8-product-authorization-v2","apply_path":"apply_letter_alternating_mix","authorization_hash":"b919483473fb98be0242db86c5eee827874f06508e2e5042b97102d50476967f","carriers":[847,65024,127,128,129,130,131,132,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,8413,78896,78897,78898,78899,78900,78901,78902,78903,78904,65529,65530,65531],"cli_algorithm_version":"release-cli-v12","detect_mechanism_id":"fuckmark-carrier-scan-v1","do_not_generate_950000":true,"do_not_retag_v030":true,"fail_closed":["source_already_contains_approved_carriers","visible_projection_mismatch","carrier_insertion_mismatch","apply_error","no_eligible_letter_sites"],"identities":{"gate_v2_confirmation_scorecard_hash":"3df98598fa1f9fb3951029f105b43dfd5f3e83a9ec69fd5c160b31686b0ad6c9","gate_v2_hash":"66da1101bc6621023a0bb2b98d40f37ada0384f19a750d4c8b0c48de1c2cae68","mix_confirmation_scorecard_hash":"a4911189af7f38d34252452821d90df1188bfe05025fe33c028c4b670eecbcce","mix_confirmation_scorecard_version":"cycle8-mix-confirmation-scorecard-v1","mix_freeze_hash":"2286aa201bd9cb70136f2895740489136aa1ba7cfd9471c6e233fe201af41986","mix_freeze_version":"cycle8-mix-freeze-v1","mix_publishability_hash":"52e25a1e4078f7bb55de729b4ab85ba4d4aa26f544fe4fa56bf3bb79d3a4bdc8","product_contract_hash":"5afd79586f82e31d0d673acbebebf0ac00804cff74b9f644f000bddfd3dc07d1"},"live":{"already_mixed_identity":true,"approved_carriers":[127,128,129,130,131,132,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,847,8413,65024,65529,65530,65531,78896,78897,78898,78899,78900,78901,78902,78903,78904],"curly_apostrophe_leftover_reported":true,"detector_finds_live_mix":true,"detector_rejects_plain_source":true,"di_me_us_cf_does_not_restore_source":true,"di_me_us_does_not_restore_source":true,"di_strip_does_not_restore_source":true,"di_then_us_does_not_restore_source":true,"emoji_only_first_unsupported_empty":true,"emoji_only_processed":true,"han_only_first_unsupported_empty":true,"han_syllable_processed":true,"latin_letter_only_processed":true,"latin_only_first_unsupported_empty":true,"mixed_unicode_processed":true,"mn_me_cc_does_not_restore_source":true,"mn_me_us_cf_does_not_restore_source":true,"mn_me_us_does_not_restore_source":true,"mn_strip_does_not_restore_source":true,"mn_then_us_does_not_restore_source":true,"nfd_cluster_inserts_after_combining_mark":true,"nfd_cluster_keeps_base_then_mark":true,"no_letter_identity":true,"process_text_equals_mix":true,"required_bundle_then_us_does_not_restore_source":true,"supported_domain":true,"visible_projection_equals_source":true},"max_selected":4096,"mechanism_id":"u034f-ufe00-cc-me-cf-ia-letter-alt-v1","mix_sanitizer_gate_v1":"PASS","notes":"Product authorization of five-layer u034f-ufe00-cc-me-cf-ia-letter-alt-v1. Mark plus Cc plus enclosing Me (U+20DD) plus cycling Egyptian hieroglyph format controls (U+13430-U+13438) plus cycling interlinear annotation controls (U+FFF9-U+FFFB) keep Mn-strip, default-ignorable-strip, UnicodeSanitizer orderings, Mn then Me then UnicodeSanitizer, Mn then Me then UnicodeSanitizer then frozen cf_strip, and the required sanitizer bundle from restoring the source. UnicodeSanitizer turns the annotation controls into spaces, so cf_strip after it cannot rebuild the original spacing. Visible projection strips approved carriers; Me may decorate glyphs in some renderers. Live sites are grapheme clusters of Latin, Greek, Cyrillic, Han, Kana, Hangul syllables, and emoji, including NFD Latin where insertions follow the combining sequence. Historical Gate v2 confirmation remains the frozen GPT-2 evidence for the prior mark-only arm. first_unsupported reports leftover non-ASCII that is not inside an eligible letter or emoji cluster. --detect is a closed-set scan of approved insertions, not a general AI-watermark detector.","product_authorized":true,"release_registry_empty":true,"required_sanitizer_bundle_not_weakened":true} diff --git a/tests/test_cli.py b/tests/test_cli.py index 622b3ca9..e1502ccd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -326,8 +326,8 @@ def test_cli_version_reports_project_identity(capsys) -> None: assert result.value.code == 0 rendered = capsys.readouterr().out.strip() assert rendered == f"FuckMark {__version__}" - assert RELEASE_CLI_ALGORITHM_VERSION == "release-cli-v11" - assert "release-cli-v11" not in rendered + assert RELEASE_CLI_ALGORITHM_VERSION == "release-cli-v12" + assert "release-cli-v12" not in rendered assert "transform-registry" not in rendered @@ -436,6 +436,7 @@ def test_cli_help_documents_file_pipe_clipboard_and_visible_contract(capsys) -> assert "--inspect" in rendered assert "--detect" in rendered assert "Fhelp@q1z.org" in rendered + assert "fuckmark web" in rendered assert "standard input" in rendered.casefold() or "--stdin" in rendered assert ":done" in rendered assert "clipboard" in rendered.casefold() diff --git a/tests/test_cycle8_mix_publishability.py b/tests/test_cycle8_mix_publishability.py index 9fd0624c..371e5c23 100644 --- a/tests/test_cycle8_mix_publishability.py +++ b/tests/test_cycle8_mix_publishability.py @@ -50,7 +50,7 @@ def test_mix_publishability_spec_fail_closes_product_promotion() -> None: assert_mix_publishability_committed() assert release_transform_registry().rules == () assert product_approved_carriers_v1() == frozenset(LETTER_MIX_APPROVED_CARRIERS) - assert RELEASE_CLI_ALGORITHM_VERSION == "release-cli-v11" + assert RELEASE_CLI_ALGORITHM_VERSION == "release-cli-v12" assert process_text("I do not agree.") != "I do not agree." diff --git a/tests/test_demo_page.py b/tests/test_demo_page.py index 53c22488..877b3579 100644 --- a/tests/test_demo_page.py +++ b/tests/test_demo_page.py @@ -39,8 +39,10 @@ def test_demo_page_is_self_contained_and_honest() -> None: assert "Fhelp@q1z.org" in html assert "mailto:Fhelp@q1z.org" in html assert "id=\"detector\"" in html - assert "\u6211\u4eec\u6ca1\u6709\u4ece\u8fd9\u91cc\u9762\u68c0\u6d4b\u5230\u6c34\u5370" in html - assert "\u8054\u7cfb\u6211\u4eec" in html + assert "We did not detect a watermark in this text." in html + assert "What? You think there is a watermark in this?" in html + assert "Contact us" in html + assert "\u6211\u4eec" not in html assert "local FuckMark scan" in html or "FuckMark detector" in html assert "file://" in html or "file://" in html.casefold() assert "GPT-2" in html diff --git a/tests/test_install_release_hardening.py b/tests/test_install_release_hardening.py index 8576a04f..0593aaf3 100644 --- a/tests/test_install_release_hardening.py +++ b/tests/test_install_release_hardening.py @@ -148,12 +148,17 @@ def test_install_docs_do_not_recommend_live_main_or_pipe_installers() -> None: assert ".venv/bin/fuckmark --version" in readme assert ".venv/bin/fuckmark --visible" in readme or ".venv/bin/fuckmark --status" in readme assert "docs/demo.html" in readme - assert readme.index("No-install demo") < readme.index("Install from this repository") + assert "fuckmark web" in readme + assert "docs/mark.html" in readme + assert readme.index("Browser tool") < readme.index("Install from this repository") website = (ROOT / "docs/website.md").read_text(encoding="utf-8") assert "| sh" not in website assert "| iex" not in website assert "demo.html" in website + assert "mark.html" in website + assert "fuckmark web" in website assert (ROOT / "docs/demo.html").is_file() + assert (ROOT / "docs/mark.html").is_file() def test_unix_path_config_quotes_custom_bin_directory() -> None: diff --git a/tests/test_mark_page.py b/tests/test_mark_page.py index 5745eafc..b6254396 100644 --- a/tests/test_mark_page.py +++ b/tests/test_mark_page.py @@ -23,14 +23,16 @@ def test_mark_page_is_closed_set_detector_with_contact() -> None: assert DETECT_CONTACT_EMAIL in html assert f"mailto:{DETECT_CONTACT_EMAIL}" in html assert "Fmark@q1z.org" not in html - assert "\u6211\u4eec\u6ca1\u6709\u4ece\u8fd9\u91cc\u9762\u68c0\u6d4b\u5230\u6c34\u5370" in html - assert "\u8054\u7cfb\u6211\u4eec" in html - assert "\u4ec0\u4e48\uff1f\u60a8\u8ba4\u4e3a\u8fd9\u91cc\u9762\u6709\u6c34\u5370\uff1f" in html + assert "We did not detect a watermark in this text." in html + assert "What? You think there is a watermark in this?" in html + assert "Contact us" in html + assert "\u6211\u4eec" not in html assert "closed-set" in html.casefold() assert "not a general" in html.casefold() assert "markedSample" in html assert "FM_CCS" in html assert "FM_CFS" in html + assert "No CLI needed" in html def test_mark_page_demo_sample_is_detectable_by_python_detector() -> None: diff --git a/tests/test_product_authorization.py b/tests/test_product_authorization.py index a519236d..11d9b43a 100644 --- a/tests/test_product_authorization.py +++ b/tests/test_product_authorization.py @@ -33,7 +33,7 @@ def test_product_authorization_spec_is_committed_and_live() -> None: assert disk["algorithm_version"] == PRODUCT_AUTHORIZATION_VERSION assert disk["product_authorized"] is True assert disk["mechanism_id"] == LETTER_MIX_MECHANISM_ID - assert disk["cli_algorithm_version"] == RELEASE_CLI_ALGORITHM_VERSION == "release-cli-v11" + assert disk["cli_algorithm_version"] == RELEASE_CLI_ALGORITHM_VERSION == "release-cli-v12" assert disk["mix_sanitizer_gate_v1"] == "PASS" assert disk["required_sanitizer_bundle_not_weakened"] is True assert disk["release_registry_empty"] is True diff --git a/tests/test_web.py b/tests/test_web.py new file mode 100644 index 00000000..df8855ec --- /dev/null +++ b/tests/test_web.py @@ -0,0 +1,71 @@ +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from urllib.request import urlopen + +from fuckmark.cli import RELEASE_CLI_ALGORITHM_VERSION, main +from fuckmark.web import mark_html_path, serve_mark_web, web_root + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_packaged_mark_html_matches_docs_source() -> None: + docs = (ROOT / "docs" / "mark.html").read_text(encoding="utf-8") + packaged = (ROOT / "fuckmark" / "webui" / "mark.html").read_text(encoding="utf-8") + assert packaged == docs + assert mark_html_path().is_file() + assert web_root() == mark_html_path().parent + + +def test_mark_html_miss_copy_is_english() -> None: + html = mark_html_path().read_text(encoding="utf-8") + assert "We did not detect a watermark in this text." in html + assert "What? You think there is a watermark in this?" in html + assert "Contact us" in html + assert "Fhelp@q1z.org" in html + assert "\u6211\u4eec" not in html + assert "\u8054\u7cfb" not in html + + +def test_fuckmark_web_serves_mark_page() -> None: + errors = StringIO() + seen: dict[str, object] = {} + + def on_ready(url: str, port: int) -> None: + seen["url"] = url + seen["port"] = port + with urlopen(url, timeout=2) as response: + seen["body"] = response.read().decode("utf-8") + with urlopen(f"http://127.0.0.1:{port}/", timeout=2) as response: + seen["index"] = response.read().decode("utf-8") + + url, port = serve_mark_web( + host="127.0.0.1", + port=0, + open_browser=False, + errors=errors, + serve_seconds=0.05, + on_ready=on_ready, + ) + assert port > 0 + assert url == seen["url"] + assert url.endswith("/mark.html") + assert "FuckMark web:" in errors.getvalue() + body = str(seen["body"]) + assert "FuckMark" in body + assert "detectFuckMark" in body + assert "We did not detect a watermark in this text." in body + assert "detectFuckMark" in str(seen["index"]) + + +def test_cli_web_help_documents_browser_tool() -> None: + assert RELEASE_CLI_ALGORITHM_VERSION == "release-cli-v12" + out = StringIO() + errors = StringIO() + with redirect_stdout(out): + status = main(StringIO(""), StringIO(), error_stream=errors, argv=("web", "--help")) + assert status == 0 + help_text = out.getvalue() + errors.getvalue() + assert "fuckmark web" in help_text + assert "browser" in help_text.casefold() or "beginner" in help_text.casefold() From a62f5900875424c25eec31f4f8949f4d40da7db2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 29 Aug 2026 18:49:14 +0000 Subject: [PATCH 18/19] Wire fuckmark web detect/strip to the Python API. The paste UI calls GET /api/health and POST /api/remove-marks so local fuckmark web uses detect_fuckmark_insertions and project_visible_v1. file:// and static mark.q1z.org keep the in-browser fallback. CLI identity stays release-cli-v12. Co-authored-by: Byte271 --- CHANGELOG.md | 2 +- README.md | 4 +- docs/cli.md | 2 +- docs/install.md | 2 +- docs/mark.html | 90 ++++++++++++++++++++-- docs/website.md | 11 ++- fuckmark/cli.py | 1 + fuckmark/web.py | 162 ++++++++++++++++++++++++++++++++++++++- fuckmark/webui/mark.html | 90 ++++++++++++++++++++-- tests/test_mark_page.py | 4 + tests/test_web.py | 129 ++++++++++++++++++++++++++++++- 11 files changed, 475 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f917338..e517cacb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Audit findings D01-D16, E01-E04, L01-L06, P01, and G01 against `d7dc98b7ee99fc46 - Latin letters outside ASCII, Han syllables, and emoji-only input are processed. Curly apostrophes stay in the visible text and are reported as `first_unsupported`. Processed letters and emoji are not leftovers. Joining scripts such as Arabic stay unchanged. Site cap is 4096 (five insertions per site). - `--detect` and the no-install demo scan for FuckMark insertion characters only. A miss prints that no watermark was detected and points to `Fhelp@q1z.org`. That scan is not a general AI-watermark detector. - Website tool source is now in-repo: `docs/mark.html` (plus `docs/rec.html` gate), packaged as `fuckmark/webui/mark.html`. It uses the same closed-set scan, strips approved insertions on a hit, and on a miss shows an English no-watermark card with mailto `Fhelp@q1z.org`. It no longer strips emoji VS / ZWJ ranges. -- `fuckmark web` serves that local browser tool for beginners who prefer a page over the CLI (`http://127.0.0.1:8765/mark.html`). +- `fuckmark web` serves that local browser tool for beginners who prefer a page over the CLI (`http://127.0.0.1:8765/mark.html`). Detect and strip go through a local Python API (`GET /api/health`, `POST /api/remove-marks`) that calls `detect_fuckmark_insertions` and `project_visible_v1`. Static `mark.q1z.org` has no Python process and keeps the in-browser fallback. - Product authorization v2 records the five-layer path. Gate v2 confirmation hashes stay historical. The v1 mix publishability report records the historical triple-layer fixture measurements and remains `product_authorized: false`. - Public copy, installer messages, and this source tree describe carrier insertion. They no longer say the CLI returns text unchanged. `docs/website.md` is the controlled website/installer copy. Do not pipe `https://d.q1z.org/mark` into a shell. - Package version is 0.4.1. The published v0.4.0 wheel remains the last GitHub Release artifact and does not implement `--text` / `--file`. diff --git a/README.md b/README.md index bf6c2996..9fb09f17 100644 --- a/README.md +++ b/README.md @@ -23,10 +23,12 @@ fuckmark web Opens a local page at `http://127.0.0.1:8765/mark.html`. Paste text in the browser. No other CLI flags required. Use `--no-open` to print the URL only, or `--port 0` to pick a free port. +`fuckmark web` also serves a local Python API. Detect and strip POST to `/api/remove-marks` and run the same engine as `fuckmark --detect`. Opening `mark.html` with `file://` or the static website has no Python process, so those paths keep the in-browser fallback. + - Product paste UI source: [`docs/mark.html`](docs/mark.html) (packaged as `fuckmark/webui/mark.html`) — after deploy: [mark.q1z.org/mark.html](https://mark.q1z.org/mark.html) - Research demo: [`docs/demo.html`](docs/demo.html) — `file://` works; after deploy: [mark.q1z.org/demo.html](https://mark.q1z.org/demo.html) -`mark.html` runs a local closed-set FuckMark scan, strips approved insertions when found, and on a miss shows an English no-watermark card with contact to `Fhelp@q1z.org`. +`mark.html` runs a local closed-set FuckMark scan, strips approved insertions when found, and on a miss shows an English no-watermark card with contact to `Fhelp@q1z.org`. Under `fuckmark web` that scan is the Python detector; the JS copy is the fallback. The research demo also includes: diff --git a/docs/cli.md b/docs/cli.md index c4ac3a04..4beb59ca 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -105,7 +105,7 @@ Existing files are read as UTF-8 bytes with no newline conversion. LF, CRLF, CR, ### `fuckmark web` -Open the local browser tool (same UI as `docs/mark.html`). Aimed at beginners who prefer a page over pipes and flags. +Open the local browser tool (same UI as `docs/mark.html`). Aimed at beginners who prefer a page over pipes and flags. The server also exposes a Python API: `GET /api/health` and `POST /api/remove-marks`. Detect and strip on that page use `detect_fuckmark_insertions` and `project_visible_v1` when the API is up. ```text fuckmark web diff --git a/docs/install.md b/docs/install.md index 38ebcfcb..ec735ddd 100644 --- a/docs/install.md +++ b/docs/install.md @@ -22,7 +22,7 @@ python3 -m venv .venv printf 'I do not agree.\n' | .venv/bin/fuckmark --visible ``` -`fuckmark web` opens `http://127.0.0.1:8765/mark.html` for beginners who do not want pipes or flags. +`fuckmark web` opens `http://127.0.0.1:8765/mark.html` for beginners who do not want pipes or flags. Detect and strip on that page talk to the local Python API. Keep using `.venv/bin/fuckmark` unless you activate the venv: diff --git a/docs/mark.html b/docs/mark.html index a34fd0e1..9902a535 100644 --- a/docs/mark.html +++ b/docs/mark.html @@ -637,7 +637,7 @@
    @@ -681,7 +681,7 @@

    Success

    -

    Runs entirely in your browser. Local FuckMark scan only — nothing is uploaded.

    +

    Uses the local FuckMark Python server when available. Falls back to in-browser detect if the API is down. Contact Fhelp@q1z.org if a miss is wrong.

    @@ -801,6 +801,7 @@

    Please paste your text.

    let lastSrc = ""; let lastMailto = ""; let demoRun = false; +let backendReady = false; const copyMorphs = []; function sleep(ms) { @@ -909,6 +910,69 @@

    Please paste your text.

    return { detected: found > 0, found, mark, cc, me, cf, ia, first }; } +async function probeBackend() { + try { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), 800); + const response = await fetch("/api/health", { cache: "no-store", signal: ctrl.signal }); + clearTimeout(timer); + if (!response.ok) return false; + const payload = await response.json(); + return !!(payload && payload.ok === true && payload.backend === "python"); + } catch { + return false; + } +} + +function applyBackendCopy(connected) { + fine.textContent = connected + ? "Talks to the local FuckMark Python server. Detect and strip use the same engine as fuckmark --detect. Contact Fhelp@q1z.org if a miss is wrong." + : "Uses the local FuckMark Python server when available. Falls back to in-browser detect if the API is down. Contact Fhelp@q1z.org if a miss is wrong."; + const sheetCopy = document.getElementById("sheetCopy"); + if (sheetCopy) { + sheetCopy.textContent = connected + ? "Talks to the local FuckMark Python server. Same engine as fuckmark --detect." + : "Uses the local FuckMark Python server when available. Falls back to in-browser detect."; + } +} + +function scanLocal(src) { + const scan = detectFuckMark(src); + if (!scan.detected) return { detected: false, text: src, removed: 0 }; + const stripped = stripMarks(src); + return { detected: true, text: stripped.text, removed: stripped.removed }; +} + +async function scanViaPython(src) { + const response = await fetch("/api/remove-marks", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ text: src }), + }); + const payload = await response.json(); + if (!payload || payload.backend !== "python") throw new Error("not-python"); + if (payload.reason === "too-large") throw new Error("too-large"); + if (payload.reason === "not-detected") { + return { detected: false, text: src, removed: 0 }; + } + if (payload.ok === true && payload.reason === "stripped" && typeof payload.text === "string") { + return { detected: true, text: payload.text, removed: Number(payload.removed) || 0 }; + } + throw new Error("bad"); +} + +async function scanText(src) { + if (backendReady) { + try { + return await scanViaPython(src); + } catch (err) { + if (String(err && err.message) === "too-large") throw err; + return scanLocal(src); + } + } + return scanLocal(src); +} + function morphIcon(m, icon, spring) { if (!m) return; if (reduce) m.set(icon); @@ -1441,8 +1505,20 @@

    Please paste your text.

    await sleep(reduce ? 0 : 480); - const scan = detectFuckMark(src); - if (!scan.detected) { + let outcome; + try { + outcome = await scanText(src); + } catch (err) { + field.classList.remove("is-work"); + go.classList.remove("is-load"); + await setLabel("Remove marks"); + go.disabled = false; + busy = false; + tip(String(err && err.message) === "too-large" ? "Text is too large" : "Could not scan this text"); + return; + } + + if (!outcome.detected) { field.classList.remove("is-work"); go.classList.remove("is-load"); await setLabel("Remove marks"); @@ -1452,7 +1528,8 @@

    Please paste your text.

    return; } - const { text, removed } = stripMarks(src); + const text = outcome.text; + const removed = outcome.removed; const copied = await copyText(text, { avoidPrompt: demoRun }); const list = loadHist(); @@ -1522,6 +1599,9 @@

    Please paste your text.

    } async function boot() { + backendReady = await probeBackend(); + applyBackendCopy(backendReady); + const list = loadHist(); if (list.length) renderHist(list); diff --git a/docs/website.md b/docs/website.md index d1e9cf6f..5eb46388 100644 --- a/docs/website.md +++ b/docs/website.md @@ -16,14 +16,19 @@ Deploy these static files (repository CI cannot push them; copy after each relea ### `mark.html` (product paste UI) -Runs in the browser. On **Remove marks** it first runs the same closed-set FuckMark insertion scan as `fuckmark --detect` (U+034F, U+FE00, C0/C1, U+20DD, U+13430-U+13438, U+FFF9-U+FFFB). +On **Remove marks** it runs the same closed-set FuckMark insertion scan as `fuckmark --detect` (U+034F, U+FE00, C0/C1, U+20DD, U+13430-U+13438, U+FFF9-U+FFFB). - Hit: strip those insertions, copy the cleaned text, keep visible words. - Miss: show the English no-watermark card and the contact form that mails `Fhelp@q1z.org`. -Do not strip emoji variation selectors, ZWJ, or other non-approved characters. Do not query GPT-2, SynthID, or any remote detector. Icon scripts may load from a CDN; the scan itself is local. +Do not strip emoji variation selectors, ZWJ, or other non-approved characters. Do not query GPT-2, SynthID, or any remote detector. Icon scripts may load from a CDN. -Local beginners can run `fuckmark web` after install. That serves the same `mark.html` tool from the package (`fuckmark/webui/mark.html`, kept identical to `docs/mark.html`) on `http://127.0.0.1:8765/mark.html`. +Local beginners can run `fuckmark web` after install. That serves the same `mark.html` tool from the package (`fuckmark/webui/mark.html`, kept identical to `docs/mark.html`) on `http://127.0.0.1:8765/mark.html`, plus a local Python API: + +- `GET /api/health` reports `backend: "python"` +- `POST /api/remove-marks` with `{ "text": "..." }` calls `detect_fuckmark_insertions` and `project_visible_v1` + +The page uses that API when the local server is up. Static `https://mark.q1z.org/mark.html` and `file://` have no Python process, so those deploys keep the in-browser scan. ### `demo.html` (research walkthrough) diff --git a/fuckmark/cli.py b/fuckmark/cli.py index 66eb70d2..a9b1a2f0 100644 --- a/fuckmark/cli.py +++ b/fuckmark/cli.py @@ -234,6 +234,7 @@ def _parser() -> argparse.ArgumentParser: "If --detect finds none, the report tells you to contact " f"{DETECT_CONTACT_EMAIL}.\n" "Use fuckmark web to open the local browser tool (beginner-friendly).\n" + "That server also runs the Python detect/strip API.\n" "Mn-strip, default-ignorable strip, UnicodeSanitizer combinations, and Cf-strip after UnicodeSanitizer leave Me/Cc/Cf residuals and spaces." ), formatter_class=argparse.RawDescriptionHelpFormatter, diff --git a/fuckmark/web.py b/fuckmark/web.py index 021a68fb..39e9b7ae 100644 --- a/fuckmark/web.py +++ b/fuckmark/web.py @@ -3,6 +3,7 @@ import argparse import functools import http.server +import json import socketserver import threading import time @@ -10,10 +11,17 @@ from collections.abc import Callable from pathlib import Path from typing import TextIO +from urllib.parse import urlparse + +from . import __version__ +from .product.detect import DETECT_CONTACT_EMAIL, detect_fuckmark_insertions +from .product.domain import PRODUCT_MAX_INPUT_CHARS +from .product.visible_projection import product_approved_carriers_v1, project_visible_v1 DEFAULT_WEB_HOST = "127.0.0.1" DEFAULT_WEB_PORT = 8765 +_MAX_BODY_BYTES = 8_000_000 def mark_html_path() -> Path: @@ -30,11 +38,86 @@ def web_root() -> Path: return mark_html_path().resolve().parent +def _detect_dict(text: str) -> dict[str, object]: + scan = detect_fuckmark_insertions(text) + return { + "detected": scan.detected, + "found": scan.found, + "mark": scan.mark_count, + "cc": scan.cc_count, + "me": scan.me_count, + "cf": scan.cf_count, + "ia": scan.ia_count, + "first": scan.first_hit, + "source_length": scan.source_length, + } + + +def remove_marks_payload(text: str) -> dict[str, object]: + if not isinstance(text, str): + raise TypeError("text must be a string") + if len(text) > PRODUCT_MAX_INPUT_CHARS: + return { + "ok": False, + "reason": "too-large", + "backend": "python", + "max": PRODUCT_MAX_INPUT_CHARS, + "detect": { + "detected": False, + "found": 0, + "mark": 0, + "cc": 0, + "me": 0, + "cf": 0, + "ia": 0, + "first": "", + "source_length": len(text), + }, + "text": "", + "removed": 0, + "contact": DETECT_CONTACT_EMAIL, + } + detect = _detect_dict(text) + if detect["detected"] is not True: + return { + "ok": False, + "reason": "not-detected", + "backend": "python", + "detect": detect, + "text": text, + "removed": 0, + "contact": DETECT_CONTACT_EMAIL, + } + cleaned = project_visible_v1(text, product_approved_carriers_v1()) + return { + "ok": True, + "reason": "stripped", + "backend": "python", + "detect": detect, + "text": cleaned, + "removed": int(detect["found"]), + "contact": DETECT_CONTACT_EMAIL, + } + + +def health_payload() -> dict[str, object]: + from .cli import RELEASE_CLI_ALGORITHM_VERSION + + return { + "ok": True, + "backend": "python", + "version": __version__, + "cli_algorithm_version": RELEASE_CLI_ALGORITHM_VERSION, + "contact": DETECT_CONTACT_EMAIL, + } + + def _web_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="fuckmark web", description=( "Open the FuckMark browser tool locally. " + "Serves mark.html and a Python API for detect/strip. " "For beginners who prefer a page over the CLI." ), ) @@ -61,11 +144,87 @@ class _MarkHandler(http.server.SimpleHTTPRequestHandler): def log_message(self, format: str, *args) -> None: return + def _json_response(self, status: int, payload: dict[str, object]) -> None: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _read_json_object(self) -> dict[str, object]: + raw_length = self.headers.get("Content-Length", "0") + try: + length = int(raw_length) + except ValueError as error: + raise ValueError("invalid Content-Length") from error + if length < 0 or length > _MAX_BODY_BYTES: + raise ValueError("request body too large") + data = self.rfile.read(length) if length else b"{}" + try: + payload = json.loads(data.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("body must be UTF-8 JSON") from error + if not isinstance(payload, dict): + raise ValueError("JSON body must be an object") + return payload + def do_GET(self) -> None: - if self.path in {"/", "/index.html"}: + path = urlparse(self.path).path + if path == "/api/health": + self._json_response(200, health_payload()) + return + if path in {"/", "/index.html"}: self.path = "/mark.html" return super().do_GET() + def do_POST(self) -> None: + path = urlparse(self.path).path + if path == "/api/detect": + try: + payload = self._read_json_object() + text = payload.get("text", "") + if not isinstance(text, str): + raise ValueError("text must be a string") + if len(text) > PRODUCT_MAX_INPUT_CHARS: + self._json_response( + 413, + { + "ok": False, + "reason": "too-large", + "backend": "python", + "max": PRODUCT_MAX_INPUT_CHARS, + }, + ) + return + detect = _detect_dict(text) + self._json_response( + 200, + { + "ok": True, + "backend": "python", + "detect": detect, + "contact": DETECT_CONTACT_EMAIL, + }, + ) + except ValueError as error: + self._json_response(400, {"ok": False, "reason": "bad-request", "error": str(error)}) + return + if path == "/api/remove-marks": + try: + payload = self._read_json_object() + text = payload.get("text", "") + if not isinstance(text, str): + raise ValueError("text must be a string") + result = remove_marks_payload(text) + status = 200 if result.get("reason") != "too-large" else 413 + self._json_response(status, result) + except ValueError as error: + self._json_response(400, {"ok": False, "reason": "bad-request", "error": str(error)}) + return + self.send_error(404, "Not Found") + def serve_mark_web( *, @@ -94,6 +253,7 @@ class _Server(socketserver.TCPServer): try: if errors is not None: errors.write(f"FuckMark web: {url}\n") + errors.write("FuckMark web: Python API at /api/health and /api/remove-marks\n") errors.write("FuckMark web: press Ctrl+C to stop.\n") errors.flush() if on_ready is not None: diff --git a/fuckmark/webui/mark.html b/fuckmark/webui/mark.html index a34fd0e1..9902a535 100644 --- a/fuckmark/webui/mark.html +++ b/fuckmark/webui/mark.html @@ -637,7 +637,7 @@
    @@ -681,7 +681,7 @@

    Success

    -

    Runs entirely in your browser. Local FuckMark scan only — nothing is uploaded.

    +

    Uses the local FuckMark Python server when available. Falls back to in-browser detect if the API is down. Contact Fhelp@q1z.org if a miss is wrong.

    @@ -801,6 +801,7 @@

    Please paste your text.

    let lastSrc = ""; let lastMailto = ""; let demoRun = false; +let backendReady = false; const copyMorphs = []; function sleep(ms) { @@ -909,6 +910,69 @@

    Please paste your text.

    return { detected: found > 0, found, mark, cc, me, cf, ia, first }; } +async function probeBackend() { + try { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), 800); + const response = await fetch("/api/health", { cache: "no-store", signal: ctrl.signal }); + clearTimeout(timer); + if (!response.ok) return false; + const payload = await response.json(); + return !!(payload && payload.ok === true && payload.backend === "python"); + } catch { + return false; + } +} + +function applyBackendCopy(connected) { + fine.textContent = connected + ? "Talks to the local FuckMark Python server. Detect and strip use the same engine as fuckmark --detect. Contact Fhelp@q1z.org if a miss is wrong." + : "Uses the local FuckMark Python server when available. Falls back to in-browser detect if the API is down. Contact Fhelp@q1z.org if a miss is wrong."; + const sheetCopy = document.getElementById("sheetCopy"); + if (sheetCopy) { + sheetCopy.textContent = connected + ? "Talks to the local FuckMark Python server. Same engine as fuckmark --detect." + : "Uses the local FuckMark Python server when available. Falls back to in-browser detect."; + } +} + +function scanLocal(src) { + const scan = detectFuckMark(src); + if (!scan.detected) return { detected: false, text: src, removed: 0 }; + const stripped = stripMarks(src); + return { detected: true, text: stripped.text, removed: stripped.removed }; +} + +async function scanViaPython(src) { + const response = await fetch("/api/remove-marks", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ text: src }), + }); + const payload = await response.json(); + if (!payload || payload.backend !== "python") throw new Error("not-python"); + if (payload.reason === "too-large") throw new Error("too-large"); + if (payload.reason === "not-detected") { + return { detected: false, text: src, removed: 0 }; + } + if (payload.ok === true && payload.reason === "stripped" && typeof payload.text === "string") { + return { detected: true, text: payload.text, removed: Number(payload.removed) || 0 }; + } + throw new Error("bad"); +} + +async function scanText(src) { + if (backendReady) { + try { + return await scanViaPython(src); + } catch (err) { + if (String(err && err.message) === "too-large") throw err; + return scanLocal(src); + } + } + return scanLocal(src); +} + function morphIcon(m, icon, spring) { if (!m) return; if (reduce) m.set(icon); @@ -1441,8 +1505,20 @@

    Please paste your text.

    await sleep(reduce ? 0 : 480); - const scan = detectFuckMark(src); - if (!scan.detected) { + let outcome; + try { + outcome = await scanText(src); + } catch (err) { + field.classList.remove("is-work"); + go.classList.remove("is-load"); + await setLabel("Remove marks"); + go.disabled = false; + busy = false; + tip(String(err && err.message) === "too-large" ? "Text is too large" : "Could not scan this text"); + return; + } + + if (!outcome.detected) { field.classList.remove("is-work"); go.classList.remove("is-load"); await setLabel("Remove marks"); @@ -1452,7 +1528,8 @@

    Please paste your text.

    return; } - const { text, removed } = stripMarks(src); + const text = outcome.text; + const removed = outcome.removed; const copied = await copyText(text, { avoidPrompt: demoRun }); const list = loadHist(); @@ -1522,6 +1599,9 @@

    Please paste your text.

    } async function boot() { + backendReady = await probeBackend(); + applyBackendCopy(backendReady); + const list = loadHist(); if (list.length) renderHist(list); diff --git a/tests/test_mark_page.py b/tests/test_mark_page.py index b6254396..8ba882cc 100644 --- a/tests/test_mark_page.py +++ b/tests/test_mark_page.py @@ -33,6 +33,10 @@ def test_mark_page_is_closed_set_detector_with_contact() -> None: assert "FM_CCS" in html assert "FM_CFS" in html assert "No CLI needed" in html + assert "/api/health" in html + assert "/api/remove-marks" in html + assert "scanViaPython" in html + assert "scanLocal" in html def test_mark_page_demo_sample_is_detectable_by_python_detector() -> None: diff --git a/tests/test_web.py b/tests/test_web.py index df8855ec..5cc1e771 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -1,13 +1,42 @@ +import json from contextlib import redirect_stdout from io import StringIO from pathlib import Path -from urllib.request import urlopen +from urllib.error import HTTPError +from urllib.request import Request, urlopen -from fuckmark.cli import RELEASE_CLI_ALGORITHM_VERSION, main -from fuckmark.web import mark_html_path, serve_mark_web, web_root +from fuckmark.cli import RELEASE_CLI_ALGORITHM_VERSION, main, process_text +from fuckmark.product.detect import DETECT_CONTACT_EMAIL +from fuckmark.product.domain import PRODUCT_MAX_INPUT_CHARS +from fuckmark.product.visible_projection import product_approved_carriers_v1, project_visible_v1 +from fuckmark.web import health_payload, mark_html_path, remove_marks_payload, serve_mark_web, web_root ROOT = Path(__file__).resolve().parents[1] +DEMO_SOURCE = "Hello from Q1z. Visible text stays." + + +def _get_json(url: str) -> tuple[int, dict[str, object]]: + with urlopen(url, timeout=2) as response: + payload = json.loads(response.read().decode("utf-8")) + return response.status, payload + + +def _post_json(url: str, body: dict[str, object]) -> tuple[int, dict[str, object]]: + encoded = json.dumps(body, ensure_ascii=False).encode("utf-8") + request = Request( + url, + data=encoded, + headers={"Content-Type": "application/json; charset=utf-8"}, + method="POST", + ) + try: + with urlopen(request, timeout=5) as response: + payload = json.loads(response.read().decode("utf-8")) + return response.status, payload + except HTTPError as error: + payload = json.loads(error.read().decode("utf-8")) + return error.code, payload def test_packaged_mark_html_matches_docs_source() -> None: @@ -28,6 +57,49 @@ def test_mark_html_miss_copy_is_english() -> None: assert "\u8054\u7cfb" not in html +def test_mark_html_calls_python_remove_marks_api() -> None: + html = mark_html_path().read_text(encoding="utf-8") + assert "/api/health" in html + assert "/api/remove-marks" in html + assert 'payload.backend === "python"' in html + assert "scanViaPython" in html + assert "scanLocal" in html + assert "detectFuckMark" in html + + +def test_health_and_remove_marks_payloads_use_python_detector() -> None: + health = health_payload() + assert health["ok"] is True + assert health["backend"] == "python" + assert health["cli_algorithm_version"] == RELEASE_CLI_ALGORITHM_VERSION == "release-cli-v12" + assert health["contact"] == DETECT_CONTACT_EMAIL + + miss = remove_marks_payload(DEMO_SOURCE) + assert miss["ok"] is False + assert miss["reason"] == "not-detected" + assert miss["backend"] == "python" + assert miss["text"] == DEMO_SOURCE + assert miss["detect"]["detected"] is False + + mixed = process_text(DEMO_SOURCE) + assert mixed != DEMO_SOURCE + hit = remove_marks_payload(mixed) + assert hit["ok"] is True + assert hit["reason"] == "stripped" + assert hit["backend"] == "python" + assert hit["text"] == DEMO_SOURCE + assert hit["text"] == project_visible_v1(mixed, product_approved_carriers_v1()) + assert int(hit["removed"]) > 0 + assert hit["detect"]["detected"] is True + assert hit["contact"] == DETECT_CONTACT_EMAIL + + huge = remove_marks_payload("a" * (PRODUCT_MAX_INPUT_CHARS + 1)) + assert huge["ok"] is False + assert huge["reason"] == "too-large" + assert huge["text"] == "" + assert huge["max"] == PRODUCT_MAX_INPUT_CHARS + + def test_fuckmark_web_serves_mark_page() -> None: errors = StringIO() seen: dict[str, object] = {} @@ -39,6 +111,23 @@ def on_ready(url: str, port: int) -> None: seen["body"] = response.read().decode("utf-8") with urlopen(f"http://127.0.0.1:{port}/", timeout=2) as response: seen["index"] = response.read().decode("utf-8") + health_status, health = _get_json(f"http://127.0.0.1:{port}/api/health") + seen["health_status"] = health_status + seen["health"] = health + mixed = process_text(DEMO_SOURCE) + hit_status, hit = _post_json(f"http://127.0.0.1:{port}/api/remove-marks", {"text": mixed}) + seen["hit_status"] = hit_status + seen["hit"] = hit + miss_status, miss = _post_json( + f"http://127.0.0.1:{port}/api/remove-marks", {"text": DEMO_SOURCE} + ) + seen["miss_status"] = miss_status + seen["miss"] = miss + detect_status, detect = _post_json( + f"http://127.0.0.1:{port}/api/detect", {"text": mixed} + ) + seen["detect_status"] = detect_status + seen["detect"] = detect url, port = serve_mark_web( host="127.0.0.1", @@ -51,12 +140,43 @@ def on_ready(url: str, port: int) -> None: assert port > 0 assert url == seen["url"] assert url.endswith("/mark.html") - assert "FuckMark web:" in errors.getvalue() + log = errors.getvalue() + assert "FuckMark web:" in log + assert "Python API" in log + assert "/api/remove-marks" in log body = str(seen["body"]) assert "FuckMark" in body assert "detectFuckMark" in body + assert "scanViaPython" in body + assert "/api/remove-marks" in body assert "We did not detect a watermark in this text." in body assert "detectFuckMark" in str(seen["index"]) + assert seen["health_status"] == 200 + health = seen["health"] + assert isinstance(health, dict) + assert health["backend"] == "python" + assert health["ok"] is True + assert seen["hit_status"] == 200 + hit = seen["hit"] + assert isinstance(hit, dict) + assert hit["backend"] == "python" + assert hit["ok"] is True + assert hit["reason"] == "stripped" + assert hit["text"] == DEMO_SOURCE + assert int(hit["removed"]) > 0 + miss = seen["miss"] + assert isinstance(miss, dict) + assert seen["miss_status"] == 200 + assert miss["ok"] is False + assert miss["reason"] == "not-detected" + detect = seen["detect"] + assert isinstance(detect, dict) + assert seen["detect_status"] == 200 + assert detect["backend"] == "python" + assert detect["ok"] is True + inner = detect["detect"] + assert isinstance(inner, dict) + assert inner["detected"] is True def test_cli_web_help_documents_browser_tool() -> None: @@ -68,4 +188,5 @@ def test_cli_web_help_documents_browser_tool() -> None: assert status == 0 help_text = out.getvalue() + errors.getvalue() assert "fuckmark web" in help_text + assert "Python API" in help_text assert "browser" in help_text.casefold() or "beginner" in help_text.casefold() From 1c9024956597586f6fa055437d588a04d45d15cf Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 29 Aug 2026 19:03:36 +0000 Subject: [PATCH 19/19] Disable caching of mark.html from fuckmark web. Serve Cache-Control: no-store on the local paste page so a prior static copy cannot hide the Python API UI. Co-authored-by: Byte271 --- fuckmark/web.py | 9 +++++++++ tests/test_web.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/fuckmark/web.py b/fuckmark/web.py index 39e9b7ae..b9210617 100644 --- a/fuckmark/web.py +++ b/fuckmark/web.py @@ -144,6 +144,15 @@ class _MarkHandler(http.server.SimpleHTTPRequestHandler): def log_message(self, format: str, *args) -> None: return + def end_headers(self) -> None: + buffer = getattr(self, "_headers_buffer", None) + already = False + if buffer: + already = any(line.lower().startswith(b"cache-control:") for line in buffer) + if not already: + self.send_header("Cache-Control", "no-store") + return super().end_headers() + def _json_response(self, status: int, payload: dict[str, object]) -> None: body = json.dumps(payload, ensure_ascii=False).encode("utf-8") self.send_response(status) diff --git a/tests/test_web.py b/tests/test_web.py index 5cc1e771..8b18183d 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -109,6 +109,7 @@ def on_ready(url: str, port: int) -> None: seen["port"] = port with urlopen(url, timeout=2) as response: seen["body"] = response.read().decode("utf-8") + seen["cache"] = response.headers.get("Cache-Control") with urlopen(f"http://127.0.0.1:{port}/", timeout=2) as response: seen["index"] = response.read().decode("utf-8") health_status, health = _get_json(f"http://127.0.0.1:{port}/api/health") @@ -151,6 +152,7 @@ def on_ready(url: str, port: int) -> None: assert "/api/remove-marks" in body assert "We did not detect a watermark in this text." in body assert "detectFuckMark" in str(seen["index"]) + assert "no-store" in str(seen.get("cache", "")).casefold() assert seen["health_status"] == 200 health = seen["health"] assert isinstance(health, dict)