diff --git a/.github/actions/check-quality/action.yml b/.github/actions/check-quality/action.yml index 9f735e8eb..9c24eb47e 100644 --- a/.github/actions/check-quality/action.yml +++ b/.github/actions/check-quality/action.yml @@ -61,6 +61,10 @@ runs: ${SHFMT} -d -ln bash -i 4 -ci \ .github/specula/setup-runner.sh guest/ubuntu/nvx-bashrc + - name: Check Markdown links + shell: bash + run: python scripts/check_markdown_links.py + - name: Test Specula CI wiring shell: bash working-directory: .github/specula diff --git a/.github/actions/validate-nvx/action.yml b/.github/actions/validate-nvx/action.yml index d26841a18..68ad94130 100644 --- a/.github/actions/validate-nvx/action.yml +++ b/.github/actions/validate-nvx/action.yml @@ -10,7 +10,7 @@ runs: run: | set -euo pipefail python3 -m compileall -q scripts - python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_guest_ingress.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py -v + python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_guest_ingress.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py scripts/test_markdown_links.py -v python3 scripts/test_adversarial.py -v python3 .github/skills/nvx-host-connect/scripts/test_hosts.py -v python3 scripts/nvx.py --help @@ -30,7 +30,7 @@ runs: $ErrorActionPreference = "Stop" python -m compileall -q scripts if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_guest_ingress.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py -v + python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_guest_ingress.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py scripts/test_markdown_links.py -v if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python scripts\test_adversarial.py -v if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/aci_edge_sandboxes/README.md b/aci_edge_sandboxes/README.md index 616aaadc6..fcc2cc1ff 100644 --- a/aci_edge_sandboxes/README.md +++ b/aci_edge_sandboxes/README.md @@ -1030,7 +1030,7 @@ An MXC `StatefulSandboxBackend` adapter maps onto this crate as follows: | `policy.network_egress` rules | `EgressPolicy` rules, field for field | | `working_directory` | `ExecRequest::with_cwd(nvx::guest_path(...))`; a directory that the workload cannot enter ends with `Failed(WorkingDirectory)` | | `process.env`, `process.inheritDefaultEnv` | `ProcessSpec::env`, `None` when omitted, and `inherit_default_env`, or `ExecRequest::with_envs` and `with_inherit_default_env`; see [Environment](#environment) | -| `runtimeConfig.networkProxy` | `ProvisionRequest::with_network_proxy` with the `agent` backend; see [Native proxy and forwarded ports](#native-proxy-and-forwarded-ports) | +| `runtimeConfig.networkProxy` | `ProvisionRequest::with_network_proxy` with the `agent` backend; see [Native proxy and host loopback](#native-proxy-and-host-loopback) | A proof-of-concept MXC adapter, `nvx_backend`, implements this mapping. It runs each phase in its own process against a real VM and consumes exec pipes diff --git a/doc/contribute.md b/doc/contribute.md index f8e01f278..4aeb0243c 100644 --- a/doc/contribute.md +++ b/doc/contribute.md @@ -66,13 +66,14 @@ and WHP runner pools, so: CI runs these checks on GitHub-hosted runners. Run the ones that cover your change before you push; [Setup](setup.md#development) lists the commands for -the Python and shell checks. +the Python, shell, and link checks. | Code | Checks | Definition | | --- | --- | --- | | Python in `scripts/` and `.github/specula/` | Ruff lint and format with 88-column lines, Pyright in strict mode over `scripts/` against Linux and Windows APIs, and the Specula wiring tests. The code targets Python 3.10. | [check-quality](../.github/actions/check-quality/action.yml), [pyproject.toml](../pyproject.toml) | | Guest and setup shell scripts | ShellCheck, and shfmt with four-space indentation and indented `case` branches. Scripts are POSIX `sh` unless check-quality checks them as Bash. | [check-quality](../.github/actions/check-quality/action.yml) | | `scripts/setup/setup-windows-whp.ps1` | PowerShell syntax | [check-quality](../.github/actions/check-quality/action.yml) | +| Markdown files and `doc/` paths | Every relative link and image in a tracked Markdown file resolves to a tracked file, and every anchor to a heading slug that GitHub generates, ignoring code. Every `doc/` path that a tracked file mentions resolves too, except the test fixtures that `DOC_PATH_FIXTURES` lists. | [check-quality](../.github/actions/check-quality/action.yml), [check_markdown_links.py](../scripts/check_markdown_links.py) | | `aci_edge_sandboxes/` | rustfmt, Clippy and rustdoc with warnings denied, tests, and the minimum supported Rust version; see [Rust crates](ci.md#rust-crates) | [check-aci-edge-sandboxes](../.github/actions/check-aci-edge-sandboxes/action.yml) | | `nvx-rs/` | rustfmt, Clippy and rustdoc with warnings denied, tests including a bundled-release fixture, the minimum supported Rust version, a macOS check, and, through `nvx.py`, the musl lint, tests, build, digest manifest, and static-ELF check of the guest binaries; see [Rust crates](ci.md#rust-crates) | [check-nvx-rs](../.github/actions/check-nvx-rs/action.yml) | | NVX CLI | Compiles `scripts/`, runs every `scripts/test_*.py` module, and checks command help, on Linux and Windows | [validate-nvx](../.github/actions/validate-nvx/action.yml) | diff --git a/doc/project-structure.md b/doc/project-structure.md index bbfb149d4..818ce554d 100644 --- a/doc/project-structure.md +++ b/doc/project-structure.md @@ -154,6 +154,7 @@ nvx/ | | |-- setup-linux-runner.sh Linux/KVM or Linux/MSHV Actions runner bootstrap | | |-- setup-windows-whp.ps1 Windows/WHP development-host and Actions runner bootstrap | | `-- tool-versions.conf Shared tool release and checksum pins +| |-- check_markdown_links.py Markdown link, anchor, and doc/ path check run by CI | |-- nvx-hosts.example.json Template for the ignored .nvx-hosts.json | |-- nvx.py Supported command-line entry point | |-- nvx_adversarial_executor.py Restricted adversarial executor entry point diff --git a/doc/setup.md b/doc/setup.md index b44e66bd3..40c15572d 100644 --- a/doc/setup.md +++ b/doc/setup.md @@ -212,9 +212,11 @@ shfmt -d -ln posix -i 4 -ci \ scripts/setup/setup-linux-mshv.sh scripts/setup/setup-linux-runner.sh shfmt -d -ln bash -i 4 -ci \ .github/specula/setup-runner.sh guest/ubuntu/nvx-bashrc +python3 scripts/check_markdown_links.py ``` -Pyright runs in strict mode for both Linux and Windows platform APIs. +Pyright runs in strict mode for both Linux and Windows platform APIs. The link +check reads the files that Git tracks, so stage a new file before you run it. ### Adversarial campaign controller diff --git a/scripts/check_markdown_links.py b/scripts/check_markdown_links.py new file mode 100644 index 000000000..4bee350a3 --- /dev/null +++ b/scripts/check_markdown_links.py @@ -0,0 +1,2532 @@ +#!/usr/bin/env python3 +"""Check the relative links and anchors of the tracked Markdown files. + +Every relative link and image target of a tracked Markdown file outside the +submodules, whether an inline link, a reference definition, or the href of an +HTML link or src of an image, must name a tracked file or directory, and the +anchor of every link must match, in lowercase as GitHub looks it up, a heading +slug that GitHub generates, or the id or name of an HTML element that GitHub +keeps, in the target document; other files have only line anchors such as +#L10. An image's anchor, such as GitHub's #gh-dark-mode-only, selects nothing +to check, and an empty target links to its own document. As on GitHub, code +blocks, code spans, HTML blocks, and front matter hold no Markdown links, an +HTML comment hides what renders up to its end, the tags that GitHub filters +render as text, and block quotes and list items decide where the blocks inside +them end. A link into a submodule resolves against its pinned revision when the +submodule is checked out and is skipped otherwise. + +Every doc/ path, with any anchor, that a tracked text file mentions must +resolve the same way from the repository root, except the test fixtures that +DOC_PATH_FIXTURES lists. +""" + +from __future__ import annotations + +import argparse +import bisect +import html.entities +import os +import posixpath +import re +import subprocess +import sys +import unicodedata +import urllib.parse +from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import NamedTuple + +REPO_ROOT = Path(__file__).resolve().parents[1] + +# Test fixtures that mention doc/ paths that do not exist, by the file that +# mentions them. An entry that its file no longer needs fails the check, and +# this file may mention every path that the entries list. +DOC_PATH_FIXTURES: Mapping[str, Collection[str]] = { + "scripts/test_markdown_links.py": frozenset({"doc/guide.md", "doc/missing.md"}), + "scripts/test_nvx_tools.py": frozenset({"doc/input.patch", "doc/notes.md"}), +} + +_ASCII_PUNCTUATION = frozenset("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") +_ESCAPED = re.compile(r"\\([!-/:-@\[-`{-~])") +# Character references, as cmark decodes them in Markdown, and as GitHub +# decodes them in HTML attributes, where a number needs no semicolon. +_ENTITY = re.compile( + r"&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|[A-Za-z][A-Za-z0-9]{1,30});" +) +_HTML_ENTITY = re.compile(r"&(?:#[0-9]+;?|#[xX][0-9A-Fa-f]+;?|[A-Za-z][A-Za-z0-9]*;)") +_EXTERNAL = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*:|//") +_SPACE = re.compile(r"[ \t]*(?:\n[ \t]*)?") +_LINE_BREAK = re.compile(r"\n[ \t]*") +_BLANK_LINE = re.compile(r"\n[ \t]*(?:\n|$)") +_BACKTICKS = re.compile(r"`+") +# Block syntax, which starts at a line's first character that is not a space +# or tab, fewer than four columns into its container. +_LIST_MARKER = re.compile(r"(?:[-+*]|([0-9]{1,9})[.)])(?=[ \t]|$)") +_FENCE_OPENER = re.compile(r"(`{3,}|~{3,})(.*)") +_FENCE_CLOSER = re.compile(r"(`{3,}|~{3,})[ \t]*") +_THEMATIC_BREAK = re.compile(r"(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,}") +_ATX_HEADING = re.compile(r"#{1,6}(?:[ \t]+(.*))?") +_CLOSING_SEQUENCE = re.compile(r"(?:^|[ \t]+)#+[ \t]*$") +_SETEXT_UNDERLINE = re.compile(r"(?:=+|-+)[ \t]*") +# A table needs a delimiter row whose cells match its header row's. +_TABLE_DELIMITER = re.compile( + r"[ \t]*\|?(?:[ \t]*:?-+:?[ \t]*\|)*[ \t]*:?-+:?[ \t]*\|?[ \t]*" +) +_FOOTNOTE = re.compile(r"\[\^([^\]\s]+)\]:") +_FOOTNOTE_LABEL = re.compile(r"\^([^\]\s]+)") +# Raw HTML, after CommonMark: tags are atomic inline, and their attribute +# values are literal. +_TAG_ATTRIBUTE = re.compile( + r"""\s+([A-Za-z_:][A-Za-z0-9_.:-]*)""" + r"""(?:\s*=\s*(?:([^\s"'=<>`]+)|'([^']*)'|"([^"]*)"))?""" +) +_OPEN_TAG = re.compile( + r"<[A-Za-z][A-Za-z0-9-]*(?:" + _TAG_ATTRIBUTE.pattern + r")*\s*/?>" +) +_CLOSE_TAG = re.compile(r"") +# The start conditions of the HTML blocks of types 1 to 6, whose lines GitHub +# passes through without parsing Markdown, as cmark finds them, and the end +# conditions of types 1 to 5, which, unlike the others, no blank line ends. +_HTML_STARTS = ( + re.compile(r"<(?:pre|script|style|textarea)(?:[ \t>]|$)", re.I), + re.compile(""), + re.compile(r"\?>"), + re.compile(">"), + re.compile(r"\]\]>"), +) +_HTML_TAG_LINE = re.compile( + "(?:" + _OPEN_TAG.pattern + "|" + _CLOSE_TAG.pattern + r")[ \t]*" +) +# What an HTML parser finds in raw HTML: a comment ends at "-->" or "--!>", or +# at once as "" or "", while other markup that starts with "". +_MARKUP = re.compile(r"<(?:!--|[!?/A-Za-z])") +_COMMENT_END = re.compile("--!?>") +_ABRUPT_COMMENT_END = re.compile("-?>") +# What the marks of marked text, as parse_markdown makes it, render. +_UNMARK = str.maketrans({"\x00": "", "\x01": " ", "\x02": " ", "\x03": " "}) +# GitHub escapes the "<" of these tags, which then render as text. +_FILTERED_TAG = re.compile( + r"]|/>)", + re.I, +) +# What GitHub's sanitizer keeps: the targets of links and images, and the +# elements whose id and name attributes are anchors. +_TARGET_ATTRIBUTES = frozenset((("a", "href"), ("img", "src"))) +_ANCHOR_ELEMENTS = frozenset( + "a b blockquote br code dd del details div dl dt em h1 h2 h3 h4 h5 h6 hr i" + " img ins kbd li mark ol p picture pre q rp rt ruby s samp section source" + " span strike strong sub summary sup table tbody td tfoot th thead tr tt" + " ul var".split() +) +# The HTML headings, whose text GitHub slugs too. +_HEADINGS = frozenset(("h1", "h2", "h3", "h4", "h5", "h6")) +# GitHub slugs the text of each heading in the tree that libxml2's HTML parser +# builds from its HTML, as an HTML5 parser rebuilds it. libxml2 closes the +# current element while a start tag that this table lists for it follows, as +# its htmlStartClose table does for each element that can stay open here. +_CLOSED_BY: Mapping[str, frozenset[str]] = { + old: frozenset(new.split()) + for old, new in { + "a": "a fieldset table td th", + "address": "dd dl dt form li ul", + "b": "center p td th", + "big": "p", + "caption": "col colgroup tbody tfoot thead tr", + "colgroup": "colgroup tbody tfoot thead tr", + "dd": "dt", + "dir": "dd dl dt form ul", + "dl": "form li", + "dt": "dd dl", + "font": "center td th", + "form": "form", + **dict.fromkeys(_HEADINGS, "fieldset form li p table"), + "i": "center p td th", + "legend": "fieldset", + "li": "li", + "listing": "dd dl dt fieldset form li table ul", + "menu": "dd dl dt form ul", + "ol": "form", + "option": "optgroup option", + "p": "address blockquote body caption center col colgroup dd dir div dl dt" + " fieldset form frameset h1 h2 h3 h4 h5 h6 head hr li listing menu ol p pre" + " table tbody td tfoot th tr ul", + "pre": "dd dl dt fieldset form li table ul", + "s": "p", + "small": "p", + "span": "td th", + "strike": "p", + "tbody": "tbody tfoot", + "td": "tbody td tfoot th tr", + "tfoot": "tbody", + "th": "tbody td tfoot th tr", + "thead": "tbody tfoot", + "tr": "tbody tfoot tr", + "tt": "p", + "u": "p td th", + "ul": "address form menu pre", + }.items() +} +# An end tag closes the elements after its own only if none of them has a +# higher priority than its own, of 100 unless listed here; libxml2 ignores it +# otherwise, as it does an end tag of no open element. +_END_PRIORITY: Mapping[str, int] = { + "div": 150, + "td": 160, + "th": 160, + "tr": 170, + "thead": 180, + "tbody": 180, + "tfoot": 180, + "table": 190, +} +# The elements that libxml2 never leaves open, and those whose start tags it +# discards after they close others. +_VOID_ELEMENTS = frozenset( + "area base basefont br col frame hr img input isindex link meta param".split() +) +_DISCARDED_ELEMENTS = frozenset(("body", "head", "html")) +# The elements that GitHub renders for block quotes, the items of bullet and +# ordered lists, and footnote definitions, around what they hold. +_CONTAINER_ELEMENTS: Mapping[str, tuple[str, ...]] = { + "quote": ("blockquote",), + "item": ("ul", "li"), + "ordered": ("ol", "li"), + "footnote": ("section", "ol", "li"), +} +_TAG_START = re.compile("\s]*>") +_EMAIL_AUTOLINK = re.compile( + r"<[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?" + r"(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*>" +) +# Comments, processing instructions, declarations, and CDATA sections. +_RAW_HTML = re.compile( + r"||<\?.*?\?>|]*>" + r"|", + re.S, +) +# GitHub keeps letters, marks, digits, connector punctuation such as "_", +# hyphens, and spaces, which become hyphens. +_SLUG_CATEGORIES = frozenset( + ("Ll", "Lm", "Lo", "Lt", "Lu", "Mc", "Me", "Mn", "Nd", "Nl", "Pc") +) +_LINE_ANCHOR = re.compile(r"L([0-9]+)(?:C[0-9]+)?(?:-L([0-9]+)(?:C[0-9]+)?)?") +_DOC_PATH = re.compile(r"(?|\\") + + +class CheckError(RuntimeError): + """Raised when the tracked files of the work tree cannot be listed.""" + + +@dataclass(frozen=True, order=True) +class Problem: + """A link target or a doc/ path that does not resolve.""" + + path: str + line: int + message: str + + def __str__(self) -> str: + location = f"{self.path}:{self.line}" if self.line else self.path + return f"{location}: {self.message}" + + def annotation(self) -> str: + """Formats the problem as a GitHub Actions error annotation.""" + properties = f"file={_escape_property(self.path)}" + if self.line: + properties += f",line={self.line}" + return f"::error {properties}::{_escape_data(str(self))}" + + +@dataclass(frozen=True) +class Link: + """A link or image target, as a browser receives it, and its span in the + text.""" + + line: int + start: int + end: int + target: str + image: bool = False + + +@dataclass(frozen=True) +class MarkdownDocument: + """The link targets and anchors of a Markdown document, each anchor being + the id of an element that GitHub renders, without its user-content- + prefix.""" + + links: tuple[Link, ...] + anchors: frozenset[str] + + +def _escape_data(value: str) -> str: + return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def _escape_property(value: str) -> str: + return _escape_data(value).replace(":", "%3A").replace(",", "%2C") + + +def normalize_newlines(text: str) -> str: + return text.replace("\r\n", "\n").replace("\r", "\n") + + +def github_slug(text: str) -> str: + """Returns the anchor that GitHub derives from a heading's text.""" + return "".join( + "-" if char == " " else char + for char in text.lower() + if char in " -" or unicodedata.category(char) in _SLUG_CATEGORIES + ) + + +def parse_markdown(text: str) -> MarkdownDocument: + """Finds the link targets and anchors of Markdown as GitHub renders it. + + Offsets and line numbers refer to the text after normalize_newlines. + """ + text = normalize_newlines(text) + lines = text.split("\n") + starts = [0, *(match.end() for match in re.finditer("\n", text))] + layout = _layout(lines) + # Blank what renders no Markdown and, as CommonMark strips them before + # parsing inline content, the block quote and list item markers. The + # text of HTML headings keeps the indentation after those markers, so + # marked holds each marker, and each control character that renders + # nothing in a slug, as "\x00", and each tab that a container consumed in + # part as the control character whose code counts the spaces it renders. + chars = list(text) + marks = ["\x00" if char < "\x04" else char for char in text] + for number, line in enumerate(lines): + end = len(line) if layout.hidden[number] else layout.content[number] + _blank(chars, starts[number], starts[number] + end) + prefix = layout.prefixes[number] + marks[starts[number] : starts[number] + prefix] = ["\x00"] * prefix + if layout.pads[number] and not layout.hidden[number]: + marks[starts[number] + prefix] = chr(layout.pads[number]) + blanked = "".join(chars) + marked = "".join(marks) + definitions, firsts = _definitions(blanked, lines, layout, starts) + labels = {definition[0] for definition in definitions} + footnotes: dict[str, str] = {} + for label, _, _ in layout.footnotes: + footnotes.setdefault(label.casefold(), label) + # The raw HTML, by the index of each HTML block where it ends, the inline + # links, each as the offset where it renders and the span of its + # destination, the footnote citations, and the block and the span of the + # inline content that holds each item of inline raw HTML, by where the + # item ends. + regions = { + starts[end - 1] + len(lines[end - 1]): index + for index, (kind, _, end) in enumerate(layout.blocks) + if kind == "html" + } + raw = [(starts[layout.blocks[index][1]], end) for end, index in regions.items()] + inline: list[tuple[int, int, int, bool]] = [] + citations: list[tuple[int, str]] = [] + spans: dict[int, tuple[int, int, int]] = {} + # Each use of a reference, as its offset, whether an image uses it, and + # its label, since a target that only rendered images use holds no anchor. + uses: list[tuple[int, bool, str]] = [] + for index, start, end in _inline_ranges(lines, layout, starts, firsts): + content = blanked[start:end] + items = _inline(content, labels, footnotes) + for item in items: + if item.kind in ("tag", "html"): + raw.append((start + item.start, start + item.end)) + spans[start + item.end] = (index, start, end) + elif item.kind == "footnote": + label = blanked[start + item.start + 2 : start + item.close] + citations.append((start + item.start, label.casefold())) + elif item.destination is not None: + first, last = item.destination + inline.append( + ( + start + item.start, + start + first, + start + last, + item.kind == "image", + ) + ) + elif item.label is not None: + uses.append((start + item.start, item.kind == "image", item.label)) + citations.sort() + # GitHub renders the cited footnotes after the rest of the document, in + # the order of their first citations, and the others not at all. + ranks: dict[str, int] = {} + for _, key in citations: + ranks.setdefault(key, len(ranks) + 1) + rank = [0] * len(lines) + for label, first, end in layout.footnotes: + rank[first:end] = [ranks.get(label.casefold(), -1)] * (end - first) + + def place(offset: int) -> tuple[int, int]: + return rank[bisect.bisect_right(starts, offset) - 1], offset + + tags, comments = _raw_html( + blanked, + sorted( + (span for span in raw if place(span[0])[0] >= 0), + key=lambda span: place(span[0]), + ), + ) + hidden = [ + (place(start), (len(ranks) + 1, 0) if end is None else place(end)) + for start, end in comments + ] + openings = [start for start, _ in hidden] + + def shown(offset: int) -> bool: + """Whether GitHub renders what is at offset, outside any comment.""" + at = place(offset) + index = bisect.bisect_right(openings, at) - 1 + return at[0] >= 0 and (index < 0 or hidden[index][1] <= at) + + attributes = [ + (tag.name, key, start, end) + for tag in tags + for key, start, end in tag.attributes + ] + references: dict[bool, set[str]] = {True: set(), False: set()} + for offset, image, label in uses: + if shown(offset): + references[image].add(label) + # A definition counts where it renders, as do the definitions that the + # rendered references use, wherever they are, such as in hidden footnotes. + # The text of each target is blanked, without the markers of the + # containers that its lines continue. + targets = [ + ( + start, + end, + _destination(blanked[start:end]), + label in references[True] and label not in references[False], + ) + for label, start, end, _ in definitions + if shown(start) or label in references[True] | references[False] + ] + targets.extend( + (start, end, _destination(blanked[start:end]), image) + for offset, start, end, image in inline + if shown(offset) + ) + targets.extend( + (start, end, _url(blanked[start:end]), element == "img") + for element, name, start, end in attributes + if (element, name) in _TARGET_ATTRIBUTES + ) + links = tuple( + Link(bisect.bisect_right(starts, start), start, end, target, image) + for start, end, target, image in sorted(targets) + ) + anchors = { + _attribute(blanked[start:end]).removeprefix(_PREFIX) + for element, name, start, end in attributes + if element in _ANCHOR_ELEMENTS and name in ("id", "name") + } + counts: dict[str, int] = {} + for offset, key in citations: + counts[key] = counts.get(key, 0) + 1 + if shown(offset): + suffix = "" if counts[key] == 1 else f"-{counts[key]}" + anchors.add(f"fnref-{footnotes[key]}{suffix}") + anchors.update( + f"fn-{footnotes[label.casefold()]}" + for label, first, _ in layout.footnotes + if shown(starts[first]) + ) + # GitHub numbers the repeated slugs of Markdown and HTML headings alike, + # in the order that it renders them. + texts = _Texts( + lines, + starts, + layout, + blanked, + marked, + firsts, + labels, + ranks, + regions, + spans, + place, + shown, + _tight(layout, firsts), + ) + headings: list[tuple[tuple[int, int], str]] = [] + for index, (_, first, _) in enumerate(layout.blocks): + offset = starts[first] + layout.content[first] + if shown(offset) and (heading := texts.markdown(index)) is not None: + headings.append((place(offset), heading)) + for tag in tags: + if tag.name in _HEADINGS and (heading := texts.html(tag)) is not None: + headings.append((place(tag.start), heading)) + anchors.update(_slugs(heading for _, heading in sorted(headings))) + anchors.discard("") + return MarkdownDocument(links, frozenset(anchors)) + + +def _blank(chars: list[str], start: int, end: int) -> None: + """Replaces the characters in a span with spaces, keeping line endings.""" + chars[start:end] = (" " if char != "\n" else char for char in chars[start:end]) + + +def _definitions( + blanked: str, lines: Sequence[str], layout: _Layout, starts: Sequence[int] +) -> tuple[list[tuple[str, int, int, int]], list[int]]: + """Parses the link reference definitions at the start of each paragraph, + within that paragraph. + + Returns the definitions, and the first line of each block after them, + where its inline content starts. + """ + definitions: list[tuple[str, int, int, int]] = [] + firsts: list[int] = [] + for kind, first, end in layout.blocks: + number = first + if kind in ("paragraph", "setext"): + base = starts[first] + paragraph = blanked[base : starts[end - 1] + len(lines[end - 1])] + while number < end: + offset = starts[number] + layout.content[number] - base + definition = _definition(paragraph, offset) + if definition is None: + break + label, start, stop, last = definition + definitions.append((label, base + start, base + stop, base + last)) + number = bisect.bisect_right(starts, base + last) + firsts.append(number) + return definitions, firsts + + +def _definition(text: str, index: int) -> tuple[str, int, int, int] | None: + """Parses a link reference definition at index of text whose container + markers are blank. + + Returns its normalized label, the span of its destination, and the end of + its last line, or None when the text there holds no definition. Like + GitHub, it accepts any destination without spaces, even one whose + parentheses do not balance. + """ + label = _LABEL.match(text, index) + if label is None or not label[1].strip(): + return None + start = _skip_space(text, label.end()) + if text.startswith("<", start): + end = _pointy(text, start) + if end is None: + return None + destination = (start + 1, end) + end += 1 + else: + end = start + while end < len(text) and text[end] > " ": + end += 1 + if end == start: + return None + destination = (start, end) + name = _label(label[1]) + title = _skip_space(text, end) + if title > end and text[title : title + 1] in ('"', "'", "("): + # A title in parentheses holds no other unescaped parenthesis. + stops = "()" if text[title] == "(" else text[title] + close = title + 1 + while close < len(text) and text[close] not in stops: + if _BLANK_LINE.match(text, close): + break + close += 2 if text[close] == "\\" else 1 + after = _LINE_END.match(text, close + 1) + if text.startswith(stops[-1], close) and after is not None: + return name, destination[0], destination[1], after.end() + # A title that does not end its line leaves the definition before it. + after = _LINE_END.match(text, end) + if after is None: + return None + return name, destination[0], destination[1], after.end() + + +def _raw_html( + text: str, regions: Iterable[tuple[int, int]] +) -> tuple[list[_Tag], list[tuple[int, int | None]]]: + """Tokenizes raw HTML as an HTML parser does, given its spans of text in + the order that GitHub renders them. + + Returns the open tags, and the span of each comment, where a comment that + one span leaves open hides everything that renders after it up to the end + of the comment in a later span, or with no end, None. + """ + tags: list[_Tag] = [] + comments: list[tuple[int, int | None]] = [] + opened: int | None = None + for start, end in regions: + index = start + while index < end: + if opened is not None: + close = _COMMENT_END.search(text, index, end) + if close is None: + break + comments.append((opened, close.end())) + opened, index = None, close.end() + continue + markup = _MARKUP.search(text, index, end) + if markup is None: + break + index = markup.start() + if _FILTERED_TAG.match(text, index, end): + index += 1 + elif markup[0] == " \\_c\\_\n" + ) + self.assertEqual( + _anchors(document), + { + "nvx", + "hello-world--more-copy", + "x", + "same", + "same-1", + "bold-x", + "a-", + "b", + "a--_c_", + }, + ) + + def test_html_headings_lose_only_the_markers_of_their_containers(self): + # While the indentation after the markers stays in the text, where + # the columns of a tab that a marker consumes in part are spaces. + cases = { + ">

\n> Usage

\n": {"usage"}, + "-

\n Usage

\n": {"usage"}, + ">

\n> Usage

\n": {"--usage"}, + ">

\n>\tUsage

\n": {"--usage"}, + "-

\n \tUsage

\n": {"--usage"}, + ">

\n> \tUsage

\n": {"usage"}, + } + for document, anchors in cases.items(): + with self.subTest(document=document): + self.assertEqual(_anchors(document), anchors) + + def test_html_headings_run_on_until_their_elements_close(self): + # GitHub slugs the text of each heading in the tree that libxml2 + # builds from its HTML, where few tags close a heading, such as the + # start tag of a paragraph, so one runs on across blocks. + cases = { + "

First\n\n\nSecond\n\n

\n": {"firstsecond"}, + "-

First\n\n \n Second\n \n": {"firstsecond"}, + ">

First\n\n\nSecond\n\n": {"first"}, + ( + "

First\n\n code\n\n***\n\n- item\n\n> quote\n\n" + "\nLast\n\n" + ): {"firstcodeitemquotelast"}, + "

First\n\nPara\n": {"first"}, + "

A B\n\nPara\n": {"a-bpara"}, + "

A \n\n| b |\n| - |\n| c |\n": {"a-bc"}, + "## Title
\n\nPara\n": {"title-para"}, + "x

A\n\n- item\n": {"aitem"}, + "

First Second

\n
\n": {"first-second"}, + } + for document, anchors in cases.items(): + with self.subTest(document=document): + self.assertEqual(_anchors(document), anchors) + + def test_tags_close_headings_as_libxml2_closes_their_elements(self): + # A start tag closes the current element only if libxml2 lists it for + # that element, and an end tag closes no element whose priority, such + # as a div's, is higher than its own. + cases: dict[str, set[str]] = { + "

A

x

y

\n": {"a-"}, + "

B
x
y

\n": {"b-"}, + "

C
  • x
    • y

    \n": {"c-"}, + "

    D
    • x
    y

    \n": {"d-x-y"}, + "

    E x

    y

    z

    \n": {"e-x-y-z"}, + "

    F x

    y

    z

    \n": {"f-x-"}, + "

    G
    x

    y z

    \n": {"g-x-y-z"}, + "

    H

    x

    \n": {"h"}, + "

    M

    \n": set(), + "## I

    x

    y\n": {"i-"}, + "## J x\n": {"j--x"}, + "## [K

    x

    y](README.md) z\n": {"k-x-y-z"}, + "## *L x*

    y

    \n": {"l-x-"}, + } + for document, anchors in cases.items(): + with self.subTest(document=document): + self.assertEqual(_anchors(document), anchors) + + def test_elements_that_earlier_blocks_leave_open_hold_later_headings(self): + # Neither a blank line nor the end tag of a block quote, list item, or + # paragraph closes a div, whose end tag then closes a heading in it. + cases = { + "
    \n\n

    H

    x\n": {"h"}, + "
    \n\n## H
    x\n": {"h-"}, + ">
    \n\n

    H

    x\n": {"h"}, + "-
    \n-

    H

    x\n": {"h"}, + "Text
    more\n\n

    H

    x\n": {"h"}, + "
    \n\nText[^1]\n\n[^1]:

    N

    x\n": { + "n", + "fn-1", + "fnref-1", + }, + "
    \n\n
    \n\n

    H x

    \n": {"h-x"}, + } + for document, anchors in cases.items(): + with self.subTest(document=document): + self.assertEqual(_anchors(document), anchors) + + def test_tight_lists_render_their_paragraphs_without_tags(self): + # So a heading inside an item runs on through the item's next + # paragraphs, unless a blank line separates two items of its list or + # two blocks of one item. Blank lines in fenced code, or that end the + # list, separate nothing, and neither does a paragraph that holds only + # definitions, unless a block of the list's parent interrupts it. + cases = { + "- text

    A\n ***\n tail\n": {"atail"}, + "1. text

    A\n ***\n tail\n": {"atail"}, + "- text

    A\n\n ***\n\n tail\n": {"a"}, + "- x

    A\n ***\n tail\n- next\n": {"atail"}, + "- x

    A\n ***\n tail\n\n- next\n": {"a"}, + "- x

    A\n ***\n tail\n\nafter\n": {"atail"}, + "- x

    A\n ***\n tail\n-\n\n- c\n": {"a"}, + "- a

    A\n - b\n\n - c\n": {"abc"}, + "- > q

    A\n > ***\n > tail\n": {"a"}, + "- x

    A\n ```\n code\n\n more\n ```\n tail\n": {"acodemoretail"}, + "[d]\n\n- x

    A\n ***\n tail\n\n [d]: d.md\n": {"atail"}, + "- x

    A\n ***\n tail\n\n [d]: d.md\n# H\n\n[d]\n": {"a", "h"}, + } + for document, anchors in cases.items(): + with self.subTest(document=document): + self.assertEqual(_anchors(document), anchors) + + def test_ordered_list_items_render_in_ol_elements(self): + # Whose end tags close only the elements of ordered lists. + cases = { + "1.

    Head tail

    \n": {"head"}, + "3.

    Head tail

    \n": {"head"}, + "1)

    Head tail

    \n": {"head"}, + "1. -

    Head tail

    \n": {"head"}, + "1.

    Head tail

    \n": {"head-tail"}, + "-

    Head tail

    \n": {"head"}, + "-

    Head tail

    \n": {"head-tail"}, + } + for document, anchors in cases.items(): + with self.subTest(document=document): + self.assertEqual(_anchors(document), anchors) + + def test_inline_raw_html_renders_as_an_html_parser_reads_it(self): + # Markup other than a tag or a comment ends at its first ">", a + # comment at "--!>" too, and markup in the attributes of the tags that + # GitHub filters is live, so the rest renders, headings included. + cases = { + "# A B ]]> C\n": {"a--b--c"}, + "# A B ?> C\n": {"a--b--c"}, + "# A B --> C\n": {"a--b----c"}, + "# A C\n": {"a--c"}, + '# A end\n\n' + "