diff --git a/docs/implementation/media/nephrology-fidelity-chart.png b/docs/implementation/media/nephrology-fidelity-chart.png new file mode 100644 index 0000000..34a1e13 Binary files /dev/null and b/docs/implementation/media/nephrology-fidelity-chart.png differ diff --git a/docs/implementation/media/nephrology-fidelity-grouping.png b/docs/implementation/media/nephrology-fidelity-grouping.png new file mode 100644 index 0000000..5781fb7 Binary files /dev/null and b/docs/implementation/media/nephrology-fidelity-grouping.png differ diff --git a/wikify/engine/__init__.py b/wikify/engine/__init__.py index bf14427..fc7a842 100644 --- a/wikify/engine/__init__.py +++ b/wikify/engine/__init__.py @@ -73,6 +73,7 @@ def parse_pdf( with fitz.open(pdf_path) as doc: total = doc.page_count + repeated = images.repeated_images(doc) sd = store.create_document( title=title, import_name=import_name, @@ -85,11 +86,11 @@ def parse_pdf( png = pdf_utils.render_png(page, dpi=dpi) kind = pdf_utils.classify_page(page, min_chars, min_drawings) markdown = baseline.parse_page(pdf_path, page_no) - # Rescue genuine graphics the baseline parser omits (diagrams/photos with no - # recoverable text) by embedding them inline; text-image placeholders are left - # for the cleanup pass. No-op on the common page with no genuine figure. - markdown = images.embed_genuine_images(page, markdown, sd, page_no, kind) page_name = store.add_page(sd, page_no, kind, png, markdown) + figures = images.page_figures(page, repeated, page_name, page_no) + if figures: + markdown = images.place_figures(markdown, figures, page_no) + store.set_baseline_markdown(page_name, markdown) # Visual pages MUST be judged (text GT is unreliable on diagrams); text # pages are judged only when the operator opted in (cost/latency). diff --git a/wikify/engine/generate.py b/wikify/engine/generate.py index fb5126d..70f36e7 100644 --- a/wikify/engine/generate.py +++ b/wikify/engine/generate.py @@ -32,6 +32,9 @@ from wikify.engine.refs import smallest_covering DATA_MAX_LENGTH = 140 +OVERVIEW_TITLE = "Overview" +CONTENTS_TITLE = "Contents" +PRINTED_CONTENTS_TITLES = {"contents", "table of contents"} def _upsert_wiki_document( @@ -92,6 +95,18 @@ def bounded_route(prefix: str, title: str, identifier: str) -> tuple[str, str]: return f"{prefix}/{slug}", slug +def overview_route(prefix: str, section_name: str) -> tuple[str, str]: + return bounded_route(prefix, OVERVIEW_TITLE, section_name) + + +def contents_route(prefix: str, source_document: str) -> tuple[str, str]: + return bounded_route(prefix, CONTENTS_TITLE, source_document) + + +def has_own_body(section: dict) -> bool: + return bool(section["is_group"] and (section["markdown"] or "").strip()) + + class _WikiGenerator: """Projects one approved Source Document tree into a Wiki Space (see module doc).""" @@ -113,6 +128,8 @@ def __init__( self.included = [s for s in self.sections if s["include_in_wiki"]] self.wiki_name: dict[str, str] = {} # section name → wiki document name self.wiki_route: dict[str, str] = {} # section name → wiki route + self.page_name: dict[str, str] = {} # section name → wiki document showing its content + self.page_route: dict[str, str] = {} self.content: dict[str, str] = {} # section name → content written self.deleted = 0 self.links = 0 @@ -124,6 +141,7 @@ def run(self) -> dict: self._stage("Building wiki pages") self._build_structure() self._rollup_empty_groups() + self._add_contents() self._stage("Resolving page references") self._rewrite_links() store.set_document_wiki(self.sd.name, self.space.name, self.root_group.name, status="Wiki-Generated") @@ -152,7 +170,26 @@ def _sweep_stale(self) -> None: rest under the root group deepest-first (NestedSet needs leaves gone first). """ kept = {self.root_group.name} + if self._needs_contents(): + kept |= set( + frappe.get_all( + "Wiki Document", + filters={"route": contents_route(self.root_group.route, self.sd.name)[0], "is_group": 0}, + pluck="name", + ) + ) kept |= {s["wiki_document"] for s in self.included if s["wiki_document"]} + overview_routes = [ + overview_route(self.root_group.route, s["name"])[0] for s in self.included if has_own_body(s) + ] + if overview_routes: + kept |= set( + frappe.get_all( + "Wiki Document", + filters={"route": ["in", overview_routes], "is_group": 0}, + pluck="name", + ) + ) descendants = get_descendants_of("Wiki Document", self.root_group.name, ignore_permissions=True) stale = ( frappe.get_all( @@ -189,7 +226,7 @@ def _content_for(section: dict) -> str: return "" if section["is_group"] else f"# {section['title']}\n" def _build_structure(self) -> None: - sort_counter: dict[str, int] = {} # parent wiki name → next sort_order + sort_counter: dict[str, int] = {self.root_group.name: 1 if self._needs_contents() else 0} total = len(self.included) for i, s in enumerate(self.included): parent_name = self._parent_for(s) @@ -200,7 +237,7 @@ def _build_structure(self) -> None: doc = _upsert_wiki_document( s["wiki_document"], title=s["title"], - content=content, + content="" if has_own_body(s) else content, is_group=bool(s["is_group"]), parent=parent_name, route=route, @@ -209,11 +246,34 @@ def _build_structure(self) -> None: ) self.wiki_name[s["name"]] = doc.name self.wiki_route[s["name"]] = doc.route + self.page_name[s["name"]] = doc.name + self.page_route[s["name"]] = doc.route self.content[s["name"]] = content + if has_own_body(s): + self._add_overview(s, doc) + sort_counter[doc.name] = 1 store.set_section_wiki_document(s["name"], doc.name) if self.progress_cb: self.progress_cb(i + 1, total) + def _add_overview(self, section: dict, group) -> None: + """The live wiki redirects a group to its first page, so a group's own text would + never be shown; it goes on an Overview page sorted first instead.""" + route, slug = overview_route(self.root_group.route, section["name"]) + existing = frappe.db.get_value("Wiki Document", {"route": route, "is_group": 0}, "name") + doc = _upsert_wiki_document( + existing, + title=OVERVIEW_TITLE, + content=self.content[section["name"]], + is_group=False, + parent=group.name, + route=route, + slug=slug, + sort_order=0, + ) + self.page_name[section["name"]] = doc.name + self.page_route[section["name"]] = doc.route + def _rollup_empty_groups(self) -> None: """Give container pages (groups with no own body) a Contents list linking to their direct children, so a section landing page renders links instead of a bare heading. @@ -236,6 +296,42 @@ def _rollup_empty_groups(self) -> None: "Wiki Document", self.wiki_name[s["name"]], "content", toc, update_modified=False ) + def _needs_contents(self) -> bool: + return bool(self.included) and not any( + s["title"].strip().lower() in PRINTED_CONTENTS_TITLES for s in self.included + ) + + def _add_contents(self) -> None: + """The PDF printed no contents pages, so the space gets one built from the section tree.""" + if not self._needs_contents(): + return + depth: dict[str, int] = {} + lines = [] + for s in self.included: + parent = self.by_name.get(s["parent_source_section"]) + while parent is not None and parent["name"] not in depth: + parent = self.by_name.get(parent["parent_source_section"]) + depth[s["name"]] = depth[parent["name"]] + 1 if parent else 0 + pages = ( + f"p. {s['page_start']}" + if s["page_end"] in (None, s["page_start"]) + else f"pp. {s['page_start']}-{s['page_end']}" + ) + lines.append( + f"{' ' * depth[s['name']]}- [{s['title']}](/{self.wiki_route[s['name']]}) — {pages}\n" + ) + route, slug = contents_route(self.root_group.route, self.sd.name) + doc = _upsert_wiki_document( + frappe.db.get_value("Wiki Document", {"route": route, "is_group": 0}, "name"), + title=CONTENTS_TITLE, + content=f"# {CONTENTS_TITLE}\n\n" + "".join(lines), + is_group=False, + parent=self.root_group.name, + route=route, + slug=slug, + ) + frappe.db.set_value("Wiki Document", doc.name, "sort_order", 0, update_modified=False) + def _route_for_page(self, n: int) -> str | None: """Smallest-span included section whose PDF page range contains n → its route.""" best = smallest_covering(self.included, n) @@ -249,11 +345,11 @@ def _rewrite_links(self) -> None: self.content[s["name"]], page_count, self._route_for_page, - current_route=self.wiki_route[s["name"]], + current_route=self.page_route[s["name"]], ) if links: frappe.db.set_value( - "Wiki Document", self.wiki_name[s["name"]], "content", new_md, update_modified=False + "Wiki Document", self.page_name[s["name"]], "content", new_md, update_modified=False ) self.links += links @@ -340,11 +436,16 @@ def route_for_page(n: int) -> str | None: best = smallest_covering(routed, n) return routes[best["name"]] if best else None - own_route = routes.get(section_name) + target, own_route = sec.wiki_document, routes.get(section_name) + if has_own_body(sec): + own_route = overview_route(own_route.rpartition("/")[0], section_name)[0] + target = frappe.db.get_value("Wiki Document", {"route": own_route, "is_group": 0}, "name") + if not target: + return {"synced": False, "reason": "needs_regenerate"} page_count = frappe.db.get_value("Source Document", sec.source_document, "page_count") content, links = rewrite_page_refs(content, page_count or 10**9, route_for_page, current_route=own_route) - frappe.db.set_value("Wiki Document", sec.wiki_document, "content", content, update_modified=False) + frappe.db.set_value("Wiki Document", target, "content", content, update_modified=False) return {"synced": True, "chars": len(content), "links": links, "route": own_route} diff --git a/wikify/engine/images.py b/wikify/engine/images.py index 49f0b9e..7e500e1 100644 --- a/wikify/engine/images.py +++ b/wikify/engine/images.py @@ -1,102 +1,365 @@ -"""Extract genuine embedded graphics from a PDF page and embed them inline. +from __future__ import annotations -pymupdf4llm omits every image as a `==> picture [W x H] intentionally omitted <==` -placeholder. For an *image of text* it also recovers the text (a "picture text" block, -which the cleanup pass turns into real Markdown), so those don't need the image. But a -genuine graphic (diagram, flowchart, photo, chart) has no recoverable text — the -placeholder is all that's left, and cleanup strips even that, silently losing the figure. +import re +import unicodedata +from collections import Counter +from dataclasses import dataclass -This replaces such genuine-graphic placeholders with an embedded image link to a private -Frappe File, so the figure survives into the section tree + wiki. Text-image placeholders -are left untouched for cleanup. Deterministic — no LLM, no key needed. -""" +import fitz -from __future__ import annotations +from wikify.engine import pdf_utils, regions, store + +FIGURE_DPI = 200 +CROP_PADDING = 4.0 +MERGE_GAP = 2.0 +MIN_FIGURE_SIDE = 72.0 +MIN_FIGURE_AREA = 0.02 +MAX_FIGURE_AREA = 0.85 +MIN_REPEATED_PAGES = 3 +REPEATED_PAGE_SHARE = 0.2 +CAPTION_GAP = 24.0 +MAX_CAPTION_CHARS = 120 +ANCHOR_LENGTHS = (60, 25) +MIN_ANCHOR_CHARS = 12 +EDGE_TOLERANCE = 2.0 +MAX_ECHO_CHARS = 120 +MIN_TEXT_LAYER_CHARS = 200 +_IMAGE_LINE_RE = re.compile(r"^\s*!\[[^\]]*\]\([^)]*\)\s*$") +_ECHO_BLOCKER_RE = re.compile(r"^\s*(?:[|]|```|[-*+]\s|\d+[.)]\s)") +_WORD_RE = re.compile(r"\w{3,}") +_TRANSCRIPTION_START_RE = re.compile(r"^\s*(?:\||]*>|&#?\w+;") +_FENCE_RE = re.compile(r"^[ \t]*```", re.MULTILINE) + + +@dataclass +class Figure: + bbox: tuple[float, float, float, float] + page_size: tuple[float, float] = (1.0, 1.0) + caption: str = "" + text_before: str = "" + text_after: str = "" + url: str = "" -import re -import fitz # PyMuPDF -from frappe.utils.file_manager import save_file - -# Below this pixel area an "image" is almost always a rule, divider, bullet, icon, or -# logo — not a figure worth embedding. -_MIN_AREA = 200_000 - -# A page whose extractable text is below this is "sparse" — when it also carries a big -# image, the content lives in the image (a flowchart/diagram pymupdf dropped), so embed it -# even without a placeholder. Text-dense pages never trip this, so we don't re-embed page -# scans whose text the parser already recovered. -_SPARSE_TEXT = 700 - -_OMIT_RE = re.compile(r"\*{0,2}==>\s*picture \[(\d+) x (\d+)\] intentionally omitted <==\*{0,2}") -_PICTURE_TEXT = "Start of picture text" - - -def _is_text_image(markdown: str, match_end: int) -> bool: - """A "picture text" marker right after the placeholder means pymupdf recovered the - image's text — keep the text (cleanup handles it), don't embed the image.""" - return _PICTURE_TEXT in markdown[match_end : match_end + 200] - - -def _save_png(page, xref: int, source_document: str, page_no: int, idx: int) -> str | None: - """Extract one embedded image to PNG and host it as a private File on the document.""" - try: - pix = fitz.Pixmap(page.parent, xref) - if pix.n - pix.alpha >= 4: # CMYK / DeviceN → RGB so PNG encodes - pix = fitz.Pixmap(fitz.csRGB, pix) - png = pix.tobytes("png") - except Exception: - return None - file_doc = save_file( - f"page-{page_no:04d}-fig-{idx + 1}.png", - png, - "Source Document", - source_document, - is_private=1, +def repeated_images(pdf_document) -> set[bytes]: + pages_seen: Counter = Counter() + for page in pdf_document: + pages_seen.update({info["digest"] for info in page.get_image_info(hashes=True)}) + threshold = max(MIN_REPEATED_PAGES, pdf_document.page_count * REPEATED_PAGE_SHARE) + return {digest for digest, count in pages_seen.items() if count >= threshold} + + +def boxes_touch(box: tuple, other: tuple) -> bool: + return ( + box[0] - MERGE_GAP <= other[2] + and other[0] - MERGE_GAP <= box[2] + and box[1] - MERGE_GAP <= other[3] + and other[1] - MERGE_GAP <= box[3] ) - return file_doc.file_url - - -def _big_images(page) -> list[int]: - """Substantial embedded image xrefs, de-duplicated, in page order.""" - imgs, seen = [], set() - for im in page.get_images(full=True): - xref = im[0] - if xref not in seen and im[2] * im[3] >= _MIN_AREA: - seen.add(xref) - imgs.append(xref) - return imgs - - -def embed_genuine_images(page, markdown: str, source_document: str, page_no: int, kind: str = "text") -> str: - """Embed genuine graphics the baseline parser would otherwise drop. - - Two cases: - 1. An omitted-picture placeholder with no recovered text → replace it inline with the - matching page image (precise placement). - 2. No placeholder, but a sparse-text page carrying a big image → a flowchart/diagram - pymupdf dropped silently; append it as a figure. - Returns the markdown unchanged on the common page (text-dense, or no substantial image, - or only text-image placeholders). Visual pages are left to the VLM re-parse. - """ - imgs = _big_images(page) - if not imgs: - return markdown - # Case 1: genuine-graphic placeholders (omitted, no "picture text"), matched in order. - genuine = [m for m in _OMIT_RE.finditer(markdown) if not _is_text_image(markdown, m.end())] - if genuine: - out, cursor = [], 0 - for idx, m in enumerate(genuine): - out.append(markdown[cursor : m.start()]) - url = _save_png(page, imgs[idx], source_document, page_no, idx) if idx < len(imgs) else None - out.append(f"![Figure {page_no}.{idx + 1}]({url})" if url else m.group(0)) - cursor = m.end() - out.append(markdown[cursor:]) - return "".join(out) - - # Case 2: silently-dropped figure on a sparse text page (the content is in the image). - if kind == "visual" or "![" in markdown or len(page.get_text("text")) >= _SPARSE_TEXT: + +def merge_touching(boxes: list[tuple]) -> list[tuple]: + merged: list[tuple] = [] + for box in boxes: + while touching := next((other for other in merged if boxes_touch(box, other)), None): + merged.remove(touching) + box = ( + min(box[0], touching[0]), + min(box[1], touching[1]), + max(box[2], touching[2]), + max(box[3], touching[3]), + ) + merged.append(box) + return merged + + +def is_figure_sized(box: tuple, page_box: tuple) -> bool: + if box[2] - box[0] < MIN_FIGURE_SIDE or box[3] - box[1] < MIN_FIGURE_SIDE: + return False + return MIN_FIGURE_AREA <= regions.area_fraction(box, page_box) <= MAX_FIGURE_AREA + + +def body_text_blocks(page, page_box: tuple) -> list[tuple[tuple, str]]: + blocks = [] + for block in page.get_text("blocks"): + text = " ".join((block[4] or "").split()) + box = tuple(float(value) for value in block[:4]) + if block[6] == 0 and text and not regions.is_furniture(box, page_box): + blocks.append((box, text)) + return blocks + + +def overlaps_horizontally(box: tuple, other: tuple) -> bool: + return min(box[2], other[2]) > max(box[0], other[0]) + + +def nearest_block(candidates: list[tuple[tuple, str]], figure_box: tuple, key) -> str: + aligned = [block for block in candidates if overlaps_horizontally(block[0], figure_box)] + pool = aligned or candidates + return min(pool, key=key)[1] if pool else "" + + +def find_caption(blocks: list[tuple[tuple, str]], figure_box: tuple) -> str: + for box, text in blocks: + near_below = 0 <= box[1] - figure_box[3] <= CAPTION_GAP + near_above = 0 <= figure_box[1] - box[3] <= CAPTION_GAP + if ( + (near_below or near_above) + and overlaps_horizontally(box, figure_box) + and len(text) <= MAX_CAPTION_CHARS + and _CAPTION_RE.match(text) + ): + return text + return "" + + +def find_figures(page, repeated: set[bytes]) -> list[Figure]: + page_box = tuple(float(value) for value in page.rect) + boxes = [] + for info in page.get_image_info(hashes=True): + if info["digest"] in repeated: + continue + box = tuple(float(value) for value in (page.rect & info["bbox"])) + if regions.area(box) and not regions.is_furniture(box, page_box): + boxes.append(box) + figure_boxes = sorted( + (box for box in merge_touching(boxes) if is_figure_sized(box, page_box)), + key=lambda box: (box[1], box[0]), + ) + + blocks = body_text_blocks(page, page_box) + figures = [] + for figure_box in figure_boxes: + outside = [block for block in blocks if regions.overlap_fraction(block[0], [figure_box]) < 0.5] + above = [block for block in outside if block[0][3] <= figure_box[1] + EDGE_TOLERANCE] + below = [block for block in outside if block[0][1] >= figure_box[3] - EDGE_TOLERANCE] + figures.append( + Figure( + bbox=figure_box, + page_size=(page.rect.width, page.rect.height), + caption=find_caption(outside, figure_box), + text_before=nearest_block(above, figure_box, key=lambda block: -block[0][3]), + text_after=nearest_block(below, figure_box, key=lambda block: block[0][1]), + ) + ) + return figures + + +def text_line_boxes(page) -> list[tuple]: + return [ + tuple(float(value) for value in line["bbox"]) + for block in page.get_text("dict")["blocks"] + if block["type"] == 0 + for line in block["lines"] + if "".join(span["text"] for span in line["spans"]).strip() + ] + + +def figure_clip(page, figure: Figure) -> fitz.Rect: + """The padded figure box, moved off any text line its top or bottom edge would cut in half: a line + mostly inside the picture is taken in whole, a line outside it (a title above) is left out whole.""" + box = figure.bbox + top, bottom = box[1] - CROP_PADDING, box[3] + CROP_PADDING + for line in text_line_boxes(page): + if not overlaps_horizontally(line, box): + continue + inside = max(0.0, min(line[3], box[3]) - max(line[1], box[1])) / max(line[3] - line[1], 1.0) + if line[1] < top < line[3]: + top = line[1] if inside >= 0.5 else line[3] + if line[1] < bottom < line[3]: + bottom = line[3] if inside >= 0.5 else line[1] + return page.rect & fitz.Rect(box[0] - CROP_PADDING, top, box[2] + CROP_PADDING, bottom) + + +def crop_figure(page, figure: Figure) -> bytes: + return pdf_utils.render_png(page, dpi=FIGURE_DPI, clip=figure_clip(page, figure)) + + +def page_figures(page, repeated: set[bytes], page_name: str, page_no: int) -> list[Figure]: + figures = find_figures(page, repeated) + for figure in figures: + figure.url = store.save_crop_file(page_name, page_no, crop_figure(page, figure)).file_url + return figures + + +def figure_hint(figures: list[Figure]) -> str: + if not figures: + return "" + lines = [] + for number, figure in enumerate(figures, start=1): + width, height = figure.page_size + left, top, right, bottom = figure.bbox + down = f"{top / height:.0%} to {bottom / height:.0%} down the page" + across = f"{left / width:.0%} to {right / width:.0%} across" + lines.append(f"- Figure {number}: {down}, {across}") + positions = "\n".join(lines) + return ( + f"\n\nFIGURES: this page has {len(figures)} picture(s) that are cropped from the PDF and shown " + f"as images, numbered top to bottom:\n{positions}\n" + "Mark where each picture sits in the reading order with its token on a line of its own, " + "followed by the picture's printed title when it has one: [[FIGURE 1]] or " + "[[FIGURE 1: Admissions per year]]. Use every token exactly once and never write Markdown " + "image tags.\n" + "- A chart, graph, plot, photo, map or illustration is fully shown by its image: write only " + "its token. Never turn its bars, lines, axis ticks, legend or data labels into a table, a list " + "or numbers — the page does not print those values, so anything you write would be invented.\n" + "- A picture that is mostly text (a flowchart, form, notice, or a table printed as an image): " + "write its token, then transcribe its text below the token following the rules above.\n" + "- Text printed outside the pictures is transcribed as usual." + ) + + +def figure_tag(figure: Figure, number: int, page_no: int, title: str = "") -> str: + alt = " ".join((figure.caption or title or f"Figure {page_no}.{number}").split()) + return f"![{alt.replace('[', '(').replace(']', ')')}]({figure.url})" + + +def searchable(markdown: str) -> tuple[str, list[int]]: + skipped = [match.span() for match in _SKIPPED_MARKUP_RE.finditer(markdown)] + characters, positions = [], [] + span_index = 0 + for index, character in enumerate(markdown): + while span_index < len(skipped) and skipped[span_index][1] <= index: + span_index += 1 + if span_index < len(skipped) and skipped[span_index][0] <= index: + continue + if character.isalnum(): + characters.append(character.lower()) + positions.append(index) + return "".join(characters), positions + + +def normalized(text: str) -> str: + return "".join(character.lower() for character in text if character.isalnum()) + + +def layer_text(text: str) -> str: + return unicodedata.normalize("NFKC", text) + + +def is_echo(line: str, page_text: str, page_words: set[str]) -> bool: + """A short line the parser read off the picture itself: neither the line nor most of its words are + anywhere in the page's text layer, so it can only be a label or caption drawn inside the image.""" + text = line.strip() + if not text or len(text) > MAX_ECHO_CHARS or _ECHO_BLOCKER_RE.match(text): + return False + if normalized(layer_text(text)) in page_text: + return False + words = {word.lower() for word in _WORD_RE.findall(layer_text(text))} + return len(words & page_words) * 2 < len(words) or not words + + +def drop_echoes_after_figures(markdown: str, page_text: str) -> str: + page_words = {word.lower() for word in _WORD_RE.findall(layer_text(page_text))} + page_text = normalized(layer_text(page_text)) + lines = markdown.splitlines() + drop: set[int] = set() + for index, line in enumerate(lines): + if not _IMAGE_LINE_RE.match(line): + continue + echoes = [] + following = index + 1 + while following < len(lines) and ( + not lines[following].strip() or is_echo(lines[following], page_text, page_words) + ): + if lines[following].strip(): + echoes.append(following) + following += 1 + if following == len(lines) or not _TRANSCRIPTION_START_RE.match(lines[following]): + drop.update(echoes) + if not drop: return markdown - url = _save_png(page, imgs[0], source_document, page_no, 0) - return f"{markdown.rstrip()}\n\n![Figure {page_no}.1]({url})\n" if url else markdown + return re.sub(r"\n{3,}", "\n\n", "\n".join(line for index, line in enumerate(lines) if index not in drop)) + + +def drop_figure_echoes(pages: list[tuple[int, str]], pdf_path: str) -> list[tuple[int, str]]: + """Drop the labels and captions a parser copied out of a picture into the text after it. Only on pages + with a text layer to check against, since on a scanned page every line would look drawn.""" + with fitz.open(pdf_path) as pdf_document: + cleaned = [] + for page_no, markdown in pages: + page_text = pdf_document[page_no - 1].get_text() if 0 < page_no <= pdf_document.page_count else "" + if "![" in markdown and len(normalized(layer_text(page_text))) >= MIN_TEXT_LAYER_CHARS: + markdown = drop_echoes_after_figures(markdown, page_text) + cleaned.append((page_no, markdown)) + return cleaned + + +def enclosing_block(markdown: str, index: int) -> tuple[int, int] | None: + fences = [match.start() for match in _FENCE_RE.finditer(markdown)] + for opening, closing in zip(fences[::2], fences[1::2], strict=False): + if opening <= index <= closing: + line_end = markdown.find("\n", closing) + return opening, len(markdown) if line_end == -1 else line_end + table_start = markdown.rfind("", 0, index) < table_start: + table_end = markdown.find("", index) + if table_end != -1: + return table_start, table_end + len("") + return None + + +def line_bounds(markdown: str, index: int) -> tuple[int, int]: + start = markdown.rfind("\n", 0, index) + 1 + end = markdown.find("\n", index) + end = len(markdown) if end == -1 else end + block = enclosing_block(markdown, index) + if block: + return markdown.rfind("\n", 0, block[0]) + 1, block[1] + if markdown[start:end].lstrip().startswith("|"): + while start > 0 and markdown[markdown.rfind("\n", 0, start - 1) + 1 : start].lstrip().startswith("|"): + start = markdown.rfind("\n", 0, start - 1) + 1 + while end < len(markdown): + next_end = markdown.find("\n", end + 1) + next_end = len(markdown) if next_end == -1 else next_end + if not markdown[end + 1 : next_end].lstrip().startswith("|"): + break + end = next_end + return start, end + + +def anchor_position(markdown: str, figure: Figure) -> int | None: + plain, positions = searchable(markdown) + before, after = normalized(figure.text_before), normalized(figure.text_after) + for length in ANCHOR_LENGTHS: + tail, head = before[-length:], after[:length] + if len(tail) >= MIN_ANCHOR_CHARS and (found := plain.find(tail)) != -1: + return line_bounds(markdown, positions[found + len(tail) - 1])[1] + if len(head) >= MIN_ANCHOR_CHARS and (found := plain.find(head)) != -1: + return line_bounds(markdown, positions[found])[0] + return None + + +def insert_at(markdown: str, position: int, tag: str) -> str: + return f"{markdown[:position].rstrip()}\n\n{tag}\n\n{markdown[position:].lstrip()}" + + +def place_figures(markdown: str, figures: list[Figure], page_no: int) -> str: + if not figures: + return FIGURE_TOKEN_RE.sub("", markdown or "") + placed: set[int] = set() + + def swap_token(match: re.Match) -> str: + number = int(match.group(1)) + if not 1 <= number <= len(figures) or number in placed or figures[number - 1].url in markdown: + return "" + placed.add(number) + return figure_tag(figures[number - 1], number, page_no, match.group(2) or "") + + text = FIGURE_TOKEN_RE.sub(swap_token, markdown or "") + for number, figure in enumerate(figures, start=1): + if number in placed or not figure.url or figure.url in text: + continue + tag = figure_tag(figure, number, page_no) + position = anchor_position(text, figure) + if position is None: + position = 0 if figure.bbox[1] < figure.page_size[1] / 2 else len(text) + text = insert_at(text, position, tag) + return re.sub(r"\n{3,}", "\n\n", text).strip() diff --git a/wikify/engine/lint.py b/wikify/engine/lint.py index 617546f..8cabe38 100644 --- a/wikify/engine/lint.py +++ b/wikify/engine/lint.py @@ -132,3 +132,111 @@ def fix_table_separators(markdown: str) -> str: if i in insertions: out.append(insertions[i]) return "\n".join(out) + + +_BLANK_RUN_RE = re.compile(r"(? str: + """Escape form blanks (runs of 3+ underscores) so markdown can't read them as emphasis and eat + the text between two blanks. Idempotent; a line that is only underscores (a rule) is left alone.""" + lines = (markdown or "").split("\n") + for index in _prose_lines(lines): + if not _RULE_LINE_RE.match(lines[index]): + lines[index] = _BLANK_RUN_RE.sub(lambda match: "\\_" * len(match.group(0)), lines[index]) + return "\n".join(lines) + + +def _fix_bold_span(before: str, content: str, after: str) -> tuple[str, str, str]: + stripped = content.strip() + if not any(char.isalnum() for char in stripped): + return before + content, "", after + if opener := _TRAILING_OPENER_RE.search(stripped): + stripped, after = stripped[: opener.start()], stripped[opener.start() :] + after + if len(stripped) == 1 and stripped.isalpha() and after[:1].islower() and not before[-1:].isalnum(): + return before + stripped, "", after + if len(stripped) > 2 and stripped[-2] == " " and stripped[-1].isupper() and after[:1].islower(): + stripped, after = stripped[:-2], f" {stripped[-1]}{after}" + if content[:1].isspace() or (before[-1:].isalpha() and stripped[:1].isalpha() and len(stripped) > 1): + before += " " + if (content[-1:].isspace() and not after[:1].isspace()) or ( + _LETTER_START_RE.match(after) and len(stripped) > 1 + ): + after = f" {after}" + return before, f"**{stripped}**", after + + +def _fix_bold_spans(text: str) -> str: + parts = text.split("**") + if len(parts) % 2 == 0: + return _fix_bold_spans(text.rstrip()[:-2]) if text.rstrip().endswith("**") else text + for index in range(1, len(parts), 2): + parts[index - 1], parts[index], parts[index + 1] = _fix_bold_span( + parts[index - 1], parts[index], parts[index + 1] + ) + return "".join(parts) + + +def fix_glued_emphasis(markdown: str) -> str: + """Repair bold the PDF extraction glued to its neighbours: `**To**confirm`, `Helps**to**gain`, + `**P**revents`, `**Post: H**ypo`, `efficiency**.**`, a stray trailing `**`. Cell by cell in tables.""" + lines = (markdown or "").split("\n") + for index in _prose_lines(lines): + line = lines[index] + if "**" not in line: + continue + if _is_table_row(line): + lines[index] = "|".join(_fix_bold_spans(cell) for cell in _UNESCAPED_PIPE_RE.split(line)) + else: + lines[index] = _fix_bold_spans(line) + return "\n".join(lines) + + +def drop_box_bullets(markdown: str) -> str: + """Drop a box a parser read off a bullet glyph (`- ☐ Peritonitis rates`): a bulleted item never needs + a second marker. A list where any item is ticked or offers several boxes is a form and is kept.""" + lines = (markdown or "").split("\n") + prose = set(_prose_lines(lines)) + index = 0 + while index < len(lines): + end = index + while end < len(lines) and end in prose and _LIST_LINE_RE.match(lines[end]): + end += 1 + if end == index: + index += 1 + continue + if not any(_FORM_MARK_RE.search(lines[line]) for line in range(index, end)): + for line in range(index, end): + lines[line] = _BOX_BULLET_RE.sub(r"\1", lines[line]) + index = end + return "\n".join(lines) + + +def repair_markdown(markdown: str) -> str: + """Every mechanical repair, in one pass, for the assembled section product.""" + return drop_box_bullets(fix_glued_emphasis(escape_fill_in_blanks(fix_table_separators(markdown)))) diff --git a/wikify/engine/loader/cleanup.py b/wikify/engine/loader/cleanup.py index 6ff8236..387ee85 100644 --- a/wikify/engine/loader/cleanup.py +++ b/wikify/engine/loader/cleanup.py @@ -1,9 +1,9 @@ """Strip repeated page headers/footers before sectionizing. Real manuals repeat a running header/footer on every page (doc title, doc code, -"Pg X of Y", version/date). Left in, each becomes a fake heading. We remove lines -that recur across many pages, plus a few varying-boilerplate patterns (page -numbers, doc codes) that won't match exactly page-to-page. +"Pg X of Y", version/date). Left in, each becomes a fake heading. We remove the run of +lines at each page edge that recur at the edges of many pages, plus a few +varying-boilerplate patterns (page numbers, doc codes) that won't match exactly page-to-page. Ported verbatim from the POC `loader/cleanup.py` (pure markdown, no I/O). Wired into the pipeline at sectionize time (Slice 4); shipped here per the Slice 3 cleanup port. @@ -13,16 +13,33 @@ import re from collections import Counter +from itertools import groupby -_NORM = re.compile(r"[#*_`>\-\s]+") -_VARYING = [ +from wikify.engine.loader.table_stitch import ( + html_tables_to_markdown, + merge_continuation_rows, + stitch_cross_page_tables, +) + +_NORM = re.compile(r"[#*_`>|\-\s]+") +_MARKUP = re.compile(r"(?:|[*_`|]|^[\s>#]+)") +_PAGE_OF = [ re.compile(r"(?i)\bpg\.?\s*\d+\s*of\s*\d+"), re.compile(r"(?i)\bpage\s*\d+\s*of\s*\d+"), +] +_HEADER_FIELDS = [ re.compile(r"(?i)^man/[a-z0-9/]+"), re.compile(r"(?i)\bver\.?\s*:"), re.compile(r"(?i)\bissue\s*:\s*\d"), re.compile(r"(?i)^\s*date\s*:"), ] +_STRUCTURAL_LINE = re.compile(r"^\s*(?:<|```|~~~|!\[)") +_HTML_TABLE = re.compile(r"]*>.*?", re.IGNORECASE | re.DOTALL) +_HTML_CELL = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) +_HTML_TAG = re.compile(r"<[^>]+>") +_TABLE_CELL_SPLIT = re.compile(r"(?") +_IMAGE_LINE = re.compile(r"^\s*!\[[^\]]*\]\([^)]*\)\s*$") +EDGE_LINES = 4 # Approval / sign-off footer block — QMS-manual page furniture rendered as a one- or # two-row Markdown table, e.g. `|Prepared by - Dr X|Issued by: QMC|Approved by - Dr Y|`. @@ -31,24 +48,59 @@ # carrying >=2 distinct sign-off phrases (a lone "approved by" in a data row is kept). _SEP_ONLY = re.compile(r"^\s*\|[\s:|-]+\|\s*$") _SIGNOFF = ("prepared by", "issued by", "approved by", "reviewed by", "authorized by") +_SIGNOFF_LABEL = re.compile(r"(?i)\b(prepared|issued|approved|reviewed|authori[sz]ed) by\s*[:\-\u2013]") +_SIGNOFF_LINE = re.compile(r"(?i)^[*_\s]*(prepared|issued|approved|reviewed|authori[sz]ed) by\s*[:\-\u2013]") +_PAGE_NUMBER = re.compile(r"^\s*\d{1,4}\s*$") +_SENTENCE_START = re.compile(r"^[a-z]") +# A sentence cut by a page break ran to the page edge, so its last line is never a short label. +_MIN_BROKEN_LINE_LENGTH = 40 +BOILERPLATE_MAX_PAGES = 10 +# A numbered heading repeated as a running sub-header is still the real section start on its first +# page; the sectionizer folds the repeats, so stripping them here would lose the section itself. +_NUMBERED_HEADING = re.compile(r"^\s*#{1,6}\s+[*_]*\d+(?:\.\d+)*\.?\s*[A-Za-z]") +_LIST_ITEM = re.compile(r"^(\s*)([-*+]|\d+[.)])\s+\S") +_ENUMERATED_TEXT = re.compile(r"^(\d+)[.)]\s") +_CONTENTS_LINE = re.compile(r"^(\d+(?:\.\d+)+\.?|\d+)\s+(\S.*?)\s*(?:\.{2,}\s*)?(\d{1,4})\s*$") +MIN_CONTENTS_LINES = 5 +_MERMAID_OPEN = re.compile(r"^\s*```\s*mermaid\b") +_FENCE_CLOSE = re.compile(r"^\s*```\s*$") +MAX_TABLE_TITLE_LINES = 2 +MAX_TABLE_TITLE_CHARS = 120 +MIN_TRANSCRIBED_ROWS = 4 + + +def _plain(line: str) -> str: + """The line's visible text: emphasis, table pipes,
, blockquote and heading markers removed.""" + return " ".join(_MARKUP.sub(" ", line).split()) def _norm(line: str) -> str: - return _NORM.sub(" ", line).strip().lower() + return _NORM.sub(" ", _plain(line)).strip().lower() def _is_signoff_footer_row(line: str) -> bool: s = line.strip() if not (s.startswith("|") and s.endswith("|")): - return False + return len({label.lower() for label in _SIGNOFF_LABEL.findall(s)}) >= 2 low = s.lower() return sum(kw in low for kw in _SIGNOFF) >= 2 +def _signoff_line_runs(lines: list[str]) -> set[int]: + """Sign-off labels laid out one per paragraph, e.g. "Prepared by: …" then "Issued by: …".""" + drop: set[int] = set() + non_blank = [index for index, line in enumerate(lines) if line.strip()] + for is_signoff, group in groupby(non_blank, key=lambda index: bool(_SIGNOFF_LINE.match(lines[index]))): + run = list(group) + if is_signoff and len({_SIGNOFF_LINE.match(lines[index]).group(1).lower() for index in run}) >= 2: + drop.update(run) + return drop + + def _strip_footer_blocks(md: str) -> str: """Drop sign-off footer rows and any separator row orphaned next to them.""" lines = md.splitlines() - drop: set[int] = set() + drop = _signoff_line_runs(lines) for i, line in enumerate(lines): if _is_signoff_footer_row(line): drop.add(i) @@ -60,34 +112,324 @@ def _strip_footer_blocks(md: str) -> str: return "\n".join(line for k, line in enumerate(lines) if k not in drop) +def _is_table_row(line: str) -> bool: + return line.strip().startswith("|") + + +def _strip_page_residue(md: str) -> str: + """Drop separator rows left headless by a stripped header box, and the closing page number.""" + lines = md.splitlines() + kept = [ + line + for index, line in enumerate(lines) + if not (_SEP_ONLY.match(line) and "-" in line and not (index and _is_table_row(lines[index - 1]))) + ] + while kept and not kept[-1].strip(): + kept.pop() + while kept and not kept[0].strip(): + kept.pop(0) + if kept and _PAGE_NUMBER.match(_plain(kept[-1])): + kept.pop() + return "\n".join(kept) + + +def _edge_lines(md: str) -> list[str]: + content = [line for line in md.splitlines() if _norm(line) and not _STRUCTURAL_LINE.match(line)] + return content[:EDGE_LINES] + content[-EDGE_LINES:] + + def find_boilerplate(pages: list[tuple[int, str]]) -> set[str]: - """Normalized lines that recur on a large fraction of pages.""" + """Normalized lines that recur at the top or bottom edge of a large fraction of pages.""" counts: Counter[str] = Counter() for _, md in pages: - for nl in {_norm(line) for line in md.splitlines() if _norm(line)}: + for nl in {_norm(line) for line in _edge_lines(md)}: counts[nl] += 1 - threshold = max(3, int(0.30 * len(pages))) + # Capped so a header that alternates between styles across a long document still counts. + threshold = max(3, min(int(0.30 * len(pages)), BOILERPLATE_MAX_PAGES)) return {line for line, c in counts.items() if c >= threshold and len(line) <= 90} -def _is_varying(line: str) -> bool: - return any(p.search(line) for p in _VARYING) +def _is_page_of(line: str) -> bool: + plain = _plain(line) + return any(pattern.search(plain) for pattern in _PAGE_OF) + + +def _is_page_furniture(line: str, boilerplate: set[str], at_top: bool) -> bool: + plain = _plain(line) + return ( + (_norm(line) in boilerplate and not _NUMBERED_HEADING.match(line)) + or any(pattern.search(plain) for pattern in _HEADER_FIELDS) + or bool(_PAGE_NUMBER.match(plain)) + or (at_top and "-" in line and bool(_SEP_ONLY.match(line))) + ) + + +def _edge_furniture(lines: list[str], boilerplate: set[str]) -> set[int]: + """The unbroken header/footer run at each page edge; a label that also recurs mid-page survives.""" + drop: set[int] = set() + for at_top, indexes in ((True, range(len(lines))), (False, range(len(lines) - 1, -1, -1))): + for index in indexes: + line = lines[index] + if not line.strip() or _IMAGE_LINE.match(line): + continue + if not _is_page_furniture(line, boilerplate, at_top): + break + drop.add(index) + return drop + + +def _is_header_text(text: str) -> bool: + plain = _plain(text) + return _is_page_of(text) or any(pattern.search(plain) for pattern in _HEADER_FIELDS) + + +def _is_header_cell(cell: str, boilerplate: set[str]) -> bool: + return _is_header_text(cell) or _is_page_furniture(cell, boilerplate, False) + + +def _is_header_block(cells: list[str], boilerplate: set[str]) -> bool: + """Cells that only carry a running header (title, doc code, version, page x of y) and name at least + one field of it, so a body table that merely repeats the document title is never mistaken for one.""" + filled = [cell for cell in cells if _plain(cell)] + return any(_is_header_text(cell) for cell in filled) and all( + _is_header_cell(cell, boilerplate) for cell in filled + ) + + +def _row_cells(line: str) -> list[str]: + return [cell for cell in _TABLE_CELL_SPLIT.split(line.strip()) if cell.strip()] + + +def _is_header_row(line: str, boilerplate: set[str]) -> bool: + cells = [cell for cell in _row_cells(line) if not _SEP_ONLY.match(f"|{cell}|")] + return all(_is_header_cell(cell, boilerplate) for cell in cells) + + +def _strip_html_header_tables(md: str, boilerplate: set[str]) -> str: + def strip(match: re.Match) -> str: + cells = [_HTML_TAG.sub(" ", cell) for cell in _HTML_CELL.findall(match.group(0))] + return "" if _is_header_block(cells, boilerplate) else match.group(0) + + return _HTML_TABLE.sub(strip, md) + + +def _split_at_header_rows(md: str, boilerplate: set[str]) -> list[str]: + """Split a page where a running header sits inside a table, which happens when an earlier pass + stitched the next page's table onto this one before its header was stripped.""" + lines = md.splitlines() + pieces: list[str] = [] + start = index = 0 + while index < len(lines): + end = index + while end < len(lines) and _is_table_row(lines[end]) and _is_header_row(lines[end], boilerplate): + end += 1 + if end > index and _is_header_block( + [cell for line in lines[index:end] for cell in _row_cells(line)], boilerplate + ): + pieces.append("\n".join(lines[start:index])) + start = end + index = max(end, index + 1) + pieces.append("\n".join(lines[start:])) + return pieces + + +def _strip_header_rows(md: str, boilerplate: set[str]) -> str: + pieces = _split_at_header_rows(md, boilerplate) + if len(pieces) == 1: + return md + rejoined = stitch_cross_page_tables(list(enumerate(pieces))) + return "\n\n".join(piece for _, piece in rejoined if piece.strip()) def strip_boilerplate(pages: list[tuple[int, str]], boilerplate: set[str]) -> list[tuple[int, str]]: out: list[tuple[int, str]] = [] for pno, md in pages: - kept = [ - line - for line in md.splitlines() - if not (_norm(line) and _norm(line) in boilerplate) and not _is_varying(line) - ] - out.append((pno, _strip_footer_blocks("\n".join(kept)))) + md = _strip_header_rows(_strip_html_header_tables(md, boilerplate), boilerplate) + lines = [line for line in _strip_footer_blocks(md).splitlines() if not _is_page_of(line)] + drop = _edge_furniture(lines, boilerplate) + kept = [line for index, line in enumerate(lines) if index not in drop] + out.append((pno, _strip_page_residue("\n".join(kept)))) return out +def _ends_mid_sentence(line: str) -> bool: + text = line.strip() + return ( + len(text) >= _MIN_BROKEN_LINE_LENGTH + and text[0] not in "#|<" + and (text[-1].isalnum() or text[-1] == ",") + ) + + +def join_page_breaks(pages: list[tuple[int, str]]) -> list[tuple[int, str]]: + """Rejoin a sentence the PDF split across a page break onto the page it started on.""" + page_lines = [md.splitlines() for _, md in pages] + for index in range(len(pages) - 1): + if pages[index + 1][0] != pages[index][0] + 1: + continue + lines, next_lines = page_lines[index], page_lines[index + 1] + last = next((i for i in range(len(lines) - 1, -1, -1) if lines[i].strip()), None) + first = next((i for i, line in enumerate(next_lines) if line.strip()), None) + if last is None or first is None: + continue + if _ends_mid_sentence(lines[last]) and _SENTENCE_START.match(next_lines[first].strip()): + lines[last] = f"{lines[last].rstrip()} {next_lines.pop(first).strip()}" + return [(page_no, "\n".join(lines)) for (page_no, _), lines in zip(pages, page_lines, strict=True)] + + +def list_kind(marker: str) -> str: + return "bullet" if marker in "-*+" else "ordered" + + +def last_list_item(lines: list[str]) -> re.Match | None: + for line in reversed(lines): + if not line.strip(): + continue + item = _LIST_ITEM.match(line) + if item or not line[0].isspace(): + return item + return None + + +def continuation_prefix(item: re.Match, first_line: str) -> str: + """What a list cut by a page break needs in front of its items on the next page, where it restarts + at the margin: the nesting it had, or the bullet it had when the bullets carried their own numbers.""" + follow = _LIST_ITEM.match(first_line) + if not follow or follow.group(1): + return "" + indent, marker = item.group(1), item.group(2) + if indent and list_kind(marker) == list_kind(follow.group(2)): + return indent + numbered_text = _ENUMERATED_TEXT.match(item.string[item.end(2) :].lstrip()) + if ( + list_kind(marker) == "bullet" + and list_kind(follow.group(2)) == "ordered" + and numbered_text + and int(follow.group(2)[:-1]) == int(numbered_text.group(1)) + 1 + ): + return f"{indent}{marker} " + return "" + + +def reindent_list_continuations(pages: list[tuple[int, str]]) -> list[tuple[int, str]]: + """Put a list continued across a page break back under the item it belongs to, up to the first + block that is not part of the list.""" + page_lines = [md.splitlines() for _, md in pages] + for index in range(1, len(pages)): + if pages[index][0] != pages[index - 1][0] + 1: + continue + item = last_list_item(page_lines[index - 1]) + lines = page_lines[index] + first = next((line for line in lines if line.strip()), "") + prefix = continuation_prefix(item, first) if item else "" + if not prefix: + continue + kind = list_kind(_LIST_ITEM.match(first).group(2)) + for line_index, line in enumerate(lines): + if not line.strip(): + continue + if line[0].isspace(): + lines[line_index] = " " * len(prefix) + line + continue + follow = _LIST_ITEM.match(line) + if not follow or list_kind(follow.group(2)) != kind: + break + lines[line_index] = prefix + line + return [(page_no, "\n".join(lines)) for (page_no, _), lines in zip(pages, page_lines, strict=True)] + + +def _contents_run(lines: list[str], start: int) -> tuple[int, list[re.Match]]: + entries: list[re.Match] = [] + end = start + while end < len(lines): + entry = _CONTENTS_LINE.match(lines[end].strip()) + if lines[end].strip() and not entry: + break + if entry: + entries.append(entry) + end += 1 + return end, entries + + +def contents_lines_to_table(md: str) -> str: + """Lay out a run of table-of-contents lines (number, title, page) as a headless table, so the page + stitches onto the contents table the pages around it were parsed as.""" + lines = md.splitlines() + out: list[str] = [] + index = 0 + while index < len(lines): + end, entries = _contents_run(lines, index) + if len(entries) < MIN_CONTENTS_LINES: + out.append(lines[index]) + index += 1 + continue + out.extend(["| | | |", "|---|---|---|"]) + out.extend(f"| {entry.group(1)} | {entry.group(2)} | {entry.group(3)} |" for entry in entries) + out.append("") + index = end + return "\n".join(out) + + +def _table_rows_from(lines: list[str], start: int) -> int: + if lines[start].lstrip().startswith(" tuple[str, int, int] | None: + """What the parser wrote for the picture straight after it: ("mermaid", fence start, fence end) for a + diagram redrawn from it, ("table", first line, rows) for a table typed out under its title.""" + following = [index for index in range(figure + 1, len(lines)) if lines[index].strip()] + if not following: + return None + first = following[0] + if _MERMAID_OPEN.match(lines[first]): + close = next((index for index in following[1:] if _FENCE_CLOSE.match(lines[index])), None) + return ("mermaid", first, close) if close is not None else None + for index in following[: MAX_TABLE_TITLE_LINES + 1]: + if _is_table_row(lines[index]) or lines[index].lstrip().startswith(" MAX_TABLE_TITLE_CHARS or _STRUCTURAL_LINE.match(lines[index]): + return None + return None + + +def drop_transcribed_figures(md: str) -> str: + """Keep one copy of a picture the parser also transcribed: the crop for a diagram it redrew as mermaid, + the typed table for a table printed as an image, since that copy is exact and searchable.""" + lines = md.splitlines() + drop: set[int] = set() + for index, line in enumerate(lines): + transcription = _IMAGE_LINE.match(line) and _transcription_after_figure(lines, index) + if not transcription: + continue + kind, start, end = transcription + if kind == "mermaid": + drop.update(range(start, end + 1)) + elif end >= MIN_TRANSCRIBED_ROWS: + drop.add(index) + if not drop: + return md + return "\n".join(line for index, line in enumerate(lines) if index not in drop) + + def clean_pages(pages: list[tuple[int, str]]) -> list[tuple[int, str]]: - return strip_boilerplate(pages, find_boilerplate(pages)) + pages = [(page_no, html_tables_to_markdown(md)) for page_no, md in pages] + stripped = [ + (page_no, contents_lines_to_table(drop_transcribed_figures(md))) + for page_no, md in strip_boilerplate(pages, find_boilerplate(pages)) + ] + stitched = stitch_cross_page_tables(stripped) + joined = join_page_breaks([(page_no, merge_continuation_rows(md)) for page_no, md in stitched]) + return reindent_list_continuations(joined) def strip_outer_markdown_fence(text: str) -> str: diff --git a/wikify/engine/loader/cleanup_llm.py b/wikify/engine/loader/cleanup_llm.py index a08dce8..f257150 100644 --- a/wikify/engine/loader/cleanup_llm.py +++ b/wikify/engine/loader/cleanup_llm.py @@ -28,6 +28,10 @@ "- Headings must be plain text: strip bold/italic emphasis from heading lines " "(e.g. `## _**Verbal Orders**_` → `## Verbal Orders`); never wrap a heading in * or _.\n" "- Keep ALL substantive text. Do NOT summarize, add, translate, or drop real content.\n" + "- Keep form checkboxes (☐ ☑; a symbol-font box such as \uf0a8 becomes ☐) and blank fill-in " + "lines (runs of underscores).\n" + "- Keep list markers as printed: letters (a. b.), roman numerals (i. ii.) and ✓ ticks are never " + "renumbered into 1. 2. 3.\n" "- Keep Markdown image embeds ![alt](url) exactly as they are — they are real figures.\n" "- Output ONLY the cleaned Markdown — no commentary, no code fences.\n\nMARKDOWN:\n" ) diff --git a/wikify/engine/loader/page_plan.py b/wikify/engine/loader/page_plan.py new file mode 100644 index 0000000..b16048a --- /dev/null +++ b/wikify/engine/loader/page_plan.py @@ -0,0 +1,445 @@ +"""Decide which sections become wiki pages; every other section folds into its nearest +page ancestor as an in-page heading. Generation mirrors sections 1:1, so this is where +wiki page granularity is set. +""" + +from __future__ import annotations + +import json +import re +from collections import Counter, defaultdict + +import frappe + +from wikify.engine import llm, settings +from wikify.engine.loader.context import context_block +from wikify.engine.loader.sectionizer import PREAMBLE_TITLE, Section, _section_number +from wikify.engine.store import resolve_parent_indexes + +FALLBACK_PAGE_NUMBER_DEPTH = 3 +FALLBACK_PAGE_TREE_DEPTH = 1 +FRAGMENT_WORDS = 40 +STUB_WORDS = 15 +LEADING_FRAGMENT_WORDS = 60 +MIN_SPLIT_PART_WORDS = 150 +MAX_PAGE_WORDS = 2500 +MAX_PAGE_PDF_PAGES = 6 +OWN_PAGE_WORDS = 150 +OUTLINE_BATCH_LINES = 250 + +_LIST_ITEM_TITLE = re.compile(r"^(?:\d+[.)]?|[A-Za-z][.)]|[IVXivx]+[.)]|[Ss]tep\s+\d+[:.)]?)\s") + +_PROMPT = ( + "You are planning how a PDF document becomes a wiki. Below is its section outline in " + "document order, one section per line: index | depth | title | PDF pages | own words | " + "words including subsections | child sections.\n\n" + "Decide which sections start their own wiki page. Every other section is merged into its " + "nearest ancestor page as an in-page heading, so no text is lost.\n" + "- A wiki page is one coherent topic a reader would bookmark, mirroring how the document's " + "own table of contents presents topics.\n" + "- Sub-points fold into their parent topic: deeper numbering (e.g. 4.2.1.3 under 4.2.1), " + "bold or unnumbered sub-headings, and list-like headings (e.g. 'Class I', 'Type 2', " + "'1. Dosage').\n" + "- Sections that start on the same PDF page and belong to the same parent topic should " + "end up on the same wiki page.\n" + "- Avoid tiny pages (under ~150 words including their subsections) unless the section is " + "a standalone policy or topic.\n" + "- Split a very long topic (over ~4000 words) only at its natural numbered sub-sections.\n" + "- Numbered depth-1 sections are always pages; an unnumbered depth-1 fragment (a form label, a " + "caption, a stray sentence) is not, and merges into the page before it. A section can only be a " + "page if its parent is one.\n\n" + 'Respond ONLY as JSON: {"pages": []}\n\n' +) + + +def plan_pages(sections: list[Section], project_context: str = "", use_llm: bool = False) -> list[Section]: + parents = resolve_parent_indexes(sections) + chosen = llm_pages(sections, parents, project_context) if use_llm else fallback_pages(sections, parents) + pages, parents = enforce_invariants(sections, parents, chosen) + return fold_sections(sections, parents, pages) + + +def tree_depths(parents: list[int | None]) -> list[int]: + depths: list[int] = [] + for parent in parents: + depths.append(0 if parent is None else depths[parent] + 1) + return depths + + +def outline_numbers(sections: list[Section], parents: list[int | None]) -> list[tuple[int, ...] | None]: + """A heading's number only counts when it extends its nearest numbered ancestor's, so a + list item like "1. Dosage" under 4.2.1 reads as unnumbered.""" + numbers: list[tuple[int, ...] | None] = [] + numbered_ancestor: list[int | None] = [] + for index, section in enumerate(sections): + parent = parents[index] + ancestor = None if parent is None else (parent if numbers[parent] else numbered_ancestor[parent]) + number = _section_number(section.title) + if number and ancestor is not None: + prefix = numbers[ancestor] + if len(number) <= len(prefix) or number[: len(prefix)] != prefix: + number = None + numbers.append(number) + numbered_ancestor.append(ancestor) + return numbers + + +def fallback_pages(sections: list[Section], parents: list[int | None]) -> set[int]: + numbers = outline_numbers(sections, parents) + depths = tree_depths(parents) + pages: set[int] = set() + for index in range(len(sections)): + number = numbers[index] + has_numbered_ancestor = any(numbers[ancestor] for ancestor in ancestors(parents, index)) + if ( + parents[index] is None + or (number and len(number) <= FALLBACK_PAGE_NUMBER_DEPTH) + or (not number and not has_numbered_ancestor and depths[index] <= FALLBACK_PAGE_TREE_DEPTH) + ): + pages.add(index) + return pages + + +def ancestors(parents: list[int | None], index: int): + parent = parents[index] + while parent is not None: + yield parent + parent = parents[parent] + + +def enforce_invariants( + sections: list[Section], parents: list[int | None], chosen: set[int] +) -> tuple[set[int], list[int | None]]: + numbers = outline_numbers(sections, parents) + child_counts = Counter(parent for parent in parents if parent is not None) + pages: set[int] = set() + for index, parent in enumerate(parents): + if parent is None: + words = len(sections[index].markdown.split()) + fragment = index not in child_counts and words < FRAGMENT_WORDS + leading_fragment = ( + index == 0 + and len(sections) > 1 + and index not in child_counts + and not numbers[index] + and words < LEADING_FRAGMENT_WORDS + ) + if not leading_fragment and (index == 0 or numbers[index] or (index in chosen and not fragment)): + pages.add(index) + elif index in chosen and parent in pages: + pages.add(index) + pages = add_numbered_siblings(parents, numbers, pages) + pages, parents = split_long_pages(sections, parents, numbers, pages) + pages, parents = keep_sibling_order(sections, parents, numbers, pages) + pages, parents = keep_pdf_pages_whole(sections, parents, numbers, pages) + child_counts = Counter(parent for parent in parents if parent is not None) + pages = { + index + for index in pages + if parents[index] is None + or index in child_counts + or (sections[index].markdown.strip() and child_counts[parents[index]] > 1) + } + return pages, parents + + +def add_numbered_siblings( + parents: list[int | None], numbers: list[tuple[int, ...] | None], pages: set[int] +) -> set[int]: + """Numbered siblings are pages alike: once most of them are, the rest follow.""" + numbered_children: dict[int, list[int]] = defaultdict(list) + for index, parent in enumerate(parents): + if parent is not None and numbers[index]: + numbered_children[parent].append(index) + pages = set(pages) + for parent in sorted(numbered_children): + siblings = numbered_children[parent] + if parent in pages and 2 * len(pages.intersection(siblings)) > len(siblings): + pages.update(siblings) + return pages + + +def keep_sibling_order( + sections: list[Section], parents: list[int | None], numbers: list[tuple[int, ...] | None], pages: set[int] +) -> tuple[set[int], list[int | None]]: + """A numbered sibling after a page must not fold into the parent, which would show it before that + page: it becomes a page, or joins the last page before it when it is a stub or a list-item fragment.""" + parents = list(parents) + subtree_words = [len(section.markdown.split()) for section in sections] + for index in reversed(range(len(sections))): + if parents[index] is not None: + subtree_words[parents[index]] += subtree_words[index] + numbered_children: dict[int, list[int]] = defaultdict(list) + for index, parent in enumerate(parents): + if parent in pages and numbers[index]: + numbered_children[parent].append(index) + pages = set(pages) + for siblings in numbered_children.values(): + previous_page = None + for sibling in siblings: + if sibling in pages: + previous_page = sibling + elif previous_page is not None and ( + subtree_words[sibling] >= FRAGMENT_WORDS + or ( + (subtree_words[sibling] >= STUB_WORDS or opens_pdf_page(sections, sibling)) + and not _LIST_ITEM_TITLE.match(sections[sibling].title) + ) + ): + pages.add(sibling) + previous_page = sibling + elif previous_page is not None: + parents[sibling] = max(page for page in pages if page < sibling) + return pages, parents + + +def keep_pdf_pages_whole( + sections: list[Section], parents: list[int | None], numbers: list[tuple[int, ...] | None], pages: set[int] +) -> tuple[set[int], list[int | None]]: + """A short section that shares a PDF page joins the wiki page before it, unless that makes the page too + long to read in one scroll. A numbered section long enough to stand alone keeps its own page.""" + parents = list(parents) + page_parents = {parents[page] for page in pages} + words: Counter[int] = Counter() + last_pdf_page: dict[int, int] = {} + for index, owner in enumerate(page_owners(parents, pages)): + if owner is not None: + words[owner] += len(sections[index].markdown.split()) + last_pdf_page[owner] = max(last_pdf_page.get(owner, 0), sections[index].page_end) + pages = set(pages) + for index in sorted(pages): + if ( + parents[index] is None + or index in page_parents + or (numbers[index] and words[index] >= OWN_PAGE_WORDS) + or opens_pdf_page(sections, index) + ): + continue + target = max((page for page in pages if page < index), default=None) + if target is None: + continue + joined_pdf_pages = max(last_pdf_page[target], last_pdf_page[index]) - sections[target].page_start + 1 + if words[target] + words[index] > MAX_PAGE_WORDS or joined_pdf_pages > MAX_PAGE_PDF_PAGES: + continue + pages.discard(index) + parents[index] = target + words[target] += words[index] + last_pdf_page[target] = max(last_pdf_page[target], last_pdf_page[index]) + return pages, parents + + +def page_owners(parents: list[int | None], pages: set[int]) -> list[int | None]: + owners: list[int | None] = [] + for index, parent in enumerate(parents): + if index in pages: + owners.append(index) + elif parent is not None: + owners.append(owners[parent]) + else: + owners.append(owners[-1] if owners else None) + return owners + + +def opens_pdf_page(sections: list[Section], index: int) -> bool: + page_start = sections[index].page_start + for earlier in reversed(sections[:index]): + if earlier.page_end < page_start: + return True + if earlier.markdown.strip(): + return False + return True + + +def split_long_pages( + sections: list[Section], parents: list[int | None], numbers: list[tuple[int, ...] | None], pages: set[int] +) -> tuple[set[int], list[int | None]]: + """A page too long to read in one scroll is split at its numbered children, recursively; tiny + children stay folded. Without numbered children it splits at its sub-headings instead.""" + parents = list(parents) + children: dict[int, list[int]] = defaultdict(list) + subtree_words = [len(section.markdown.split()) for section in sections] + for index in reversed(range(len(sections))): + parent = parents[index] + if parent is not None: + children[parent].insert(0, index) + subtree_words[parent] += subtree_words[index] + pages = set(pages) + queue = sorted(pages) + while queue: + page = queue.pop(0) + folded, pending = [], [page] + while pending: + index = pending.pop() + folded.append(index) + pending.extend(child for child in children[index] if child not in pages) + words = sum(len(sections[index].markdown.split()) for index in folded) + pdf_pages = max(sections[index].page_end for index in folded) - sections[page].page_start + 1 + if words <= MAX_PAGE_WORDS and pdf_pages <= MAX_PAGE_PDF_PAGES: + continue + split = [ + child + for child in children[page] + if numbers[child] and child not in pages and subtree_words[child] >= FRAGMENT_WORDS + ] + if not split: + for part in heading_parts(sections, children[page], subtree_words): + split.append(part[0]) + for member in part[1:]: + parents[member] = part[0] + children[part[0]].append(member) + children[page] = [child for child in children[page] if child not in part[1:]] + pages.update(split) + queue.extend(split) + return pages, parents + + +def heading_parts(sections: list[Section], children: list[int], subtree_words: list[int]) -> list[list[int]]: + """Runs of children that each start at a sub-heading and hold at least MIN_SPLIT_PART_WORDS; a + tiny run joins the run before it, or the run after it when it is a bare heading. A title the + document repeats, or a stack of bare headings, is a form letterhead, never a part's title.""" + title_counts = Counter(section.title.lower() for section in sections) + bare = [not sections[child].markdown.strip() for child in children] + starts = [ + child + for position, child in enumerate(children) + if not _LIST_ITEM_TITLE.match(sections[child].title) + and title_counts[sections[child].title.lower()] == 1 + and not in_bare_stack(bare, position) + ] + if len(starts) < 2: + starts = children + parts: list[list[int]] = [] + for child in children: + if child in starts: + parts.append([child]) + elif parts: + parts[-1].append(child) + merged: list[list[int]] = [] + carried: list[int] = [] + for part in parts: + part = carried + part + carried = [] + if sum(subtree_words[child] for child in part) >= MIN_SPLIT_PART_WORDS: + merged.append(part) + elif merged and sections[part[0]].markdown.strip(): + merged[-1].extend(part) + else: + carried = part + if carried and merged: + merged[-1].extend(carried) + elif carried: + merged.append(carried) + return merged if len(merged) > 1 else [] + + +def in_bare_stack(bare: list[bool], position: int) -> bool: + before = position > 0 and bare[position - 1] + after = position + 1 < len(bare) and bare[position + 1] + return bare[position] and (before or after) + + +def fold_sections(sections: list[Section], parents: list[int | None], pages: set[int]) -> list[Section]: + owner: dict[int, int] = {} + depth_below_page: dict[int, int] = {} + page_parents = {parents[index] for index in pages} + leading: list[str] = [] + for index, section in enumerate(sections): + if index in pages: + owner[index], depth_below_page[index] = index, 0 + if leading and index not in page_parents: + section.markdown = "\n\n".join(part for part in (*leading, section.markdown.strip()) if part) + section.page_start = min(section.page_start, sections[0].page_start) + leading = [] + continue + parent = parents[index] + if parent is None and index - 1 not in owner: + title = "" if section.title == PREAMBLE_TITLE else f"**{section.title}**" + leading.extend(part for part in (title, section.markdown.strip()) if part) + continue + if parent is None: + owner[index], depth_below_page[index] = owner[index - 1], 1 + else: + owner[index], depth_below_page[index] = owner[parent], depth_below_page[parent] + 1 + page = sections[owner[index]] + heading = f"{'#' * min(6, depth_below_page[index] + 1)} {section.title}" + page.markdown = "\n\n".join( + part for part in (page.markdown.strip(), heading, section.markdown.strip()) if part + ) + page.page_end = max(page.page_end, section.page_end) + return [section for index, section in enumerate(sections) if index in pages] + + +def llm_pages(sections: list[Section], parents: list[int | None], project_context: str) -> set[int]: + chosen: set[int] = set() + for batch in outline_batches(parents): + chosen |= ask_for_pages(sections, parents, batch, project_context) + return chosen + + +def outline_batches(parents: list[int | None]) -> list[list[int]]: + chapters: list[list[int]] = [] + for index, parent in enumerate(parents): + if parent is None: + chapters.append([]) + chapters[-1].append(index) + batches: list[list[int]] = [] + for chapter in chapters: + if batches and len(batches[-1]) + len(chapter) <= OUTLINE_BATCH_LINES: + batches[-1].extend(chapter) + else: + batches.append(list(chapter)) + return batches + + +def format_outline(sections: list[Section], parents: list[int | None], batch: list[int]) -> str: + depths = tree_depths(parents) + own_words = [len(section.markdown.split()) for section in sections] + total_words = list(own_words) + child_counts = [0] * len(sections) + for index in reversed(range(len(sections))): + parent = parents[index] + if parent is not None: + total_words[parent] += total_words[index] + child_counts[parent] += 1 + lines = [] + for index in batch: + section = sections[index] + fields = ( + index, + depths[index] + 1, + section.title, + f"p{section.page_start}-{section.page_end}", + own_words[index], + total_words[index], + child_counts[index], + ) + lines.append(" " * depths[index] + " | ".join(str(field) for field in fields)) + return "\n".join(lines) + + +def ask_for_pages( + sections: list[Section], parents: list[int | None], batch: list[int], project_context: str +) -> set[int]: + prompt = ( + context_block(project_context) + _PROMPT + "OUTLINE:\n" + format_outline(sections, parents, batch) + ) + try: + response = llm.chat_completion( + settings.get("classifier_model"), + [{"role": "user", "content": prompt}], + label="page_plan", + response_format={"type": "json_object"}, + max_tokens=4096, + timeout=300, + ) + pages = json.loads(response["choices"][0]["message"]["content"] or "{}").get("pages") + if isinstance(pages, str): + # The Claude CLI backend's open object schema returns the list JSON-encoded. + pages = json.loads(pages) + except Exception: + frappe.log_error(title="Wikify page plan failed") + pages = None + batch_indexes = set(batch) + if not isinstance(pages, list): + return fallback_pages(sections, parents) & batch_indexes + return {index for index in pages if type(index) is int and index in batch_indexes} diff --git a/wikify/engine/loader/sectionizer.py b/wikify/engine/loader/sectionizer.py index fe80407..74ee8f9 100644 --- a/wikify/engine/loader/sectionizer.py +++ b/wikify/engine/loader/sectionizer.py @@ -1,12 +1,16 @@ from __future__ import annotations import re +from bisect import bisect_right from collections import defaultdict from dataclasses import dataclass +from itertools import pairwise +from wikify.engine.loader.cleanup import _SEP_ONLY from wikify.engine.loader.toc import correct_level MAX_TITLE_LENGTH = 140 +PREAMBLE_TITLE = "Preamble" RUNNING_HEADER_MIN_PAGES = 3 TOP_OF_PAGE_LINES = 3 @@ -15,10 +19,27 @@ _NUM_RE = re.compile(r"^(\d+(?:\.\d+)*)\.?\s+\S") _LEADING_NUM = re.compile(r"^(\d+)\b") _DOUBLE_NUM = re.compile(r"^\d+\.\s+\d") +_GLUED_NUM = re.compile(r"^(\d+(?:\.\d+)+\.?)(?=[A-Z])") +_BOLD_LINE = re.compile(r"^\*\*([^*]+)\*\*$") +_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)") +_HTML_TAG = re.compile(r"]*>") +_LETTERED_NUM = re.compile(r"^(\d+(?:\.\d+)+)\.[a-z](?![a-z])\s*") +_TOC_NUMBERED = re.compile(r"^(\d+(?:\.\d+)*)\.?\s+(.*?)(?:\s*\.{3,})?(?:\s+\d{1,4})?$") +_CONTENTS_TITLE = re.compile(r"(?i)^(?:table of )?contents$|^index$") +_TOC_ENTRY = re.compile( + r"^(?:[-*+]\s+)?\|?\s*\d+(?:\.\d+)*\.?\s*\|?\s*[A-Za-z(].*?(?:\.{3,}|\s[-\u2013\u2014]\s|\||
|\s)\s*(\d{1,4})\s*\|?$" +) +_TOC_LEADER = re.compile(r"[A-Za-z].*\.{4,}\s*(\d{1,4})$") +TOC_MIN_ENTRIES = 5 +TOC_MIN_ENTRY_SHARE = 0.6 +TOC_MIN_ASCENDING_SHARE = 0.75 def _clean_title(raw: str) -> str: - title = raw.strip().strip("*_").strip() + text = _HTML_TAG.sub("", raw).replace("**", "").strip().strip("*_").strip() + title = _GLUED_NUM.sub(r"\1 ", _LINK.sub(r"\1", text).strip()) + if title.endswith(".") and not title.endswith(".."): + title = title[:-1].rstrip() if len(title) <= MAX_TITLE_LENGTH: return title clipped = title[: MAX_TITLE_LENGTH - 1] @@ -33,6 +54,12 @@ def _infer_level(title: str, fallback: int) -> int: return fallback +def _is_label(title: str, level_map: dict[str, int]) -> bool: + return ( + title not in level_map and not _NUM_RE.match(title) and (title[:1].islower() or title.endswith(":")) + ) + + def _chapter_num(title: str) -> int | None: m = _LEADING_NUM.match(title) return int(m.group(1)) if m else None @@ -42,6 +69,266 @@ def _looks_like_list_item(title: str) -> bool: return "*" in title or bool(_DOUBLE_NUM.match(title)) +def _section_number(title: str) -> tuple[int, ...] | None: + m = _NUM_RE.match(title) + return tuple(int(part) for part in m.group(1).split(".")) if m else None + + +def _title_words(title: str) -> list[str]: + m = _NUM_RE.match(title) + text = title[m.end(1) :] if m else title + return [word.removesuffix("s") for word in re.findall(r"[a-z0-9]+", text.lower())] + + +def _repeats_open_section(title: str, stack: list[tuple[int, str, bool]]) -> bool: + number = _section_number(title) + if number is None: + return False + words = _title_words(title) + return any( + numbered + and _section_number(open_title)[: len(number)] == number + and _title_words(open_title) == words + for _, open_title, numbered in stack + ) + + +def extends_number(number: tuple[int, ...] | None, prefix: tuple[int, ...]) -> bool: + return bool(number) and len(number) > len(prefix) and number[: len(prefix)] == prefix + + +def dotted_heading_positions(pages: list[tuple[int, str]]) -> list[tuple[int, int, tuple[int, ...]]]: + positions = [] + for page_index, (_, md) in enumerate(pages): + for line_index, line in enumerate(md.splitlines()): + match = _HEADING_RE.match(line) + number = _section_number(_clean_title(match.group(2))) if match else None + if number and len(number) > 1: + positions.append((page_index, line_index, number)) + return positions + + +def opens_next_chapter( + chapter: int, last_number: tuple[int, ...], next_dotted: tuple[int, ...] | None, title: str +) -> bool: + """Inside a numbered sub-section, "4." is a list item unless it is the next chapter: the next dotted + heading no longer belongs to the current chapter, or, with none left to read, it is set in capitals + like a chapter title.""" + if chapter != last_number[0] + 1: + return False + return next_dotted[0] >= chapter if next_dotted else title.isupper() + + +def adopt_preceding_descendants( + sections: list[Section], + current: Section | None, + stack: list[tuple[int, str, bool]], + title: str, + level: int, + page_no: int, +) -> None: + """A parent heading that resurfaces after its sub-sections were read (an excerpt that starts + mid-section) becomes their group instead of a new section after them.""" + number = _section_number(title) + + def descends(section: Section) -> bool: + return any( + extends_number(_section_number(path_title), number) for path_title in section.hierarchy_path + ) + + run = [*sections, current] if current else list(sections) + start = len(run) + while start and descends(run[start - 1]): + start -= 1 + if start == len(run): + return + for section in run[start:]: + depth = next( + index + for index, path_title in enumerate(section.hierarchy_path) + if extends_number(_section_number(path_title), number) + ) + section.hierarchy_path.insert(depth, title) + group_path = run[start].hierarchy_path[: run[start].hierarchy_path.index(title) + 1] + sections.insert( + start, + Section( + title=title, + level=level, + hierarchy_path=list(group_path), + page_start=run[start].page_start, + page_end=page_no, + ), + ) + stack_index = next( + ( + index + for index, (_, open_title, numbered) in enumerate(stack) + if numbered and extends_number(_section_number(open_title), number) + ), + len(stack), + ) + stack.insert(stack_index, (level, title, True)) + + +def _continues_numbering(number: tuple[int, ...], last_number: tuple[int, ...]) -> bool: + if number == (*last_number, 1): + return True + depth = len(number) + return ( + depth <= len(last_number) + and number[:-1] == last_number[: depth - 1] + and number[-1] == last_number[depth - 1] + 1 + ) + + +def _promote_numbered_bold_line( + line: str, last_number: tuple[int, ...], next_dotted: tuple[int, ...] | None +) -> str: + bold = _BOLD_LINE.match(line.strip()) + if not (bold and last_number): + return line + title = bold.group(1).strip() + number = _section_number(title) + if number is None or not _continues_numbering(number, last_number): + return line + if ( + len(number) == 1 + and len(last_number) > 1 + and not opens_next_chapter(number[0], last_number, next_dotted, title) + ): + return line + return f"{'#' * min(6, len(number))} {title}" + + +def next_dotted_number( + dotted_headings: list[tuple[int, int, tuple[int, ...]]], page_index: int, line_index: int +) -> tuple[int, ...] | None: + index = bisect_right(dotted_headings, (page_index, line_index), key=lambda heading: heading[:2]) + return dotted_headings[index][2] if index < len(dotted_headings) else None + + +def correct_chapter_typo(title: str, last_number: tuple[int, ...]) -> str: + """A deep number whose chapter jumps while its tail carries on the current numbering ("1.5.3.1.5" + after 6.5.3.1.4) is a misprint of the current chapter.""" + match = _NUM_RE.match(title) + if not (match and last_number): + return title + number = tuple(int(part) for part in match.group(1).split(".")) + corrected = (last_number[0], *number[1:]) + if len(number) < 3 or number[0] == last_number[0] or not _continues_numbering(corrected, last_number): + return title + return ".".join(str(part) for part in corrected) + title[match.end(1) :] + + +def toc_section_titles(pages: list[tuple[int, str]]) -> dict[tuple[int, ...], str]: + titles: dict[tuple[int, ...], str] = {} + for _, md in pages: + lines = md.splitlines() + for line in lines[: toc_end_line(lines) + 1]: + text = " ".join(_HTML_TAG.sub(" ", line).replace("|", " ").split()).lstrip("-*+ ") + entry = _TOC_NUMBERED.match(text) + if entry and entry.group(2): + number = tuple(int(part) for part in entry.group(1).split(".")) + titles.setdefault(number, _clean_title(f"{entry.group(1)} {entry.group(2)}")) + return titles + + +def recurring_section_titles(pages: list[tuple[int, str]]) -> dict[tuple[int, ...], list[str]]: + """Numbered heading titles printed on more than one page, in reading order: running headers that + name a section even where the body never opens it.""" + pages_seen: dict[str, set[int]] = defaultdict(set) + for page_no, md in pages: + for line in md.splitlines(): + match = _HEADING_RE.match(line) + if match: + pages_seen[_clean_title(match.group(2))].add(page_no) + recurring: dict[tuple[int, ...], list[str]] = defaultdict(list) + for title, seen in pages_seen.items(): + number = _section_number(title) + if number and len(seen) > 1: + recurring[number].append(title) + return recurring + + +def repeats_running_header( + title: str, stack: list[tuple[int, str, bool]], recurring: dict[tuple[int, ...], list[str]] +) -> bool: + """A lettered number ("6.2.1.b POLICIES…") under its open section 6.2.1 is that section's running + header "6.2.1 POLICIES…" printed with a suffix.""" + lettered = _LETTERED_NUM.match(title) + if not lettered: + return False + base = tuple(int(part) for part in lettered.group(1).split(".")) + words = _title_words(title[lettered.end() :]) + return any(numbered and _section_number(open_title) == base for _, open_title, numbered in stack) and any( + _title_words(running) == words for running in recurring.get(base, ()) + ) + + +def missing_ancestors( + number: tuple[int, ...], stack: list[tuple[int, str, bool]], last_at_depth: dict[int, tuple[int, ...]] +) -> list[tuple[int, ...]]: + """Numeric ancestors of a heading that the body never opened, where the numbering shows they belong: + the last number read at that depth is the ancestor's predecessor (6.1 after chapter 5).""" + deepest = max( + (len(_section_number(open_title)) for _, open_title, numbered in stack if numbered), default=0 + ) + missing = [] + for depth in range(deepest + 1, len(number)): + prefix = number[:depth] + previous = last_at_depth.get(depth) + if previous and previous[:-1] == prefix[:-1] and previous[-1] + 1 == prefix[-1]: + missing.append(prefix) + return missing + + +def ancestor_title( + prefix: tuple[int, ...], + recurring: dict[tuple[int, ...], list[str]], + toc_titles: dict[tuple[int, ...], str], + child_title: str, +) -> str: + if recurring.get(prefix): + return recurring[prefix][0] + if prefix in toc_titles: + return toc_titles[prefix] + label = ".".join(str(part) for part in prefix) + child = _NUM_RE.match(child_title) + if _section_number(child_title) == (*prefix, 0): + return label + child_title[child.end(1) :] + return label + + +def is_bare_heading_for(section: Section, title: str) -> bool: + return ( + not section.markdown + and not _NUM_RE.match(section.title) + and _title_words(section.title) == _title_words(title) + ) + + +def toc_end_line(lines: list[str]) -> int: + """Index of the last table-of-contents entry when the page is a TOC page, else -1. TOC entries name + real headings, so reading them as headings opens phantom sections and advances chapter numbering.""" + entries: list[tuple[int, int]] = [] + content_lines = 0 + for index, line in enumerate(lines): + text = line.strip() + if not text or _SEP_ONLY.match(text): + continue + content_lines += 1 + entry = _TOC_ENTRY.match(text) or _TOC_LEADER.search(text) + if entry: + entries.append((index, int(entry.group(1)))) + if len(entries) < TOC_MIN_ENTRIES or len(entries) < TOC_MIN_ENTRY_SHARE * content_lines: + return -1 + ascending = sum(later >= earlier for (_, earlier), (_, later) in pairwise(entries)) + if ascending < TOC_MIN_ASCENDING_SHARE * (len(entries) - 1): + return -1 + return entries[-1][0] + + def running_header_titles(pages: list[tuple[int, str]]) -> set[str]: occurrences: dict[str, list[int]] = defaultdict(list) pages_seen: dict[str, set[int]] = defaultdict(set) @@ -81,19 +368,73 @@ def sectionize(pages: list[tuple[int, str]], level_map: dict[str, int] | None = max_chapter = 0 capital_chapters = True last_list_item = (0, 0) + last_number: tuple[int, ...] = () running_headers = running_header_titles(pages) opened_headers: set[str] = set() + dotted_headings = dotted_heading_positions(pages) + recurring_titles = recurring_section_titles(pages) + toc_titles = toc_section_titles(pages) + last_at_depth: dict[int, tuple[int, ...]] = {} def flush(): if current is not None: current.markdown = "\n".join(buf).strip() sections.append(current) - for page_no, md in pages: - for line in md.splitlines(): - m = _HEADING_RE.match(line) + for page_index, (page_no, md) in enumerate(pages): + # Without a blank line, GFM reads the next page's first line as another row of a table, more + # raw HTML, or a lazy continuation of a list item that ended the previous page. + if buf and buf[-1].strip(): + buf.append("") + lines = md.splitlines() + toc_end = toc_end_line(lines) + first_line = next((index for index, line in enumerate(lines) if line.strip()), -1) + for line_index, line in enumerate(lines): + in_toc = line_index <= toc_end and bool(line.strip()) + next_dotted = next_dotted_number(dotted_headings, page_index, line_index) + m = _HEADING_RE.match( + line if in_toc else _promote_numbered_bold_line(line, last_number, next_dotted) + ) + title = _clean_title(m.group(2)) if m else "" + if in_toc: + if not (current and _CONTENTS_TITLE.match(current.title)): + flush() + buf = [] + contents_title = title if _CONTENTS_TITLE.match(title) else "Contents" + stack = [(1, contents_title, False)] + current = Section( + title=contents_title, + level=1, + hierarchy_path=[contents_title], + page_start=page_no, + page_end=page_no, + ) + if contents_title == title: + continue + m = None + elif m and _is_label(title, level_map): + line, m = f"**{title}**", None if m: - title = _clean_title(m.group(2)) + title = correct_chapter_typo(title, last_number) + if _repeats_open_section(title, stack) or repeats_running_header( + title, stack, recurring_titles + ): + if current is not None: + current.page_end = page_no + continue + number = _section_number(title) + if number and extends_number(last_number, number): + # Numbering that steps back to an ancestor is that ancestor's running header. + if len(number) > 1: + if not any(_section_number(open_title) == number for _, open_title, _ in stack): + level = correct_level(title, _infer_level(title, len(m.group(1))), level_map) + adopt_preceding_descendants(sections, current, stack, title, level, page_no) + if current is not None: + current.page_end = page_no + continue + next_line = next((text for text in lines[line_index + 1 :] if text.strip()), "") + if line_index == first_line and _HEADING_RE.match(next_line): + continue if title in running_headers: if title in opened_headers: if current is not None: @@ -110,6 +451,12 @@ def flush(): if ( _looks_like_list_item(title) or cnum <= max_chapter + or (last_number and cnum <= last_number[0]) + or ( + len(last_number) > 1 + and (next_dotted or not max_chapter) + and not opens_next_chapter(cnum, last_number, next_dotted, title) + ) or (max_chapter and capital_chapters and not title.isupper()) or (page_no, cnum - 1) == last_list_item ): @@ -123,9 +470,39 @@ def flush(): anchor = next((lvl for lvl, _, numbered in reversed(stack) if numbered), None) if anchor is not None: level = min(6, max(level, anchor + 1)) - while stack and stack[-1][0] >= level: + numbered_heading = not demoted and bool(_NUM_RE.match(title)) + while stack and ( + stack[-1][0] >= level + or ( + numbered_heading + and not (stack[-1][2] and extends_number(number, _section_number(stack[-1][1]))) + ) + ): stack.pop() - stack.append((level, title, not demoted and bool(_NUM_RE.match(title)))) + if numbered_heading: + for prefix in missing_ancestors(number, stack, last_at_depth): + group_title = ancestor_title(prefix, recurring_titles, toc_titles, title) + group_level = max(1, level - len(number) + len(prefix)) + group_start = page_no + if sections and is_bare_heading_for(sections[-1], group_title): + group_start = sections.pop().page_start + stack.append((group_level, group_title, True)) + sections.append( + Section( + title=group_title, + level=group_level, + hierarchy_path=[t for _, t, _ in stack], + page_start=group_start, + page_end=page_no, + ) + ) + last_at_depth[len(prefix)] = prefix + if len(prefix) == 1: + max_chapter = max(max_chapter, prefix[0]) + stack.append((level, title, numbered_heading)) + if numbered_heading: + last_number = number + last_at_depth[len(number)] = number current = Section( title=title, level=level, @@ -134,6 +511,14 @@ def flush(): page_end=page_no, ) else: + if current is None and line.strip(): + current = Section( + title=PREAMBLE_TITLE, + level=1, + hierarchy_path=[PREAMBLE_TITLE], + page_start=page_no, + page_end=page_no, + ) if current is not None: current.page_end = page_no buf.append(line) @@ -157,19 +542,4 @@ def flush(): merged.append(sec) if is_numbered: numbered[key] = sec - sections = merged - - if not sections and pages: - whole = "\n".join(md for _, md in pages).strip() - if whole: - sections.append( - Section( - title="Preamble", - level=1, - hierarchy_path=["Preamble"], - page_start=pages[0][0], - page_end=pages[-1][0], - markdown=whole, - ) - ) - return sections + return merged diff --git a/wikify/engine/loader/table_stitch.py b/wikify/engine/loader/table_stitch.py index 89aa693..75e92fb 100644 --- a/wikify/engine/loader/table_stitch.py +++ b/wikify/engine/loader/table_stitch.py @@ -1,8 +1,9 @@ """Merge tables that pymupdf split across a page boundary. If page N ends with a Markdown table and page N+1 begins with one of the same -column count, the rows are joined into a single table (a repeated header on the -continuation page is dropped). Heuristic, but cheap and reversible. +column count, the rows are joined into a single table (a repeated or empty header on +the continuation page is dropped; any other header is kept as a row, since the parser +often promotes the page's first data row to a header). Heuristic, but cheap and reversible. Ported verbatim from the POC `loader/table_stitch.py` (no I/O — pure markdown). """ @@ -10,13 +11,232 @@ from __future__ import annotations import re +from html.parser import HTMLParser _ROW = re.compile(r"^\s*\|.*\|\s*$") _SEP = re.compile(r"^\s*\|[\s:|-]+\|\s*$") +_CELL_SPLIT = re.compile(r"(?]*>.*?", re.IGNORECASE | re.DOTALL) +_INLINE_MARKS = {"strong": "**", "b": "**", "em": "*", "i": "*"} +_KEPT_INLINE_TAGS = ("sup", "sub") +_WRAPPER_TAGS = ("thead", "tbody", "tfoot", "span", "p", "u") +_LIST_TAGS = ("ul", "ol") +MIN_NUMBERED_ROWS = 2 + + +class _TableReader(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.rows: list[list[str]] = [] + self.header_flags: list[list[bool]] = [] + self.cell: list[str] | None = None + self.row_open = False + self.saw_row_tag = False + self.table_depth = 0 + self.convertible = True + self.list_counters: list[int | None] = [] + + def handle_starttag(self, tag, attrs): + if tag == "table": + self.table_depth += 1 + self.convertible = self.convertible and self.table_depth == 1 + elif tag == "tr": + self.close_cell() + self.saw_row_tag = True + self.start_row() + elif tag in ("td", "th"): + self.start_cell(tag, attrs) + elif tag == "br": + self.append("
") + elif tag in _INLINE_MARKS: + self.append(_INLINE_MARKS[tag]) + elif tag in _KEPT_INLINE_TAGS: + self.append(f"<{tag}>") + elif tag in (*_LIST_TAGS, "li") and self.cell is not None: + self.start_list_tag(tag) + elif tag not in _WRAPPER_TAGS: + self.convertible = False + + def handle_endtag(self, tag): + if tag in ("td", "th"): + self.close_cell() + elif tag == "tr": + self.close_cell() + self.row_open = False + elif tag in _INLINE_MARKS: + self.append(_INLINE_MARKS[tag]) + elif tag in _KEPT_INLINE_TAGS: + self.append(f"") + elif tag in _LIST_TAGS and self.list_counters: + self.list_counters.pop() + + def handle_data(self, data): + if self.cell is not None: + self.cell.append(data.replace("<", "<")) + elif data.strip(): + self.convertible = False + + def append(self, text: str): + if self.cell is not None: + self.cell.append(text) + + def start_cell(self, tag: str, attrs): + if any(name in ("rowspan", "colspan") and (value or "1").strip() != "1" for name, value in attrs): + self.convertible = False + self.close_cell() + if not self.row_open: + self.start_row() + self.cell = [] + self.header_flags[-1].append(tag == "th") + + def start_list_tag(self, tag: str): + if tag in _LIST_TAGS: + self.list_counters.append(0 if tag == "ol" else None) + elif self.list_counters: + self.start_list_item() + + def start_list_item(self): + counter = self.list_counters[-1] + if counter is None: + marker = "•" + else: + self.list_counters[-1] = counter = counter + 1 + marker = f"{counter}." + text = "".join(self.cell or []).rstrip() + self.cell = [f"{text}
{marker} " if text else f"{marker} "] + + def start_row(self): + self.rows.append([]) + self.header_flags.append([]) + self.row_open = True + + def close_cell(self): + if self.cell is None: + return + text = " ".join("".join(self.cell).split()) + self.rows[-1].append(text.replace("|", "\\|")) + self.cell = None + + +def _format_row(cells: list[str]) -> str: + return "| " + " | ".join(cells) + " |" + + +def _html_table_to_markdown(html: str) -> str | None: + reader = _TableReader() + reader.feed(html) + reader.close() + reader.close_cell() + if not reader.convertible: + return None + rows, header_flags = reader.rows, reader.header_flags + if not reader.saw_row_tag: + columns = sum(header_flags[0]) if header_flags else 0 + if not columns: + return None + cells = rows[0] + rows = [cells[start : start + columns] for start in range(0, len(cells), columns)] + header_flags = [[index < columns for index in range(len(row))] for row in rows] + rows_with_flags = [(row, flags) for row, flags in zip(rows, header_flags, strict=True) if row] + if not rows_with_flags: + return None + columns = max(len(row) for row, _ in rows_with_flags) + padded = [row + [""] * (columns - len(row)) for row, _ in rows_with_flags] + if all(rows_with_flags[0][1]): + header, body = padded[0], padded[1:] + else: + header, body = [""] * columns, padded + return "\n".join([_format_row(header), "|" + "---|" * columns, *(_format_row(row) for row in body)]) + + +def html_tables_to_markdown(md: str) -> str: + """Rewrite flat HTML tables (no merged cells, inline content only) as pipe tables, repairing + unclosed cells/rows on the way, so they can be stitched and cleaned like any other table.""" + + def replace(match: re.Match) -> str: + converted = _html_table_to_markdown(match.group(0)) + if not converted: + return match.group(0) + before, after = md[: match.start()], md[match.end() :] + prefix = "" if not before or before.endswith("\n\n") else "\n" * (1 + (not before.endswith("\n"))) + suffix = "" if not after or after.startswith("\n\n") else "\n" * (1 + (not after.startswith("\n"))) + return f"{prefix}{converted}{suffix}" + + return _HTML_TABLE.sub(replace, md) + + +def _cells(row: str) -> list[str]: + text = row.strip() + text = text[1:] if text.startswith("|") else text + text = text[:-1] if text.endswith("|") and not text.endswith("\\|") else text + return [cell.strip() for cell in _CELL_SPLIT.split(text)] def _ncols(row: str) -> int: - return len(row.strip().strip("|").split("|")) + return len(_cells(row)) + + +def _plain_cell(cell: str) -> str: + return _EMPHASIS.sub("", cell).strip() + + +def _header_key(row: str) -> str: + return " ".join(_plain_cell(cell).lower() for cell in _cells(row)).strip() + + +def _is_numbered(rows: list[str]) -> bool: + return sum(1 for row in rows if _STEP.match(_plain_cell(_cells(row)[0]))) >= MIN_NUMBERED_ROWS + + +def _continues(row: str, after_page_break: bool) -> bool: + """A row that carries on the previous step instead of starting one: no step number and, in the + first non-empty cell, a lowercase start (or, right after a page break, any non-capital start).""" + cells = [_plain_cell(cell) for cell in _cells(row)] + if _STEP.match(cells[0]): + return False + if after_page_break: + text = next((cell for cell in cells if cell), "") + return bool(text) and not text[0].isupper() + return bool(cells[0]) and cells[0][0].islower() + + +def _merge_rows(previous: str, row: str) -> str: + merged = [ + " ".join(part for part in (first, second) if part) + for first, second in zip(_cells(previous), _cells(row), strict=False) + ] + return _format_row(merged) + + +def merge_continuation_rows(md: str) -> str: + """Fold a row a page break cut off (lowercase, unnumbered) back into the step it continues.""" + lines = md.split("\n") + out: list[str] = [] + index = 0 + while index < len(lines): + if not _ROW.match(lines[index]): + out.append(lines[index]) + index += 1 + continue + end = index + while end < len(lines) and _ROW.match(lines[end]): + end += 1 + block = lines[index:end] + body_start = 2 if len(block) > 1 and _SEP.match(block[1]) else 0 + rows = block[:body_start] + if _is_numbered(block[body_start:]): + for row in block[body_start:]: + if len(rows) > body_start and _ncols(row) == _ncols(rows[-1]) and _continues(row, False): + rows[-1] = _merge_rows(rows[-1], row) + else: + rows.append(row) + else: + rows = block + out.extend(rows) + index = end + return "\n".join(out) def _trailing_table(md: str): @@ -39,6 +259,29 @@ def _leading_table(md: str): return lines[:j], lines[j:] # (table_lines, after) +def _empty_column(table: list[str]) -> int | None: + """The first column with no text in any body row: the column a parser drops when a table goes on + over a page break, since nothing on the continuation page shows it is there.""" + body = table[2:] if len(table) > 1 and _SEP.match(table[1]) else table + rows = [_cells(row) for row in body] + if not rows: + return None + return next( + ( + column + for column in range(_ncols(table[0])) + if not any(_plain_cell(row[column]) for row in rows if column < len(row)) + ), + None, + ) + + +def _with_blank_cell(row: str, column: int) -> str: + cells = _cells(row) + cells.insert(column, "") + return _format_row(cells) + + def stitch_cross_page_tables(pages: list[tuple[int, str]]) -> list[tuple[int, str]]: out = [[pno, md] for pno, md in pages] for k in range(len(out) - 1): @@ -48,10 +291,20 @@ def stitch_cross_page_tables(pages: list[tuple[int, str]]) -> list[tuple[int, st continue a_before, a_tbl = a b_tbl, b_after = b - if not a_tbl or not b_tbl or _ncols(a_tbl[0]) != _ncols(b_tbl[0]): + if not a_tbl or not b_tbl: + continue + dropped_column = _empty_column(a_tbl) if _ncols(b_tbl[0]) == _ncols(a_tbl[0]) - 1 else None + if _ncols(a_tbl[0]) != _ncols(b_tbl[0]) and dropped_column is None: continue - # Drop a repeated header+separator at the start of the continuation table. - cont = b_tbl[2:] if len(b_tbl) >= 2 and _SEP.match(b_tbl[1]) else b_tbl + cont = b_tbl + if len(b_tbl) >= 2 and _SEP.match(b_tbl[1]): + repeated = _header_key(b_tbl[0]) in ("", _header_key(a_tbl[0])) + cont = b_tbl[2:] if repeated else [b_tbl[0], *b_tbl[2:]] + if dropped_column is not None: + cont = [_with_blank_cell(row, dropped_column) for row in cont] + if cont and not _SEP.match(a_tbl[-1]) and _is_numbered(a_tbl) and _continues(cont[0], True): + a_tbl = [*a_tbl[:-1], _merge_rows(a_tbl[-1], cont[0])] + cont = cont[1:] out[k][1] = "\n".join(a_before + a_tbl + cont).strip() out[k + 1][1] = "\n".join(b_after).strip() return [(pno, md) for pno, md in out] diff --git a/wikify/engine/parsers/vlm.py b/wikify/engine/parsers/vlm.py index eac3a20..4764490 100644 --- a/wikify/engine/parsers/vlm.py +++ b/wikify/engine/parsers/vlm.py @@ -31,6 +31,10 @@ 'and statutory references (e.g. "u/s 115BAC", "section 44AD", "First Proviso") character ' "for character. Never round, re-word, convert or summarise a number, and never invent " "content.\n" + "Forms keep their furniture: write every checkbox as ☐ (empty) or ☑ (ticked) in front of its " + "option, and keep blank fill-in lines as runs of underscores.\n" + "Keep list markers as printed: letters (a. b. c.), roman numerals (i. ii.) and ✓ ticks are " + "never renumbered into 1. 2. 3.\n" "Output only the Markdown — no commentary, and no code fences except ```mermaid." ) @@ -41,6 +45,7 @@ def parse_page_image( project_context: str = "", instruction: str = "", shape_hint: str = "", + figure_hint: str = "", ) -> str: preamble = context_block(project_context) + instruction_block(instruction) + shape_hint resp = llm.chat_completion( @@ -49,7 +54,7 @@ def parse_page_image( { "role": "user", "content": [ - {"type": "text", "text": preamble + _PROMPT}, + {"type": "text", "text": preamble + _PROMPT + figure_hint}, {"type": "image_url", "image_url": {"url": image_data_url}}, ], } diff --git a/wikify/engine/remediate.py b/wikify/engine/remediate.py index 9059306..43b23ac 100644 --- a/wikify/engine/remediate.py +++ b/wikify/engine/remediate.py @@ -5,9 +5,8 @@ import fitz -from wikify.engine import diagrams, llm, pdf_utils, regions, settings, store +from wikify.engine import diagrams, images, llm, pdf_utils, regions, settings, store from wikify.engine.loader.cleanup_llm import clean_markdown -from wikify.engine.loader.table_stitch import stitch_cross_page_tables from wikify.engine.parsers import vlm from wikify.engine.sectionize import rebuild_and_classify from wikify.engine.verify import deterministic as det @@ -93,6 +92,7 @@ def remediate_pdf( # noqa: C901 with fitz.open(pdf_path) as doc: furniture = det.find_furniture_lines([doc[p["page_no"] - 1].get_text("text") for p in pages]) + repeated = images.repeated_images(doc) doc_cost = 0.0 for i, p in enumerate(targets): @@ -102,6 +102,7 @@ def remediate_pdf( # noqa: C901 page_image = p["image"] or "" data_url = pdf_utils.png_to_data_url(pdf_utils.render_png(page, dpi=dpi)) page_regions = regions.find_regions(page) + figures = images.page_figures(page, repeated, p["name"], p["page_no"]) use_judge = judge_all or kind == "visual" img = data_url if use_judge else None @@ -122,7 +123,9 @@ def remediate_pdf( # noqa: C901 project_context=project_context, instruction=instruction, shape_hint=regions.shape_hint(page_regions), + figure_hint=images.figure_hint(figures), ) + vlm_md = images.place_figures(vlm_md, figures, p["page_no"]) vlm_md, diagram_notes = diagrams.remove_unverified_diagrams(vlm_md, page_image) errors.extend(diagram_notes) vlm_ps = score_page( @@ -134,7 +137,11 @@ def remediate_pdf( # noqa: C901 if kind != "visual": try: clean_md, clean_notes = diagrams.remove_unverified_diagrams( - clean_markdown(base_md, project_context=project_context, instruction=instruction), + images.place_figures( + clean_markdown(base_md, project_context=project_context, instruction=instruction), + figures, + p["page_no"], + ), page_image, ) errors.extend(clean_notes) @@ -175,6 +182,7 @@ def remediate_pdf( # noqa: C901 canon_src[p["page_no"]] = method else: canon_md[p["page_no"]] = base_md + canon_md[p["page_no"]] = images.place_figures(canon_md[p["page_no"]], figures, p["page_no"]) canon_md[p["page_no"]] = repair_broken_image_tags(canon_md[p["page_no"]], page_image) canon_md[p["page_no"]] = with_page_crop(canon_md[p["page_no"]], page_image) @@ -193,11 +201,10 @@ def remediate_pdf( # noqa: C901 if progress_cb: progress_cb(i + 1, total) - stitched = dict(stitch_cross_page_tables([(p["page_no"], canon_md[p["page_no"]]) for p in pages])) with events.suspended_indexing(): for p in pages: pno = p["page_no"] - store.set_canonical(p["name"], stitched[pno], canon_comp[pno], canon_src[pno]) + store.set_canonical(p["name"], canon_md[pno], canon_comp[pno], canon_src[pno]) comps = [c for c in canon_comp.values() if c is not None] canonical_mean = round(sum(comps) / len(comps), 3) if comps else None diff --git a/wikify/engine/reparse.py b/wikify/engine/reparse.py index 4b80f51..6b6c430 100644 --- a/wikify/engine/reparse.py +++ b/wikify/engine/reparse.py @@ -2,7 +2,7 @@ import fitz -from wikify.engine import diagrams, llm, pdf_utils, regions, remediate, settings, store +from wikify.engine import diagrams, images, llm, pdf_utils, regions, remediate, settings, store from wikify.engine.loader.cleanup_llm import clean_markdown from wikify.engine.parsers import vlm from wikify.engine.tags import find_tag_spans @@ -36,6 +36,7 @@ def reparse_page( fpage = doc[page_no - 1] gt = fpage.get_text("text") shape_hint = regions.shape_hint(regions.find_regions(fpage)) + figures = images.page_figures(fpage, images.repeated_images(doc), page["name"], page_no) data_url = pdf_utils.png_to_data_url(pdf_utils.render_png(fpage, dpi=dpi)) kind = page["kind"] @@ -47,11 +48,16 @@ def reparse_page( llm.reset_metrics() new_md = ( vlm.parse_page_image( - data_url, project_context=project_context, instruction=instruction, shape_hint=shape_hint + data_url, + project_context=project_context, + instruction=instruction, + shape_hint=shape_hint, + figure_hint=images.figure_hint(figures), ) if method == "vlm" else clean_markdown(base_md, project_context=project_context, instruction=instruction) ) + new_md = images.place_figures(new_md, figures, page_no) page_image = store.get_page_image(page["name"]) or "" new_md, diagram_notes = diagrams.remove_unverified_diagrams(new_md, page_image) new_md = remediate.repair_broken_image_tags(new_md, page_image) diff --git a/wikify/engine/sectionize.py b/wikify/engine/sectionize.py index 7473e5d..12ad56d 100644 --- a/wikify/engine/sectionize.py +++ b/wikify/engine/sectionize.py @@ -20,23 +20,28 @@ import frappe -from wikify.engine import store +from wikify.engine import images, llm, store from wikify.engine.classify import classify_document -from wikify.engine.lint import fix_table_separators +from wikify.engine.lint import repair_markdown from wikify.engine.loader.cleanup import clean_pages +from wikify.engine.loader.page_plan import plan_pages from wikify.engine.loader.sectionizer import _HEADING_RE, _clean_title, sectionize from wikify.engine.loader.toc import toc_level_map -def sectionize_document(source_document: str, pdf_path: str) -> int: +def sectionize_document(source_document: str, pdf_path: str, project_context: str = "") -> int: """Rebuild the Source Section tree from the doc's canonical pages. Returns the count.""" level_map = toc_level_map(str(pdf_path)) - pages = clean_pages(store.get_canonical_pages(source_document)) - sections = sectionize(pages, level_map) - # 0.6 auto-fix boundary: repair separator-less tables in the assembled section + pages = clean_pages(images.drop_figure_echoes(store.get_canonical_pages(source_document), pdf_path)) + sections = plan_pages( + sectionize(pages, level_map), + project_context, + use_llm=llm.has_openrouter() and not frappe.flags.in_test, + ) + # 0.6 auto-fix boundary: mechanical markdown repairs in the assembled section # product, pre-review. Page canonical stays untouched — pages are evidence. for sec in sections: - sec.markdown = fix_table_separators(sec.markdown) + sec.markdown = repair_markdown(sec.markdown) return store.replace_sections(source_document, sections) @@ -55,7 +60,7 @@ def rebuild_and_classify( """ if stage_cb: stage_cb("Building section tree") - count = sectionize_document(source_document, pdf_path) + count = sectionize_document(source_document, pdf_path, project_context) if stage_cb: stage_cb("Classifying sections") classify_document( @@ -153,7 +158,7 @@ def rebuild_section_markdown(section_name: str) -> dict: raise ValueError(f"No pages found in range {sec.page_start}-{sec.page_end}.") cleaned = clean_pages(pages) - markdown = fix_table_separators("\n\n".join(md.strip() for _, md in cleaned if md.strip())) + markdown = repair_markdown("\n\n".join(md.strip() for _, md in cleaned if md.strip())) first_line, _, body = markdown.partition("\n") heading = _HEADING_RE.match(first_line) if heading and _clean_title(heading.group(2)) == sec.title: diff --git a/wikify/engine/store.py b/wikify/engine/store.py index 36b28c2..5e55ee0 100644 --- a/wikify/engine/store.py +++ b/wikify/engine/store.py @@ -1,7 +1,7 @@ from __future__ import annotations import frappe -from frappe.utils.file_manager import save_file +from frappe.utils.file_manager import get_content_hash, save_file from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random from wikify.engine.verify.harness import get_verdict @@ -51,6 +51,10 @@ def add_page( return page.name +def set_baseline_markdown(page_name: str, markdown: str) -> None: + frappe.db.set_value("Source Page", page_name, "baseline_markdown", markdown) + + def set_page_count(source_document: str, count: int) -> None: frappe.db.set_value("Source Document", source_document, "page_count", count) @@ -166,6 +170,17 @@ def get_page(source_document: str, page_no: int) -> dict | None: def save_crop_file(page_name: str, page_no: int, png_bytes: bytes): + existing = frappe.db.get_value( + "File", + { + "attached_to_doctype": "Source Page", + "attached_to_name": page_name, + "content_hash": get_content_hash(png_bytes), + }, + "name", + ) + if existing: + return frappe.get_doc("File", existing) return save_file(f"page-{page_no:04d}-crop.png", png_bytes, "Source Page", page_name, is_private=1) diff --git a/wikify/tests/test_generate.py b/wikify/tests/test_generate.py index b77264a..2d8e68e 100644 --- a/wikify/tests/test_generate.py +++ b/wikify/tests/test_generate.py @@ -6,10 +6,11 @@ import frappe from frappe.tests.utils import FrappeTestCase from frappe.utils.nestedset import get_descendants_of +from wiki.frappe_wiki.doctype.wiki_document.wiki_document import get_landing_page_for_route from wikify.api import imports as imports_api from wikify.engine import generate_wiki, preview_wiki, store -from wikify.engine.generate import DATA_MAX_LENGTH +from wikify.engine.generate import DATA_MAX_LENGTH, sync_section from wikify.engine.loader.sectionizer import Section from wikify.engine.loader.wiki import rewrite_page_refs, slugify from wikify.jobs import generate as generate_job @@ -205,6 +206,66 @@ def test_regenerate_after_rename_updates_route(self): self.assertIn("1.1 Goals", docs) self.assertTrue(docs["1.1 Goals"].route.endswith(f"{slugify('1.1 Goals')}-{purpose}")) + def test_group_body_shows_as_its_first_page(self): + res = self._generate() + docs = self._by_title(res["root_group"]) + intro, overview = docs["1. Intro"], docs["Overview"] + self.assertEqual(intro.content, "") + self.assertEqual(overview.is_group, 0) + self.assertEqual(overview.parent_wiki_document, intro.name) + self.assertEqual(overview.content, "overview") + self.assertLess(overview.sort_order, docs["1.1 Purpose"].sort_order) + self.assertEqual(get_landing_page_for_route(intro.route)["route"], overview.route) + + intro_section = frappe.db.get_value( + "Source Section", {"title": "1. Intro", "source_document": self.sd.name}, "name" + ) + frappe.db.set_value("Source Section", intro_section, "markdown", "edited overview") + self.assertTrue(sync_section(intro_section)["synced"]) + self.assertEqual(frappe.db.get_value("Wiki Document", overview.name, "content"), "edited overview") + + def test_regenerate_keeps_the_overview_and_drops_it_once_the_group_body_is_gone(self): + res = self._generate() + overview = self._by_title(res["root_group"])["Overview"].name + generate_wiki(self.sd.name, wiki_space=res["space"]) + self.assertEqual(self._by_title(res["root_group"])["Overview"].name, overview) + + frappe.db.set_value( + "Source Section", {"title": "1. Intro", "source_document": self.sd.name}, "markdown", "" + ) + generate_wiki(self.sd.name, wiki_space=res["space"]) + docs = self._by_title(res["root_group"]) + self.assertNotIn("Overview", docs) + self.assertIn("## Contents", docs["1. Intro"].content) + + def test_contents_page_is_built_when_the_pdf_prints_none(self): + res = self._generate() + docs = self._by_title(res["root_group"]) + contents = docs["Contents"] + self.assertEqual(contents.is_group, 0) + self.assertEqual(contents.parent_wiki_document, res["root_group"]) + self.assertLess(contents.sort_order, docs["1. Intro"].sort_order) + self.assertIn(f"- [1. Intro](/{docs['1. Intro'].route}) — pp. 1-3", contents.content) + self.assertIn(f" - [1.2 Scope](/{docs['1.2 Scope'].route}) — p. 2", contents.content) + self.assertEqual(get_landing_page_for_route(docs["Gen Test"].route)["route"], contents.route) + + generate_wiki(self.sd.name, wiki_space=res["space"]) + self.assertEqual(self._by_title(res["root_group"])["Contents"].name, contents.name) + + def test_printed_contents_replaces_the_built_one(self): + res = self._generate() + store.replace_sections( + self.sd.name, + [ + _sec("Contents", 1, ["Contents"], 1, 1, "1. Intro .... 2"), + _sec("1. Intro", 1, ["1. Intro"], 2, 5), + ], + ) + generate_wiki(self.sd.name, wiki_space=res["space"]) + docs = self._by_title(res["root_group"]) + self.assertEqual(docs["Contents"].content, "1. Intro .... 2") + self.assertEqual(len(docs), 3) + def test_data_max_length_matches_the_column_width(self): self.assertEqual(DATA_MAX_LENGTH, frappe.db.VARCHAR_LEN) @@ -216,7 +277,7 @@ def test_deep_tree_with_long_titles_stays_within_route_limit(self): res = self._generate() docs = self._docs_under(res["root_group"]) - self.assertEqual(len(docs), len(titles) + 1) + self.assertEqual(len(docs), 2 * len(titles) + 1) self.assertTrue(all(len(doc.route) <= DATA_MAX_LENGTH for doc in docs.values())) self.assertEqual(len({doc.route for doc in docs.values()}), len(docs)) diff --git a/wikify/tests/test_images.py b/wikify/tests/test_images.py new file mode 100644 index 0000000..0307b04 --- /dev/null +++ b/wikify/tests/test_images.py @@ -0,0 +1,266 @@ +# Copyright (c) 2026, BWH and contributors +# For license information, please see license.txt + +import tempfile +from pathlib import Path +from unittest.mock import patch + +import fitz +import frappe +from frappe.tests.utils import FrappeTestCase + +from wikify.engine import images, parse_pdf, remediate_pdf +from wikify.engine import settings as engine_settings +from wikify.engine.loader.cleanup import drop_transcribed_figures +from wikify.tests import _cleanup +from wikify.tests.test_remediate_pipeline import _fake_chat + +CHART_RECT = fitz.Rect(100, 300, 400, 500) +TEXT_BEFORE = "Admissions rose steadily over the decade." +TEXT_AFTER = "Discharge planning starts on the first day." + + +def solid_pixmap(width: int, height: int, colour: tuple[int, int, int]) -> fitz.Pixmap: + pixmap = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, width, height), False) + pixmap.set_rect(pixmap.irect, colour) + return pixmap + + +def make_report_pdf(path: str) -> None: + logo = solid_pixmap(80, 80, (20, 40, 160)) + chart = solid_pixmap(600, 400, (200, 60, 30)) + document = fitz.open() + for page_no in range(1, 4): + page = document.new_page(width=612, height=792) + page.insert_image(fitz.Rect(72, 100, 152, 180), pixmap=logo) + page.insert_text((72, 230), f"Annual report section {page_no}", fontsize=14) + if page_no == 2: + page.insert_text((100, 280), TEXT_BEFORE, fontsize=11) + page.insert_image(CHART_RECT, pixmap=chart) + page.insert_text((100, 540), TEXT_AFTER, fontsize=11) + else: + page.insert_text((72, 280), "Plain text page with no pictures at all.", fontsize=11) + document.save(path) + document.close() + + +def chart_figure(**values) -> images.Figure: + return images.Figure( + bbox=tuple(CHART_RECT), + page_size=(612.0, 792.0), + text_before=TEXT_BEFORE, + text_after=TEXT_AFTER, + url="/private/files/chart.png", + **values, + ) + + +class TestFigureDetection(FrappeTestCase): + def test_finds_the_body_chart_and_skips_the_logo_repeated_on_every_page(self): + path = Path(tempfile.mkdtemp()) / "report.pdf" + make_report_pdf(str(path)) + with fitz.open(str(path)) as document: + repeated = images.repeated_images(document) + found = [images.find_figures(page, repeated) for page in document] + + self.assertEqual([len(figures) for figures in found], [0, 1, 0]) + chart = found[1][0] + self.assertEqual(fitz.Rect(chart.bbox), CHART_RECT) + self.assertEqual(chart.text_before, TEXT_BEFORE) + self.assertEqual(chart.text_after, TEXT_AFTER) + + def test_the_crop_leaves_out_a_title_line_its_padding_would_cut_in_half(self): + document = fitz.open() + page = document.new_page(width=612, height=792) + page.insert_text( + (100, CHART_RECT.y0 - 2), "DOSES OF IP ANTIBIOTICS (ISPD 2016 GUIDELINES)", fontsize=11 + ) + page.insert_image(CHART_RECT, pixmap=solid_pixmap(600, 400, (200, 60, 30))) + title_bottom = images.text_line_boxes(page)[0][3] + + clip = images.figure_clip(page, images.find_figures(page, set())[0]) + + self.assertGreater(title_bottom, CHART_RECT.y0 - images.CROP_PADDING) + self.assertEqual(clip.y0, title_bottom) + self.assertEqual(clip.y1, CHART_RECT.y1 + images.CROP_PADDING) + + +class TestFigurePlacement(FrappeTestCase): + def test_places_the_figure_after_the_text_that_precedes_it(self): + markdown = f"# Annual report\n\n{TEXT_BEFORE}\n\n## Discharge\n\n{TEXT_AFTER}" + + placed = images.place_figures(markdown, [chart_figure()], 2) + + self.assertEqual( + placed, + f"# Annual report\n\n{TEXT_BEFORE}\n\n![Figure 2.1](/private/files/chart.png)\n\n" + f"## Discharge\n\n{TEXT_AFTER}", + ) + + def test_a_figure_after_a_table_row_lands_below_the_whole_table(self): + markdown = f"\n\n\n
{TEXT_BEFORE}
Second row
\n\nClosing note." + + placed = images.place_figures(markdown, [chart_figure()], 2) + + self.assertTrue(placed.index("![Figure 2.1]") > placed.index("")) + self.assertTrue(placed.index("![Figure 2.1]") < placed.index("Closing note.")) + + def test_swaps_the_vlm_token_for_the_crop_titled_by_the_vlm(self): + markdown = f"{TEXT_BEFORE}\n\n[[FIGURE 1: Admissions per year]]\n\n[[FIGURE 2]]\n\n{TEXT_AFTER}" + + placed = images.place_figures(markdown, [chart_figure()], 2) + + self.assertEqual( + placed, f"{TEXT_BEFORE}\n\n![Admissions per year](/private/files/chart.png)\n\n{TEXT_AFTER}" + ) + + def test_a_printed_caption_wins_over_the_vlm_title(self): + placed = images.place_figures( + "[[FIGURE 1: Bar chart]]", [chart_figure(caption="Figure 3: Admissions")], 2 + ) + + self.assertEqual(placed, "![Figure 3: Admissions](/private/files/chart.png)") + + def test_inserts_a_figure_the_vlm_left_out_after_its_preceding_text(self): + markdown = f"## Admissions\n\n{TEXT_BEFORE}\n\n{TEXT_AFTER}" + + placed = images.place_figures(markdown, [chart_figure()], 2) + + self.assertEqual( + placed, + f"## Admissions\n\n{TEXT_BEFORE}\n\n![Figure 2.1](/private/files/chart.png)\n\n{TEXT_AFTER}", + ) + + def test_a_figure_already_in_the_markdown_is_not_added_twice(self): + markdown = f"{TEXT_BEFORE}\n\n![Admissions per year](/private/files/chart.png)\n\n{TEXT_AFTER}" + + self.assertEqual(images.place_figures(markdown, [chart_figure()], 2), markdown) + + +class TestTranscribedFigures(FrappeTestCase): + def test_keeps_the_crop_of_a_redrawn_chart_and_the_typed_copy_of_a_table_image(self): + org_chart = ( + "![Figure 28.1](/private/files/page-0028-crop.png)\n\n" + '```mermaid\nflowchart TD\nA["Head of the Department"]\nB["Clinical"]\nA --> B\n```\n\n28' + ) + table_image = ( + "## DOSES OF IP ANTIBIOTICS\n\n![Figure 251.1](/private/files/page-0251-crop.png)\n\n" + "**TABLE 5**\nIntraperitoneal Antibiotic Dosing Recommendations\n\n" + "| | Intermittent | Continuous |\n|---|---|---|\n" + "| Amikacin | 2 mg/kg daily | LD 25 mg/L |\n| Gentamicin | 0.6 mg/kg daily | LD 8 mg/L |\n" + "| Cefazolin | 15 mg/kg daily | LD 500 mg/L |" + ) + + self.assertEqual( + drop_transcribed_figures(org_chart), "![Figure 28.1](/private/files/page-0028-crop.png)\n\n\n28" + ) + self.assertEqual( + drop_transcribed_figures(table_image), + "## DOSES OF IP ANTIBIOTICS\n\n\n**TABLE 5**\nIntraperitoneal Antibiotic Dosing Recommendations\n\n" + "| | Intermittent | Continuous |\n|---|---|---|\n" + "| Amikacin | 2 mg/kg daily | LD 25 mg/L |\n| Gentamicin | 0.6 mg/kg daily | LD 8 mg/L |\n" + "| Cefazolin | 15 mg/kg daily | LD 500 mg/L |", + ) + + def test_keeps_a_chart_followed_by_a_paragraph_and_a_table(self): + markdown = ( + "![Figure 3: Admissions](/private/files/chart.png)\n\n" + f"{TEXT_BEFORE} {TEXT_AFTER} {TEXT_BEFORE} {TEXT_AFTER}\n\n" + "| Year | Admissions |\n|---|---|\n| 2019 | 40 |\n| 2020 | 52 |\n| 2021 | 61 |" + ) + + self.assertEqual(drop_transcribed_figures(markdown), markdown) + + +class TestFigureEchoes(FrappeTestCase): + def test_drops_labels_read_off_the_picture_and_keeps_the_printed_text(self): + body = [ + "Figure 2 . Anatomy of lumbar spine", + "Prep and drape the area after identifying landmarks. Use lidocaine 1% with or without", + "epinephrine to anesthetize the skin and the deeper tissues under the insertion site", + "Assemble needle and manometer. Attach the 3-way stopcock to manometer", + ] + path = Path(tempfile.mkdtemp()) / "procedure.pdf" + document = fitz.open() + page = document.new_page(width=612, height=792) + page.insert_image(CHART_RECT, pixmap=solid_pixmap(600, 400, (200, 60, 30))) + for offset, line in enumerate(body): + page.insert_text((72, 530 + 16 * offset), line, fontsize=10) + document.save(str(path)) + markdown = ( + "![Figure 2 . Anatomy of lumbar spine](/private/files/page-0348-crop.png)\n\n" + "Puncture site (L4-5 interspace)\n\nLevel of posterior superior iliac crests\n\nL5 L4 L3\n\n" + "Figure 2 . Anatomy of lumbar spine\n\n" + "Prep and drape the area after identifying landmarks. Use lidocaine 1% with or without epinephrine " + "to anesthetize the skin and the deeper tissues under the insertion site\n\n" + "Assemble needle and manometer. Attach the 3-way stopcock to manometer" + ) + + [(_, cleaned)] = images.drop_figure_echoes([(1, markdown)], str(path)) + + self.assertEqual( + cleaned, + "![Figure 2 . Anatomy of lumbar spine](/private/files/page-0348-crop.png)\n\n" + "Figure 2 . Anatomy of lumbar spine\n\n" + "Prep and drape the area after identifying landmarks. Use lidocaine 1% with or without epinephrine " + "to anesthetize the skin and the deeper tissues under the insertion site\n\n" + "Assemble needle and manometer. Attach the 3-way stopcock to manometer", + ) + + +class TestFigureRemediation(FrappeTestCase): + def setUp(self): + real_get = engine_settings.get + patcher = patch( + "wikify.engine.settings.get", + side_effect=lambda field: 0 if field == "judge_all_pages" else real_get(field), + ) + patcher.start() + self.addCleanup(patcher.stop) + + def test_remediation_keeps_the_chart_as_its_crop_instead_of_a_transcription(self): + path = Path(tempfile.mkdtemp()) / "report.pdf" + make_report_pdf(str(path)) + vlm_markdown = f"{TEXT_BEFORE}\n\n[[FIGURE 1: Admissions per year]]\n\n{TEXT_AFTER}" + with ( + patch("wikify.engine.llm.has_openrouter", return_value=True), + patch("wikify.engine.llm.chat_completion", side_effect=_fake_chat), + ): + source_document = parse_pdf(str(path), title="Figure Remediation Test") + self.addCleanup(_cleanup.delete_document, source_document) + with ( + patch("wikify.engine.remediate.clean_markdown", side_effect=RuntimeError("offline")), + patch( + "wikify.engine.remediate.vlm.parse_page_image", return_value=vlm_markdown + ) as parse_page_image, + ): + remediate_pdf(source_document, str(path), scope="all") + + hints = [call.kwargs["figure_hint"] for call in parse_page_image.call_args_list] + self.assertEqual([bool(hint) for hint in hints], [False, True, False]) + self.assertIn("this page has 1 picture(s)", hints[1]) + self.assertIn("[[FIGURE 1]]", hints[1]) + self.assertIn("Never turn its bars, lines, axis ticks, legend or data labels into a table", hints[1]) + + page = frappe.db.get_value( + "Source Page", + {"source_document": source_document, "page_no": 2}, + ["name", "image", "remediation_method", "remediation_markdown", "canonical_markdown"], + as_dict=True, + ) + crop_urls = frappe.get_all( + "File", + filters={ + "attached_to_doctype": "Source Page", + "attached_to_name": page.name, + "file_url": ["!=", page.image], + }, + pluck="file_url", + ) + self.assertEqual(len(crop_urls), 1) + self.assertEqual(page.remediation_method, "vlm") + self.assertEqual( + page.remediation_markdown, + f"{TEXT_BEFORE}\n\n![Admissions per year]({crop_urls[0]})\n\n{TEXT_AFTER}", + ) + self.assertEqual(page.canonical_markdown.count(crop_urls[0]), 1) diff --git a/wikify/tests/test_lint.py b/wikify/tests/test_lint.py index 1eca042..f950afc 100644 --- a/wikify/tests/test_lint.py +++ b/wikify/tests/test_lint.py @@ -17,7 +17,14 @@ from wikify.api import sections as api from wikify.engine import store -from wikify.engine.lint import fix_table_separators, lint_markdown, table_artifacts +from wikify.engine.lint import ( + escape_fill_in_blanks, + fix_glued_emphasis, + fix_table_separators, + lint_markdown, + repair_markdown, + table_artifacts, +) from wikify.engine.loader.sectionizer import Section from wikify.engine.sectionize import rebuild_section_markdown from wikify.engine.verify.deterministic import parser_artifacts @@ -83,6 +90,70 @@ def test_leaves_lone_rows_and_fenced_tables_alone(self): self.assertEqual(fix_table_separators(fenced), fenced) +class TestFormBlanks(FrappeTestCase): + def test_fill_in_blanks_are_escaped_outside_code_and_html(self): + row = "| | Replacement fluid | ______________________@_____ ___mL/h Additives_______ |" + untouched = "___\n\n`a___b`\n\nRate_____" + blank = "\\_" + fixed = escape_fill_in_blanks(f"{row}\n\n{untouched}") + self.assertEqual( + fixed, + f"| | Replacement fluid | {blank * 22}@{blank * 5} {blank * 3}mL/h Additives{blank * 7} |\n\n{untouched}", + ) + self.assertEqual(escape_fill_in_blanks(fixed), fixed) + + +class TestGluedEmphasis(FrappeTestCase): + def test_bold_glued_to_its_neighbours_is_separated(self): + md = ( + "|1. Inspect the dressing|**To**confirm the need.|\n" + "|3. Explain the procedure|Helps**to**gain co-operation.|\n" + "|8. Clean the site|**P**revents contamination.|\n" + "- **Postpheresis: H**ypokalemia, Bleeding tendencies\n" + "Saves time and improves efficiency**.**\n" + "**Date:10/11/2023 **Pg. 337\n" + "Continuous Renal Replacement Therapy Order sheet :**" + ) + self.assertEqual( + fix_glued_emphasis(md), + "|1. Inspect the dressing|**To** confirm the need.|\n" + "|3. Explain the procedure|Helps **to** gain co-operation.|\n" + "|8. Clean the site|Prevents contamination.|\n" + "- **Postpheresis:** Hypokalemia, Bleeding tendencies\n" + "Saves time and improves efficiency.\n" + "**Date:10/11/2023** Pg. 337\n" + "Continuous Renal Replacement Therapy Order sheet :", + ) + self.assertEqual( + fix_glued_emphasis("**AVF**s and **Artero Venous Fistula (AVF**) is"), + "**AVF**s and **Artero Venous Fistula (AVF**) is", + ) + + +class TestFormattingLeaks(FrappeTestCase): + def test_bold_closing_after_a_bracket_and_boxes_read_off_bullets_are_repaired(self): + indicators = ( + "PATIENT RELATED INDICATORS (TARGET)\n\n" + "1. **Haemoglobin > 11 g (**In > 80% of patients on PD)\n" + "2. **MEASUREMENT AND REPORTING OF PERITONITIS**\n" + " - ☐ Culture - negative peritonitis (< 15 % of all peritonitis episodes)\n" + " - ☐ Pre - PD peritonitis - % of total\n\n" + "- ☐ Yes ☐ No\n- ☐ Consent filed\n\n" + "- ☑ Catheter flushed\n- ☐ Dressing changed" + ) + + self.assertEqual( + repair_markdown(indicators), + "PATIENT RELATED INDICATORS (TARGET)\n\n" + "1. **Haemoglobin > 11 g** (In > 80% of patients on PD)\n" + "2. **MEASUREMENT AND REPORTING OF PERITONITIS**\n" + " - Culture - negative peritonitis (< 15 % of all peritonitis episodes)\n" + " - Pre - PD peritonitis - % of total\n\n" + "- ☐ Yes ☐ No\n- ☐ Consent filed\n\n" + "- ☑ Catheter flushed\n- ☐ Dressing changed", + ) + + class TestPageArtifacts(FrappeTestCase): def test_parser_artifacts_include_table_breakage(self): self.assertIn("table missing separator row", parser_artifacts(MISSING_SEPARATOR)) diff --git a/wikify/tests/test_page_plan.py b/wikify/tests/test_page_plan.py new file mode 100644 index 0000000..b7e82cf --- /dev/null +++ b/wikify/tests/test_page_plan.py @@ -0,0 +1,438 @@ +# Copyright (c) 2026, BWH and contributors +# For license information, please see license.txt +import json +from unittest.mock import patch + +from frappe.tests.utils import FrappeTestCase + +from wikify.engine.loader import page_plan +from wikify.engine.loader.page_plan import plan_pages +from wikify.engine.loader.sectionizer import Section + + +def _sec(path, page_start, page_end=None, markdown=None): + return Section( + title=path[-1], + level=len(path), + hierarchy_path=path, + page_start=page_start, + page_end=page_end or page_start, + markdown=markdown if markdown is not None else f"body of {path[-1]}", + ) + + +def _words(count): + return " ".join(["word"] * count) + + +def _outline(): + return [ + _sec(["Preamble"], 1, markdown=_words(60)), + _sec(["5.8 Care"], 2, markdown=""), + _sec(["5.8 Care", "5.8.1 Chaperone"], 2), + _sec(["5.8 Care", "5.8.4 Sedation"], 3), + _sec(["5.8 Care", "5.8.4 Sedation", "5.8.4.1 Indications"], 4), + _sec(["5.8 Care", "5.8.4 Sedation", "5.8.4.1 Indications", "Contraindications"], 4), + _sec(["5.8 Care", "5.8.4 Sedation", "5.8.4.2 Consents"], 5, 6), + _sec(["5.8 Care", "5.8.9 Grievances"], 7, markdown=_words(30)), + _sec(["5.8 Care", "5.8.9 Grievances", "1. Drug Regimen"], 7), + _sec(["6.1 Protocols"], 8, markdown=""), + _sec(["6.1 Protocols", "6.1.2 Lupus nephritis"], 8), + _sec(["6.1 Protocols", "6.1.2 Lupus nephritis", "Class IV"], 9, markdown=""), + _sec(["6.1 Protocols", "6.1.2 Lupus nephritis", "Class V"], 10), + ] + + +def _llm_reply(payload): + content = payload if isinstance(payload, str) else json.dumps(payload) + return {"choices": [{"message": {"content": content}}]} + + +def _titles(sections): + return [section.title for section in sections] + + +class TestFallbackPagePlan(FrappeTestCase): + def test_pages_are_the_top_two_numbered_levels_below_the_chapter(self): + pages = plan_pages(_outline()) + self.assertEqual( + _titles(pages), + [ + "Preamble", + "5.8 Care", + "5.8.1 Chaperone", + "5.8.4 Sedation", + "5.8.9 Grievances", + "6.1 Protocols", + "6.1.2 Lupus nephritis", + ], + ) + + def test_unnumbered_document_keeps_its_top_two_heading_levels(self): + sections = [ + _sec(["Guide"], 1), + _sec(["Guide", "Setup"], 2), + _sec(["Guide", "Setup", "Prerequisites"], 3), + _sec(["Guide", "Usage"], 4), + ] + self.assertEqual(_titles(plan_pages(sections)), ["Guide", "Setup", "Usage"]) + + def test_a_lone_sub_section_folds_into_its_parent(self): + sections = [ + _sec(["5.7 Billing"], 1, markdown=""), + _sec(["5.7 Billing", "5.7.2 Inpatient billing"], 1, 2), + _sec(["5.8 Care"], 2), + _sec(["5.8 Care", "5.8.1 Chaperone"], 3), + _sec(["5.8 Care", "5.8.1 Chaperone", "5.8.1.1 Escort"], 4), + ] + pages = plan_pages(sections) + self.assertEqual(_titles(pages), ["5.7 Billing", "5.8 Care", "5.8.1 Chaperone"]) + self.assertEqual(pages[0].markdown, "## 5.7.2 Inpatient billing\n\nbody of 5.7.2 Inpatient billing") + + def test_folded_sections_become_headings_in_document_order(self): + sedation = plan_pages(_outline())[3] + self.assertEqual( + sedation.markdown, + "body of 5.8.4 Sedation\n\n" + "## 5.8.4.1 Indications\n\nbody of 5.8.4.1 Indications\n\n" + "### Contraindications\n\nbody of Contraindications\n\n" + "## 5.8.4.2 Consents\n\nbody of 5.8.4.2 Consents", + ) + self.assertEqual((sedation.page_start, sedation.page_end), (3, 6)) + + def test_an_empty_folded_section_keeps_its_heading(self): + lupus = plan_pages(_outline())[-1] + self.assertEqual( + lupus.markdown, + "body of 6.1.2 Lupus nephritis\n\n## Class IV\n\n## Class V\n\nbody of Class V", + ) + self.assertEqual(lupus.page_end, 10) + + def test_heading_depth_is_capped_at_six(self): + path = [f"Level {depth}" for depth in range(1, 9)] + sections = [_sec(path[:depth], min(depth, 2)) for depth in range(1, 9)] + child = plan_pages(sections)[1] + self.assertEqual(child.title, "Level 2") + self.assertIn("###### Level 7\n\nbody of Level 7\n\n###### Level 8", child.markdown) + + def test_hierarchy_of_the_remaining_pages_is_unchanged(self): + pages = plan_pages(_outline()) + self.assertEqual(pages[-1].hierarchy_path, ["6.1 Protocols", "6.1.2 Lupus nephritis"]) + + +class TestLlmPagePlan(FrappeTestCase): + def _plan(self, reply, sections=None): + with patch.object(page_plan.llm, "chat_completion", return_value=reply) as chat: + pages = plan_pages(sections or _outline(), "Hospital manual", use_llm=True) + return pages, chat + + def test_llm_choice_decides_the_pages(self): + pages, chat = self._plan(_llm_reply({"pages": [2, 3, 7, 10]})) + self.assertEqual( + _titles(pages), + [ + "Preamble", + "5.8 Care", + "5.8.1 Chaperone", + "5.8.4 Sedation", + "5.8.9 Grievances", + "6.1 Protocols", + "6.1.2 Lupus nephritis", + ], + ) + self.assertEqual(chat.call_args.kwargs["label"], "page_plan") + self.assertEqual(chat.call_args.kwargs["response_format"], {"type": "json_object"}) + prompt = chat.call_args.args[1][0]["content"] + self.assertIn("Hospital manual", prompt) + self.assertIn("4 | 3 | 5.8.4.1 Indications | p4-4 | 4 | 7 | 1", prompt) + + def test_a_json_encoded_page_list_is_accepted(self): + pages, _ = self._plan(_llm_reply({"pages": "[3, 7]"})) + self.assertEqual( + _titles(pages), + [ + "Preamble", + "5.8 Care", + "5.8.1 Chaperone", + "5.8.4 Sedation", + "5.8.9 Grievances", + "6.1 Protocols", + ], + ) + + def test_top_level_sections_are_pages_even_when_the_llm_skips_them(self): + pages, _ = self._plan(_llm_reply({"pages": []})) + self.assertEqual(_titles(pages), ["Preamble", "5.8 Care", "6.1 Protocols"]) + self.assertIn("## 5.8.1 Chaperone", pages[1].markdown) + self.assertIn("### 5.8.4.1 Indications", pages[1].markdown) + + def test_a_page_under_a_folded_parent_folds_with_it(self): + pages, _ = self._plan(_llm_reply({"pages": [4, 5]})) + self.assertEqual(_titles(pages), ["Preamble", "5.8 Care", "6.1 Protocols"]) + + def test_an_empty_page_without_page_children_folds_into_its_parent(self): + pages, _ = self._plan(_llm_reply({"pages": [10, 11]})) + self.assertEqual(_titles(pages)[-2:], ["6.1 Protocols", "6.1.2 Lupus nephritis"]) + self.assertIn("## Class IV\n\n## Class V", pages[-1].markdown) + + def test_unusable_indices_are_ignored(self): + pages, _ = self._plan(_llm_reply({"pages": ["2", 99, -1, True, 3]})) + self.assertEqual( + _titles(pages), ["Preamble", "5.8 Care", "5.8.4 Sedation", "5.8.9 Grievances", "6.1 Protocols"] + ) + + def test_invalid_reply_falls_back_to_the_numbering_rule(self): + expected = _titles(plan_pages(_outline())) + for reply in (_llm_reply("not json"), _llm_reply({"pages": "all"}), _llm_reply({})): + with patch.object(page_plan.frappe, "log_error"): + pages, _ = self._plan(reply) + self.assertEqual(_titles(pages), expected) + + def test_llm_error_falls_back_to_the_numbering_rule(self): + with ( + patch.object(page_plan.llm, "chat_completion", side_effect=RuntimeError("down")), + patch.object(page_plan.frappe, "log_error") as log_error, + ): + pages = plan_pages(_outline(), use_llm=True) + self.assertEqual(_titles(pages), _titles(plan_pages(_outline()))) + log_error.assert_called_once() + + def test_large_outlines_are_sent_per_chapter(self): + replies = [_llm_reply({"pages": [2]}), _llm_reply({"pages": [10]})] + with ( + patch.object(page_plan, "OUTLINE_BATCH_LINES", 9), + patch.object(page_plan.llm, "chat_completion", side_effect=replies) as chat, + ): + pages = plan_pages(_outline(), use_llm=True) + self.assertEqual(chat.call_count, 2) + self.assertNotIn("6.1 Protocols", chat.call_args_list[0].args[1][0]["content"]) + self.assertEqual( + _titles(pages), + [ + "Preamble", + "5.8 Care", + "5.8.1 Chaperone", + "5.8.4 Sedation", + "5.8.9 Grievances", + "6.1 Protocols", + "6.1.2 Lupus nephritis", + ], + ) + + def test_an_unnumbered_top_level_fragment_folds_into_the_page_before_it(self): + sections = [ + _sec(["Preamble"], 1, markdown=_words(60)), + _sec(["IMMEDIATE PRE-OPERATIVE PROTOCOLS"], 7, 9, markdown=_words(60)), + _sec(["Pre transplant day"], 9, markdown=""), + _sec(["PRE-TRANSPLANT RECIPIENT ORDER SHEET"], 9, 10, markdown=_words(60)), + _sec(["6.3.3.2 Donor evaluation"], 11), + ] + pages, _ = self._plan(_llm_reply({"pages": [1, 2]}), sections) + self.assertEqual( + _titles(pages), ["Preamble", "IMMEDIATE PRE-OPERATIVE PROTOCOLS", "6.3.3.2 Donor evaluation"] + ) + self.assertIn("## Pre transplant day\n\n## PRE-TRANSPLANT RECIPIENT ORDER SHEET", pages[1].markdown) + self.assertEqual(pages[1].page_end, 10) + self.assertEqual( + _titles(plan_pages(sections)), + [ + "Preamble", + "IMMEDIATE PRE-OPERATIVE PROTOCOLS", + "PRE-TRANSPLANT RECIPIENT ORDER SHEET", + "6.3.3.2 Donor evaluation", + ], + ) + + def test_a_page_spanning_too_many_pdf_pages_splits_at_its_numbered_children(self): + capd = "6.2.2 CAPD" + initiation = "6.2.2.1 Initiation" + sections = [ + _sec([capd], 4, markdown=""), + _sec([capd, "6.2.2.0 Introduction"], 4, markdown=_words(80)), + _sec([capd, initiation], 4, markdown=""), + _sec([capd, initiation, "6.2.2.1.1 Counselling"], 4, 5, markdown=_words(200)), + _sec([capd, initiation, "6.2.2.1.2 Pre-op evaluation"], 5, 17, markdown=_words(2000)), + _sec([capd, initiation, "6.2.2.1.3 Intra-op"], 17, 20, markdown=_words(800)), + _sec([capd, "6.2.2.2 Note"], 20, markdown=_words(10)), + ] + pages, _ = self._plan(_llm_reply({"pages": []}), sections) + self.assertEqual( + _titles(pages), + [ + capd, + "6.2.2.0 Introduction", + initiation, + "6.2.2.1.1 Counselling", + "6.2.2.1.2 Pre-op evaluation", + "6.2.2.1.3 Intra-op", + ], + ) + self.assertIn("6.2.2.2 Note", pages[-1].markdown) + self.assertNotIn("6.2.2.2 Note", pages[0].markdown) + + def test_a_long_page_without_numbered_children_splits_at_its_sub_headings(self): + access = "6.2.1.4 PROTOCOL FOR VASCULAR ACCESS" + sections = [ + _sec([access], 3, markdown=""), + _sec([access, "AV FISTULA/AV GRAFT"], 3, markdown=_words(60)), + _sec([access, "A. Skin Preparation"], 3, 4, markdown=_words(230)), + _sec([access, "CENTRAL VENOUS CATHETER"], 5, markdown=""), + _sec([access, "CATHETER CARE"], 5, markdown=_words(260)), + _sec([access, "I. EXIT SITE CARE"], 6, 7, markdown=_words(140)), + _sec([access, "DIALYZER REUSE"], 7, markdown=_words(30)), + _sec([access, "IDENTIFYING AND MANAGING COMPLICATIONS"], 7, markdown=""), + _sec([access, "1. INTRA DIALYTIC HYPOTENSION"], 7, 8, markdown=_words(110)), + _sec([access, "2. HYPERKALEMIA"], 9, 10, markdown=_words(80)), + _sec([access, "Peritoneal Dialysis"], 11, markdown=""), + _sec([access, "1 Dialysis Prescription"], 11, markdown=_words(30)), + _sec([access, "2 Peritonitis"], 11, markdown=_words(90)), + _sec([access, "4 Exit Site Infection"], 11, markdown=_words(40)), + ] + pages, _ = self._plan(_llm_reply({"pages": []}), sections) + self.assertEqual( + _titles(pages), + [ + access, + "AV FISTULA/AV GRAFT", + "CENTRAL VENOUS CATHETER", + "Peritoneal Dialysis", + ], + ) + self.assertEqual(pages[0].markdown, "") + self.assertIn("## CATHETER CARE", pages[2].markdown) + self.assertIn( + f"## DIALYZER REUSE\n\n{_words(30)}\n\n## IDENTIFYING AND MANAGING COMPLICATIONS", + pages[2].markdown, + ) + self.assertEqual(pages[3].hierarchy_path, [access, "Peritoneal Dialysis"]) + self.assertEqual((pages[3].page_start, pages[3].page_end), (11, 11)) + + def test_a_short_leading_preamble_joins_the_page_after_it(self): + sections = [ + _sec(["Preamble"], 1, markdown="Saline dialysis is reserved for bleeding patients."), + _sec(["6.2.1 POLICIES"], 2, markdown=""), + _sec(["6.2.1 POLICIES", "6.2.1.3 ASSESSMENT"], 2), + _sec(["6.2.1 POLICIES", "6.2.1.4 VASCULAR ACCESS"], 3), + ] + pages, _ = self._plan(_llm_reply({"pages": [0, 1, 2, 3]}), sections) + self.assertEqual(_titles(pages), ["6.2.1 POLICIES", "6.2.1.3 ASSESSMENT", "6.2.1.4 VASCULAR ACCESS"]) + self.assertEqual(pages[0].markdown, "") + self.assertEqual( + pages[1].markdown, + "Saline dialysis is reserved for bleeding patients.\n\nbody of 6.2.1.3 ASSESSMENT", + ) + + def test_numbered_siblings_follow_the_majority_into_pages(self): + sections = [ + _sec(["5.8 Policies on patient care"], 1, markdown=_words(20)), + _sec(["5.8 Policies on patient care", "5.8.1 Chaperone"], 2, markdown=_words(90)), + _sec(["5.8 Policies on patient care", "5.8.2 Antibiotic policy"], 3, markdown=_words(29)), + _sec(["5.8 Policies on patient care", "5.8.3 Consent"], 4, markdown=_words(200)), + _sec(["5.8 Policies on patient care", "5.8.4 Sedation"], 5, markdown=_words(300)), + _sec(["5.8 Policies on patient care", "5.8.4 Sedation", "Contraindications"], 5), + ] + pages, _ = self._plan(_llm_reply({"pages": [1, 3, 4]}), sections) + self.assertEqual( + _titles(pages), + [ + "5.8 Policies on patient care", + "5.8.1 Chaperone", + "5.8.2 Antibiotic policy", + "5.8.3 Consent", + "5.8.4 Sedation", + ], + ) + pages, _ = self._plan(_llm_reply({"pages": [4]}), sections) + self.assertEqual(_titles(pages), ["5.8 Policies on patient care", "5.8.4 Sedation"]) + + def test_numbered_siblings_after_a_page_are_pages_even_when_short(self): + sections = [ + _sec(["2. Organogram"], 1, markdown=_words(20)), + _sec(["2. Organogram", "2.1 The staff"], 1, markdown=_words(200)), + _sec(["2. Organogram", "2.2 Hierarchy"], 2, markdown=_words(200)), + _sec(["2. Organogram", "2.3 Staff list"], 3, markdown=_words(200)), + _sec(["2. Organogram", "2.4 Laboratory staff"], 4, markdown=_words(60)), + _sec(["2. Organogram", "2.5 Office staff"], 5, markdown=_words(60)), + _sec(["2. Organogram", "2.6 Support staff"], 6, markdown=_words(25)), + _sec(["2. Organogram", "2.7 Attendants"], 6, markdown=_words(5)), + ] + pages, _ = self._plan(_llm_reply({"pages": [0, 2]}), sections) + self.assertEqual( + _titles(pages), + [ + "2. Organogram", + "2.2 Hierarchy", + "2.3 Staff list", + "2.4 Laboratory staff", + "2.5 Office staff", + "2.6 Support staff", + ], + ) + self.assertNotIn("2.6 Support staff", pages[0].markdown + pages[-2].markdown) + self.assertIn("## 2.7 Attendants", pages[-1].markdown) + + def test_a_long_page_never_splits_at_a_form_letterhead_or_a_step(self): + training = "6.2.2.1.5 PD TRAINING PROCEDURE" + sections = [ + _sec([training], 227, markdown=""), + _sec([training, "CHRISTIAN MEDICAL COLLEGE"], 227, markdown=""), + _sec([training, "DEPARTMENT OF NEPHROLOGY"], 227, markdown=""), + _sec([training, "PD TRAINING OBJECTIVES"], 227, 228, markdown=_words(200)), + _sec([training, "DAY WISE SCHEDULE OF TRAINING"], 229, 231, markdown=_words(300)), + _sec([training, "CHRISTIAN MEDICAL COLLEGE VELLORE"], 232, markdown=""), + _sec([training, "PROCEDURE MANUAL - NEPHROLOGY"], 232, markdown=""), + _sec([training, "PD EXCHANGE PROCEDURE"], 232, 233, markdown=_words(229)), + _sec([training, "Step 11: Check, measure, Record and Discard"], 233, 234, markdown=_words(191)), + _sec([training, "DISCHARGE CHECKLIST"], 239, 240, markdown=_words(193)), + _sec([training, "CONTACT INFORMATION SHEET"], 240, markdown=""), + _sec([training, "CHRISTIAN MEDICAL COLLEGE"], 240, markdown=""), + _sec([training, "DEPARTMENT OF NEPHROLOGY"], 240, markdown=_words(178)), + ] + pages, _ = self._plan(_llm_reply({"pages": []}), sections) + self.assertEqual( + _titles(pages), + [ + training, + "PD TRAINING OBJECTIVES", + "DAY WISE SCHEDULE OF TRAINING", + "PD EXCHANGE PROCEDURE", + "DISCHARGE CHECKLIST", + ], + ) + self.assertIn("## Step 11: Check, measure, Record and Discard", pages[3].markdown) + self.assertIn("## CONTACT INFORMATION SHEET", pages[4].markdown) + + def test_sections_sharing_a_pdf_page_share_a_wiki_page(self): + sections = [ + _sec(["2. Organogram"], 25, markdown=_words(12)), + _sec(["2. Organogram", "2.1 The staff"], 25, markdown=_words(50)), + _sec(["2. Organogram", "2.2 Staff hierarchy"], 26, 28, markdown=_words(390)), + _sec(["2. Organogram", "2.3 Staff list"], 29, 32, markdown=_words(1000)), + _sec(["2. Organogram", "2.4 Laboratory staff"], 33, markdown=_words(5)), + _sec(["2. Organogram", "2.5 Research staff"], 33, markdown=_words(5)), + _sec(["2. Organogram", "2.6 Office staff"], 33, markdown=_words(25)), + _sec(["2. Organogram", "2.7 Support staff"], 33, markdown=_words(30)), + ] + pages, _ = self._plan(_llm_reply({"pages": [0, 1, 2, 3, 4, 5, 6, 7]}), sections) + self.assertEqual( + _titles(pages), + ["2. Organogram", "2.2 Staff hierarchy", "2.3 Staff list", "2.4 Laboratory staff"], + ) + self.assertIn("## 2.1 The staff", pages[0].markdown) + for title in ("2.5 Research staff", "2.6 Office staff", "2.7 Support staff"): + self.assertIn(f"## {title}", pages[-1].markdown) + + def test_a_long_numbered_section_keeps_its_page_when_it_starts_mid_page(self): + sections = [ + _sec(["3. Job descriptions"], 34, markdown=_words(80)), + _sec(["3. Job descriptions", "3.1 Head of the department"], 34, 36, markdown=_words(600)), + _sec(["3. Job descriptions", "3.2 Head of the unit"], 37, 38, markdown=_words(300)), + _sec(["3. Job descriptions", "3.3 Consultant"], 38, 39, markdown=_words(180)), + _sec(["3. Job descriptions", "3.4 Registrar"], 39, markdown=_words(90)), + ] + pages, _ = self._plan(_llm_reply({"pages": [0, 1, 2, 3, 4]}), sections) + self.assertEqual( + _titles(pages), + ["3. Job descriptions", "3.1 Head of the department", "3.2 Head of the unit", "3.3 Consultant"], + ) + self.assertIn("## 3.4 Registrar", pages[-1].markdown) diff --git a/wikify/tests/test_sectionize.py b/wikify/tests/test_sectionize.py index b9a248d..e147b57 100644 --- a/wikify/tests/test_sectionize.py +++ b/wikify/tests/test_sectionize.py @@ -132,6 +132,90 @@ def test_content_before_any_heading_becomes_preamble(self): self.assertEqual(secs[0].title, "Preamble") self.assertEqual(secs[0].level, 1) + def test_content_before_the_first_heading_is_kept(self): + pages = [ + ( + 1, + "- ✓ Concessions of Rs 2,500 are discussed.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\n## 5.7 Billing\nbill body", + ), + ] + secs = sectionize(pages) + self.assertEqual((secs[0].title, secs[0].hierarchy_path), ("Preamble", ["Preamble"])) + self.assertIn("Concessions of Rs 2,500", secs[0].markdown) + self.assertIn("| 1 | 2 |", secs[0].markdown) + self.assertEqual(secs[1].hierarchy_path, ["5.7 Billing"]) + + def test_running_sub_header_variants_fold_into_the_open_section(self): + running = "## 5.8 Policies on patient care" + pages = [ + (1, f"{running}\n### 5.8.1 Chaperone\na\n{running}\n### 5.8.2 Antibiotics\nb"), + (2, f"{running}\n### 5.8.3 Pain\nc"), + (3, f"{running}\n### 5.8.4 Sedation\nd\n## 5.8Policies on patient care\n### 5.8.5 Restraints\ne"), + ] + secs = sectionize(clean_pages(pages)) + self.assertEqual([s.title for s in secs].count("5.8 Policies on patient care"), 1) + self.assertEqual(secs[0].title, "5.8 Policies on patient care") + self.assertTrue(all(s.hierarchy_path[0] == "5.8 Policies on patient care" for s in secs)) + + def test_repeated_chapter_title_with_another_number_or_plural_is_not_a_new_section(self): + protocols = [ + (1, "# 6.1 PROTOCOLS FOR MANAGEMENT\n## 6.1.1 IgA nephropathy\na"), + (2, "# 6. PROTOCOLS FOR MANAGEMENT\n## 6.1.2 Lupus nephritis\n#### Hydroxychloroquine.\nb"), + ] + paths = {s.title: s.hierarchy_path for s in sectionize(protocols)} + self.assertNotIn("6. PROTOCOLS FOR MANAGEMENT", paths) + self.assertEqual( + paths["6.1.2 Lupus nephritis"], ["6.1 PROTOCOLS FOR MANAGEMENT", "6.1.2 Lupus nephritis"] + ) + self.assertIn("Hydroxychloroquine", paths) + + jobs = [ + (1, "# 3. JOB DESCRIPTIONS\n## 3.1 Head\na"), + (2, "## 3. JOB DESCRIPTION\n## 3.2 Unit head\nb"), + ] + paths = {s.title: s.hierarchy_path for s in sectionize(jobs)} + self.assertNotIn("3. JOB DESCRIPTION", paths) + self.assertEqual(paths["3.2 Unit head"], ["3. JOB DESCRIPTIONS", "3.2 Unit head"]) + + def test_bold_line_continuing_the_numbering_is_a_heading(self): + markdown = ( + "## 5.8.6 Blood Donation\na\n### 5.8.6.2 Donor criteria\nb\n" + "**5.8.6.3 Requesting blood from blood bank**\n**For use in ward**\nc\n" + "**5.8.6.4 For use in OR**\nd\n**1. Not a heading**\ne" + ) + pages = [(1, markdown)] + secs = sectionize(pages) + self.assertEqual( + [(s.title, s.level) for s in secs], + [ + ("5.8.6 Blood Donation", 3), + ("5.8.6.2 Donor criteria", 4), + ("5.8.6.3 Requesting blood from blood bank", 4), + ("5.8.6.4 For use in OR", 4), + ], + ) + self.assertNotIn("Requesting blood", secs[1].markdown) + self.assertIn("**For use in ward**", secs[2].markdown) + self.assertIn("**1. Not a heading**", secs[3].markdown) + + def test_unnumbered_heading_never_parents_a_numbered_heading(self): + pages = [(1, "# Contraindications\na\n# 5.8.4.2 Patient Assessment\nb\n### 5.8.5 Restraints\nc")] + paths = {s.title: s.hierarchy_path for s in sectionize(pages)} + self.assertEqual(paths["5.8.4.2 Patient Assessment"], ["5.8.4.2 Patient Assessment"]) + self.assertEqual(paths["5.8.5 Restraints"], ["5.8.5 Restraints"]) + + def test_numbered_list_headings_nest_under_a_deep_numbered_section(self): + pages = [ + (1, "#### 5.9.2.2 Staff tested for HIV\na\n# 1. Drug Regimen\nb\n# 2. Testing for HBsAg\nc"), + (2, "# 6.1 PROTOCOLS\n## 6.1.1 IgA nephropathy\nd"), + ] + paths = {s.title: s.hierarchy_path for s in sectionize(pages)} + self.assertEqual(paths["1. Drug Regimen"], ["5.9.2.2 Staff tested for HIV", "1. Drug Regimen"]) + self.assertEqual( + paths["2. Testing for HBsAg"], ["5.9.2.2 Staff tested for HIV", "2. Testing for HBsAg"] + ) + self.assertEqual(paths["6.1.1 IgA nephropathy"], ["6.1 PROTOCOLS", "6.1.1 IgA nephropathy"]) + def test_emphasis_is_stripped_from_heading_titles(self): pages = [ (1, "## _**Verbal Orders**_\nbody"), @@ -150,6 +234,13 @@ def test_clean_pages_strips_varying_boilerplate_before_sectionizing(self): self.assertTrue(all("Pg" not in s.markdown for s in secs)) self.assertEqual([s.title for s in secs], ["1. Intro", "2. Next"]) + def test_clean_pages_strips_a_header_style_that_covers_only_part_of_a_long_document(self): + pages = [ + (number, f"{'NEPHROLOGY MANUAL' if number % 4 == 0 else 'PROCEDURE MANUAL'}\n\nbody {number}") + for number in range(1, 101) + ] + self.assertTrue(all("MANUAL" not in markdown for _, markdown in clean_pages(pages))) + def test_clean_pages_strips_signoff_footer_block(self): footer = "|**Prepared by - Dr. A**|**Issued by: QMC**|**Approved by - Dr. B**|\n|---|---|---|" pages = [(1, f"## 1. Intro\nreal body\n{footer}"), (2, f"## 2. Next\nmore body\n{footer}")] @@ -161,6 +252,38 @@ def test_clean_pages_strips_signoff_footer_block(self): self.assertIn("real body", cleaned[1]) self.assertIn("more body", cleaned[2]) + def test_clean_pages_strips_page_numbers_and_headless_separator_rows(self): + header = "NEPHROLOGY MANUAL\n\n| MAN/RPT/NEPH/001 | Ver.: 1 | Pg. {page} of 398 |\n|---|---|---|" + table = "| Health worker | Action |\n|---|---|\n| Antibody 100 | Reassure |" + pages = [ + (1, header.format(page=121) + f"\n\n## 5.7 Billing\n\n- Pay 2500 by noon.\n\n{table}\n\n121"), + (2, header.format(page=122) + "\n\n- Second page item.\n\n122"), + (3, header.format(page=123) + "\n\n- Third page item.\n\n123"), + ] + cleaned = dict(clean_pages(pages)) + self.assertFalse(cleaned[1].startswith("|")) + self.assertEqual(cleaned[2].strip(), "- Second page item.") + self.assertNotIn("121", cleaned[1]) + self.assertIn(f"- Pay 2500 by noon.\n\n{table}", cleaned[1]) + + def test_clean_pages_rejoins_a_sentence_split_by_a_page_break(self): + pages = [ + (1, "- Organs are retrieved after withdrawal of life-sustaining\n\n132"), + ( + 2, + "measures; the kidneys may be recovered.\n\n## 5.8.12 Grievances\n\n- Last item ends.\n\n133", + ), + (3, "- New item starts the page.\n\n134"), + ] + cleaned = dict(clean_pages(pages)) + self.assertEqual( + cleaned[1], + "- Organs are retrieved after withdrawal of life-sustaining measures; the kidneys may be recovered.", + ) + self.assertNotIn("measures", cleaned[2]) + self.assertIn("## 5.8.12 Grievances", cleaned[2]) + self.assertEqual(cleaned[3], "- New item starts the page.") + def test_title_is_clipped_to_the_storable_length(self): long_title = "DETERMINATION OF RESIDENTIAL STATUS OF " + "HINDU UNDIVIDED FAMILY " * 8 secs = sectionize([(1, f"## {long_title}\nbody")]) @@ -214,6 +337,543 @@ def test_clean_pages_keeps_data_row_mentioning_approved_by_once(self): self.assertIn("approved by committee", cleaned[1]) self.assertIn("|---|---|", cleaned[1]) + def test_table_of_contents_is_one_section_that_never_opens_chapters(self): + table_rows = "\n".join(f"| 1.{n} | Scope of service {n} | {18 + n} |" for n in range(1, 7)) + procedure_rows = "\n".join(f"| 5.{n} | Policies for topic {n} | {95 + n} |" for n in range(1, 7)) + list_rows = "\n".join(f" - 6.1.{n} Protocol {n} — {200 + n}" for n in range(1, 7)) + pages = [ + (1, "# PROCEDURE MANUAL\n\n## REVISION HISTORY\nrevisions"), + (2, f"## Contents\n\n| S.No | CONTENTS | Pg No |\n|---|---|---|\n{table_rows}"), + ( + 3, + f"## 5 Procedures for access of the patients\n\n| No. | Topic | Page |\n|---|---|---|\n{procedure_rows}", + ), + (4, f"- 6.1 Protocols — 199\n{list_rows}\n\n**4**"), + (5, "# 1. PROFILE OF THE DEPARTMENT\n\n## Overview\nIn the year 1900 the institution began."), + ] + secs = sectionize(pages) + self.assertEqual( + [(s.title, s.level) for s in secs], + [ + ("PROCEDURE MANUAL", 1), + ("REVISION HISTORY", 2), + ("Contents", 1), + ("1. PROFILE OF THE DEPARTMENT", 1), + ("Overview", 2), + ], + ) + contents = secs[2] + self.assertEqual((contents.page_start, contents.page_end), (2, 4)) + self.assertIn("5 Procedures for access of the patients", contents.markdown) + self.assertIn("6.1.6 Protocol 6 — 206", contents.markdown) + + def test_link_markup_is_stripped_from_heading_titles(self): + secs = sectionize([(1, "# [PROCEDURE MANUAL - NEPHROLOGY](#)\n\n## REVISION HISTORY\nrows")]) + self.assertEqual(secs[0].title, "PROCEDURE MANUAL - NEPHROLOGY") + + def test_lowercase_or_colon_heading_is_a_bold_label(self): + markdown = ( + "# **IMMEDIATE PRE-OPERATIVE PROTOCOLS**\nintro\n# **are carried out as below:**\n" + "- PreHD: WBC\n# **Drug list:**\n- MMF 8 am\n# 5.1 Definitions:\nterms" + ) + pages = [(1, markdown)] + secs = sectionize(pages) + self.assertEqual([s.title for s in secs], ["IMMEDIATE PRE-OPERATIVE PROTOCOLS", "5.1 Definitions:"]) + self.assertIn("**are carried out as below:**\n- PreHD: WBC\n**Drug list:**", secs[0].markdown) + + def test_clean_pages_strips_signoff_footer_written_as_text(self): + pipes = "Prepared by: Dr. A, Dr. B | Issued by: QMC | Approved by: Dr. C" + merged = "**Prepared by: Dr. A, Dr. Issued by: QMC Approved by: Dr. C B** " + paragraphs = "Prepared by: Dr. A, Dr. B\n\nIssued by: QMC\n\nApproved by: Dr. C" + pages = [ + (1, f"## DONOR WITH OBESITY\nreal body\n\n{pipes}"), + (2, f"- more body\n\n{merged}\n"), + (3, f"## DIABETIC PATIENT\n\n{paragraphs}\n\nThe protocol is approved by the HOD."), + ] + cleaned = dict(clean_pages(pages)) + for md in cleaned.values(): + self.assertNotIn("Prepared by", md) + self.assertNotIn("Issued by", md) + self.assertIn("real body", cleaned[1]) + self.assertIn("- more body", cleaned[2]) + self.assertIn("approved by the HOD", cleaned[3]) + + def test_clean_pages_keeps_html_tables_that_recur_on_every_page(self): + header = "PROCEDURE MANUAL - NEPHROLOGY\n\nMAN/RPT/NEPH/001 | Ver.: 1 | Pg. {page} of 398\n\n" + table = ( + "\n\n\n\n\n" + "\n\n\n\n" + "\n
Nursing ActionRationale
1. Check doctor's order.Gains cooperation
2. Administer inj. calcium gluconate.Prevents hypocalcemia\n" + "
3. Remove sutures.Avoids blood loss
" + ) + pages = [ + (page, header.format(page=page) + f"## Step {page}\n\n{table}\n\nNotes {page}.") + for page in (1, 2, 3) + ] + cleaned = dict(clean_pages(pages)) + self.assertIn( + "| Nursing Action | Rationale |\n|---|---|\n" + "| 1. Check doctor's order. | Gains cooperation |\n" + "| 2. Administer inj. calcium gluconate. | Prevents hypocalcemia |\n" + "| 3. Remove sutures. | Avoids blood loss |", + cleaned[2], + ) + + def test_clean_pages_stitches_a_table_row_split_by_a_page_break(self): + header = "PROCEDURE MANUAL - NEPHROLOGY\n\nMAN/RPT/NEPH/001 | Ver.: 1 | Pg. {page} of 398\n\n" + first = ( + "|**Nursing Action**|**Rationale**|\n|---|---|\n" + "|1. Inspect the previous dressing|Confirms the need.|\n" + "|2. Explain the procedure and
position him according to the|Helps gain co-operation.|\n" + "|**MAN/RPT/NEPH/001 Ver.:**|**1 Issue :1**
Pg.**2 of 398**|\n" + "|convenience to access the site.||\n" + "|3. Send blood for investigations|Electrolytes are removed during the process of|" + ) + second = ( + "\n\n\n\n\n" + "\n\n\n\n
(Ca, PO4, Na, K)plasmapheresis.
4. Check BP.Identifies complications.
\n\n" + "# **Specific Instructions:**\n\n- Avoid swimming." + ) + pages = [ + (1, header.format(page=1) + first), + (2, header.format(page=2) + second), + (3, header.format(page=3)), + ] + secs = sectionize(clean_pages(pages)) + self.assertEqual( + secs[0].markdown, + "|**Nursing Action**|**Rationale**|\n|---|---|\n" + "|1. Inspect the previous dressing|Confirms the need.|\n" + "| 2. Explain the procedure and
position him according to the convenience to access the site. " + "| Helps gain co-operation. |\n" + "| 3. Send blood for investigations (Ca, PO4, Na, K) " + "| Electrolytes are removed during the process of plasmapheresis. |\n" + "| 4. Check BP. | Identifies complications. |\n\n" + "**Specific Instructions:**\n\n- Avoid swimming.", + ) + + def test_clean_pages_strips_running_headers_wrapped_in_markup(self): + pages = [ + ( + 1, + "PROCEDURE MANUAL\n\n> **MAN/RPT/NEPH/001 Ver.: 1**\n\n> **Date:10/11/2023** Pg. **321 of 398**\n\n- First.", + ), + ( + 2, + "||PROCEDURE MANUAL||\n|---|---|---|\n|**MAN/RPT/NEPH/001**|
**Ver.: 1**|Pg.**322 of 398**|\n\n- Second.", + ), + ( + 3, + "PROCEDURE MANUAL\n\n**MAN/RPT/NEPH/001 Ver.: 1 Date:10/11/2023** Pg. **323 of 398**\n\n- Third.", + ), + ( + 4, + "PROCEDURE MANUAL **MAN/RPT/NEPH/001 Ver.: 1 Date:10/11/2023** Pg. **324 of 398**\n\n- Fourth.\n\n**324**", + ), + ] + cleaned = dict(clean_pages(pages)) + self.assertEqual(cleaned, {1: "- First.", 2: "- Second.", 3: "- Third.", 4: "- Fourth."}) + + def test_clean_pages_keeps_a_label_that_recurs_mid_page(self): + header = "PROCEDURE MANUAL\n\nMAN/RPT/NEPH/001 | Ver.: 1 | Pg. {page} of 398\n\n" + body = "## 6.4.{page} Step {page}\n\n**Definition**\n\nText {page}.\n\n## Procedure\n\n1. Do {page}.\n\nFooter note" + pages = [(page, header.format(page=page) + body.format(page=page)) for page in range(1, 6)] + cleaned = dict(clean_pages(pages)) + for page in range(1, 6): + self.assertEqual( + cleaned[page], + f"## 6.4.{page} Step {page}\n\n**Definition**\n\nText {page}.\n\n## Procedure\n\n1. Do {page}.", + ) + + def test_stray_bold_markers_are_stripped_from_titles(self): + secs = sectionize([(1, "# 3. **Other equipment:\n- Gloves\n# **CRRT Order sheet** :\n- Form")]) + self.assertEqual([s.title for s in secs], ["3. Other equipment:"]) + self.assertIn("**CRRT Order sheet :**", secs[0].markdown) + + def test_numbered_list_items_inside_a_sub_section_stay_under_it(self): + extra_duties = ( + "# 4. Extra procedures to be performed by the CAPD nurse educator:\n\n- Assist PET\n\n" + "# 5. Other responsibilities of the CAPD Nurse educator\n\n- Attend meetings" + ) + administration = ( + "## 11. Administration\n\n- Plan duty rosters\n\n" + "## 3.6.9 RENAL TRANSPLANT CO-ORDINATOR\n\n**Qualification:** nurse" + ) + pages = [ + (5, "## 3.6.8 CAPD Nurse Educator\n\nThe CAPD nurse educator trains patients."), + ( + 7, + "# 3. Responsibilities of the CAPD Nurse Educator towards inpatients\n\n- Check the log-book", + ), + (8, extra_duties), + (9, administration), + (13, "## 3.7 Allied Health Staff\n\n### 3.7.1 Pharmacist\n\n- Dispensing"), + (14, "# 4. PROTOCOLS\n\n## 4.1 Dialysis\n\n- Prime the circuit"), + ] + self.assertEqual( + [section.hierarchy_path for section in sectionize(pages)], + [ + ["3.6.8 CAPD Nurse Educator"], + [ + "3.6.8 CAPD Nurse Educator", + "3. Responsibilities of the CAPD Nurse Educator towards inpatients", + ], + [ + "3.6.8 CAPD Nurse Educator", + "4. Extra procedures to be performed by the CAPD nurse educator:", + ], + ["3.6.8 CAPD Nurse Educator", "5. Other responsibilities of the CAPD Nurse educator"], + ["3.6.8 CAPD Nurse Educator", "11. Administration"], + ["3.6.9 RENAL TRANSPLANT CO-ORDINATOR"], + ["3.7 Allied Health Staff"], + ["3.7 Allied Health Staff", "3.7.1 Pharmacist"], + ["4. PROTOCOLS"], + ["4. PROTOCOLS", "4.1 Dialysis"], + ], + ) + + def test_a_parent_heading_met_after_its_sub_sections_becomes_their_group(self): + policies = "6.2.1 POLICIES FOR MANAGEMENT OF HEMODIALYSIS" + first_page = ( + "Saline dialysis is reserved for bleeding patients.\n\n" + "## 6.2.1.3ASSESSMENT AND MONITORING\n\nPatients develop complications." + ) + pages = [ + (1, first_page), + (3, "## 6.2.1.4 PROTOCOL FOR VASCULAR ACCESS\n\n### AV FISTULA\n\nCannulate with care."), + (19, f"# {policies}\n\n## 6.2.1.8 Emergency Dialysis\n\nDialyse within an hour."), + (20, f"## {policies}\n\n### 6.2.1.9 Temporary Vascular Access\n\nUse a femoral catheter."), + ] + sections = sectionize(pages) + self.assertEqual( + [section.hierarchy_path for section in sections], + [ + ["Preamble"], + [policies], + [policies, "6.2.1.3 ASSESSMENT AND MONITORING"], + [policies, "6.2.1.4 PROTOCOL FOR VASCULAR ACCESS"], + [policies, "6.2.1.4 PROTOCOL FOR VASCULAR ACCESS", "AV FISTULA"], + [policies, "6.2.1.8 Emergency Dialysis"], + [policies, "6.2.1.9 Temporary Vascular Access"], + ], + ) + self.assertEqual((sections[1].level, sections[1].markdown, sections[1].page_start), (3, "", 1)) + self.assertEqual(sections[4].markdown, "Cannulate with care.") + + def test_running_chapter_header_at_a_page_top_is_dropped(self): + pages = [ + (10, "## 3.6.9 RENAL TRANSPLANT CO-ORDINATOR\n\n- Maintain the registry."), + ( + 13, + "# 3. JOB DESCRIPTIONS\n\n## 3.7 Allied Health Staff\n\n### 3.7.1 Pharmacist\n\n- Dispensing", + ), + (14, "### 3.7.2 Transplant Coordinator\n\n- Counsel donors."), + (15, "# 3. JOB DESCRIPTION\n\n## 3.7.3 Dialysis therapists\n\n- Prime the circuit."), + ] + sections = sectionize(pages) + self.assertEqual( + [section.title for section in sections], + [ + "3.6.9 RENAL TRANSPLANT CO-ORDINATOR", + "3.7 Allied Health Staff", + "3.7.1 Pharmacist", + "3.7.2 Transplant Coordinator", + "3.7.3 Dialysis therapists", + ], + ) + self.assertFalse(any("JOB DESCRIPTION" in section.markdown for section in sections)) + + def test_a_numbered_section_never_nests_under_another_chapter(self): + markdown = ( + "# 3. **Other equipment:** \n\n- Gloves\n\n## Patient Preparation\n\nExplain the procedure.\n\n" + "# **6.4.2. CENTRAL VENOUS CATHETERIZATION** \n\n# **Definition** \n\nA double lumen catheter." + ) + pages = [(1, markdown)] + self.assertEqual( + [section.hierarchy_path for section in sectionize(pages)], + [ + ["3. Other equipment:"], + ["3. Other equipment:", "Patient Preparation"], + ["6.4.2. CENTRAL VENOUS CATHETERIZATION"], + ["6.4.2. CENTRAL VENOUS CATHETERIZATION", "Definition"], + ], + ) + + def test_clean_pages_keeps_a_list_continued_on_the_next_page_in_its_place(self): + hyperkalemia = ( + "## 7. HYPERKALEMIA\n\n4. **Drugs:** given on verbal order\n" + " - Administer Inj. Dextrose 50% with Inj. Actrapid.\n" + " - Administer Salbutamol nebulization 5mg.\n\n169" + ) + pages = [ + (169, hyperkalemia), + ( + 170, + "- Keep Inj.Calcium Gluconate 10% ready.\n\n## CENTRAL VENOUS CATHETER BLOCK\n\n- Flush it.\n\n170", + ), + ( + 323, + "# **2. Patient/family**\n\n- 1) Explain the cost to the patient\n\n- 2) Obtain consent\n\n323", + ), + ( + 324, + "3) Instruct patient not to touch the sterile area\n4) Explain it is a temporary access\n\n## Procedure", + ), + ] + cleaned = dict(clean_pages(pages)) + self.assertEqual( + cleaned[170], + " - Keep Inj.Calcium Gluconate 10% ready.\n\n## CENTRAL VENOUS CATHETER BLOCK\n\n- Flush it.", + ) + self.assertEqual( + cleaned[324], + "- 3) Instruct patient not to touch the sterile area\n- 4) Explain it is a temporary access\n\n## Procedure", + ) + + def test_a_chapter_the_body_never_opens_becomes_the_group_of_its_sections(self): + chapter = "6. PROTOCOLS FOR CLINICAL NEPHROLOGY" + contents = "\n".join( + [ + "| S.No | CONTENTS | Pg No |", + "|---|---|---|", + "| 5 | Procedures for access | 96 |", + "| 5.9 | Policies on reporting | 136 |", + "| 6.1 | Protocols for clinical nephrology | 140 |", + "| 6.2 | Procedures in dialysis | 158 |", + "| 6.3 | Transplantation | 267 |", + "| 6.3.3 | Policies for pre-transplant evaluation | 270 |", + ] + ) + transplantation = ( + f"## {chapter}\n\n### 6.3.0 TRANSPLANTATION\n\nKidney transplants.\n\n" + "## 6.3.2 Organ donation\n\nBrain death.\n\n## POLICIES FOR PRE-TRANSPLANT EVALUATION\n\n" + "### 6.3.3.1 RECIPIENT WORKUP\n\nPreliminary evaluation." + ) + pages = [ + (2, contents), + (91, "# 5. PROCEDURES FOR ACCESS\n\n## 5.9 Policies on reporting\n\nReport incidents."), + (137, "# 6.1 PROTOCOLS FOR CLINICAL NEPHROLOGY\n\n## 6.1.1 IgA nephropathy\n\nGive steroids."), + (138, f"# {chapter}\n\n## 6.1.2 Lupus nephritis\n\nBiopsy first."), + (155, f"## {chapter}\n\n### 6.2 PROCEDURES IN DIALYSIS\n\nThe dialysis unit."), + (275, transplantation), + ] + protocols = [chapter, "6.1 PROTOCOLS FOR CLINICAL NEPHROLOGY"] + transplant = [chapter, "6.3 Transplantation"] + evaluation = [*transplant, "6.3.3 Policies for pre-transplant evaluation"] + self.assertEqual( + [section.hierarchy_path for section in sectionize(pages)[1:]], + [ + ["5. PROCEDURES FOR ACCESS"], + ["5. PROCEDURES FOR ACCESS", "5.9 Policies on reporting"], + [chapter], + protocols, + [*protocols, "6.1.1 IgA nephropathy"], + [*protocols, "6.1.2 Lupus nephritis"], + [chapter, "6.2 PROCEDURES IN DIALYSIS"], + transplant, + [*transplant, "6.3.0 TRANSPLANTATION"], + [*transplant, "6.3.2 Organ donation"], + evaluation, + [*evaluation, "6.3.3.1 RECIPIENT WORKUP"], + ], + ) + + def test_a_list_heading_numbered_like_the_next_chapter_stays_in_its_section(self): + complications = ( + "## 6.2.1.4 PROTOCOL FOR VASCULAR ACCESS\n\n# IDENTIFYING AND MANAGING COMPLICATIONS\n\n" + "## 4. AIR EMBOLISM\n\nClamp the line.\n\n# 7. HYPERKALEMIA\n\nGive calcium.\n\n" + "## CENTRAL VENOUS CATHETER BLOCK\n\nUse urokinase." + ) + pages = [ + (91, "# 5. PROCEDURES FOR ACCESS\n\n## 5.1 Policies for access\n\nRegister patients."), + (137, "# 6. PROTOCOLS\n\n## 6.2 PROCEDURES IN DIALYSIS\n\n### 6.2.1 HEMODIALYSIS\n\nThe unit."), + (167, complications), + (172, "## 6.2.1.5 HEPATITIS B VACCINATION\n\nVaccinate."), + (179, "## 6.2.1POLICIES FOR HEMODIALYSIS\n\n### 6.2.1.8 Emergency Dialysis\n\nDialyse now."), + (180, "## 6.2.1 POLICIES FOR HEMODIALYSIS\n\n### 6.2.1.9 Temporary Access\n\nFemoral line."), + (371, "# 7. PATIENT RIGHTS AND EDUCATION\n\n## 7.1 Privacy\n\nScreens are used."), + ] + hemodialysis = ["6. PROTOCOLS", "6.2 PROCEDURES IN DIALYSIS", "6.2.1 HEMODIALYSIS"] + access = [*hemodialysis, "6.2.1.4 PROTOCOL FOR VASCULAR ACCESS"] + paths = {section.title: section.hierarchy_path for section in sectionize(pages)} + self.assertEqual(paths["7. HYPERKALEMIA"], [*access, "7. HYPERKALEMIA"]) + self.assertEqual(paths["CENTRAL VENOUS CATHETER BLOCK"], [*access, "CENTRAL VENOUS CATHETER BLOCK"]) + self.assertEqual( + paths["6.2.1.5 HEPATITIS B VACCINATION"], [*hemodialysis, "6.2.1.5 HEPATITIS B VACCINATION"] + ) + self.assertEqual(paths["6.2.1.9 Temporary Access"], [*hemodialysis, "6.2.1.9 Temporary Access"]) + self.assertNotIn("6.2.1 POLICIES FOR HEMODIALYSIS", paths) + + def test_a_lettered_running_header_of_the_open_section_is_dropped(self): + policies = "6.2.1 POLICIES FOR MANAGEMENT OF HEMODIALYSIS" + introduction = ( + "### 6.2.1HEMODIALYSIS\n\n#### 6.2.1.aIntroduction to the unit\n\nThe unit has 40 machines." + ) + icu = ( + "# ICU dialysis\n\nBedside dialysis.\n\n# 6.2.1.b POLICIES FOR MANAGEMENT OF HEMODIALYSIS\n\n" + "## 6.2.1.1 PROTOCOL FOR INITIATING\n\nCheck consent." + ) + pages = [ + (155, introduction), + (157, icu), + (179, f"## {policies}\n\n### 6.2.1.8 Emergency Dialysis\n\nDialyse within an hour."), + (180, f"## {policies}\n\n### 6.2.1.9 Temporary Vascular Access\n\nUse a femoral line."), + ] + sections = sectionize(pages) + self.assertEqual( + [section.title for section in sections], + [ + "6.2.1 HEMODIALYSIS", + "6.2.1.aIntroduction to the unit", + "ICU dialysis", + "6.2.1.1 PROTOCOL FOR INITIATING", + "6.2.1.8 Emergency Dialysis", + "6.2.1.9 Temporary Vascular Access", + ], + ) + self.assertEqual(sections[2].markdown, "Bedside dialysis.") + + def test_a_bold_list_item_numbered_like_the_next_chapter_does_not_take_the_chapter(self): + pages = [ + (91, "# 5. PROCEDURES FOR ACCESS\n\n## 5.1 Policies for access\n\nRegister patients."), + (137, "# 6. PROTOCOLS\n\n## 6.2.1.4 PROTOCOL FOR VASCULAR ACCESS\n\nCannulate with care."), + (169, "**6. PULMONARY EDEMA**\n\nGive oxygen.\n\n**7. HYPERKALEMIA**\n\nGive calcium."), + (170, "## 6.2.1.5 HEPATITIS B VACCINATION\n\nVaccinate."), + ( + 371, + "# **7. PATIENT RIGHTS AND EDUCATION** \n\nRights are displayed.\n\n# **7.1 Privacy**\n\nScreens.", + ), + ] + sections = sectionize(pages) + self.assertEqual( + [section.hierarchy_path for section in sections if section.title.startswith("7")], + [["7. PATIENT RIGHTS AND EDUCATION"], ["7. PATIENT RIGHTS AND EDUCATION", "7.1 Privacy"]], + ) + self.assertIn("**7. HYPERKALEMIA**", sections[3].markdown) + + def test_a_misprinted_chapter_in_a_deep_number_is_read_as_the_current_chapter(self): + kinship = ( + "## 6.5.3. PROCEDURES FOR TESTING\n\n### 6.5.3.1. KINSHIP TESTING PROTOCOL\n\nDNA test.\n\n" + "#### 6.5.3.1.4. Turn-Around-Time:\n\nTwo weeks.\n\n" + "#### 1.5.3.1.5. PATIENT'S RIGHTS & CONFIDENTIALITY\n\nReports are sealed." + ) + endotoxin = ( + "## 6.5.3.2.0. Endotoxin Testing Protocol\n\nLAL assay.\n\n" + "## 1.5.3.2.1 Pre Analytical Phase :\n\nCollect water.\n\n## 6.5.4 RECORDS MAINTAINED\n\nRegisters." + ) + testing = "6.5.3. PROCEDURES FOR TESTING" + protocol = [testing, "6.5.3.1. KINSHIP TESTING PROTOCOL"] + endotoxin_group = [testing, "6.5.3.2. Endotoxin Testing Protocol"] + self.assertEqual( + [section.hierarchy_path for section in sectionize([(356, kinship), (361, endotoxin)])], + [ + [testing], + protocol, + [*protocol, "6.5.3.1.4. Turn-Around-Time:"], + [*protocol, "6.5.3.1.5. PATIENT'S RIGHTS & CONFIDENTIALITY"], + endotoxin_group, + [*endotoxin_group, "6.5.3.2.0. Endotoxin Testing Protocol"], + [*endotoxin_group, "6.5.3.2.1 Pre Analytical Phase :"], + ["6.5.4 RECORDS MAINTAINED"], + ], + ) + + def test_html_tags_are_stripped_from_heading_titles(self): + markdown = ( + "## 2.3. STAFF LIST as on 24th May 2018\n\nNames.\n\n" + "## **2.4 CODE RED PROTOCOL** \n\nEvacuate." + ) + self.assertEqual( + [section.title for section in sectionize([(29, markdown)])], + ["2.3. STAFF LIST as on 24th May 2018", "2.4 CODE RED PROTOCOL"], + ) + + def test_clean_pages_drops_a_running_header_stitched_inside_a_table(self): + header = "NEPHROLOGY MANUAL\n\nMAN/RPT/NEPH/001 | Ver.: 1 | Issue : 1 | Pg. {page} of 398\n\n" + timeline = ( + "| Year | Milestone |\n|---|---|\n" + "| 2005 | Nephrology Laboratory Started |\n" + "| 2007 | DNA fingerprinting facility established to ascertain genetic relationship between |\n" + "||NEPHROLOGY MANUAL
|\n" + "||**MAN/RPT/NEPH/001**
**Ver.: 1**
**Issue : 1**
**Date:10/11/2023**
Pg.**14 of 398**|\n" + "||transplant patient & prospective donor|\n" + "|2008|First CAPD catheterization done by percutaneous method|" + ) + header_box = ( + '\n\n\n\n\n' + "\n\n\n\n" + "\n
NEPHROLOGY MANUAL
MAN/RPT/NEPH/001Ver.: 1Issue : 1Pg. 15 of 398
\n\n" + ) + pages = [ + (13, header.format(page=13) + timeline), + (14, header.format(page=14) + "# Education\n\nTwo programmes."), + (15, header_box + "Every staff member keeps this manual correct."), + (16, header.format(page=16) + "Closing notes."), + ] + cleaned = dict(clean_pages(pages)) + self.assertEqual( + cleaned[13], + "| Year | Milestone |\n|---|---|\n" + "| 2005 | Nephrology Laboratory Started |\n" + "| 2007 | DNA fingerprinting facility established to ascertain genetic relationship between " + "transplant patient & prospective donor |\n" + "|2008|First CAPD catheterization done by percutaneous method|", + ) + self.assertEqual(cleaned[15], "Every staff member keeps this manual correct.") + + def test_clean_pages_stitches_a_continuation_that_lost_an_empty_column_and_its_header(self): + checklist = ( + "### PRE-OPERATIVE CHECKLIST FOR PD NURSE\n\n\n\n\n\n\n\n" + "\n\n" + "\n\n\n" + "\n\n\n\n\n
Tick (√)
PD education material given in patient's own language\n
    \n" + "
  • Introductory PD pamphlet
    • \n
  • PD patient information sheet
    • \n
\n
Patient's ability to do CAPD ensured
\n\n212" + ) + continuation = ( + "| Care-giver identified | Specify |\n|---|---|\n" + "| Dedicated PD area at home identified | |\n" + "| Date of surgery decided | Date: |\n\n" + "Signature of the PD nurse." + ) + cleaned = dict(clean_pages([(212, checklist), (213, continuation)])) + self.assertEqual( + cleaned[212], + "### PRE-OPERATIVE CHECKLIST FOR PD NURSE\n\n| | Tick (√) | |\n|---|---|---|\n" + "| PD education material given in patient's own language
• Introductory PD pamphlet" + "
• PD patient information sheet | | |\n" + "| Patient's ability to do CAPD ensured | | |\n" + "| Care-giver identified | | Specify |\n" + "| Dedicated PD area at home identified | | |\n" + "| Date of surgery decided | | Date: |", + ) + self.assertEqual(cleaned[213], "Signature of the PD nurse.") + + def test_clean_pages_lays_contents_lines_out_as_the_contents_table_they_continue(self): + contents_table = ( + "# CONTENTS\n\n| S.No | CONTENTS | Pg No |\n|---|---|---|\n" + "| 3.5.4 | Fellow in Interventional Nephrology | 46 |\n" + "| 3.5.5 | Responsibilities of Dialysis Registrar | 47 |" + ) + contents_lines = ( + "3.5.6 Transplant registrar 50\n3.5.7 Fellow in Renal Transplantation 51\n3.6 Nursing staff 59\n" + "3.6.1 Nurse manager 57\n3.9.1 Hospital Attendant -Posted in clinical Area (A.K. Lab) 88\n" + "4 Records maintained 93" + ) + cleaned = dict(clean_pages([(2, contents_table), (3, contents_lines)])) + self.assertEqual( + cleaned[2], + "# CONTENTS\n\n| S.No | CONTENTS | Pg No |\n|---|---|---|\n" + "| 3.5.4 | Fellow in Interventional Nephrology | 46 |\n" + "| 3.5.5 | Responsibilities of Dialysis Registrar | 47 |\n" + "| 3.5.6 | Transplant registrar | 50 |\n" + "| 3.5.7 | Fellow in Renal Transplantation | 51 |\n" + "| 3.6 | Nursing staff | 59 |\n" + "| 3.6.1 | Nurse manager | 57 |\n" + "| 3.9.1 | Hospital Attendant -Posted in clinical Area (A.K. Lab) | 88 |\n" + "| 4 | Records maintained | 93 |", + ) + self.assertEqual(cleaned[3], "") + class TestEmptySectionsAreFlagged(FrappeTestCase): def test_section_without_markdown_is_chunked_as_title_only(self):