From e6aee02be0670a4d0c93602d684da019112f2e21 Mon Sep 17 00:00:00 2001 From: vishwajeet-13 Date: Sun, 11 Oct 2026 02:32:35 +0530 Subject: [PATCH 1/2] fix(import): stop short-section pdfs collapsing and step numbers swallowing chapters - cleanup: a "Page N" label at the page edge is stripped like a bare page number - cleanup: a running "Chapter > Section" breadcrumb becomes the chapter heading, with the page's heading nested under it - sectionize: a "Topic (continued)" or "Continued from the previous page" heading folds into the open section - sectionize: a numbered heading that continues a run of bare step numbers on its page is a list step, not a chapter - page plan: when no top-level section qualifies as a page, the ones that open a pdf page still are --- wikify/engine/loader/cleanup.py | 62 +++++++++++++++++++++--- wikify/engine/loader/page_plan.py | 11 +++++ wikify/engine/loader/sectionizer.py | 27 +++++++++-- wikify/tests/test_page_plan.py | 10 ++++ wikify/tests/test_sectionize.py | 74 +++++++++++++++++++++++++++++ 5 files changed, 173 insertions(+), 11 deletions(-) diff --git a/wikify/engine/loader/cleanup.py b/wikify/engine/loader/cleanup.py index 64d42c2..5b892d9 100644 --- a/wikify/engine/loader/cleanup.py +++ b/wikify/engine/loader/cleanup.py @@ -14,7 +14,7 @@ import re import unicodedata from collections import Counter -from itertools import groupby +from itertools import groupby, pairwise from wikify.engine.loader.table_stitch import ( html_tables_to_markdown, @@ -51,7 +51,7 @@ _SIGNOFF = ("prepared by", "issued by", "approved by", "reviewed by", "authorized by") _SIGNOFF_LABEL = re.compile(r"(?i)\b(prepared|issued|approved|reviewed|authori[sz]ed) by\s*[:\-\u2013]") _SIGNOFF_LINE = re.compile(r"(?i)^[*_\s]*(prepared|issued|approved|reviewed|authori[sz]ed) by\s*[:\-\u2013]") -_PAGE_NUMBER = re.compile(r"^\s*\d{1,4}\s*$") +_PAGE_NUMBER = re.compile(r"(?i)^\s*(?:(?:page|pg\.?)\s*)?\d{1,4}\s*$") _SENTENCE_START = re.compile(r"^[a-z]") # A sentence cut by a page break ran to the page edge, so its last line is never a short label. _MIN_BROKEN_LINE_LENGTH = 40 @@ -70,6 +70,10 @@ MIN_TRANSCRIBED_ROWS = 4 TYPED_TABLE_WORD_SHARE = 0.6 _TABLE_WORD = re.compile(r"[^\W_]{3,}") +_HEADING_RE = re.compile(r"^(#{1,6})\s+(.*\S)\s*$") +_BREADCRUMB_SPLIT = re.compile(r"\s+>\s+") +_NOT_BREADCRUMB_START = ("#", "|", ">", "-", "*", "+") +BREADCRUMB_MIN_PAGES = 2 def _plain(line: str) -> str: @@ -265,6 +269,50 @@ def strip_boilerplate(pages: list[tuple[int, str]], boilerplate: set[str]) -> li return out +def markdown_heading(level: int, title: str) -> str: + return f"{'#' * min(6, level)} {title}" + + +def page_breadcrumb(lines: list[str]) -> tuple[int, int, str, list[str]] | None: + content = [index for index, line in enumerate(lines) if line.strip()] + for index, next_index in pairwise(content[:EDGE_LINES]): + text = lines[index].strip() + if text.startswith(_NOT_BREADCRUMB_START) or _STRUCTURAL_LINE.match(text): + continue + parts = [part.strip() for part in _BREADCRUMB_SPLIT.split(_plain(text))] + heading = _HEADING_RE.match(lines[next_index]) + if len(parts) > 1 and all(parts) and heading and _norm(heading.group(2)) == _norm(parts[-1]): + return index, next_index, heading.group(2), parts + return None + + +def breadcrumbs_to_headings(pages: list[tuple[int, str]]) -> list[tuple[int, str]]: + page_lines = [md.splitlines() for _, md in pages] + crumbs = [page_breadcrumb(lines) for lines in page_lines] + if sum(1 for crumb in crumbs if crumb) < BREADCRUMB_MIN_PAGES: + return pages + previous: list[str] = [] + for lines, crumb in zip(page_lines, crumbs, strict=True): + if not crumb: + continue + crumb_index, heading_index, title, parts = crumb + trail = [_norm(part) for part in parts] + opened = 0 + for old, new in zip(previous, trail[:-1], strict=False): + if old != new: + break + opened += 1 + lines[heading_index] = markdown_heading(len(parts), title) + lines[crumb_index] = "\n\n".join( + markdown_heading(depth + 1, parts[depth]) for depth in range(opened, len(parts) - 1) + ) + previous = trail + return [ + (page_no, "\n".join(lines).strip() if crumb else md) + for (page_no, md), lines, crumb in zip(pages, page_lines, crumbs, strict=True) + ] + + def _ends_mid_sentence(line: str) -> bool: text = line.strip() return ( @@ -449,10 +497,12 @@ def clean_pages( ) -> list[tuple[int, str]]: pages = [(page_no, html_tables_to_markdown(md)) for page_no, md in pages] page_texts = page_texts or {} - stripped = [ - (page_no, contents_lines_to_table(drop_transcribed_figures(md, page_texts.get(page_no, "")))) - for page_no, md in strip_boilerplate(pages, find_boilerplate(pages)) - ] + stripped = breadcrumbs_to_headings( + [ + (page_no, contents_lines_to_table(drop_transcribed_figures(md, page_texts.get(page_no, "")))) + for page_no, md in strip_boilerplate(pages, find_boilerplate(pages)) + ] + ) stitched = stitch_cross_page_tables(stripped) joined = join_page_breaks([(page_no, merge_continuation_rows(md)) for page_no, md in stitched]) return reindent_list_continuations(joined) diff --git a/wikify/engine/loader/page_plan.py b/wikify/engine/loader/page_plan.py index ab279c4..59af509 100644 --- a/wikify/engine/loader/page_plan.py +++ b/wikify/engine/loader/page_plan.py @@ -128,6 +128,8 @@ def enforce_invariants( pages.add(index) elif index in chosen and parent in pages: pages.add(index) + if not pages: + pages = pdf_page_openers(sections, parents) pages = add_numbered_siblings(parents, numbers, pages) pages, parents = split_long_pages(sections, parents, numbers, pages) pages, parents = keep_sibling_order(sections, parents, numbers, pages) @@ -143,6 +145,15 @@ def enforce_invariants( return pages or {0}, parents +def pdf_page_openers(sections: list[Section], parents: list[int | None]) -> set[int]: + openers = { + index + for index, parent in enumerate(parents) + if index and parent is None and (index == 1 or opens_pdf_page(sections, index)) + } + return openers if len(openers) > 1 else set() + + def add_numbered_siblings( parents: list[int | None], numbers: list[tuple[int, ...] | None], pages: set[int] ) -> set[int]: diff --git a/wikify/engine/loader/sectionizer.py b/wikify/engine/loader/sectionizer.py index 2618397..f2fe222 100644 --- a/wikify/engine/loader/sectionizer.py +++ b/wikify/engine/loader/sectionizer.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from itertools import pairwise -from wikify.engine.loader.cleanup import _SEP_ONLY +from wikify.engine.loader.cleanup import _HEADING_RE, _SEP_ONLY from wikify.engine.loader.toc import correct_level MAX_TITLE_LENGTH = 140 @@ -15,7 +15,6 @@ RUNNING_HEADER_MIN_PAGES = 3 TOP_OF_PAGE_LINES = 3 -_HEADING_RE = re.compile(r"^(#{1,6})\s+(.*\S)\s*$") _NUM_RE = re.compile(r"^(\d+(?:\.\d+)*)\.?\s+\S") _LEADING_NUM = re.compile(r"^(\d+)\b") _DOUBLE_NUM = re.compile(r"^\d+\.\s+\d") @@ -24,6 +23,8 @@ _LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)") _HTML_TAG = re.compile(r"]*>") _LETTERED_NUM = re.compile(r"^(\d+(?:\.\d+)+)\.[a-z](?![a-z])\s*") +_CONTINUED = re.compile(r"(?i)^(.*?)\s*\((?:continued|cont'd|cont\.?)\)$") +_CONTINUED_FROM = re.compile(r"(?i)^continued(?: from (?:the )?previous page)?$") _TOC_NUMBERED = re.compile(r"^(\d+(?:\.\d+)*)\.?\s+(.*?)(?:\s*\.{3,})?(?:\s+\d{1,4})?$") _CONTENTS_TITLE = re.compile(r"(?i)^(?:table of )?contents$|^index$") _TOC_ENTRY = re.compile( @@ -94,6 +95,16 @@ def _repeats_open_section(title: str, stack: list[tuple[int, str, bool]]) -> boo ) +def continues_open_section(title: str, stack: list[tuple[int, str, bool]]) -> bool: + if _CONTINUED_FROM.match(title): + return bool(stack) + continued = _CONTINUED.match(title) + if not continued: + return False + words = _title_words(continued.group(1)) + return any(words == _title_words(open_title) for _, open_title, _ in stack) + + def extends_number(number: tuple[int, ...] | None, prefix: tuple[int, ...]) -> bool: return bool(number) and len(number) > len(prefix) and number[: len(prefix)] == prefix @@ -374,6 +385,7 @@ def sectionize(pages: list[tuple[int, str]], level_map: dict[str, int] | None = max_chapter = 0 capital_chapters = True last_list_item = (0, 0) + last_step_number = (0, 0) last_number: tuple[int, ...] = () running_headers = running_header_titles(pages) opened_headers: set[str] = set() @@ -422,10 +434,15 @@ def flush(): m = None elif m and _is_label(title, level_map): line, m = f"**{title}**", None + step = title if m else line.strip() + if step.isdigit(): + last_step_number = (page_no, int(step)) if m: title = correct_chapter_typo(title, last_number) - if _repeats_open_section(title, stack) or repeats_running_header( - title, stack, recurring_titles + if ( + _repeats_open_section(title, stack) + or repeats_running_header(title, stack, recurring_titles) + or continues_open_section(title, stack) ): if current is not None: current.page_end = page_no @@ -466,7 +483,7 @@ def flush(): and not opens_next_chapter(cnum, last_number, next_dotted, title) ) or (max_chapter and capital_chapters and not title.isupper()) - or (page_no, cnum - 1) == last_list_item + or (page_no, cnum - 1) in (last_list_item, last_step_number) ): level = 2 demoted = True diff --git a/wikify/tests/test_page_plan.py b/wikify/tests/test_page_plan.py index 8ff365a..582c23d 100644 --- a/wikify/tests/test_page_plan.py +++ b/wikify/tests/test_page_plan.py @@ -126,6 +126,16 @@ def test_a_short_memo_of_unnumbered_headings_keeps_its_text(self): for text in ("Memo to all renal staff.", "## Clinic closure", "Call extension 4455."): self.assertIn(text, pages[0].markdown) + def test_short_unnumbered_sections_each_opening_a_pdf_page_stay_pages(self): + topics = ["Five rights", "IV administration", "Covert medication", "Self-administration"] + sections = [_sec(["Preamble"], 1, markdown="Repeat Prescribing Protocol")] + sections += [ + _sec([topic], 2 * n + 1, 2 * n + 2, markdown=_words(20)) for n, topic in enumerate(topics) + ] + pages = plan_pages(sections) + self.assertEqual(_titles(pages), topics) + self.assertIn("Repeat Prescribing Protocol", pages[0].markdown) + def test_hierarchy_of_the_remaining_pages_is_unchanged(self): pages = plan_pages(_outline()) self.assertEqual(pages[-1].hierarchy_path, ["6.1 Protocols", "6.1.2 Lupus nephritis"]) diff --git a/wikify/tests/test_sectionize.py b/wikify/tests/test_sectionize.py index f1bad61..601c558 100644 --- a/wikify/tests/test_sectionize.py +++ b/wikify/tests/test_sectionize.py @@ -919,6 +919,80 @@ def test_clean_pages_lays_contents_lines_out_as_the_contents_table_they_continue ) self.assertEqual(cleaned[3], "") + def _breadcrumb_pages(self): + topics = [ + ("Administering Medicines", "Five rights"), + ("Administering Medicines", "IV administration"), + ("Controlled Drugs", "Storage and keys"), + ("Controlled Drugs", "Register entries"), + ] + pages = [] + for index, (chapter, topic) in enumerate(topics): + opening = 2 * index + 1 + pages.append( + ( + opening, + f"Repeat Prescribing Protocol\n\nPage {opening}\n\n{chapter} > {topic}\n\n" + f"# {topic}\n\nPractice for {topic.lower()}.", + ) + ) + pages.append((opening + 1, f"# {topic} (continued)\n\nMore on {topic.lower()}.")) + return pages + + def test_clean_pages_strips_a_page_label_at_the_page_edge(self): + cleaned = dict(clean_pages(self._breadcrumb_pages())) + self.assertNotIn("Page 1", cleaned[1]) + self.assertNotIn("Page 5", cleaned[5]) + + def test_a_breadcrumb_running_header_nests_the_section_under_its_chapter(self): + secs = sectionize(clean_pages(self._breadcrumb_pages())) + paths = {s.title: s.hierarchy_path for s in secs} + self.assertEqual(paths["Five rights"], ["Administering Medicines", "Five rights"]) + self.assertEqual(paths["IV administration"], ["Administering Medicines", "IV administration"]) + self.assertEqual(paths["Register entries"], ["Controlled Drugs", "Register entries"]) + self.assertEqual([s.title for s in secs].count("Administering Medicines"), 1) + + def test_a_breadcrumb_needs_a_matching_heading_below_it(self): + pages = [ + (1, "Open Settings > Users to add staff.\n\n# Users\n\nbody"), + (2, "Open Settings > Roles to add roles.\n\n# Roles\n\nbody"), + ] + cleaned = dict(clean_pages(pages)) + self.assertIn("Open Settings > Users to add staff.", cleaned[1]) + + def test_a_continued_heading_folds_into_the_open_section(self): + secs = sectionize(clean_pages(self._breadcrumb_pages())) + titles = [s.title for s in secs] + self.assertFalse(any("continued" in title for title in titles)) + five_rights = next(s for s in secs if s.title == "Five rights") + self.assertIn("More on five rights.", five_rights.markdown) + self.assertEqual((five_rights.page_start, five_rights.page_end), (1, 2)) + + def test_a_continued_from_previous_page_heading_is_not_a_section(self): + pages = [ + (1, "# Abbreviations\n\nIV: intravenous"), + (2, "# Continued from the previous page.\n\nPO: oral"), + ] + secs = sectionize(pages) + self.assertEqual([s.title for s in secs], ["Abbreviations"]) + self.assertIn("PO: oral", secs[0].markdown) + + def test_a_numbered_heading_after_bare_step_numbers_is_a_step_not_a_chapter(self): + pages = [ + ( + 2, + "# The Method\n\nIntro.\n\n### 1\n\n## Build a fresh bench\n\nNever upgrade in place.\n\n" + "### 2\n\n## Restore a backup\n\nAlways restore fresh.\n\n## 3 Run bench migrate\n\n" + "Expect failures.\n\n4\n\n## Read the whole log\n\nA clean exit is not enough.", + ), + (3, "# Preparing the New Bench\n\nSet up the environment.\n\n## The Apps\n\nList the apps."), + ] + secs = sectionize(pages) + paths = {s.title: s.hierarchy_path for s in secs} + self.assertEqual(paths["3 Run bench migrate"], ["The Method", "3 Run bench migrate"]) + self.assertEqual(paths["Preparing the New Bench"], ["Preparing the New Bench"]) + self.assertEqual(paths["The Apps"], ["Preparing the New Bench", "The Apps"]) + class TestEmptySectionsAreFlagged(FrappeTestCase): def test_section_without_markdown_is_chunked_as_title_only(self): From 1d28edbf6179ba55a89e21fe38ac94a467763271 Mon Sep 17 00:00:00 2001 From: vishwajeet-13 Date: Sun, 11 Oct 2026 02:38:49 +0530 Subject: [PATCH 2/2] test(sectionize): avoid implicit string concat inside list Co-Authored-By: Claude Opus 5.5 --- wikify/tests/test_sectionize.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wikify/tests/test_sectionize.py b/wikify/tests/test_sectionize.py index 601c558..c8f8e97 100644 --- a/wikify/tests/test_sectionize.py +++ b/wikify/tests/test_sectionize.py @@ -978,13 +978,13 @@ def test_a_continued_from_previous_page_heading_is_not_a_section(self): self.assertIn("PO: oral", secs[0].markdown) def test_a_numbered_heading_after_bare_step_numbers_is_a_step_not_a_chapter(self): + method = ( + "# The Method\n\nIntro.\n\n### 1\n\n## Build a fresh bench\n\nNever upgrade in place.\n\n" + "### 2\n\n## Restore a backup\n\nAlways restore fresh.\n\n## 3 Run bench migrate\n\n" + "Expect failures.\n\n4\n\n## Read the whole log\n\nA clean exit is not enough." + ) pages = [ - ( - 2, - "# The Method\n\nIntro.\n\n### 1\n\n## Build a fresh bench\n\nNever upgrade in place.\n\n" - "### 2\n\n## Restore a backup\n\nAlways restore fresh.\n\n## 3 Run bench migrate\n\n" - "Expect failures.\n\n4\n\n## Read the whole log\n\nA clean exit is not enough.", - ), + (2, method), (3, "# Preparing the New Bench\n\nSet up the environment.\n\n## The Apps\n\nList the apps."), ] secs = sectionize(pages)