diff --git a/build_jurisprudence_shards.py b/build_jurisprudence_shards.py index 8d579073..3093bf79 100644 --- a/build_jurisprudence_shards.py +++ b/build_jurisprudence_shards.py @@ -27,9 +27,11 @@ import json import re from collections import Counter, defaultdict -from datetime import datetime, timezone +from datetime import date, datetime, timezone from pathlib import Path +import jur_paragraph_ids as pids + ROOT = Path(__file__).resolve().parent DEFAULT_INFO = ROOT / 'mysite_pythonanywhere' / 'jurisprudence_info.json' @@ -71,6 +73,26 @@ def paragraph_number(raw: str): return s +MONTH_NUMBERS = {m: i for i, m in enumerate( + ('jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'), 1)} + + +def iso_date(raw: str | None) -> str | None: + """'6 Apr 2018' / '17 March 2025' / '2018-04-06' -> '2018-04-06'; else None.""" + s = re.sub(r'\s+', ' ', (raw or '').strip()) + m = re.fullmatch(r'(\d{4})-(\d{2})-(\d{2})', s) + if m: + return s + m = re.fullmatch(r'(\d{1,2})\.? ([A-Za-z]{3,9})\.?,? (\d{4})', s) + if not m or m.group(2)[:3].lower() not in MONTH_NUMBERS: + return None + day, month, year = int(m.group(1)), MONTH_NUMBERS[m.group(2)[:3].lower()], int(m.group(3)) + try: + return date(year, month, day).isoformat() + except ValueError: + return None + + def compact_document(doc: dict) -> dict: """Keep Tier-1 fields useful to the browser; drop body-only bookkeeping.""" keys = [ @@ -100,8 +122,29 @@ def compact_document(doc: dict) -> dict: 'jurisSubstantiveArticles', 'jurisProceduralArticles', 'jurisDownloads', 'jurisLastCheckedAt', 'firstAddedAt', 'lastVerifiedAt', + 'outcomeFlags', 'outcomeOperativeIds', + # docIds of records merged into this one; old links resolve through them. + 'alternativeIds', ] - return {k: doc[k] for k in keys if k in doc and doc[k] not in (None, '', [])} + out = {k: doc[k] for k in keys if k in doc and doc[k] not in (None, '', [])} + out.update(iso_dates(doc)) + return out + + +def iso_dates(doc: dict) -> dict: + """ISO 8601 twins of the free-text dates ("06 Apr 2018", "17 March 2025"). + + Derived here, on every build, so they cannot be lost the way the earlier + one-off normalisation of documents.json was. + """ + out = {} + adopted = iso_date(doc.get('jurisDecisionDate')) or iso_date(doc.get('adoptionDate')) + if adopted: + out['adoptionDateIso'] = adopted + submitted = iso_date(doc.get('submittedDate')) + if submitted: + out['submittedDateIso'] = submitted + return out def is_placeholder_title(value: str | None) -> bool: @@ -152,9 +195,10 @@ def lite_document(doc: dict) -> dict: 'jurisCountry', 'jurisSubmissionDate', 'jurisDecisionDate', 'jurisSubstantiveIssues', 'jurisProceduralIssues', 'jurisSubstantiveArticles', 'jurisProceduralArticles', - 'firstAddedAt', 'lastVerifiedAt', + 'firstAddedAt', 'lastVerifiedAt', 'alternativeIds', ] out = {k: doc[k] for k in keys if k in doc and doc[k] not in (None, '', [])} + out.update(iso_dates(doc)) title = public_title(doc) out['name'] = title out['nameShort'] = title @@ -268,7 +312,7 @@ def load_paragraphs(doc: dict) -> list[dict]: original_id = item.get('ID') original_source_id = item.get('OriginalID') rows.append({ - 'id': f'{doc["docId"]}-{idx:04d}', + 'id': None, # label-based, set below 'docId': doc['docId'], 'idx': idx, 'n': paragraph_number(original_id), @@ -293,9 +337,47 @@ def load_paragraphs(doc: dict) -> list[dict]: 'year': doc.get('year'), 'outcome': doc.get('outcome'), }) + for row, para_id in zip(rows, pids.label_ids(doc['docId'], [r['n'] for r in rows])): + row['id'] = para_id return rows +def read_previous(shard_dir: Path, doc_ids: set[str]) -> dict[str, list[dict]]: + """The previous build's paragraphs ({id, text}) of the documents being rebuilt.""" + previous: dict[str, list[dict]] = defaultdict(list) + for path in sorted(shard_dir.glob('*.json')) if shard_dir.exists() else []: + for p in read_json(path).get('paragraphs', []): + if p.get('docId') in doc_ids: + previous[p['docId']].append({'id': p['id'], 'text': p.get('text') or '', 'idx': p.get('idx') or 0}) + for paras in previous.values(): + paras.sort(key=lambda p: p['idx']) + return previous + + +def build_redirects(docs: list[dict], by_doc: dict[str, list[dict]], previous: dict[str, list[dict]], + redirects_path: Path, decisions: dict) -> tuple[dict, list[dict], list[str]]: + """Align the previous build to this one and update the cumulative map.""" + update: dict = {} + review: list[dict] = [] + for doc in docs: + old = previous.get(doc['docId']) + if not old: + continue + new = [{'id': p['id'], 'text': p['text']} for p in by_doc.get(doc['docId'], [])] + doc_redirects, doc_review = pids.align(old, new) + update.update(doc_redirects) + for item in doc_review: + item['docId'] = doc['docId'] + review.extend(doc_review) + update = pids.apply_decisions(update, decisions) + reviewed = set(decisions) + review = [item for item in review if item['oldId'] not in reviewed] + existing = read_json(redirects_path).get('redirects', {}) if redirects_path.exists() else {} + live_ids = {p['id'] for paras in by_doc.values() for p in paras} + redirects, dangling = pids.merge_redirects(existing, update, live_ids, set(by_doc)) + return redirects, review, dangling + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument('--info', type=Path, default=DEFAULT_INFO) @@ -303,6 +385,12 @@ def main() -> int: ap.add_argument('--treaty', help='Restrict to one treaty body, e.g. CRPD') ap.add_argument('--all', action='store_true', help='Build every treaty present in jurisprudence_info.json') ap.add_argument('--pretty', action='store_true', help='Pretty-print shard JSON for inspection') + ap.add_argument('--previous', type=Path, + help='Shards of the previous build to redirect from (default: OUT/shards, read before overwriting)') + ap.add_argument('--decisions', type=Path, + help='Reviewed redirects, {oldId: newId or null}, overriding provisional matches') + ap.add_argument('--review-out', type=Path, default=ROOT / 'jur_paragraph_redirect_review.json', + help='Where to write redirects the text alignment could not settle') args = ap.parse_args() if not args.all and not args.treaty: @@ -317,6 +405,7 @@ def main() -> int: shard_paragraphs: dict[str, list[dict]] = defaultdict(list) shard_docs: dict[str, list[str]] = defaultdict(list) all_paragraphs = [] + by_doc: dict[str, list[dict]] = {} diagnostics = [] for doc in docs: @@ -331,12 +420,23 @@ def main() -> int: doc['wordCount'] = sum(len(p['text'].split()) for p in paragraphs) doc['labelCount'] = sum(len(p['labels']) for p in paragraphs) doc['articlesCited'] = extract_articles_cited(paragraphs) + by_doc[doc['docId']] = paragraphs shard_docs[shard_id].append(doc['docId']) shard_paragraphs[shard_id].extend(paragraphs) all_paragraphs.extend(paragraphs) out = args.out shard_out = out / 'shards' + previous = read_previous(args.previous or shard_out, set(by_doc)) + decisions = read_json(args.decisions) if args.decisions else {} + redirects_path = out / 'paragraph-redirects.json' + redirects, review, dangling = build_redirects(docs, by_doc, previous, redirects_path, decisions) + legacy = pids.legacy_index(redirects) + for p in all_paragraphs: + if p['id'] in legacy: + p['legacyIds'] = legacy[p['id']] + if dangling: + diagnostics.append(f'{len(dangling)} redirects point at ids that no longer exist, e.g. {dangling[:3]}') if shard_out.exists(): for old_shard in shard_out.glob('*.json'): old_shard.unlink() @@ -345,6 +445,13 @@ def main() -> int: write_json(out / 'documents-lite.json', docs_lite, pretty=args.pretty) facets = build_facets(docs, all_paragraphs) write_json(out / 'facets.json', facets, pretty=True) + write_json(redirects_path, { + 'scheme': '{docId}:{label}; a repeated label gets ~2, ~3; unlabelled records u1, u2', + 'note': 'Every jurisprudence paragraph id ever published -> its current id; null: the old record was not a paragraph of the decision.', + 'count': len(redirects), + 'redirects': redirects, + }) + write_json(args.review_out, review, pretty=True) built_at = datetime.now(timezone.utc).replace(microsecond=0).isoformat() shard_files = {} @@ -393,11 +500,16 @@ def main() -> int: 'sha': sha256_file(out / 'facets.json'), 'bytes': (out / 'facets.json').stat().st_size, }, + 'paragraph-redirects.json': { + 'sha': sha256_file(redirects_path), + 'bytes': redirects_path.stat().st_size, + 'redirects': len(redirects), + }, **shard_files, }, 'schema': { 'document': ['docId', 'type', 'treaty', 'symbol', 'country', 'year', 'communicationYear?', 'adoptionYear?', 'title', 'outcome', 'adoptionDate?', 'languages', 'link', 'sourceFile', 'sourceFormat', 'shardId', 'paragraphCount', 'wordCount', 'labelCount', 'caseLabels'], - 'paragraph': ['id', 'docId', 'idx', 'n', 'paragraphId', 'originalParagraphId?', 'rawParagraphId?', 'idCorrection?', 'generatedParagraphId?', 'generatedIdReason?', 'namespace?', 'section', 'text', 'footnotes?', 'labels', 'sourceFormat?', 'ocrStatus?', 'ocrMeanConf?', 'ocrLowConfRatio?', 'type', 'treaty', 'country', 'year', 'outcome'], + 'paragraph': ['id', 'legacyIds?', 'docId', 'idx', 'n', 'paragraphId', 'originalParagraphId?', 'rawParagraphId?', 'idCorrection?', 'generatedParagraphId?', 'generatedIdReason?', 'namespace?', 'section', 'text', 'footnotes?', 'labels', 'sourceFormat?', 'ocrStatus?', 'ocrMeanConf?', 'ocrLowConfRatio?', 'type', 'treaty', 'country', 'year', 'outcome'], }, 'diagnostics': diagnostics, } @@ -407,6 +519,7 @@ def main() -> int: print(f' documents: {len(docs)}') print(f' paragraphs: {len(all_paragraphs)}') print(f' shards: {len(shard_files)}') + print(f' redirects: {len(redirects)} ({len(review)} to review -> {args.review_out})') print(f' out: {out}') if diagnostics: print(f' diagnostics: {len(diagnostics)}') diff --git a/clean_extract.py b/clean_extract.py index 3a12e6c0..ecbe2d62 100644 --- a/clean_extract.py +++ b/clean_extract.py @@ -46,9 +46,25 @@ def _is_separator_block(text: str) -> bool: # "10 Ibid., para. 11."). The "N text" pattern with two spaces is the # fingerprint we use when no underscore separator is present. _FOOTNOTE_LEAD = re.compile(r'^\s*\d{1,3}\s+[A-Z]') +_DECIMAL_PARA_LEAD = re.compile(r'^\s*\d{1,3}[.~]\d{1,2}\b') -def _clean_page_text(page: fitz.Page) -> str: +def _is_body_block_in_margin(y0: float, y1: float, text: str) -> bool: + """A margin-band block that is body text, not a running header or footer. + + PyMuPDF blocks are whole paragraphs, so a paragraph that starts near the + top of a page (y0 < TOP_MARGIN_Y) or a short one at the very bottom + (y0 > BOTTOM_MARGIN_Y) would otherwise be dropped whole. Older and + re-typeset jurisprudence PDFs put body text there (CCPR/C/50/D/488/1992 + paras. 5.1 and 8.2, CCPR/C/18/D/74/1980 paras. 1.3 and 9.2). + """ + if _DECIMAL_PARA_LEAD.match(text): + return True + lines = [l for l in text.splitlines() if l.strip()] + return y0 < TOP_MARGIN_Y and y1 > TOP_MARGIN_Y + 15 and len(lines) >= 2 + + +def _clean_page_text(page: fitz.Page, keep_body_in_margins: bool = False, blocks: list | None = None) -> str: """Return the page's body text with headers, footers and footnotes stripped. Detection strategy (combined — first applicable rule wins): @@ -61,15 +77,22 @@ def _clean_page_text(page: fitz.Page) -> str: x-indent for this page; any block whose x0 differs and whose first line matches the "N " footnote-marker pattern → treat as footnote AND drop every following block. + + With keep_body_in_margins, rule A spares margin-band blocks that look like + body text (see _is_body_block_in_margin). Jurisprudence ingestion uses it, + and passes its own `blocks` (page.get_text("blocks") tuples whose text + carries footnote markers); the GC/SP callers keep the original behaviour. """ - blocks = page.get_text("blocks") + if blocks is None: + blocks = page.get_text("blocks") # First pass: filter by y-band, keep raw block tuples. in_band = [] for b in blocks: x0, y0, x1, y1, text, *_ = b if y0 < TOP_MARGIN_Y or y0 > BOTTOM_MARGIN_Y: - continue + if not (keep_body_in_margins and text and _is_body_block_in_margin(y0, y1, text)): + continue if not (text and text.strip()): continue in_band.append((y0, x0, text)) diff --git a/docs/assets/app.css b/docs/assets/app.css index 84a1ef7c..1cbd1ae3 100644 --- a/docs/assets/app.css +++ b/docs/assets/app.css @@ -5658,6 +5658,24 @@ body.is-corpus-loading #q { cursor: progress; } outline: 0; } +/* v19.81: notes whose reference OCR garbled, listed under the paragraph. */ +.fn-unanchored { + margin-top: 6px; + padding-top: 4px; + border-top: 1px dashed var(--rule-soft); + font-size: 0.85em; + color: var(--ink-3); +} +/* In the reader the paragraph is a two-column grid (gutter | text): keep + the list under the text, not in the gutter. */ +.docs-reader-para > .fn-unanchored { grid-column: 2; margin-top: -10px; } +.fn-unanchored-label { + font-family: var(--mono); + font-size: 10px; + letter-spacing: 0.04em; + margin-right: 4px; +} + .fn-popover { position: absolute; z-index: 9000; diff --git a/docs/assets/app.js b/docs/assets/app.js index 80264904..e30ede38 100644 --- a/docs/assets/app.js +++ b/docs/assets/app.js @@ -12,6 +12,9 @@ const state = { manifest: null, paragraphs: [], paragraphById: new Map(), // id → paragraph, for O(1) FlexSearch hydration + paraIdAlias: new Map(), // v19.80: superseded JUR paragraph id → canonical id + // (null = record removed), from shard legacyIds and + // /api/paragraph redirects — see resolveParagraphId documents: new Map(), facets: null, baseFacets: null, @@ -244,6 +247,7 @@ async function _apiGet(base, path, params, timeoutMs) { const error = new Error(message); error.status = res.status; error.detail = detail; + error.payload = payload; // v19.80: /api/paragraph 404s carry {doc_id, removed} throw error; } return await res.json(); @@ -474,6 +478,7 @@ async function fetchSearchPage(params) { // shape has to satisfy paintDossier + renderResult + workspace marks // without further conditional logic in those code paths. function adaptApiHit(h) { + _registerLegacyParaIds({ id: h.para_id, legacyIds: h.legacy_ids }); // v19.80, when the server sends them const year = documentResearchYear({ type: h.type, year: h.year, @@ -584,7 +589,11 @@ const URL_KEYS = { q: 'q', scope: 'scope', tb: 'tb', g: 'g', gm: 'gm', y1: 'y1', function documentIdForParagraphId(paraId) { if (!paraId) return null; const id = String(paraId); - const positional = id.replace(/-\d{4}$/, ''); + // v19.80: JUR paragraph ids are label-based, ":