diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d45e71a..f6ba889a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project are documented in this file. +## Unreleased + +### Added +- **`parquet` is now a first-class source kind.** Table sections accept `source: {kind: parquet, local: ..., url: [...]}`: the file is scanned lazily with polars and its columns are renamed to the positional `column_1`..`column_n` convention in schema order, exactly what the headerless `text`/`excel` readers produce, so `method: column` encodings keep addressing columns with Excel-style letters (`A` is the first column). Parquet is self-describing (typed columns, real headers, no sheets or delimiters), so the source adds no format-specific fields beyond the shared `rows`/`row_slice`/`reindex` controls. The autonomous agent accepts `.parquet` payloads as tables too: fetch bucketing, previews, and row counts all dispatch on the extension. + ## 19.4.0 - 2026-09-29 ### Added diff --git a/docs/configuration/table.md b/docs/configuration/table.md index 30abb8b1..e2b304ae 100644 --- a/docs/configuration/table.md +++ b/docs/configuration/table.md @@ -107,7 +107,7 @@ source: row_slice: [1, auto] # Start at the second physical row, read to end ``` -> **Specify `kind` explicitly.** Tablassert selects the reader purely from the declared `kind`: `excel` reads a workbook (`sheet`), `text` scans delimited text (`delimiter`); the file on disk is never inspected to infer its format. Because `kind` carries a default, a source whose `kind` is omitted or does not match the actual file is still accepted and fed to the wrong reader, surfacing only later as a read error or garbled rows. Stating `kind` explicitly makes a mis-declared source fail fast. +> **Specify `kind` explicitly.** Tablassert selects the reader purely from the declared `kind`: `excel` reads a workbook (`sheet`), `text` scans delimited text (`delimiter`), `parquet` scans a parquet file (columns renamed positionally); the file on disk is never inspected to infer its format. Because `kind` carries a default, a source whose `kind` is omitted or does not match the actual file is still accepted and fed to the wrong reader, surfacing only later as a read error or garbled rows. Stating `kind` explicitly makes a mis-declared source fail fast. #### Text Source (CSV/TSV) @@ -132,6 +132,35 @@ source: row_slice: [1, auto] ``` +#### Parquet Source + +Reads a columnar `.parquet` file. Parquet is self-describing (typed columns, real headers, no +sheets or delimiters), so the source adds no format-specific fields. Columns are addressed +positionally: the reader renames the file's columns to `column_1`..`column_n` in schema order +(the same convention the headerless `text`/`excel` readers produce), so `method: column` +encodings keep using Excel-style letters (`A` is the first column, `B` the second, and so on) +and the file's own header names are never referenced from configs. Column types are read +as-is. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `kind` | String | No | Source kind. Model default is `"parquet"`, but specify it explicitly in configs. | +| `local` | Path | Yes | Local file path the source is read from. The file must already exist here; Tablassert does not download it. | +| `url` | List[URL] | Yes | One or more source URLs recorded as provenance (emitted as the primary `sources` entry's `source_record_urls` list and in the RIG; when `provenance.override.upstream_source_record_urls` is set, RIG only; the per-upstream mapping determines edge placement). At least one URL is required; supply multiple to back a single section with several links. Format-validated only; not fetched. | +| `row_slice` | List[PositiveInt\|"auto"] | No | Two-value zero-based crop bounds: `[start, stop]`. Each value may be a positive integer or `"auto"`. Mutually exclusive with `rows`. | +| `rows` | List[PositiveInt] | No | Zero-based row indices to keep after any `row_slice` crop. Mutually exclusive with `row_slice`. | +| `reindex` | List[Reindex] | No | Conditional row filtering | + +**Example:** +```yaml +source: + kind: parquet + local: ./data/mydata.parquet + url: + - https://example.com/data.parquet + row_slice: [1, auto] +``` + #### Reindexing (Conditional Filtering) Filter rows based on column values. diff --git a/src/tablassert/agent.py b/src/tablassert/agent.py index 1ecadf66..593b4ec7 100644 --- a/src/tablassert/agent.py +++ b/src/tablassert/agent.py @@ -115,7 +115,7 @@ def is_lazy() -> bool: PMC_BUCKET: str = "pmc-oa-opendata" PMC_HTTPS_BASE: str = "https://pmc-oa-opendata.s3.amazonaws.com" PMC_S3API_BASE: str = "https://pmc-oa-opendata.s3.us-east-1.amazonaws.com" -TABLE_EXTENSIONS: frozenset[str] = frozenset({".xlsx", ".xls", ".csv", ".tsv"}) +TABLE_EXTENSIONS: frozenset[str] = frozenset({".xlsx", ".xls", ".csv", ".tsv", ".parquet"}) #: Default minimum number of non-empty data rows for agent table candidates. #: #: Counts use the same header inference as the production readers, so the header is not included. @@ -662,8 +662,9 @@ def _load_table(path: Path, sheet: str | None = None) -> pl.DataFrame: """Dispatch a local table file to the right polars reader by suffix. ``.csv`` -> ``read_csv``; ``.tsv``/``.txt`` -> ``read_csv(separator="\\t")``; - ``.xlsx``/``.xls`` -> :func:`_read_excel` (``sheet`` selects a worksheet by name; - ignored for csv/tsv). Any polars parse failure becomes a clear ``ValueError``; an + ``.parquet`` -> ``read_parquet`` (typed columns read as-is); ``.xlsx``/``.xls`` -> + :func:`_read_excel` (``sheet`` selects a worksheet by name; ignored for + csv/tsv/parquet). Any polars parse failure becomes a clear ``ValueError``; an unknown suffix is a ``ValueError`` too (never a silent mis-read). """ suffix: str = path.suffix.lower() @@ -674,6 +675,8 @@ def _load_table(path: Path, sheet: str | None = None) -> pl.DataFrame: return pl.read_csv(path) if suffix in {".tsv", ".txt"}: return pl.read_csv(path, separator="\t") + if suffix == ".parquet": + return pl.read_parquet(path) except Exception as e: raise ValueError(f"Could not read table {path}: {e}") from e raise ValueError(f"Could not read table {path}: unsupported extension {suffix!r}") @@ -685,8 +688,9 @@ def _effective_rows_cached(path: str, mtime_ns: int, size: int, sheet: str | Non ``path``, ``mtime_ns``, and ``size`` form the cache key so a re-fetched or rewritten file cannot reuse a stale count. Delimited files are counted by the parser rather than physical - lines, which handles quoted embedded newlines correctly. Excel uses the same reader as - :func:`_load_table` and removes rows that are null in every column (formatted blank rows). + lines, which handles quoted embedded newlines correctly. Excel and parquet use the same + reader as :func:`_load_table` and remove rows that are null in every column (formatted + blank rows). """ table_path: Path = Path(path) suffix: str = table_path.suffix.lower() @@ -694,6 +698,10 @@ def _effective_rows_cached(path: str, mtime_ns: int, size: int, sheet: str | Non frame: pl.DataFrame = _read_excel(table_path, sheet) return frame.filter(~pl.all_horizontal(pl.all().is_null())).height + if suffix == ".parquet": + pq: pl.DataFrame = pl.read_parquet(table_path) + return pq.filter(~pl.all_horizontal(pl.all().is_null())).height + if suffix == ".csv": separator: str = "," elif suffix in {".tsv", ".txt"}: @@ -1233,7 +1241,7 @@ def visit(value: object) -> None: # hand-maintained guess table), so a model default change automatically changes # what counts as removable. Semantic guards: the ``provenance`` subtree is never # touched (legal attribution), ``kind`` is never dropped (it discriminates the -# ``Excel | Text`` source union), a non-default null such as ``taxon: null`` +# ``Excel | Text | Parquet`` source union), a non-default null such as ``taxon: null`` # (default 9606) is preserved, and in ``{template, sections}`` configs a section # entry equal to a model default is only removed when the template cannot change # the merged result (fastmerge lets section scalars override template values, so @@ -1245,7 +1253,7 @@ def visit(value: object) -> None: #: Keys compaction never removes and never recurses into: ``provenance`` values are #: the edge's legal attribution (untouched even when they equal a model default), -#: and ``kind`` discriminates the ``Excel | Text`` source union — dropping it could +#: and ``kind`` discriminates the ``Excel | Text | Parquet`` source union — dropping it could #: flip which model a re-parsed source validates as. _COMPACT_UNTOUCHED_KEYS: frozenset[str] = frozenset({"provenance", "kind"}) diff --git a/src/tablassert/enums.py b/src/tablassert/enums.py index 000ff0bd..362185c4 100644 --- a/src/tablassert/enums.py +++ b/src/tablassert/enums.py @@ -52,6 +52,7 @@ class Functions(str, Enum): class Files(str, Enum): TEXT = "text" EXCEL = "excel" + PARQUET = "parquet" class EncodingMethods(str, Enum): diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index c6501306..ccc33a7e 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -953,6 +953,26 @@ def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame: return df.lazy() +def parquet(p: Path) -> pl.LazyFrame: + """Read a parquet source as a LazyFrame with positional ``column_`` names. + + Args: + p: Path to the parquet file. + + Returns: + LazyFrame whose columns are renamed to ``column_1..`` in schema order. + + Notes: + Parquet carries its own column names, but the config contract addresses columns + positionally (Excel-style ``A-ZZ`` encoding letters resolve to ``column_``), so + the reader renames whatever names the file carries to ``column_1..`` -- the same + convention the headerless ``csv``/``excel`` readers produce. + """ + lf: pl.LazyFrame = pl.scan_parquet(source=p) + names: list[str] = list(lf.collect_schema().names()) + return lf.rename(dict(zip(names, (f"column_{i + 1}" for i in range(len(names))), strict=True))) + + def crop(lf: pl.LazyFrame, row_slice: list[NonNegativeInt | Literal[Tokens.AUTO]]) -> pl.LazyFrame: """Take a contiguous slice from a LazyFrame. @@ -1296,6 +1316,7 @@ def _source_ops(self: Self) -> list[Any]: return [ (csv, (self.source.local, self.source.delimiter)) if self.source.kind == Files.TEXT else None, # pyright: ignore (excel, (self.source.local, self.source.sheet)) if self.source.kind == Files.EXCEL else None, # pyright: ignore + (parquet, (self.source.local,)) if self.source.kind == Files.PARQUET else None, # pyright: ignore (idx, ()), (crop, (self.source.row_slice,)) if self.source.row_slice else None, (pick, (self.source.rows,)) if self.source.rows else None, @@ -1507,6 +1528,7 @@ def collect(self: Self, db: Path) -> list[tuple[Callable, tuple[Any]]] | Path: # Maps each Tcode op callable to a short lowercase phase label for progress UX. csv: "load", excel: "load", + parquet: "load", idx: "load", crop: "filter", pick: "filter", diff --git a/src/tablassert/models.py b/src/tablassert/models.py index 4241eb1e..126fe739 100644 --- a/src/tablassert/models.py +++ b/src/tablassert/models.py @@ -67,7 +67,7 @@ def _shown(name: str, target: str) -> str: return f"`{name}` (coerced to `{target}`)" if target != name else f"`{name}`" -def _section_source_label(source: Excel | Text) -> str: +def _section_source_label(source: Excel | Text | Parquet) -> str: """Render a section's source for a diagnostic message: the local path, plus the sheet for Excel. Args: @@ -174,6 +174,12 @@ class Text(BaseSource): delimiter: str | None = Field(",", description="Field delimiter for headerless text/CSV scanning.", examples=[",", "\t", "|"]) +# Parquet is self-describing (typed columns, real headers, no sheets or delimiters), so unlike +# `Text`/`Excel` it adds no format-specific fields beyond `BaseSource`. +class Parquet(BaseSource): + kind: Literal[Files.PARQUET] = Field(Files.PARQUET, description="Source kind; must be 'parquet'.") + + class Regex(TablaBase): pattern: int | float | str = Field(..., description="Regex pattern passed to string replacement.", examples=["\\s+", "\\.$"]) @@ -791,7 +797,7 @@ def warn_when_the_slot_cannot_reach_the_edge(self) -> Self: class Section(TablaBase): """Pydantic section model and coercion target for a single table configuration.""" - source: Excel | Text = Field(..., description="Input source definition for reading tabular rows.") + source: Excel | Text | Parquet = Field(..., description="Input source definition for reading tabular rows.") statement: Statement = Field(..., description="Subject-object statement mapping for this section.") provenance: Provenance = Field(..., description="Provenance metadata applied to all produced edges.") annotations: list[Annotation] | None = Field(None, description="Optional extra encoded columns added to each row.") diff --git a/src/tablassert/runcache.py b/src/tablassert/runcache.py index 299dfe10..45b98c0f 100644 --- a/src/tablassert/runcache.py +++ b/src/tablassert/runcache.py @@ -252,6 +252,7 @@ def prefix_digest(ops: list[tuple[Callable, tuple[Any, ...]]]) -> str: # to resolution. lib.csv: 2, lib.excel: 2, + lib.parquet: 2, # Filter: single lazy row predicates/slices. lib.idx: 1, lib.crop: 1, diff --git a/tests/test_agent_fetch.py b/tests/test_agent_fetch.py index efa911fa..17a83036 100644 --- a/tests/test_agent_fetch.py +++ b/tests/test_agent_fetch.py @@ -12,6 +12,7 @@ from pathlib import Path +import polars as pl import pytest from tablassert.agent import ( @@ -151,6 +152,7 @@ def test_version_prefixes_from_listing_bad(listing: str) -> None: ("a.xlsx", None, True), ("a.csv", None, True), ("a.tsv", None, True), + ("a.parquet", None, True), ("fig.jpg", "Table 1", False), # DROP extension wins over a "Table" label ("data.bin", "Table S2", True), # label-only match for an unknown extension ("data.bin", None, False), @@ -278,6 +280,18 @@ def test_effective_rows_cache_invalidates_when_file_changes(tmp_path: Path) -> N assert _effective_rows(table) == 2 +def test_effective_rows_counts_parquet_excluding_all_null_rows(tmp_path: Path) -> None: + """A ``.parquet`` table counts data rows minus rows null in every column. + + The agent shows candidates with a row threshold, so a parquet payload must count the same + way the Excel branch does (formatted blank rows are not data). A row that is null in every + column is skipped; partial-null rows still count. + """ + pp: Path = tmp_path / "counts.parquet" + pl.DataFrame({"a": ["x", None, "y", "z"], "b": ["1", None, "2", None]}).write_parquet(pp) + assert _effective_rows(pp) == 3 + + def test_excel_sheet_heights_ignores_blank_rows(tmp_path: Path) -> None: """Worksheet heights use effective rows, not formatted blank trailing rows.""" openpyxl = pytest.importorskip("openpyxl") diff --git a/tests/test_cover_agent_core.py b/tests/test_cover_agent_core.py index 635abeec..65e5eaee 100644 --- a/tests/test_cover_agent_core.py +++ b/tests/test_cover_agent_core.py @@ -18,11 +18,20 @@ from pathlib import Path from typing import Any +import polars as pl import pytest import yaml from tablassert import rs -from tablassert.agent import _count_ndjson_lines, build_and_audit, is_open_access, parse_jats_summary, read_table, supplementary_materials_from_jats +from tablassert.agent import ( + _count_ndjson_lines, + _load_table, + build_and_audit, + is_open_access, + parse_jats_summary, + read_table, + supplementary_materials_from_jats, +) # --------------------------------------------------------------------------- # # JATS parsers: _nearest_label/_nearest_caption (233) + supplementary (277) @@ -156,6 +165,38 @@ def test_read_table_unsupported_suffix_raises_valueerror(tmp_path: Path) -> None read_table(path) +def test_read_table_reads_parquet_payload(tmp_path: Path) -> None: + """``read_table`` renders a ``.parquet`` payload through ``_load_table`` -> ``read_parquet``. + + Parquet payloads downloaded from supplements must reach the agent context like csv/tsv do: + the suffix dispatches to ``pl.read_parquet`` (typed columns as-is), the preview renders the + cells, and the shape line reports the real height/width. No ``sheets:`` note appears -- + that annotation is Excel-only and parquet has no worksheets. + """ + path: Path = tmp_path / "table.parquet" + pl.DataFrame({"gene": ["brca1", "mapk1"], "score": [0.5, 0.9]}).write_parquet(path) + + rendered: str = read_table(path) + assert "shape: 2x2" in rendered + assert "brca1" in rendered + assert "mapk1" in rendered + assert "sheets:" not in rendered + + +def test_load_table_reads_parquet_directly(tmp_path: Path) -> None: + """``_load_table`` returns the parquet frame as-is (real names, no positional remap). + + Unlike the pipeline reader (``lib.parquet``, which renames to ``column_1..`` for the + positional config contract), the agent reads tables to SHOW them, so the frame keeps the + file's own column names; only the rendered preview reaches the model. + """ + path: Path = tmp_path / "table.parquet" + pl.DataFrame({"gene": ["brca1"]}).write_parquet(path) + frame: pl.DataFrame = _load_table(path) + assert frame.columns == ["gene"] + assert frame["gene"].to_list() == ["brca1"] + + # --------------------------------------------------------------------------- # # _count_ndjson_lines: absent file -> 0 (939) # --------------------------------------------------------------------------- # diff --git a/tests/test_cover_lib.py b/tests/test_cover_lib.py index 903d6136..ee60db8c 100644 --- a/tests/test_cover_lib.py +++ b/tests/test_cover_lib.py @@ -118,6 +118,25 @@ def test_excel_reads_sheet_without_header(tmp_path: Path) -> None: assert df.rows() == [("a", "b"), ("c", "d")] +def test_parquet_reads_schema_and_renames_columns_positionally(tmp_path: Path) -> None: + """``parquet`` scans lazily and maps the file's real names to positional ``column_``. + + The config contract addresses columns positionally (``method: column`` encodings accept + only Excel-style ``A-ZZ`` letters, resolved to ``column_``), so the reader must rename + whatever names the parquet carries to ``column_1..`` in schema order -- the same convention + the headerless ``csv``/``excel`` readers produce -- and stay lazy for ``Tcode`` composition. + """ + pp: Path = tmp_path / "data.parquet" + pl.DataFrame({"gene_a": ["brca1", "brca1"], "gene_b": ["mapk1", "mapk1"], "p": [0.5, 0.01]}).write_parquet(pp) + + lf: pl.LazyFrame = lib.parquet(pp) + assert isinstance(lf, pl.LazyFrame) + df: pl.DataFrame = lf.collect() + assert df.columns == ["column_1", "column_2", "column_3"] + assert df["column_1"].to_list() == ["brca1", "brca1"] + assert df["column_3"].to_list() == [0.5, 0.01] + + def test_crop_slices_contiguous_range() -> None: """Lines 435-442: ``crop`` resolves integer ``[start, stop]`` bounds to a slice.""" lf: pl.LazyFrame = pl.LazyFrame({"c": [1, 2, 3, 4]}) diff --git a/tests/test_cover_models.py b/tests/test_cover_models.py index 890de0d0..2f9bc047 100644 --- a/tests/test_cover_models.py +++ b/tests/test_cover_models.py @@ -13,10 +13,10 @@ from typing import Any import pytest -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError from tablassert.biolink import _build_str_enum, _category_name -from tablassert.models import Graph, ManualProvenance +from tablassert.models import Excel, Graph, ManualProvenance, Parquet, Section, Text, _section_source_label def test_manual_provenance_explicit_none_publications_short_circuits() -> None: @@ -69,3 +69,51 @@ class NoCategory(BaseModel): x: int = 1 assert _category_name(NoCategory) == "NoCategory" + + +def test_parquet_source_parses_and_forbids_extras() -> None: + """The ``Parquet`` source parses a minimal config and rejects format-specific fields. + + Parquet is self-describing (typed columns, real headers, no sheets or delimiters), so unlike + ``Excel`` (``sheet``) and ``Text`` (``delimiter``) it adds no fields beyond ``BaseSource``; + ``TablaBase``'s ``extra="forbid"`` must therefore reject an ``Excel``-style ``sheet`` key + instead of silently accepting (and ignoring) it. + """ + source: Parquet = Parquet.model_validate({"kind": "parquet", "local": "./data.parquet", "url": ["https://example.com/data.parquet"]}) + assert source.kind == "parquet" + assert not hasattr(source, "sheet") + assert not hasattr(source, "delimiter") + with pytest.raises(ValidationError, match="extra_forbidden"): + Parquet.model_validate({"kind": "parquet", "local": "./data.parquet", "url": ["https://example.com/data.parquet"], "sheet": "Sheet1"}) + + +def test_section_accepts_all_three_source_kinds() -> None: + """``Section.source`` validates ``kind: parquet`` alongside the existing ``text``/``excel``. + + The union is pydantic's smart union on the ``kind`` literal, so adding a third member must + keep every existing kind parsing (no union ambiguity) and resolve each config to the model + its ``kind`` names. This is the config-author-facing contract of the new source kind. + """ + base: dict[str, Any] = { + "statement": {"subject": {"method": "value", "encoding": "BRCA1"}, "object": {"method": "value", "encoding": "TP53"}}, + "provenance": {"repo": "PMC", "publication": "PMC0000000"}, + } + for kind, model in [("text", Text), ("excel", Excel), ("parquet", Parquet)]: + payload: dict[str, Any] = {**base, "source": {"kind": kind, "local": f"./table.{kind}", "url": [f"https://example.com/table.{kind}"]}} + section: Section = Section.model_validate(payload) + assert section.source.kind == kind + assert type(section.source) is model + + +def test_section_source_label_per_kind() -> None: + """The diagnostic label appends the sheet only for Excel; text and parquet render the bare path. + + ``_section_source_label`` feeds read-failure diagnostics, so a new kind must render without + hitting the Excel-only ``source.sheet`` access (parquet has no ``sheet`` attribute at all). + """ + urls: list[Any] = ["https://example.com/table"] + # `local` is a Path, so a leading "./" normalizes away in the rendered label. + assert _section_source_label(Text.model_validate({"kind": "text", "local": "./t.tsv", "url": urls})) == "`t.tsv`" + assert _section_source_label(Parquet.model_validate({"kind": "parquet", "local": "./t.parquet", "url": urls})) == "`t.parquet`" + excel: Excel = Excel.model_validate({"kind": "excel", "local": "./t.xlsx", "url": urls, "sheet": "S1"}) + assert _section_source_label(excel) == "`t.xlsx` (sheet `S1`)" diff --git a/tests/test_docs_source_of_truth.py b/tests/test_docs_source_of_truth.py index 6996546c..ad68459e 100644 --- a/tests/test_docs_source_of_truth.py +++ b/tests/test_docs_source_of_truth.py @@ -80,6 +80,7 @@ Graph, ManualProvenance, NodeEncoding, + Parquet, Provenance, Qualifier, Reindex, @@ -585,6 +586,7 @@ def location(self) -> str: CONFIG_MODEL_BINDINGS: tuple[ModelTableBinding, ...] = ( ModelTableBinding(Excel, TABLE_CONFIGURATION, ("Excel Source",)), ModelTableBinding(Text, TABLE_CONFIGURATION, ("Text Source (CSV/TSV)",)), + ModelTableBinding(Parquet, TABLE_CONFIGURATION, ("Parquet Source",)), ModelTableBinding(Reindex, TABLE_CONFIGURATION, ("Reindexing (Conditional Filtering)",)), ModelTableBinding(Statement, TABLE_CONFIGURATION, ("Statement (Triple Definition)",)), ModelTableBinding(NodeEncoding, TABLE_CONFIGURATION, ("NodeEncoding",)), diff --git a/tests/test_e2e_smoke.py b/tests/test_e2e_smoke.py index 3c1df737..fd7eec61 100644 --- a/tests/test_e2e_smoke.py +++ b/tests/test_e2e_smoke.py @@ -18,6 +18,7 @@ from pathlib import Path from typing import Any +import polars as pl import pytest from tablassert import rs @@ -112,6 +113,69 @@ def test_build_pipeline_against_real_redb(tmp_path: Path, monkeypatch: pytest.Mo assert "HGNC:6871" in edge_text +def test_build_pipeline_over_parquet_source(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rig_factory: Any) -> None: + """SMOKE (iii): the same six-stage build reads a ``kind: parquet`` source end to end. + + The strongest possible proof of the new source kind: the declared ``kind`` must select the + parquet reader, its positional ``column_1..`` convention must drive the column encodings + (the same ``A``/``B`` letters smoke (i) uses on text), and the rows must reach the real + redb and come back as resolved CURIEs in the KGX output. Mirrors smoke (i) exactly except + for the source kind, so any parquet-specific break shows up as a diff against (i). + """ + monkeypatch.chdir(tmp_path) + (tmp_path / ".tablassert" / "store").mkdir(parents=True) + + fullmap: Path = _build_real_redb(tmp_path / "fullmap") + + # Two-row parquet source: first column = subject term, second column = object term. Columns + # are addressed positionally (encoding letters A/B), the same contract as text/excel sources. + data: Path = tmp_path / "data.parquet" + pl.DataFrame({"gene_a": ["brca1", "brca1"], "gene_b": ["mapk1", "mapk1"]}).write_parquet(data) + + table: Path = tmp_path / "table.yaml" + table_config: dict[str, Any] = { + "template": { + "source": {"kind": "parquet", "local": str(data), "url": ["https://example.com/data.parquet"]}, + "statement": { + "subject": {"method": "column", "encoding": "A"}, + "predicate": "associated_with", + "object": {"method": "column", "encoding": "B"}, + }, + "provenance": {"repo": "PMC", "publication": "PMC0000000"}, + } + } + to_yaml(table, table_config) + + graph: Path = tmp_path / "graph.yaml" + graph_config: dict[str, Any] = { + "name": "PARQUET_KG", + "version": "1.0.0", + "tables": [str(table)], + "fullmap": str(fullmap), + "rig": rig_factory(tmp_path, infores_id="infores:parquet-kg", source_info={"description": "parquet source smoke graph"}), + } + to_yaml(graph, graph_config) + + build_pipeline(graph, PipelineProgress(total_stages=6)) + + nodes: Path = tmp_path / "PARQUET_KG_1.0.0.nodes.ndjson" + edges: Path = tmp_path / "PARQUET_KG_1.0.0.edges.ndjson" + assert nodes.is_file() + assert edges.is_file() + + node_text: str = nodes.read_text() + edge_text: str = edges.read_text() + assert node_text.strip() + assert edge_text.strip() + + # Only the real redb knows these CURIEs, so their presence proves the parquet rows flowed + # through load, encode, and resolve under the positional column contract. + assert "HGNC:1100" in node_text + assert "HGNC:6871" in node_text + assert "HGNC:1100" in edge_text + assert "HGNC:6871" in edge_text + + def test_validate_command_happy_path(tmp_path: Path) -> None: """SMOKE (ii): ``validate`` (cli.py wrapper) + ``validate_pipeline`` accept a valid table. diff --git a/tests/test_enums.py b/tests/test_enums.py index f1355f29..695c8a6c 100644 --- a/tests/test_enums.py +++ b/tests/test_enums.py @@ -84,6 +84,7 @@ def test_contributions_values() -> None: def test_files_values() -> None: assert Files.TEXT == "text" assert Files.EXCEL == "excel" + assert Files.PARQUET == "parquet" def test_comparisons_membership() -> None: