diff --git a/docs/configuration/table.md b/docs/configuration/table.md index 5accc16e..cc0f2f68 100644 --- a/docs/configuration/table.md +++ b/docs/configuration/table.md @@ -227,7 +227,6 @@ Defines how to extract and resolve entities. |-------|------|----------|-------------| | `method` | String | No | `"value"` (literal) or `"column"` (source column letters). Defaults to `"value"`. | | `encoding` | String\|Int\|Float | Yes | Literal value or source column letters, depending on `method` | -| `no_map` | Bool | No | Skip fullmap entity resolution: the encoded value is used verbatim as the node CURIE, for data that already carries its own CURIEs. No NLP normalization, taxon constraint, or category lookup is applied; blank cells are dropped like unresolved nodes and a non-blank non-CURIE cell fails the build. Combines with nothing resolution-only (`prioritize`, `avoid`, `exclude_prefixes`, `exclude_regex`, non-default `taxon` are rejected as dead config). See [No-map mode](#no-map-mode). | | `taxon` | PositiveInt | No | NCBI Taxon ID for filtering (e.g., `9606` for human) | | `prioritize` | List[String] | No | Preferred Biolink categories (must be valid `Categories` enum values such as `Gene`, `Protein`) | | `avoid` | List[String] | No | Excluded Biolink categories (must be valid `Categories` enum values). Setting `avoid` makes the column an allow-list by complement: categories the `Categories` enum cannot name (fullmap-emitted Biolink mixins; see `biolink.CATEGORY_OVERRIDES`) are dropped as well, so the guard has no silent hole | @@ -259,29 +258,6 @@ subject: encoding: A # Read from column A ``` -#### No-map mode - -**`no_map: true`** - Use the data's own CURIEs as-is - -Most sources name entities in plain text and need entity resolution against the fullmap to turn a -mention like `BRCA1` into a CURIE. When the table already carries CURIEs (your own identifiers or -an upstream resource's), `no_map: true` skips that lookup entirely: the encoded value becomes the -node id verbatim, no NLP normalization or taxon/category filtering is applied, and the node is -emitted with the generic `biolink:NamedThing` category (so its edges derive `biolink:Association`). - -```yaml -subject: - method: column - encoding: A - no_map: true # Column A already holds CURIEs like "CHEBI:41774" -``` - -Blank cells are dropped like unresolved nodes (an edge needs a node id), and a non-blank cell that -is not CURIE-shaped fails the build with `no-map-curie-violation`. `prefix`/`suffix` still apply, -so raw identifiers without a namespace can form CURIEs (`prefix: "CHEBI:"`). Resolution filters -(`taxon`, `prioritize`, `avoid`, `exclude_prefixes`, `exclude_regex`) are rejected alongside -`no_map` as dead config: with no lookup, nothing reads them. - At runtime those letters are converted internally to Polars column names such as `column_1`, but those internal names are not valid configuration values. #### `split_by` diff --git a/src/tablassert/errors.py b/src/tablassert/errors.py index 8d1b1ded..04140235 100644 --- a/src/tablassert/errors.py +++ b/src/tablassert/errors.py @@ -63,9 +63,6 @@ def redact_secrets(text: str, secrets: Sequence[str] = ()) -> str: "provenance-missing-publication", "provenance-publication-and-override", "encoding-list-method-removed", - "no-map-dead-config", - "no-map-bad-curie", - "no-map-curie-violation", "annotation-split-by-requires-column", "annotation-split-by-empty", "field-disabled", diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index e0206fdd..fb540a97 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -48,7 +48,7 @@ from tablassert.errors import TablassertError from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch from tablassert.log import cat -from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section, is_curie +from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section from tablassert.nlp import level_one, level_two from tablassert.qc import fullmap_audit from tablassert.rig import ( @@ -1220,65 +1220,6 @@ def qualifier_vocabulary_audit(lf: pl.LazyFrame, col: str, vocabulary: Iterable[ ) -def curie_audit(lf: pl.LazyFrame, col: str, section: str, config_file: str) -> pl.LazyFrame: - """Finalize a ``no_map`` node column: blank rows drop, non-CURIE values fail, category is synthesized. - - Columns configured with ``no_map: true`` bypass entity resolution, so nothing else - (a) drops rows whose cell is blank (a missing node id can never form an edge, exactly - like an unresolved subject/object), (b) rejects cells that are not CURIE-shaped -- the - encoded value becomes the KGX node id verbatim, so garbage would ship as an identifier -- - or (c) produces the ``_category`` column downstream stages require: without a - fullmap lookup there is no role to derive, so the column is pinned to - ``biolink:NamedThing`` and edge derivation degrades to ``biolink:Association`` exactly - like the documented no-role-lookup fallback in :func:`edge_category`. - - Args: - lf: Source LazyFrame. - col: The no_map node column (``subject``/``object`` or a CURIE-ranged qualifier key). - section: Short section hash (for log context). - config_file: Originating config file (for log context). - - Returns: - LazyFrame with blank/null-like rows dropped and ``_category`` added. - - Notes: - A non-blank value that is not CURIE-shaped raises a coded error naming the - offending values (mirroring ``qualifier_vocabulary_audit``); blank and null-like - cells (``""``, ``"NA"``, ...) are dropped with a loud per-column count instead, - mirroring how an unresolvable subject/object row is dropped, never emitted. - """ - values: pl.Series = lf.select(pl.col(col)).collect().get_column(col) - if values.dtype.base_type() in (pl.List, pl.Array): - values = values.explode() - blank_mask: pl.Series = values.is_null() | values.str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT)) - # Judge CURIE shape on the distinct non-blank values only: the audit runs once per - # section, and map_elements is a per-element Python call that must not scale with - # row count when one bad cell repeats a million times. - distinct: pl.Series = values.filter(~blank_mask).unique() - offending: pl.Series = distinct.filter(~distinct.str.strip_chars().map_elements(is_curie, return_dtype=pl.Boolean)).sort() - if offending.len() > 0: - shown: list[str] = offending.head(10).to_list() - more: str = "..." if offending.len() > len(shown) else "" - raise TablassertError( - f"{col} is configured with `no_map: true` so its values are used verbatim as node CURIEs, but " - f"{offending.len()} non-blank value(s) in the built column are not CURIE-shaped: " - f"{', '.join(repr(x) for x in shown)}{more}. Fix the source table or the encoding " - "(prefix/suffix can form a CURIE from raw cells).", - code="no-map-curie-violation", - ) - dropped: int = int(blank_mask.sum()) - if dropped: - logger.info( - "no_map column {col} in {config} ({hash}) dropped {n} blank row(s): a missing CURIE cannot form a node", - col=col, - config=config_file, - hash=section, - n=dropped, - ) - present: pl.Expr = pl.col(col).is_not_null() & ~pl.col(col).str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT)) - return lf.filter(present).with_columns(pl.lit("biolink:NamedThing").alias(f"{col}_category")) - - def to_store(lf: pl.LazyFrame, p: Path, config_name: str) -> Path: """Collect and write a section LazyFrame to a parquet file. @@ -1437,25 +1378,16 @@ def _node_ops(self: Self, db: Path) -> list[Any]: enabled. """ # Subject/object/qualifiers share one resolve_batch call instead of one per column. - # Two kinds of columns never reach it: enum-ranged qualifiers, whose range is a - # closed Biolink vocabulary (sending them through the fullmap would turn the - # required token `increased` into the CURIE `UMLS:C0205217`, which the slot - # rejects), and no_map nodes, whose values are already CURIEs used verbatim. + # Enum-ranged qualifiers are excluded from resolution: their range is a closed + # Biolink vocabulary, so sending them through the fullmap would turn the required + # token `increased` into the CURIE `UMLS:C0205217`, which the slot rejects. qualifiers: list[Qualifier] = self.statement.qualifiers or [] - subj_obj: list[tuple[NodeEncoding, str]] = [(self.statement.subject, "subject"), (self.statement.object, "object")] node_columns: list[tuple[NodeEncoding, str]] = [ - *[(x, col) for x, col in subj_obj if not x.no_map], + (self.statement.subject, "subject"), + (self.statement.object, "object"), *[(x, x.qualifier) for x in qualifiers if x.resolved], ] - # no_map subject/object plus CURIE-ranged qualifiers that opted out (a no_map - # enum-ranged qualifier is rejected at config time). Their encoded values become - # node ids verbatim: encode only, then curie_audit drops blank rows, fails on - # non-CURIE cells, and synthesizes the category column edge derivation needs. - verbatim: list[tuple[NodeEncoding, str]] = [ - *[(x, col) for x, col in subj_obj if x.no_map], - *[(x, x.qualifier) for x in qualifiers if not x.resolved and x.no_map], - ] - literals: list[Qualifier] = [x for x in qualifiers if not x.resolved and not x.no_map] + literals: list[Qualifier] = [x for x in qualifiers if not x.resolved] # A nullable qualifier keeps its edge when the cell is blank or unresolvable (the # column stays null and the null-stripper omits the key); subject/object and # strict qualifiers drop the row as before. @@ -1478,18 +1410,6 @@ def _node_ops(self: Self, db: Path) -> list[Any]: # ranged column ends with a vocabulary audit: nothing else validates its cell # values against the closed Biolink range before KGX emission. [[*self.encoding(x, x.qualifier), (qualifier_vocabulary_audit, (str(x.qualifier), sorted(x.vocabulary or ())))] for x in literals], - # no_map columns are encoded exactly like enum-ranged literals (no NLP - # normalization -- it exists only to feed entity resolution these columns - # never undergo), then finalized by curie_audit: blank rows dropped, - # non-CURIE cells failed loudly, and the pinned `biolink:NamedThing` - # category added for downstream edge derivation. The `_pre_resolution` - # snapshot rides along because `_collect_subframes` discovers node columns - # through it at graph-compile time; without it a verbatim subject/object - # would never be split out of the edges into nodes. - [ - [*self.encoding(x, col), (column, (f"{col}_pre_resolution", col)), (curie_audit, (col, self.store.stem, self.config.name))] - for x, col in verbatim - ], # Raw ``column_`` columns are dead weight once the encodings above have # copied them into named slots; trim before resolution so the frame that # resolve_batch materializes and joins stays narrow. @@ -1654,7 +1574,6 @@ def collect(self: Self, db: Path) -> list[tuple[Callable, tuple[Any]]] | Path: fullmap_audit: "qc", column: "encode", qualifier_vocabulary_audit: "encode", - curie_audit: "encode", edge_category: "edge", publications: "provenance", retrieval_sources: "provenance", diff --git a/src/tablassert/models.py b/src/tablassert/models.py index 6dd4694f..35727398 100644 --- a/src/tablassert/models.py +++ b/src/tablassert/models.py @@ -47,11 +47,20 @@ pl = LazyModule("polars") -# Deprecated config keys -> guidance. EMPTY today: the before-hook below is a silent no-op until a -# future rename registers a key here to WARN on it. The hook only warns and returns the data -# unchanged — a renamed key that is no longer a valid field is STILL rejected by extra="forbid" -# unless the future hook also pops/translates it; registering a key here supplies the warning half only. -DEPRECATED_KEYS: dict[str, str] = {} +# Deprecated config keys -> guidance. The before-hook below only warns and returns the data +# unchanged: a key that is no longer a valid field is STILL rejected by extra="forbid" unless the +# hook also pops/translates it, so registering a key here supplies the warning half only. That is +# the right half for a removed feature: the config still fails, but it says why and what to do. +DEPRECATED_KEYS: dict[str, str] = { + "no_map": ( + "`no_map` is removed: subject, object, and CURIE-ranged qualifier values resolve against the fullmap again. " + "The mode emitted node ids verbatim and pinned each one to `biolink:NamedThing`, which flattened every derived " + "edge to `biolink:Association`. Tables that already carry CURIEs need nothing else, because the fullmap indexes " + "CURIEs and their equivalent identifiers as terms, so a CURIE cell resolves back to its own CURIE. The usual " + "resolution rules still apply: `taxon`, `prioritize`/`avoid`, `exclude_prefixes`/`exclude_regex`, and a blank " + "cell still drops the row." + ) +} def _shown(name: str, target: str) -> str: @@ -94,10 +103,11 @@ def _warn_deprecated_keys(cls, data: Any) -> Any: """Soft-warn when a raw input key is registered in ``DEPRECATED_KEYS``. Pure observation: returns ``data`` unchanged and never rejects (``extra="forbid"`` - still governs truly-unknown keys). The registry is EMPTY today, so this is a silent - no-op; a future field rename registers its old key here to warn instead of hard-fail. + still governs keys that name no live field, so a removed feature's key warns and then + still fails; that pairing is the point, since silently dropping the key would build the + output the removal exists to prevent). ``no_map`` is the registered entry today. Note: with ``validate_assignment=True`` pydantic also re-runs this on attribute set, - so a registered key warns on assignment too — inert while the registry is empty. + so a registered key warns on assignment too. """ if isinstance(data, dict): for key in data: @@ -288,16 +298,6 @@ def polars_compatible_replacement(cls, remove: list[int | float | str] | None) - ) -#: CURIE shape enforced on verbatim (``no_map``) node values: non-empty ``PREFIX:local-id``. -#: Shared by config-time literal validation (models) and build-time cell audits (lib). -CURIE_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]*:[A-Za-z0-9_.\-/]+$") - - -def is_curie(value: object) -> bool: - """Whether ``value`` is a CURIE-shaped string (``PREFIX:local-id``, both halves non-empty).""" - return isinstance(value, str) and CURIE_PATTERN.match(value.strip()) is not None - - class NodeEncoding(Encoding): taxon: PositiveInt | None = Field( 9606, @@ -322,87 +322,6 @@ class NodeEncoding(Encoding): examples=[["^OMIM:\\d+$"]], ) - no_map: bool = Field( - False, - description=( - "When True, skip fullmap entity resolution: the encoded value is used verbatim as the node CURIE, " - "for data that already carries its own CURIEs. No NLP normalization, taxon constraint, or category " - "lookup is applied; blank cells are dropped like unresolved nodes and non-CURIE cells fail the build." - ), - examples=[True], - ) - - @model_validator(mode="before") - @classmethod - def no_map_rejects_explicit_taxon(cls, data: Any) -> Any: - """Reject ``no_map`` with an explicit non-default ``taxon`` (dead resolution config). - - ``taxon`` defaults to 9606 and only steers fullmap ranking, so an explicit - non-default taxon alongside ``no_map: true`` is config that can never take - effect. The default itself stays indistinguishable from an explicit 9606 and - is therefore allowed. Checked ``before`` precisely so "explicitly set" is - detectable from the raw input dict. - """ - if isinstance(data, dict) and data.get("no_map") and "taxon" in data and data["taxon"] not in (None, 9606): - raise TablassertValidationError( - f"`no_map: true` skips entity resolution, so `taxon: {data['taxon']}` can never take effect; remove the taxon constraint.", - code="no-map-dead-config", - ) - return data - - @model_validator(mode="after") - def no_map_rejects_resolution_knobs(self: Self) -> Self: - """Reject ``no_map`` combined with resolution-only knobs (dead config). - - ``prioritize``, ``avoid``, ``exclude_prefixes``, and ``exclude_regex`` only - steer fullmap ranking and filtering; with ``no_map: true`` no lookup happens, - so these fields mislead the reader. Fail loudly at config time, like the - ``nullable``-on-literal check on :class:`Qualifier`. - """ - if not self.no_map: - return self - dead: list[str] = [ - name - for name, value in ( - ("prioritize", self.prioritize), - ("avoid", self.avoid), - ("exclude_prefixes", self.exclude_prefixes), - ("exclude_regex", self.exclude_regex), - ) - if value - ] - if dead: - raise TablassertValidationError( - f"`no_map: true` skips entity resolution, so {', '.join(f'`{name}`' for name in dead)} can never take effect; remove them.", - code="no-map-dead-config", - ) - return self - - @model_validator(mode="after") - def no_map_literal_must_be_curie(self: Self) -> Self: - """Validate that a ``no_map`` literal encodes a CURIE at config time. - - A ``method: value`` node with ``no_map: true`` becomes the node id verbatim, - so the literal (after ``prefix``/``suffix`` are applied, matching the build's - ``encoding()``) must already be CURIE-shaped; otherwise every row would be - dropped at build time. ``method: column`` cells are audited per row at build - time instead. - """ - if not self.no_map or self.method != EncodingMethods.VALUE: - return self - # Enum-ranged qualifier literals are closed-vocabulary tokens (audited against - # the vocabulary, and `no_map` on them is rejected as dead config), not CURIEs. - if getattr(self, "vocabulary", None) is not None: - return self - literal: str = f"{self.prefix or ''}{self.encoding}{self.suffix or ''}".strip() - if not is_curie(literal): - raise TablassertValidationError( - f"`no_map: true` uses the encoded value verbatim as the node CURIE, but the literal {literal!r} is not CURIE-shaped " - "(expected `PREFIX:local-id`). Fix the literal, or add `prefix`/`suffix` to form a CURIE.", - code="no-map-bad-curie", - ) - return self - @field_validator("exclude_regex", mode="after") @classmethod def polars_compatible_exclude_regex(cls, exclude_regex: list[str] | None) -> list[str] | None: @@ -450,26 +369,10 @@ def vocabulary(self: Self) -> frozenset[str] | None: def resolved(self: Self) -> bool: """Whether this qualifier's values go through fullmap entity resolution. - Enum-ranged qualifiers carry a literal token from a closed Biolink vocabulary - and must be passed through verbatim. CURIE-ranged qualifiers are resolved - unless they opt out with ``no_map: true`` (their values are already CURIEs). - """ - return self.vocabulary is None and not self.no_map - - @model_validator(mode="after") - def reject_no_map_on_enum_ranged(self: Self) -> Self: - """Reject ``no_map: true`` on enum-ranged qualifiers as dead config. - - Enum-ranged qualifiers already pass their literal tokens through verbatim, so - ``no_map`` would be a no-op that misleads the reader; fail loudly at config - time, like the ``nullable``-on-literal check below. + Only CURIE-ranged qualifiers are resolved. Enum-ranged ones carry a literal + token from a closed Biolink vocabulary and must be passed through verbatim. """ - if self.no_map and self.vocabulary is not None: - raise TablassertValidationError( - f"{self.qualifier} has a closed Biolink vocabulary and already passes values through verbatim, so `no_map` is redundant; remove it.", - code="no-map-dead-config", - ) - return self + return self.vocabulary is None @model_validator(mode="after") def reject_disabled_qualifiers(self: Self) -> Self: diff --git a/src/tablassert/runcache.py b/src/tablassert/runcache.py index 40b45b85..45b98c0f 100644 --- a/src/tablassert/runcache.py +++ b/src/tablassert/runcache.py @@ -272,9 +272,6 @@ def prefix_digest(ops: list[tuple[Callable, tuple[Any, ...]]]) -> str: lib.split_list: 1, # Vocabulary audit: one eager distinct-value pass over a single enum column. lib.qualifier_vocabulary_audit: 1, - # CURIE audit: one eager pass over a single no_map column (distinct-value check, - # blank-row filter, category column synthesis). - lib.curie_audit: 1, # Clean: coerce_columns applies five coercion rules (schema resolve + renames + value # coercion) in one pass, so it outprices a single-expression clean op. lib.coerce_columns: 3, diff --git a/tests/test_e2e_smoke.py b/tests/test_e2e_smoke.py index 15319939..b73fef78 100644 --- a/tests/test_e2e_smoke.py +++ b/tests/test_e2e_smoke.py @@ -113,110 +113,6 @@ def test_build_pipeline_against_real_redb(tmp_path: Path, monkeypatch: pytest.Mo assert "HGNC:6871" in edge_text -def test_build_pipeline_with_no_map_nodes_passes_curies_through_verbatim(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rig_factory: Any) -> None: - """SMOKE: ``no_map: true`` subject/object build KGX from verbatim CURIEs, no lookup. - - The source cells carry ``CUSTOM:...`` CURIEs that the tiny real redb cannot resolve - (it only knows ``brca1``/``mapk1``), so their presence in the output proves the - pipeline used the values as-is. The blank subject row must vanish: a missing CURIE - cannot form a node, exactly like an unresolved term. Nodes carry the synthesized - ``biolink:NamedThing`` category so consumers still see a valid KGX category. - """ - monkeypatch.chdir(tmp_path) - (tmp_path / ".tablassert" / "store").mkdir(parents=True) - - fullmap: Path = _build_real_redb(tmp_path / "fullmap") - - # Two data rows: the second has a blank subject and must be dropped, not emitted. - data: Path = tmp_path / "data.tsv" - data.write_text("CUSTOM:1001\tCUSTOM:2001\n\tCUSTOM:2002\n") - - table: Path = tmp_path / "table.yaml" - table_config: dict[str, Any] = { - "template": { - "source": {"kind": "text", "local": str(data), "url": ["https://example.com/data.tsv"], "delimiter": "\t"}, - "statement": { - "subject": {"method": "column", "encoding": "A", "no_map": True}, - "predicate": "associated_with", - "object": {"method": "column", "encoding": "B", "no_map": True}, - }, - "provenance": {"repo": "PMC", "publication": "PMC0000000"}, - } - } - to_yaml(table, table_config) - - graph: Path = tmp_path / "graph.yaml" - graph_config: dict[str, Any] = { - "name": "NOMAP_KG", - "version": "1.0.0", - "tables": [str(table)], - "fullmap": str(fullmap), - "rig": rig_factory(tmp_path, infores_id="infores:nomap-kg", source_info={"description": "no_map e2e graph"}), - } - to_yaml(graph, graph_config) - - build_pipeline(graph, PipelineProgress(total_stages=6)) - - nodes: Path = tmp_path / "NOMAP_KG_1.0.0.nodes.ndjson" - edges: Path = tmp_path / "NOMAP_KG_1.0.0.edges.ndjson" - node_text: str = nodes.read_text() - edge_text: str = edges.read_text() - - # Verbatim CURIEs absent from the redb: only a true pass-through can produce them. - assert "CUSTOM:1001" in node_text - assert "CUSTOM:2001" in node_text - assert "CUSTOM:1001" in edge_text - assert "CUSTOM:2001" in edge_text - # Synthesized category keeps the nodes valid KGX without a fullmap role lookup. - assert "biolink:NamedThing" in node_text - # The blank-subject row was dropped, never emitted with an empty id. - assert "CUSTOM:2002" not in edge_text - assert "CUSTOM:2002" not in node_text - - -def test_build_pipeline_with_no_map_rejects_non_curie_cells(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rig_factory: Any) -> None: - """SMOKE (negative): a non-CURIE cell in a no_map column fails the build loudly. - - The encoded value would become the KGX node id verbatim; free text must not ship - as an identifier, so the build raises the coded no-map-curie-violation instead of - emitting KGX that fails validation far from the cause. - """ - monkeypatch.chdir(tmp_path) - (tmp_path / ".tablassert" / "store").mkdir(parents=True) - - fullmap: Path = _build_real_redb(tmp_path / "fullmap") - - data: Path = tmp_path / "data.tsv" - data.write_text("not-a-curie\tCUSTOM:2001\n") - - table: Path = tmp_path / "table.yaml" - table_config: dict[str, Any] = { - "template": { - "source": {"kind": "text", "local": str(data), "url": ["https://example.com/data.tsv"], "delimiter": "\t"}, - "statement": { - "subject": {"method": "column", "encoding": "A", "no_map": True}, - "predicate": "associated_with", - "object": {"method": "column", "encoding": "B", "no_map": True}, - }, - "provenance": {"repo": "PMC", "publication": "PMC0000000"}, - } - } - to_yaml(table, table_config) - - graph: Path = tmp_path / "graph.yaml" - graph_config: dict[str, Any] = { - "name": "NOMAPBAD_KG", - "version": "1.0.0", - "tables": [str(table)], - "fullmap": str(fullmap), - "rig": rig_factory(tmp_path, infores_id="infores:nomapbad-kg", source_info={"description": "no_map negative e2e graph"}), - } - to_yaml(graph, graph_config) - - with pytest.raises(Exception, match="no-map-curie-violation"): - build_pipeline(graph, PipelineProgress(total_stages=6)) - - def test_build_pipeline_with_linkouts_resolves_edge_ids(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rig_factory: Any) -> None: """SMOKE: ``build_pipeline(linkouts=True)`` emits the resolved EdgeLinkouts record URL. diff --git a/tests/test_lib.py b/tests/test_lib.py index 057eee19..fa084fe3 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -36,7 +36,6 @@ coerce_pvalue_columns, coerce_study_size_columns, coerced_target, - curie_audit, drop_low_number_of_cases, drop_not_significant, drop_zero_effect_size, @@ -646,123 +645,6 @@ def test_tcode_collect_audits_enum_ranged_qualifier_vocabulary(fixtures_path: Pa assert idx < trim_idx -def test_curie_audit_adds_named_thing_category_and_keeps_curies() -> None: - """curie_audit passes CURIE cells through verbatim and synthesizes the category column. - - Without the synthesized `_category`, edge_category would crash on a missing - column for every no_map node; `biolink:NamedThing` is the valid KGX category that - degrades edge derivation to `biolink:Association` like the documented no-role fallback. - """ - lf: pl.LazyFrame = pl.LazyFrame({"subject": ["CHEBI:1234", "HGNC:1100"]}) - - result: pl.DataFrame = curie_audit(lf, "subject", "sectionhash", "config.yaml").collect() - - assert result.get_column("subject").to_list() == ["CHEBI:1234", "HGNC:1100"] - assert result.get_column("subject_category").to_list() == ["biolink:NamedThing", "biolink:NamedThing"] - - -def test_curie_audit_drops_blank_and_null_like_rows() -> None: - """Blank/null-like cells in a no_map column drop the row, like an unresolved subject/object. - - A missing CURIE can never form a node, so the edge is dropped -- never emitted with a - garbage id. Null-like text ("NA", "none") is treated as blank, matching strip_nulls. - """ - lf: pl.LazyFrame = pl.LazyFrame({"subject": ["CHEBI:1234", None, "", "NA"], "row": [1, 2, 3, 4]}) - - result: pl.DataFrame = curie_audit(lf, "subject", "sectionhash", "config.yaml").collect() - - assert result.get_column("row").to_list() == [1] - assert result.get_column("subject").to_list() == ["CHEBI:1234"] - - -def test_curie_audit_rejects_non_curie_values() -> None: - """Non-blank cells that are not CURIE-shaped fail the build with a coded error. - - The encoded value becomes the KGX node id verbatim; shipping free text as an - identifier would fail validation far from the cause, so curie_audit raises where - the qualifier vocabulary audit does. - """ - lf: pl.LazyFrame = pl.LazyFrame({"subject": ["CHEBI:1234", "BRCA1"]}) - - with pytest.raises(TablassertError) as exc_info: - curie_audit(lf, "subject", "sectionhash", "config.yaml") - assert "no-map-curie-violation" in str(exc_info.value) - assert "BRCA1" in str(exc_info.value) - - -def test_tcode_collect_routes_no_map_nodes_around_resolve_batch(fixtures_path: Path) -> None: - """no_map subject is excluded from resolve_batch specs and finalized by curie_audit instead. - - The fullmap is never queried for the no_map column (it is absent from the batch - specs), while the mapped object keeps its ResolveSpec. - """ - data: Any = from_yaml(fixtures_path / "minimal_section.yaml") - store: Path = Path("/tmp/sectionhash_no_map.parquet") - data["statement"]["subject"] = {"method": "column", "encoding": "A", "no_map": True} - - tcode_model: Tcode = Tcode.model_validate( # pyright: ignore - {**data, "config": fixtures_path / "minimal_section.yaml", "store": store} - ) - - collected: list[tuple[Any, tuple[Any]]] = tcode_model.collect(Path("/tmp/fullmap.redb")) # pyright: ignore - specs: list[ResolveSpec] = next(op[1][0] for op in collected if op[0].__name__ == "resolve_batch") - spec_cols: list[str] = [spec.col for spec in specs] - audits: list[tuple[Any, ...]] = [op[1] for op in collected if op[0].__name__ == "curie_audit"] - - assert "subject" not in spec_cols - assert "object" in spec_cols - assert audits == [("subject", "sectionhash_no_map", "minimal_section.yaml")] - - -def test_tcode_collect_no_map_on_both_nodes_empties_resolve_batch(fixtures_path: Path) -> None: - """With subject and object both no_map, resolve_batch receives zero specs and early-returns. - - This is the proof that the fullmap redb is never touched for an all-verbatim - section: resolve_batch returns the frame untouched before any redb access when - `not specs`. - """ - data: Any = from_yaml(fixtures_path / "minimal_section.yaml") - store: Path = Path("/tmp/sectionhash_all_no_map.parquet") - data["statement"]["subject"] = {"method": "column", "encoding": "A", "no_map": True} - data["statement"]["object"] = {"method": "column", "encoding": "B", "no_map": True} - - tcode_model: Tcode = Tcode.model_validate( # pyright: ignore - {**data, "config": fixtures_path / "minimal_section.yaml", "store": store} - ) - - collected: list[tuple[Any, tuple[Any]]] = tcode_model.collect(Path("/tmp/does-not-exist.redb")) # pyright: ignore - specs: list[ResolveSpec] = next(op[1][0] for op in collected if op[0].__name__ == "resolve_batch") - audits: list[tuple[Any, ...]] = [op[1] for op in collected if op[0].__name__ == "curie_audit"] - - assert specs == [] - assert [args[0] for args in audits] == ["subject", "object"] - - -def test_tcode_collect_no_map_curie_ranged_qualifier_is_verbatim(fixtures_path: Path) -> None: - """A CURIE-ranged qualifier with no_map skips resolution like an enum-ranged literal does. - - It must land on curie_audit (not qualifier_vocabulary_audit, which would reject every - CURIE against the empty vocabulary, and not resolve_batch). - """ - data: Any = from_yaml(fixtures_path / "minimal_section.yaml") - store: Path = Path("/tmp/sectionhash_no_map_qual.parquet") - data["statement"]["object"] = {"method": "column", "encoding": "B", "no_map": True} - data["statement"]["qualifiers"] = [{"qualifier": "anatomical_context_qualifier", "method": "column", "encoding": "C", "no_map": True}] - - tcode_model: Tcode = Tcode.model_validate( # pyright: ignore - {**data, "config": fixtures_path / "minimal_section.yaml", "store": store} - ) - - collected: list[tuple[Any, tuple[Any]]] = tcode_model.collect(Path("/tmp/fullmap.redb")) # pyright: ignore - names: list[str] = [op[0].__name__ for op in collected] - specs: list[ResolveSpec] = next(op[1][0] for op in collected if op[0].__name__ == "resolve_batch") - audits: list[tuple[Any, ...]] = [op[1] for op in collected if op[0].__name__ == "curie_audit"] - - assert "anatomical_context_qualifier" not in [spec.col for spec in specs] - assert "qualifier_vocabulary_audit" not in names - assert [args[0] for args in audits] == ["object", "anatomical_context_qualifier"] - - def test_tcode_collect_edge_ops_follow_resolve_batch(fixtures_path: Path) -> None: """tcode collect runs predicate/edge_category after the single resolve_batch op.""" data: Any = from_yaml(fixtures_path / "minimal_section.yaml") diff --git a/tests/test_models.py b/tests/test_models.py index f5c15b75..d735e2a0 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -352,92 +352,6 @@ def test_node_encoding_explicit_null_disables_taxon() -> None: assert node.taxon is None -def test_node_encoding_no_map_defaults_false() -> None: - """no_map defaults to False so every existing config resolves exactly as before.""" - node: NodeEncoding = NodeEncoding(method="value", encoding="BRCA1") # pyright: ignore - assert node.no_map is False - - -def test_node_encoding_no_map_literal_must_be_curie() -> None: - """A no_map literal becomes the node id verbatim, so a non-CURIE literal is rejected at config time. - - Without this check every row would build-drop at the CURIE audit, turning one - config typo into a silently empty node file. - """ - with pytest.raises(ValidationError) as exc_info: - NodeEncoding(method="value", encoding="BRCA1", no_map=True) # pyright: ignore - assert "no-map-bad-curie" in str(exc_info.value) - - -def test_node_encoding_no_map_literal_curie_accepted() -> None: - """A CURIE-shaped no_map literal passes config validation.""" - node: NodeEncoding = NodeEncoding(method="value", encoding="HGNC:1100", no_map=True) # pyright: ignore - assert node.no_map is True - - -def test_node_encoding_no_map_literal_prefix_suffix_form_curie() -> None: - """prefix/suffix participate in the literal CURIE check, matching the build's encoding().""" - node: NodeEncoding = NodeEncoding(method="value", encoding="1100", prefix="HGNC:", no_map=True) # pyright: ignore - assert node.no_map is True - with pytest.raises(ValidationError) as exc_info: - NodeEncoding(method="value", encoding="1100", prefix="HGNC", no_map=True) # pyright: ignore - assert "no-map-bad-curie" in str(exc_info.value) - - -def test_node_encoding_no_map_column_needs_no_literal_curie() -> None: - """method: column no_map nodes skip literal validation; cells are audited per row at build time.""" - node: NodeEncoding = NodeEncoding(method="column", encoding="A", no_map=True) # pyright: ignore - assert node.no_map is True - - -@pytest.mark.parametrize( - "kwargs", - [ - {"prioritize": [Categories.GENE]}, - {"avoid": [Categories.DISEASE]}, - {"exclude_prefixes": ["OMIM"]}, - {"exclude_regex": ["^OMIM:\\d+$"]}, - {"taxon": 10090}, - ], -) -def test_node_encoding_no_map_rejects_resolution_knobs(kwargs: dict[str, Any]) -> None: - """Resolution-only knobs alongside no_map: true are dead config and fail loudly at config time. - - Each knob only steers fullmap ranking/filtering; with no_map no lookup happens, - so keeping them would mislead the reader into thinking they take effect. - """ - with pytest.raises(ValidationError) as exc_info: - NodeEncoding(method="column", encoding="A", no_map=True, **kwargs) # pyright: ignore - assert "no-map-dead-config" in str(exc_info.value) - - -def test_node_encoding_no_map_allows_default_taxon() -> None: - """An explicit default taxon (9606) is indistinguishable from the unset default and stays allowed.""" - node: NodeEncoding = NodeEncoding(method="column", encoding="A", no_map=True, taxon=9606) # pyright: ignore - assert node.no_map is True - - -def test_qualifier_resolved_property_truth_table() -> None: - """resolved reflects vocabulary presence AND the no_map opt-out. - - Enum-ranged qualifiers never resolve; CURIE-ranged ones resolve unless no_map - opts them out. This property is the single routing predicate in lib._node_ops. - """ - enum_ranged: Qualifier = Qualifier(qualifier="object_direction_qualifier", method="value", encoding="increased") # pyright: ignore - assert enum_ranged.resolved is False - curie_ranged: Qualifier = Qualifier(qualifier="anatomical_context_qualifier", method="column", encoding="A") # pyright: ignore - assert curie_ranged.resolved is True - curie_no_map: Qualifier = Qualifier(qualifier="anatomical_context_qualifier", method="column", encoding="A", no_map=True) # pyright: ignore - assert curie_no_map.resolved is False - - -def test_qualifier_no_map_rejected_on_enum_ranged() -> None: - """no_map on an enum-ranged qualifier is redundant (already verbatim) and is rejected as dead config.""" - with pytest.raises(ValidationError) as exc_info: - models.Qualifier(qualifier="object_direction_qualifier", method="value", encoding="increased", no_map=True) # pyright: ignore - assert "no-map-dead-config" in str(exc_info.value) - - def test_qualifier_rejects_disabled_species_context_qualifier() -> None: """species_context_qualifier is disabled and cannot be manually declared.""" with pytest.raises(ValidationError) as exc_info: @@ -1183,7 +1097,7 @@ def test_provenance_pmc_repo_with_non_pmc_publication_rejected() -> None: def test_no_deprecation_warnings_for_current_fixtures(fixtures_path: Path, recwarn: pytest.WarningsRecorder) -> None: - """US-M4: an EMPTY DEPRECATED_KEYS registry makes the before-hook a silent no-op. + """US-M4: a registry that names only removed keys makes the before-hook silent for shipped configs. Every shipped fixture and docs example must load and validate with ZERO UserWarnings, proving the deprecation scaffold never regresses today's corpus. Scoped to UserWarning so @@ -1248,8 +1162,8 @@ def test_deprecated_key_in_registry_warns_but_still_validates(monkeypatch: pytes Monkeypatching a REAL current field name (`delimiter`) into DEPRECATED_KEYS makes the before-hook emit a UserWarning while `extra="forbid"` still accepts the key and the value - round-trips intact -- isolating "warn, don't fail". monkeypatch restores the empty registry - so no other test observes the temporary entry. + round-trips intact -- isolating "warn, don't fail". monkeypatch restores the registry, so no + other test observes the temporary entry. """ monkeypatch.setitem(models.DEPRECATED_KEYS, "delimiter", "delimiter is deprecated; use 'sep' instead") with pytest.warns(UserWarning, match="sep"): @@ -1257,6 +1171,25 @@ def test_deprecated_key_in_registry_warns_but_still_validates(monkeypatch: pytes assert source.delimiter == "\t" +def test_removed_no_map_key_warns_then_still_fails() -> None: + """A 21.0.0-era config that still sets `no_map` is told why it fails, then still fails. + + The mode is registered in `DEPRECATED_KEYS` so the before-hook names the removal and the + migration, while `extra="forbid"` keeps rejecting the key. Both halves are pinned because each + fails differently: dropping the registry entry regresses the message to a bare "Extra inputs are + not permitted", and popping the key instead would silently accept a config whose output the + removal exists to prevent. Both values are covered because the hook keys on presence + (`for key in data`), so a config that was merely opting out with `no_map: false` breaks too. + """ + for value in (True, False): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with pytest.raises(ValidationError, match="no_map"): + NodeEncoding.model_validate({"method": "value", "encoding": "BRCA1", "no_map": value}) + notes: list[str] = [str(w.message) for w in caught if issubclass(w.category, UserWarning)] + assert any("`no_map` is removed" in note for note in notes), f"expected the removal note for {value}, saw {notes}" + + def test_deprecated_hook_is_noop_for_non_dict_input(recwarn: pytest.WarningsRecorder) -> None: """US-M4: the before-hook passes non-mapping raw input through untouched.