From c0dce879868dd460b8429745d3528fb961ebf39b Mon Sep 17 00:00:00 2001 From: Kevin Schaper Date: Wed, 3 Sep 2025 18:50:55 -0700 Subject: [PATCH 1/4] Add automatic Pandera schema generator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Create PanderaGenerator class in matrix_schema/generators/panderagen.py - Generate schemas for MatrixNode, MatrixEdge, UnionedNode, UnionedEdge - Integrate with Makefile via gen-pandera target (always runs) - Maintain PySpark compatibility with proper ArrayType nullable=False for list items - Preserve existing validation patterns (enum checks, unique constraints) - Auto-generate from LinkML schema with proper formatting 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- Makefile | 5 + matrix_schema/__init__.py | 1 + matrix_schema/generators/__init__.py | 1 + matrix_schema/generators/panderagen.py | 359 +++++++++++++++++++++++++ project.Makefile | 3 +- src/matrix_schema/datamodel/pandera.py | 171 +++++++----- 6 files changed, 469 insertions(+), 71 deletions(-) create mode 100644 matrix_schema/__init__.py create mode 100644 matrix_schema/generators/__init__.py create mode 100644 matrix_schema/generators/panderagen.py diff --git a/Makefile b/Makefile index 9d00d37..f937ffd 100644 --- a/Makefile +++ b/Makefile @@ -126,6 +126,11 @@ gen-examples: gen-project: $(PYMODEL) $(RUN) gen-project ${CONFIG_YAML} -d $(DEST) $(SOURCE_SCHEMA_PATH) && mv $(DEST)/*.py $(PYMODEL) +# generate pandera schemas +.PHONY: gen-pandera +gen-pandera: + python -m matrix_schema.generators.panderagen $(SOURCE_SCHEMA_PATH) -c MatrixNode -c MatrixEdge -c UnionedNode -c UnionedEdge --output $(PYMODEL)/pandera.py + # non-empty arg triggers owl (workaround https://github.com/linkml/linkml/issues/1453) ifneq ($(strip ${GEN_OWL_ARGS}),) diff --git a/matrix_schema/__init__.py b/matrix_schema/__init__.py new file mode 100644 index 0000000..a7778e2 --- /dev/null +++ b/matrix_schema/__init__.py @@ -0,0 +1 @@ +"""Matrix schema package.""" \ No newline at end of file diff --git a/matrix_schema/generators/__init__.py b/matrix_schema/generators/__init__.py new file mode 100644 index 0000000..5ad3b51 --- /dev/null +++ b/matrix_schema/generators/__init__.py @@ -0,0 +1 @@ +"""Generators for matrix_schema.""" \ No newline at end of file diff --git a/matrix_schema/generators/panderagen.py b/matrix_schema/generators/panderagen.py new file mode 100644 index 0000000..0cb1f70 --- /dev/null +++ b/matrix_schema/generators/panderagen.py @@ -0,0 +1,359 @@ +""" +Pandera schema generator for LinkML schemas. + +This generator creates Pandera schemas from LinkML schemas with customizable +class selection and specialized handling for union types. +""" + +import os +import subprocess +import tempfile +from pathlib import Path +from typing import List, Optional, Dict, Any, Set + +import click +from jinja2 import Template +from linkml_runtime.linkml_model import SchemaDefinition, ClassDefinition, SlotDefinition +from linkml_runtime.utils.schemaview import SchemaView + +try: + from linkml.generators.generator import Generator, shared_arguments +except ImportError: + # Fallback for older versions or missing linkml.generators + class Generator: + generatorname = "panderagen" + valid_formats = ["py"] + visit_all_class_slots = True + + def __init__(self, schema, **kwargs): + self.schema = schema + + def shared_arguments(cls): + def decorator(f): + f = click.argument('yamlfile')(f) + return f + return decorator + + +class PanderaGenerator(Generator): + """Generator for Pandera schemas from LinkML schemas.""" + + generatorname = os.path.basename(__file__) + valid_formats = ["py"] + visit_all_class_slots = True + + def __init__( + self, + schema: str | SchemaDefinition, + classes: Optional[List[str]] = None, + **kwargs + ): + super().__init__(schema, **kwargs) + self.schemaview = SchemaView(schema) + self.target_classes = set(classes) if classes else set() + + def serialize(self, **kwargs) -> str: + """Generate Pandera schema code.""" + # Get template + template_str = self._get_template() + template = Template(template_str) + + # Prepare data for template + template_data = self._prepare_template_data() + + # Render template + code = template.render(**template_data) + + # Format with ruff if available (disabled for now due to template indentation issues) + # return self._format_code(code) + return code + + def _format_code(self, code: str) -> str: + """Format code using ruff if available.""" + try: + # Create a temporary file to format + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + f.write(code) + temp_path = f.name + + try: + # Try to format with ruff + result = subprocess.run( + ['ruff', 'format', '--stdin-filename', temp_path, '-'], + input=code, + text=True, + capture_output=True, + check=False + ) + + if result.returncode == 0: + return result.stdout + else: + # If ruff fails, return original code + return code + + finally: + # Clean up temp file + os.unlink(temp_path) + + except (FileNotFoundError, subprocess.SubprocessError): + # If ruff is not available, return original code + return code + + def _get_template(self) -> str: + """Get the Jinja template for Pandera generation.""" + return '''# THIS FILE IS AUTOGENERATED. DO NOT EDIT MANUALLY. +""" +Auto-generated Pandera schemas from LinkML schema. +""" + +try: + import pyspark.sql.types as T + import pandera.pandas as pa + from ..utils.pandera_utils import Column, DataFrameSchema + # Import the enums from the pydantic model + from .matrix_schema_pydantic import ( + PredicateEnum, + NodeCategoryEnum, + KnowledgeLevelEnum, + AgentTypeEnum + ) + DEPENDENCIES_AVAILABLE = True +except ImportError as e: + T = None + pa = None + Column = None + DataFrameSchema = None + PredicateEnum = None + NodeCategoryEnum = None + KnowledgeLevelEnum = None + AgentTypeEnum = None + DEPENDENCIES_AVAILABLE = False + IMPORT_ERROR = str(e) + +{% for class_name, class_info in classes.items() %} +def get_{{ class_info.function_name }}(validate_enumeration_values: bool = True): + """Get the Pandera schema for {{ class_name }} validation. + + Returns a universal schema that works with both pandas and PySpark DataFrames. + The actual schema type is determined at validation time based on the DataFrame type. + """ + if not DEPENDENCIES_AVAILABLE: + error_msg = f"Dependencies not available for Pandera schema. Original error: {IMPORT_ERROR if 'IMPORT_ERROR' in globals() else 'Unknown'}. Install with: pip install 'pandera[pyspark]' pyspark" + raise ImportError(error_msg) +{%- set enum_slots = [] -%} +{%- for slot_name, slot_info in class_info.slots.items() -%} +{%- if slot_info.enum_checks -%} +{%- set _ = enum_slots.append((slot_name, slot_info.enum_checks)) -%} +{%- endif -%} +{%- endfor -%} +{%- if enum_slots %} + + {% for slot_name, enum_check in enum_slots -%} + if validate_enumeration_values: + {{ slot_name }}_checks = [{{ enum_check }}] + else: + {{ slot_name }}_checks = [] + + {% endfor -%} +{%- endif -%} + + return DataFrameSchema( + columns={ +{%- for slot_name, slot_info in class_info.slots.items() %} + "{{ slot_name }}": Column({{ slot_info.pyspark_type }}, nullable={{ slot_info.nullable | title }}{% if slot_info.enum_checks %}, + checks={{ slot_name }}_checks{% endif %}), +{%- endfor %} + }, +{%- if class_info.unique_fields %} + unique={{ class_info.unique_fields }}, +{%- endif %} + strict=True, + ) + + +{% endfor %} +''' + + def _prepare_template_data(self) -> Dict[str, Any]: + """Prepare data for the Jinja template.""" + classes_data = {} + + for class_name in self.target_classes: + class_def = self.schemaview.get_class(class_name) + if not class_def: + continue + + class_info = self._process_class(class_name, class_def) + classes_data[class_name] = class_info + + return { + "classes": classes_data, + "schema_name": self.schemaview.schema.name, + } + + def _process_class(self, class_name: str, class_def: ClassDefinition) -> Dict[str, Any]: + """Process a single class definition.""" + # Get all slots for this class + all_slots = self.schemaview.class_slots(class_name) + + slots_data = {} + has_primary_knowledge_sources = False + has_categories = False + + for slot_name in all_slots: + slot_def = self.schemaview.get_slot(slot_name) + if not slot_def: + continue + + slot_info = self._process_slot(slot_def) + slots_data[slot_name] = slot_info + + if slot_name == "primary_knowledge_sources": + has_primary_knowledge_sources = True + elif slot_name in ["category", "categories"]: + has_categories = True + + # Generate function name (convert CamelCase to snake_case) + function_name = self._to_snake_case(class_name) + "_schema" + + # Determine unique fields based on class name + unique_fields = None + if "node" in class_name.lower(): + unique_fields = '["id"]' + elif "edge" in class_name.lower(): + unique_fields = '["subject", "predicate", "object"]' + + return { + "function_name": function_name, + "slots": slots_data, + "unique_fields": unique_fields, + "description": class_def.description, + } + + def _process_slot(self, slot_def: SlotDefinition) -> Dict[str, Any]: + """Process a single slot definition.""" + # Determine PySpark type + pyspark_type = self._linkml_to_pyspark_type( + slot_def.range if slot_def.range else "string", + slot_def.multivalued + ) + + # Determine nullability + nullable = not slot_def.required if slot_def.required is not None else True + + # Generate enum checks if applicable + enum_checks = None + if slot_def.range: + range_name = slot_def.range + if range_name == "NodeCategoryEnum": + enum_checks = "pa.Check.isin([category.value for category in NodeCategoryEnum])" + elif range_name == "PredicateEnum": + enum_checks = "pa.Check.isin([predicate.value for predicate in PredicateEnum])" + elif range_name == "KnowledgeLevelEnum": + enum_checks = "pa.Check.isin([level.value for level in KnowledgeLevelEnum])" + elif range_name == "AgentTypeEnum": + enum_checks = "pa.Check.isin([agent.value for agent in AgentTypeEnum])" + + return { + "pyspark_type": pyspark_type, + "nullable": nullable, + "enum_checks": enum_checks, + "description": slot_def.description, + "multivalued": slot_def.multivalued, + } + + def _get_pandera_type(self, slot_def: SlotDefinition) -> str: + """Convert LinkML type to Pandera type.""" + if slot_def.range: + range_type = slot_def.range + else: + range_type = "string" # default + + # Handle multivalued slots + if slot_def.multivalued: + base_type = self._linkml_to_pandera_type(range_type) + return f"object" # Lists are stored as objects in pandas + else: + return self._linkml_to_pandera_type(range_type) + + def _linkml_to_pyspark_type(self, linkml_type: str, multivalued: bool = False) -> str: + """Map LinkML types to PySpark types.""" + base_type_mapping = { + "string": "T.StringType()", + "integer": "T.IntegerType()", + "float": "T.FloatType()", + "double": "T.DoubleType()", + "boolean": "T.BooleanType()", + "date": "T.StringType()", + "datetime": "T.StringType()", + "uri": "T.StringType()", + "uriorcurie": "T.StringType()", + } + + base_type = base_type_mapping.get(linkml_type.lower(), "T.StringType()") + + if multivalued: + # For arrays, the nullable=False parameter applies to the array items, not the array itself + return f"T.ArrayType({base_type}, False)" + else: + return base_type + + def _generate_checks(self, slot_def: SlotDefinition) -> List[str]: + """Generate Pandera checks for a slot.""" + checks = [] + + # Add pattern check if available + if slot_def.pattern: + checks.append(f'pa.Check.str_matches(r"{slot_def.pattern}")') + + # Add enum check if available + if hasattr(slot_def, 'permissible_values') and slot_def.permissible_values: + values = [f'"{pv.text}"' for pv in slot_def.permissible_values.values()] + checks.append(f'pa.Check.isin([{", ".join(values)}])') + + return checks + + def _to_snake_case(self, camel_str: str) -> str: + """Convert CamelCase to snake_case.""" + result = [] + for i, char in enumerate(camel_str): + if char.isupper() and i > 0: + result.append('_') + result.append(char.lower()) + return ''.join(result) + + +@click.command() +@click.argument('yamlfile') +@click.option( + "--classes", + "-c", + multiple=True, + required=True, + help="Classes to generate Pandera schemas for (can be specified multiple times)" +) +@click.option( + "--output", + "-o", + help="Output file (default: stdout)" +) +def cli(yamlfile, classes, output): + """Generate Pandera schemas from LinkML schema.""" + try: + generator = PanderaGenerator(yamlfile, classes=list(classes)) + result = generator.serialize() + + if output: + with open(output, 'w') as f: + f.write(result) + click.echo(f"Generated Pandera schemas written to {output}", err=True) + else: + click.echo(result) + + except Exception as e: + raise click.ClickException(f"Error: {e}") + + +if __name__ == "__main__": + cli() \ No newline at end of file diff --git a/project.Makefile b/project.Makefile index a33203a..020c40f 100644 --- a/project.Makefile +++ b/project.Makefile @@ -9,7 +9,6 @@ gen-valid-edge-type-table: gen-pydantic: $(RUN) gen-pydantic --meta None src/matrix_schema/schema/matrix_schema.yaml > src/matrix_schema/datamodel/matrix_schema_pydantic.py -gen-pandera: gen-pydantic - @echo "The pandera schema currently needs to be updated manually in src/matrix_schema/datamodel/pandera.py" +# gen-pandera target is now defined in main Makefile gen-project: gen-pydantic gen-pandera gen-biolink-imports gen-valid-edge-type-table diff --git a/src/matrix_schema/datamodel/pandera.py b/src/matrix_schema/datamodel/pandera.py index 66352ca..5cf576a 100644 --- a/src/matrix_schema/datamodel/pandera.py +++ b/src/matrix_schema/datamodel/pandera.py @@ -1,3 +1,8 @@ +# THIS FILE IS AUTOGENERATED. DO NOT EDIT MANUALLY. +""" +Auto-generated Pandera schemas from LinkML schema. +""" + try: import pyspark.sql.types as T import pandera.pandas as pa @@ -23,8 +28,8 @@ IMPORT_ERROR = str(e) -def get_matrix_node_schema(validate_enumeration_values: bool = True): - """Get the Pandera schema for MatrixNode validation. +def get_matrix_edge_schema(validate_enumeration_values: bool = True): + """Get the Pandera schema for MatrixEdge validation. Returns a universal schema that works with both pandas and PySpark DataFrames. The actual schema type is determined at validation time based on the DataFrame type. @@ -34,31 +39,49 @@ def get_matrix_node_schema(validate_enumeration_values: bool = True): raise ImportError(error_msg) if validate_enumeration_values: - category_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] + predicate_checks = [pa.Check.isin([predicate.value for predicate in PredicateEnum])] else: - category_checks = [] + predicate_checks = [] + + if validate_enumeration_values: + knowledge_level_checks = [pa.Check.isin([level.value for level in KnowledgeLevelEnum])] + else: + knowledge_level_checks = [] + + if validate_enumeration_values: + agent_type_checks = [pa.Check.isin([agent.value for agent in AgentTypeEnum])] + else: + agent_type_checks = [] return DataFrameSchema( columns={ - "id": Column(T.StringType(), nullable=False), - "name": Column(T.StringType(), nullable=True), - "category": Column(T.StringType(), nullable=False, - checks=category_checks), - "description": Column(T.StringType(), nullable=True), - "equivalent_identifiers": Column(T.ArrayType(T.StringType()), nullable=True), - "all_categories": Column(T.ArrayType(T.StringType()), nullable=True), - "publications": Column(T.ArrayType(T.StringType()), nullable=True), - "labels": Column(T.ArrayType(T.StringType()), nullable=True), - "international_resource_identifier": Column(T.StringType(), nullable=True), - "upstream_data_source": Column(T.ArrayType(T.StringType()), nullable=True), + "subject": Column(T.StringType(), nullable=True), + "predicate": Column(T.StringType(), nullable=True, + checks=predicate_checks), + "object": Column(T.StringType(), nullable=True), + "knowledge_level": Column(T.StringType(), nullable=True, + checks=knowledge_level_checks), + "agent_type": Column(T.StringType(), nullable=True, + checks=agent_type_checks), + "primary_knowledge_source": Column(T.StringType(), nullable=True), + "aggregator_knowledge_source": Column(T.ArrayType(T.StringType(), False), nullable=True), + "publications": Column(T.ArrayType(T.StringType(), False), nullable=True), + "subject_aspect_qualifier": Column(T.StringType(), nullable=True), + "subject_direction_qualifier": Column(T.StringType(), nullable=True), + "object_aspect_qualifier": Column(T.StringType(), nullable=True), + "object_direction_qualifier": Column(T.StringType(), nullable=True), + "upstream_data_source": Column(T.ArrayType(T.StringType(), False), nullable=True), + "num_references": Column(T.IntegerType(), nullable=True), + "num_sentences": Column(T.IntegerType(), nullable=True), }, - unique=["id"], + unique=["subject", "predicate", "object"], strict=True, ) -def get_matrix_edge_schema(validate_enumeration_values: bool = True): - """Get the Pandera schema for MatrixEdge validation. + +def get_unioned_edge_schema(validate_enumeration_values: bool = True): + """Get the Pandera schema for UnionedEdge validation. Returns a universal schema that works with both pandas and PySpark DataFrames. The actual schema type is determined at validation time based on the DataFrame type. @@ -66,35 +89,41 @@ def get_matrix_edge_schema(validate_enumeration_values: bool = True): if not DEPENDENCIES_AVAILABLE: error_msg = f"Dependencies not available for Pandera schema. Original error: {IMPORT_ERROR if 'IMPORT_ERROR' in globals() else 'Unknown'}. Install with: pip install 'pandera[pyspark]' pyspark" raise ImportError(error_msg) - + if validate_enumeration_values: predicate_checks = [pa.Check.isin([predicate.value for predicate in PredicateEnum])] - knowledge_level_checks = [pa.Check.isin([level.value for level in KnowledgeLevelEnum])] - agent_type_checks = [pa.Check.isin([agent.value for agent in AgentTypeEnum])] else: predicate_checks = [] + + if validate_enumeration_values: + knowledge_level_checks = [pa.Check.isin([level.value for level in KnowledgeLevelEnum])] + else: knowledge_level_checks = [] + + if validate_enumeration_values: + agent_type_checks = [pa.Check.isin([agent.value for agent in AgentTypeEnum])] + else: agent_type_checks = [] return DataFrameSchema( columns={ - "subject": Column(T.StringType(), nullable=False), - "predicate": Column(T.StringType(), nullable=False, - checks=predicate_checks), - "object": Column(T.StringType(), nullable=False), + "primary_knowledge_sources": Column(T.ArrayType(T.StringType(), False), nullable=True), + "subject": Column(T.StringType(), nullable=True), + "predicate": Column(T.StringType(), nullable=True, + checks=predicate_checks), + "object": Column(T.StringType(), nullable=True), "knowledge_level": Column(T.StringType(), nullable=True, - checks=knowledge_level_checks), + checks=knowledge_level_checks), "agent_type": Column(T.StringType(), nullable=True, - checks=agent_type_checks), + checks=agent_type_checks), "primary_knowledge_source": Column(T.StringType(), nullable=True), - "aggregator_knowledge_source": Column(T.ArrayType(T.StringType()), nullable=True), - "publications": Column(T.ArrayType(T.StringType()), nullable=True), + "aggregator_knowledge_source": Column(T.ArrayType(T.StringType(), False), nullable=True), + "publications": Column(T.ArrayType(T.StringType(), False), nullable=True), "subject_aspect_qualifier": Column(T.StringType(), nullable=True), "subject_direction_qualifier": Column(T.StringType(), nullable=True), "object_aspect_qualifier": Column(T.StringType(), nullable=True), "object_direction_qualifier": Column(T.StringType(), nullable=True), - "upstream_data_source": Column(T.ArrayType(T.StringType()), nullable=True), - # Additional fields from original pandera schema + "upstream_data_source": Column(T.ArrayType(T.StringType(), False), nullable=True), "num_references": Column(T.IntegerType(), nullable=True), "num_sentences": Column(T.IntegerType(), nullable=True), }, @@ -103,8 +132,9 @@ def get_matrix_edge_schema(validate_enumeration_values: bool = True): ) -def get_unioned_node_schema(validate_enumeration_values: bool = True): - """Get the Pandera schema for UnionedNode validation. + +def get_matrix_node_schema(validate_enumeration_values: bool = True): + """Get the Pandera schema for MatrixNode validation. Returns a universal schema that works with both pandas and PySpark DataFrames. The actual schema type is determined at validation time based on the DataFrame type. @@ -118,27 +148,34 @@ def get_unioned_node_schema(validate_enumeration_values: bool = True): else: category_checks = [] + if validate_enumeration_values: + all_categories_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] + else: + all_categories_checks = [] + return DataFrameSchema( columns={ - "id": Column(T.StringType(), nullable=False), + "id": Column(T.StringType(), nullable=True), "name": Column(T.StringType(), nullable=True), - "category": Column(T.StringType(), nullable=False, + "category": Column(T.StringType(), nullable=True, checks=category_checks), "description": Column(T.StringType(), nullable=True), - "equivalent_identifiers": Column(T.ArrayType(T.StringType()), nullable=True), - "all_categories": Column(T.ArrayType(T.StringType()), nullable=True), - "publications": Column(T.ArrayType(T.StringType()), nullable=True), - "labels": Column(T.ArrayType(T.StringType()), nullable=True), + "equivalent_identifiers": Column(T.ArrayType(T.StringType(), False), nullable=True), + "all_categories": Column(T.ArrayType(T.StringType(), False), nullable=True, + checks=all_categories_checks), + "publications": Column(T.ArrayType(T.StringType(), False), nullable=True), + "labels": Column(T.ArrayType(T.StringType(), False), nullable=True), "international_resource_identifier": Column(T.StringType(), nullable=True), - "upstream_data_source": Column(T.ArrayType(T.StringType()), nullable=True), + "upstream_data_source": Column(T.ArrayType(T.StringType(), False), nullable=True), }, unique=["id"], strict=True, ) -def get_unioned_edge_schema(validate_enumeration_values: bool = True): - """Get the Pandera schema for UnionedEdge validation. + +def get_unioned_node_schema(validate_enumeration_values: bool = True): + """Get the Pandera schema for UnionedNode validation. Returns a universal schema that works with both pandas and PySpark DataFrames. The actual schema type is determined at validation time based on the DataFrame type. @@ -146,38 +183,34 @@ def get_unioned_edge_schema(validate_enumeration_values: bool = True): if not DEPENDENCIES_AVAILABLE: error_msg = f"Dependencies not available for Pandera schema. Original error: {IMPORT_ERROR if 'IMPORT_ERROR' in globals() else 'Unknown'}. Install with: pip install 'pandera[pyspark]' pyspark" raise ImportError(error_msg) - + if validate_enumeration_values: - predicate_checks = [pa.Check.isin([predicate.value for predicate in PredicateEnum])] - knowledge_level_checks = [pa.Check.isin([level.value for level in KnowledgeLevelEnum])] - agent_type_checks = [pa.Check.isin([agent.value for agent in AgentTypeEnum])] + category_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] else: - predicate_checks = [] - knowledge_level_checks = [] - agent_type_checks = [] + category_checks = [] + + if validate_enumeration_values: + all_categories_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] + else: + all_categories_checks = [] return DataFrameSchema( columns={ - "primary_knowledge_sources": Column(T.ArrayType(T.StringType(), False), nullable=False), - "subject": Column(T.StringType(), nullable=False), - "predicate": Column(T.StringType(), nullable=False, - checks=predicate_checks), - "object": Column(T.StringType(), nullable=False), - "knowledge_level": Column(T.StringType(), nullable=True, - checks=knowledge_level_checks), - "agent_type": Column(T.StringType(), nullable=True, - checks=agent_type_checks), - "primary_knowledge_source": Column(T.StringType(), nullable=True), - "aggregator_knowledge_source": Column(T.ArrayType(T.StringType()), nullable=True), - "publications": Column(T.ArrayType(T.StringType()), nullable=True), - "subject_aspect_qualifier": Column(T.StringType(), nullable=True), - "subject_direction_qualifier": Column(T.StringType(), nullable=True), - "object_aspect_qualifier": Column(T.StringType(), nullable=True), - "object_direction_qualifier": Column(T.StringType(), nullable=True), - "upstream_data_source": Column(T.ArrayType(T.StringType()), nullable=True), - "num_references": Column(T.IntegerType(), nullable=True), - "num_sentences": Column(T.IntegerType(), nullable=True), + "id": Column(T.StringType(), nullable=True), + "name": Column(T.StringType(), nullable=True), + "category": Column(T.StringType(), nullable=True, + checks=category_checks), + "description": Column(T.StringType(), nullable=True), + "equivalent_identifiers": Column(T.ArrayType(T.StringType(), False), nullable=True), + "all_categories": Column(T.ArrayType(T.StringType(), False), nullable=True, + checks=all_categories_checks), + "publications": Column(T.ArrayType(T.StringType(), False), nullable=True), + "labels": Column(T.ArrayType(T.StringType(), False), nullable=True), + "international_resource_identifier": Column(T.StringType(), nullable=True), + "upstream_data_source": Column(T.ArrayType(T.StringType(), False), nullable=True), }, - unique=["subject", "predicate", "object"], + unique=["id"], strict=True, ) + + From 4960776025ca4153ff1b932e6e55beaad5dcf85f Mon Sep 17 00:00:00 2001 From: Kevin Schaper Date: Thu, 4 Sep 2025 13:17:32 -0700 Subject: [PATCH 2/4] move gen-pandera to project.Makefile, preserve order from cli to generated schema so that we get a better diff on the PR --- Makefile | 6 -- matrix_schema/generators/panderagen.py | 7 +- project.Makefile | 9 ++- src/matrix_schema/datamodel/pandera.py | 100 ++++++++++++------------- 4 files changed, 63 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index f937ffd..8f9be06 100644 --- a/Makefile +++ b/Makefile @@ -126,12 +126,6 @@ gen-examples: gen-project: $(PYMODEL) $(RUN) gen-project ${CONFIG_YAML} -d $(DEST) $(SOURCE_SCHEMA_PATH) && mv $(DEST)/*.py $(PYMODEL) -# generate pandera schemas -.PHONY: gen-pandera -gen-pandera: - python -m matrix_schema.generators.panderagen $(SOURCE_SCHEMA_PATH) -c MatrixNode -c MatrixEdge -c UnionedNode -c UnionedEdge --output $(PYMODEL)/pandera.py - - # non-empty arg triggers owl (workaround https://github.com/linkml/linkml/issues/1453) ifneq ($(strip ${GEN_OWL_ARGS}),) mkdir -p ${DEST}/owl || true diff --git a/matrix_schema/generators/panderagen.py b/matrix_schema/generators/panderagen.py index 0cb1f70..f436b7c 100644 --- a/matrix_schema/generators/panderagen.py +++ b/matrix_schema/generators/panderagen.py @@ -50,7 +50,8 @@ def __init__( ): super().__init__(schema, **kwargs) self.schemaview = SchemaView(schema) - self.target_classes = set(classes) if classes else set() + # Preserve the order of classes as specified + self.target_classes = classes if classes else [] def serialize(self, **kwargs) -> str: """Generate Pandera schema code.""" @@ -177,7 +178,9 @@ def get_{{ class_info.function_name }}(validate_enumeration_values: bool = True) def _prepare_template_data(self) -> Dict[str, Any]: """Prepare data for the Jinja template.""" - classes_data = {} + # Use OrderedDict to preserve the order of classes as specified in CLI + from collections import OrderedDict + classes_data = OrderedDict() for class_name in self.target_classes: class_def = self.schemaview.get_class(class_name) diff --git a/project.Makefile b/project.Makefile index 020c40f..8ab5bb9 100644 --- a/project.Makefile +++ b/project.Makefile @@ -1,14 +1,21 @@ ## Add your own custom Makefile targets here +.PHONY: generate_biolink_imports gen-biolink-imports: $(RUN) python src/matrix_schema/utils/generate_biolink_imports.py +.PHONY: generate_valid_edge_type_table gen-valid-edge-type-table: $(RUN) python src/matrix_schema/utils/generate_valid_edge_type_table.py +.PHONY: gen-pydantic gen-pydantic: $(RUN) gen-pydantic --meta None src/matrix_schema/schema/matrix_schema.yaml > src/matrix_schema/datamodel/matrix_schema_pydantic.py -# gen-pandera target is now defined in main Makefile +# generate pandera schemas +.PHONY: gen-pandera +gen-pandera: + python -m matrix_schema.generators.panderagen $(SOURCE_SCHEMA_PATH) -c MatrixNode -c MatrixEdge -c UnionedNode -c UnionedEdge --output $(PYMODEL)/pandera.py + gen-project: gen-pydantic gen-pandera gen-biolink-imports gen-valid-edge-type-table diff --git a/src/matrix_schema/datamodel/pandera.py b/src/matrix_schema/datamodel/pandera.py index 5cf576a..759a619 100644 --- a/src/matrix_schema/datamodel/pandera.py +++ b/src/matrix_schema/datamodel/pandera.py @@ -28,8 +28,8 @@ IMPORT_ERROR = str(e) -def get_matrix_edge_schema(validate_enumeration_values: bool = True): - """Get the Pandera schema for MatrixEdge validation. +def get_matrix_node_schema(validate_enumeration_values: bool = True): + """Get the Pandera schema for MatrixNode validation. Returns a universal schema that works with both pandas and PySpark DataFrames. The actual schema type is determined at validation time based on the DataFrame type. @@ -39,49 +39,38 @@ def get_matrix_edge_schema(validate_enumeration_values: bool = True): raise ImportError(error_msg) if validate_enumeration_values: - predicate_checks = [pa.Check.isin([predicate.value for predicate in PredicateEnum])] - else: - predicate_checks = [] - - if validate_enumeration_values: - knowledge_level_checks = [pa.Check.isin([level.value for level in KnowledgeLevelEnum])] + category_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] else: - knowledge_level_checks = [] + category_checks = [] if validate_enumeration_values: - agent_type_checks = [pa.Check.isin([agent.value for agent in AgentTypeEnum])] + all_categories_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] else: - agent_type_checks = [] + all_categories_checks = [] return DataFrameSchema( columns={ - "subject": Column(T.StringType(), nullable=True), - "predicate": Column(T.StringType(), nullable=True, - checks=predicate_checks), - "object": Column(T.StringType(), nullable=True), - "knowledge_level": Column(T.StringType(), nullable=True, - checks=knowledge_level_checks), - "agent_type": Column(T.StringType(), nullable=True, - checks=agent_type_checks), - "primary_knowledge_source": Column(T.StringType(), nullable=True), - "aggregator_knowledge_source": Column(T.ArrayType(T.StringType(), False), nullable=True), + "id": Column(T.StringType(), nullable=True), + "name": Column(T.StringType(), nullable=True), + "category": Column(T.StringType(), nullable=True, + checks=category_checks), + "description": Column(T.StringType(), nullable=True), + "equivalent_identifiers": Column(T.ArrayType(T.StringType(), False), nullable=True), + "all_categories": Column(T.ArrayType(T.StringType(), False), nullable=True, + checks=all_categories_checks), "publications": Column(T.ArrayType(T.StringType(), False), nullable=True), - "subject_aspect_qualifier": Column(T.StringType(), nullable=True), - "subject_direction_qualifier": Column(T.StringType(), nullable=True), - "object_aspect_qualifier": Column(T.StringType(), nullable=True), - "object_direction_qualifier": Column(T.StringType(), nullable=True), + "labels": Column(T.ArrayType(T.StringType(), False), nullable=True), + "international_resource_identifier": Column(T.StringType(), nullable=True), "upstream_data_source": Column(T.ArrayType(T.StringType(), False), nullable=True), - "num_references": Column(T.IntegerType(), nullable=True), - "num_sentences": Column(T.IntegerType(), nullable=True), }, - unique=["subject", "predicate", "object"], + unique=["id"], strict=True, ) -def get_unioned_edge_schema(validate_enumeration_values: bool = True): - """Get the Pandera schema for UnionedEdge validation. +def get_matrix_edge_schema(validate_enumeration_values: bool = True): + """Get the Pandera schema for MatrixEdge validation. Returns a universal schema that works with both pandas and PySpark DataFrames. The actual schema type is determined at validation time based on the DataFrame type. @@ -107,7 +96,6 @@ def get_unioned_edge_schema(validate_enumeration_values: bool = True): return DataFrameSchema( columns={ - "primary_knowledge_sources": Column(T.ArrayType(T.StringType(), False), nullable=True), "subject": Column(T.StringType(), nullable=True), "predicate": Column(T.StringType(), nullable=True, checks=predicate_checks), @@ -133,8 +121,8 @@ def get_unioned_edge_schema(validate_enumeration_values: bool = True): -def get_matrix_node_schema(validate_enumeration_values: bool = True): - """Get the Pandera schema for MatrixNode validation. +def get_unioned_node_schema(validate_enumeration_values: bool = True): + """Get the Pandera schema for UnionedNode validation. Returns a universal schema that works with both pandas and PySpark DataFrames. The actual schema type is determined at validation time based on the DataFrame type. @@ -174,8 +162,8 @@ def get_matrix_node_schema(validate_enumeration_values: bool = True): -def get_unioned_node_schema(validate_enumeration_values: bool = True): - """Get the Pandera schema for UnionedNode validation. +def get_unioned_edge_schema(validate_enumeration_values: bool = True): + """Get the Pandera schema for UnionedEdge validation. Returns a universal schema that works with both pandas and PySpark DataFrames. The actual schema type is determined at validation time based on the DataFrame type. @@ -185,31 +173,43 @@ def get_unioned_node_schema(validate_enumeration_values: bool = True): raise ImportError(error_msg) if validate_enumeration_values: - category_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] + predicate_checks = [pa.Check.isin([predicate.value for predicate in PredicateEnum])] else: - category_checks = [] + predicate_checks = [] if validate_enumeration_values: - all_categories_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] + knowledge_level_checks = [pa.Check.isin([level.value for level in KnowledgeLevelEnum])] else: - all_categories_checks = [] + knowledge_level_checks = [] + + if validate_enumeration_values: + agent_type_checks = [pa.Check.isin([agent.value for agent in AgentTypeEnum])] + else: + agent_type_checks = [] return DataFrameSchema( columns={ - "id": Column(T.StringType(), nullable=True), - "name": Column(T.StringType(), nullable=True), - "category": Column(T.StringType(), nullable=True, - checks=category_checks), - "description": Column(T.StringType(), nullable=True), - "equivalent_identifiers": Column(T.ArrayType(T.StringType(), False), nullable=True), - "all_categories": Column(T.ArrayType(T.StringType(), False), nullable=True, - checks=all_categories_checks), + "primary_knowledge_sources": Column(T.ArrayType(T.StringType(), False), nullable=True), + "subject": Column(T.StringType(), nullable=True), + "predicate": Column(T.StringType(), nullable=True, + checks=predicate_checks), + "object": Column(T.StringType(), nullable=True), + "knowledge_level": Column(T.StringType(), nullable=True, + checks=knowledge_level_checks), + "agent_type": Column(T.StringType(), nullable=True, + checks=agent_type_checks), + "primary_knowledge_source": Column(T.StringType(), nullable=True), + "aggregator_knowledge_source": Column(T.ArrayType(T.StringType(), False), nullable=True), "publications": Column(T.ArrayType(T.StringType(), False), nullable=True), - "labels": Column(T.ArrayType(T.StringType(), False), nullable=True), - "international_resource_identifier": Column(T.StringType(), nullable=True), + "subject_aspect_qualifier": Column(T.StringType(), nullable=True), + "subject_direction_qualifier": Column(T.StringType(), nullable=True), + "object_aspect_qualifier": Column(T.StringType(), nullable=True), + "object_direction_qualifier": Column(T.StringType(), nullable=True), "upstream_data_source": Column(T.ArrayType(T.StringType(), False), nullable=True), + "num_references": Column(T.IntegerType(), nullable=True), + "num_sentences": Column(T.IntegerType(), nullable=True), }, - unique=["id"], + unique=["subject", "predicate", "object"], strict=True, ) From 44229370a212613268c043ad1be165f32c0acdc9 Mon Sep 17 00:00:00 2001 From: Kevin Schaper Date: Thu, 4 Sep 2025 16:22:04 -0700 Subject: [PATCH 3/4] updated the pyproject toml to match poetry 2.x needs, made id a required slot, ran gen-project --- matrix_schema/__init__.py | 1 - poetry.lock | 25 +- project/excel/matrix_schema.xlsx | Bin 9680 -> 9682 bytes project/jsonld/matrix_schema.context.jsonld | 2 +- project/jsonld/matrix_schema.jsonld | 4 +- project/owl/matrix_schema.owl.ttl | 612 ++++++++--------- project/shacl/matrix_schema.shacl.ttl | 636 +++++++++--------- pyproject.toml | 28 +- src/matrix_schema/datamodel/matrix_schema.py | 2 +- src/matrix_schema/datamodel/pandera.py | 4 +- .../matrix_schema}/generators/__init__.py | 0 .../matrix_schema}/generators/panderagen.py | 0 src/matrix_schema/schema/matrix_kg.yaml | 1 + 13 files changed, 650 insertions(+), 665 deletions(-) delete mode 100644 matrix_schema/__init__.py rename {matrix_schema => src/matrix_schema}/generators/__init__.py (100%) rename {matrix_schema => src/matrix_schema}/generators/panderagen.py (100%) diff --git a/matrix_schema/__init__.py b/matrix_schema/__init__.py deleted file mode 100644 index a7778e2..0000000 --- a/matrix_schema/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Matrix schema package.""" \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 07a9d58..923255a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "annotated-types" @@ -175,7 +175,7 @@ version = "25.1.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "docs"] files = [ {file = "attrs-25.1.0-py3-none-any.whl", hash = "sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a"}, {file = "attrs-25.1.0.tar.gz", hash = "sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e"}, @@ -1470,7 +1470,7 @@ version = "3.1.5" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["dev", "docs"] files = [ {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, @@ -1488,7 +1488,7 @@ version = "24.4.0" description = "Command-line interface to Jinja2 for templating in shell scripts." optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["dev", "docs"] files = [ {file = "jinjanator-24.4.0-py3-none-any.whl", hash = "sha256:9447109da974ac402dd8ee5ef586b2c24358dac67c5c00d2259f98baec4d73f8"}, {file = "jinjanator-24.4.0.tar.gz", hash = "sha256:83ec06cdedceed1df4b2b7f85c227b79668621037cbe6c065bbfc6e90d038b9e"}, @@ -1508,7 +1508,7 @@ version = "24.2.0" description = "Package which provides the plugin API for the jinjanator tool" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["dev", "docs"] files = [ {file = "jinjanator_plugins-24.2.0-py3-none-any.whl", hash = "sha256:a0adfe5df0e8bd1a025602bc1867505b243eac5783464f8ce2b8600ac41e73ba"}, {file = "jinjanator_plugins-24.2.0.tar.gz", hash = "sha256:5fa8eecb6d9fbef5e7947b3820da4bd7b2ef3f102721e42382de5e71e52b409a"}, @@ -2094,7 +2094,7 @@ version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["dev", "docs"] files = [ {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, @@ -2951,7 +2951,7 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "docs"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -3607,7 +3607,7 @@ version = "1.0.1" description = "Read key-value pairs from a .env file and set them as environment variables" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["dev", "docs"] files = [ {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, @@ -3714,7 +3714,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "docs"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -4870,7 +4870,7 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "docs"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -5186,10 +5186,7 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] -[extras] -docs = ["jinjanator"] - [metadata] lock-version = "2.1" python-versions = "^3.9" -content-hash = "e0cf02a533a0ebe8ddf696506b730b38e07a85c7c0618285e141dea3b6925b14" +content-hash = "0ce9b7498feff7c40e6850f9aa8d260c40ecb2576fbddc9c15d0ab3376806576" diff --git a/project/excel/matrix_schema.xlsx b/project/excel/matrix_schema.xlsx index 320a855d1c67c7f74614cb3c0ba329a585c0b3c6..c053a24d68013f417acc1721c11f8e51932a7c2f 100644 GIT binary patch delta 837 zcmccMeaV|Qz?+#xgn@y9gJDjS%0%7+^*}26Q(p7xw?M(uj0_BdKsqHqIiM)NpjbaS zzbI9&A~$F1#Dji^4S3q(Q%~(Jops66C00+tPeIX;^N6n3p0$fS7av)1_x9y!Yo(jG zkN^1p=Vn-z@RFBGcE|M+matvUDp>1(sfE{CfR%H?8OFJOKRtdJ-YLD9DK)>4-?4r+ z|6X^2IqMH9X0G-;ta9d1Qi}Rg({7`hV`X9y%r_P}`*U{fY2fyji2b!W(NC<%6kbXc z96on4yRv+LZGu-pVCo;U$L7(2UI$Y{_xi7sSNto!@O94S?0=at@sj76LJzO64~|C( zq|L7w?bw)s!L(VM;{_v_A;$HG3B;JJ&7rc{lJ^K$bQ3=-Ggy{GC>NrvLKw<;A+i7> z)h$s0kz$jof|$@O^9-yhM|L|zMopd%!Z24z2QwxwRJ7)V$I|4die_M|<&>lu?@zW; zQUdehm86*zI3_Eoh)(wBVAT%rW@Hj!hR1XCBJR?>lS~W@Z`m0bB!CeN0}YMblMgDX z!;RqL1sfr&EC?1dRF-D?$_ExoRu)tMIigpMLDdSFOhkaDp_=qp5Ug+|RPSD(-dtg@ z(0gS;d5~U{MTNU_nHd-udC~RWoh+{cafPRfG*g2FSlfCD*2xVjJYbjft4K5cp1fK` z2FyF9B5hS6!@v+X-%$BViOBI+K^Es8t(=_mgF9xdIJoCU?CZ@2g@^fE0vS?@9hMhc3e<6wD_Z&FM2_*jkC#5(EQD$ zJ$mooZRhv-*mYm*>aGW?bN7mw$9(uzTk==_#eS~X4QJQ=Z2zbfAXN8v_AeHcAlm$f z(T*e4D z3&|@(^cH?tD|`v4*N6jM?`J-sLaE7F$~<7bmCDjgzXid1S1Ut|+Yc1V76u6!n1PZ{ zfHxzP2s1p{MVl-t+?~tJz`)3hZuafTiYn@0W4%?Rnd&6K#@4DpjGd?=&G>WjS``_v zf-@@8R%J2_4Dm&&ImP;VAQIgk8-Bs0USOJ*WMN?7hj|`IH!#LeR#cUf2ZfOc-+R@Y jK(Sw}3=C2zY9uGescM3)>Qa?v)SA3ZRgG=1.0.0", "poetry-dynamic-versioning"] build-backend = "poetry_dynamic_versioning.backend" -[project] +[tool.poetry] name = "matrix_schema" +version = "0.0.0" description = "The collected MATRIX schemas" -authors = [ - {name = "Nicolas Matentzoglu", email = "nicolas.matentzoglu@gmail.com"}, -] +authors = ["Nicolas Matentzoglu "] license = "BSD-3" readme = "README.md" +packages = [{include = "matrix_schema", from = "src"}] include = ["README.md", "src/matrix_schema/schema", "project"] -requires-python = ">=3.9" - -dynamic = ["version"] - -dependencies = [ - "linkml-runtime >=1.8.0", - "biolink-model (>=4.2.5,<5.0.0)", -] - -[tool.poetry] -requires-poetry = ">=2.0" -version = "0.0.0" - [tool.poetry.dependencies] python = "^3.9" -linkml-runtime = "^1.1.24" +linkml-runtime = ">=1.8.0" +biolink-model = ">=4.2.5,<5.0.0" -[project.optional-dependencies] -docs = ["jinjanator>=24.4.0"] +[tool.poetry.group.docs.dependencies] +jinjanator = ">=24.4.0" [tool.poetry.requires-plugins] poetry-dynamic-versioning = ">=1.5.2" diff --git a/src/matrix_schema/datamodel/matrix_schema.py b/src/matrix_schema/datamodel/matrix_schema.py index 0b84c32..b9af4eb 100644 --- a/src/matrix_schema/datamodel/matrix_schema.py +++ b/src/matrix_schema/datamodel/matrix_schema.py @@ -1,5 +1,5 @@ # Auto generated from matrix_schema.yaml by pythongen.py version: 0.0.1 -# Generation date: 2025-08-26T11:42:20 +# Generation date: 2025-09-04T16:20:54 # Schema: matrix-schema # # id: https://w3id.org/everycure-org/matrix-schema diff --git a/src/matrix_schema/datamodel/pandera.py b/src/matrix_schema/datamodel/pandera.py index 759a619..95e16fe 100644 --- a/src/matrix_schema/datamodel/pandera.py +++ b/src/matrix_schema/datamodel/pandera.py @@ -50,7 +50,7 @@ def get_matrix_node_schema(validate_enumeration_values: bool = True): return DataFrameSchema( columns={ - "id": Column(T.StringType(), nullable=True), + "id": Column(T.StringType(), nullable=False), "name": Column(T.StringType(), nullable=True), "category": Column(T.StringType(), nullable=True, checks=category_checks), @@ -143,7 +143,7 @@ def get_unioned_node_schema(validate_enumeration_values: bool = True): return DataFrameSchema( columns={ - "id": Column(T.StringType(), nullable=True), + "id": Column(T.StringType(), nullable=False), "name": Column(T.StringType(), nullable=True), "category": Column(T.StringType(), nullable=True, checks=category_checks), diff --git a/matrix_schema/generators/__init__.py b/src/matrix_schema/generators/__init__.py similarity index 100% rename from matrix_schema/generators/__init__.py rename to src/matrix_schema/generators/__init__.py diff --git a/matrix_schema/generators/panderagen.py b/src/matrix_schema/generators/panderagen.py similarity index 100% rename from matrix_schema/generators/panderagen.py rename to src/matrix_schema/generators/panderagen.py diff --git a/src/matrix_schema/schema/matrix_kg.yaml b/src/matrix_schema/schema/matrix_kg.yaml index d757d80..e9ea5ab 100644 --- a/src/matrix_schema/schema/matrix_kg.yaml +++ b/src/matrix_schema/schema/matrix_kg.yaml @@ -24,6 +24,7 @@ imports: slots: id: identifier: true + required: true slot_uri: schema:identifier range: uriorcurie description: A unique identifier for a thing From fc972247531e49dff6bcaafdab02f48728c49320 Mon Sep 17 00:00:00 2001 From: Kevin Schaper Date: Thu, 4 Sep 2025 16:53:16 -0700 Subject: [PATCH 4/4] update enum checks to check each value of a list to ensure that's valid, for multivalued fields, and generate the enum checks in a more generic way in panderagen.py --- src/matrix_schema/datamodel/pandera.py | 24 +++++++++++++--------- src/matrix_schema/generators/panderagen.py | 21 ++++++++++--------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/matrix_schema/datamodel/pandera.py b/src/matrix_schema/datamodel/pandera.py index 95e16fe..3119b9f 100644 --- a/src/matrix_schema/datamodel/pandera.py +++ b/src/matrix_schema/datamodel/pandera.py @@ -39,12 +39,14 @@ def get_matrix_node_schema(validate_enumeration_values: bool = True): raise ImportError(error_msg) if validate_enumeration_values: - category_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] + category_checks = [pa.Check.isin([enum_val.value for enum_val in NodeCategoryEnum])] else: category_checks = [] if validate_enumeration_values: - all_categories_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] + all_categories_checks = [pa.Check(lambda series: series.dropna().apply( + lambda arr: all(item in [enum_val.value for enum_val in NodeCategoryEnum] for item in (arr if isinstance(arr, list) else [])) + ).all(), element_wise=False, error="all items in all_categories must be valid NodeCategory enum values")] else: all_categories_checks = [] @@ -80,17 +82,17 @@ def get_matrix_edge_schema(validate_enumeration_values: bool = True): raise ImportError(error_msg) if validate_enumeration_values: - predicate_checks = [pa.Check.isin([predicate.value for predicate in PredicateEnum])] + predicate_checks = [pa.Check.isin([enum_val.value for enum_val in PredicateEnum])] else: predicate_checks = [] if validate_enumeration_values: - knowledge_level_checks = [pa.Check.isin([level.value for level in KnowledgeLevelEnum])] + knowledge_level_checks = [pa.Check.isin([enum_val.value for enum_val in KnowledgeLevelEnum])] else: knowledge_level_checks = [] if validate_enumeration_values: - agent_type_checks = [pa.Check.isin([agent.value for agent in AgentTypeEnum])] + agent_type_checks = [pa.Check.isin([enum_val.value for enum_val in AgentTypeEnum])] else: agent_type_checks = [] @@ -132,12 +134,14 @@ def get_unioned_node_schema(validate_enumeration_values: bool = True): raise ImportError(error_msg) if validate_enumeration_values: - category_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] + category_checks = [pa.Check.isin([enum_val.value for enum_val in NodeCategoryEnum])] else: category_checks = [] if validate_enumeration_values: - all_categories_checks = [pa.Check.isin([category.value for category in NodeCategoryEnum])] + all_categories_checks = [pa.Check(lambda series: series.dropna().apply( + lambda arr: all(item in [enum_val.value for enum_val in NodeCategoryEnum] for item in (arr if isinstance(arr, list) else [])) + ).all(), element_wise=False, error="all items in all_categories must be valid NodeCategory enum values")] else: all_categories_checks = [] @@ -173,17 +177,17 @@ def get_unioned_edge_schema(validate_enumeration_values: bool = True): raise ImportError(error_msg) if validate_enumeration_values: - predicate_checks = [pa.Check.isin([predicate.value for predicate in PredicateEnum])] + predicate_checks = [pa.Check.isin([enum_val.value for enum_val in PredicateEnum])] else: predicate_checks = [] if validate_enumeration_values: - knowledge_level_checks = [pa.Check.isin([level.value for level in KnowledgeLevelEnum])] + knowledge_level_checks = [pa.Check.isin([enum_val.value for enum_val in KnowledgeLevelEnum])] else: knowledge_level_checks = [] if validate_enumeration_values: - agent_type_checks = [pa.Check.isin([agent.value for agent in AgentTypeEnum])] + agent_type_checks = [pa.Check.isin([enum_val.value for enum_val in AgentTypeEnum])] else: agent_type_checks = [] diff --git a/src/matrix_schema/generators/panderagen.py b/src/matrix_schema/generators/panderagen.py index f436b7c..8f1983b 100644 --- a/src/matrix_schema/generators/panderagen.py +++ b/src/matrix_schema/generators/panderagen.py @@ -247,16 +247,17 @@ def _process_slot(self, slot_def: SlotDefinition) -> Dict[str, Any]: # Generate enum checks if applicable enum_checks = None - if slot_def.range: - range_name = slot_def.range - if range_name == "NodeCategoryEnum": - enum_checks = "pa.Check.isin([category.value for category in NodeCategoryEnum])" - elif range_name == "PredicateEnum": - enum_checks = "pa.Check.isin([predicate.value for predicate in PredicateEnum])" - elif range_name == "KnowledgeLevelEnum": - enum_checks = "pa.Check.isin([level.value for level in KnowledgeLevelEnum])" - elif range_name == "AgentTypeEnum": - enum_checks = "pa.Check.isin([agent.value for agent in AgentTypeEnum])" + if slot_def.range and slot_def.range.endswith("Enum"): + range_name = slot_def.range # e.g., "NodeCategoryEnum" + slot_name = slot_def.name + enum_name = range_name.replace("Enum", "") # e.g., "NodeCategory" + + if slot_def.multivalued: + enum_checks = f"""pa.Check(lambda series: series.dropna().apply( + lambda arr: all(item in [enum_val.value for enum_val in {range_name}] for item in (arr if isinstance(arr, list) else [])) + ).all(), element_wise=False, error="all items in {slot_name} must be valid {enum_name} enum values")""" + else: + enum_checks = f"pa.Check.isin([enum_val.value for enum_val in {range_name}])" return { "pyspark_type": pyspark_type,