diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/.travis.yml b/.travis.yml old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.rst b/README.rst old mode 100644 new mode 100755 diff --git a/TODO.md b/TODO.md new file mode 100755 index 0000000..3cdd8fb --- /dev/null +++ b/TODO.md @@ -0,0 +1,10 @@ +* [ ] Add validations that apply to every column in the DF equally +* [x] Fix CombinedValidations +* [x] Add replacement for allow_empty Columns +* [ ] New column() tests +* [ ] New CombinedValidation tests +* [x] Fix Negate +* [ ] Add facility for allow_empty +* [x] Fix messages +* [x] Re-implement the or/and using operators +* [ ] Allow and/or operators between Series-level and row-level validations diff --git a/UPDATE.md b/UPDATE.md new file mode 100755 index 0000000..c6c8a1d --- /dev/null +++ b/UPDATE.md @@ -0,0 +1,47 @@ +# ValidationWarnings +## Options for the ValidationWarning data +* We keep it as is, with one single ValidationWarning class that stores a `message` and a reference to the validation +that spawned it +* PREFERRED: As above, but we add a dictionary of miscellaneous kwargs to the ValidationWarning for storing stuff like the row index that failed +* We have a dataclass for each Validation type that stores things in a more structured way + * Why bother doing this if the Validation stores its own structure for the column index etc? + +## Options for the ValidationWarning message +* It's generated from the Validation as a fixed string, as it is now +* It's generated dynamically by the VW + * This means that custom messages means overriding the VW class +* PREFERRED: It's generated dynamically in the VW by calling the parent Validation with a reference to itself, e.g. + ```python + class ValidationWarning: + def __str__(self): + return self.validation.generate_message(self) + + class Validation: + def generate_message(warning: ValidationWarning) -> str: + pass + ``` + * This lets the message function use all the validation properties, and the dictionary of kwargs that it specified + * `generate_message()` will call `default_message(**kwargs)`, the dynamic class method, or `self.custom_message`, the + non-dynamic string specified by the user + * Each category of Validation will define a `create_prefix()` method, that creates the {row: 1, column: 2} prefix + that goes before each message. Thus, `generate_message()` will concatenate that with the actual message +* + +## Options for placing CombinedValidation in the inheritance hierarchy +* In order to make both CombinedValidation and BooleanSeriesValidation both share a class, so they can be chained together, +either we had to make a mixin that creates a "side path" that doesn't call `validate` (in this case, `validate_with_series`), +or we + +# Rework of Validation Indexing +## All Indexed +* All Validations now have an index and an axis +* However, this index can be none, can be column only, row only, or both +* When combined with each other, the resulting boolean series will be broadcast using numpy broadcasting rules +* e.g. + * A per-series validation might have index 0 (column 0) and return a scalar (the whole series is okay) + * A per-cell validation might have index 0 (column 0) and return a series (True, True, False) indicating that cell 0 and 1 of column 0 are okay + * A per-frame validation would have index None, and might return True if the whole frame meets the validation, or a series indicating which columns or rows match the validation + +# Rework of combinedvalidations +## Bitwise +* Could assign each validation a bit in a large bitwise enum, and `or` together a number each time that index fails a validatioin. This lets us track the origin of each warning, allowing us to slice them out by bit and generate an appropriate list of warnings \ No newline at end of file diff --git a/doc/common/introduction.rst b/doc/common/introduction.rst old mode 100644 new mode 100755 diff --git a/doc/readme/README.rst b/doc/readme/README.rst old mode 100644 new mode 100755 diff --git a/doc/readme/conf.py b/doc/readme/conf.py old mode 100644 new mode 100755 diff --git a/doc/site/Makefile b/doc/site/Makefile old mode 100644 new mode 100755 diff --git a/doc/site/conf.py b/doc/site/conf.py old mode 100644 new mode 100755 diff --git a/doc/site/index.rst b/doc/site/index.rst old mode 100644 new mode 100755 diff --git a/example/boolean.py b/example/boolean.py old mode 100644 new mode 100755 diff --git a/example/boolean.txt b/example/boolean.txt old mode 100644 new mode 100755 diff --git a/example/example.py b/example/example.py old mode 100644 new mode 100755 diff --git a/example/example.txt b/example/example.txt old mode 100644 new mode 100755 diff --git a/pandas_schema/__init__.py b/pandas_schema/__init__.py old mode 100644 new mode 100755 index 6f7ff97..fabe184 --- a/pandas_schema/__init__.py +++ b/pandas_schema/__init__.py @@ -1,4 +1,2 @@ -from .column import Column from .validation_warning import ValidationWarning -from .schema import Schema from .version import __version__ diff --git a/pandas_schema/column.py b/pandas_schema/column.py old mode 100644 new mode 100755 index 199b883..ab3b58a --- a/pandas_schema/column.py +++ b/pandas_schema/column.py @@ -1,27 +1,69 @@ import typing -import pandas as pd -from . import validation -from .validation_warning import ValidationWarning +import pandas_schema.core +from pandas_schema.index import PandasIndexer -class Column: - def __init__(self, name: str, validations: typing.Iterable['validation._BaseValidation'] = [], allow_empty=False): - """ - Creates a new Column object - :param name: The column header that defines this column. This must be identical to the header used in the CSV/Data Frame you are validating. - :param validations: An iterable of objects implementing _BaseValidation that will generate ValidationErrors - :param allow_empty: True if an empty column is considered valid. False if we leave that logic up to the Validation - """ - self.name = name - self.validations = list(validations) - self.allow_empty = allow_empty +def column( + validations: typing.Iterable['pandas_schema.core.IndexSeriesValidation'], + index: PandasIndexer = None, + override: bool = False, + allow_empty=False +): + """ + A utility method for setting the index data on a set of Validations + :param validations: A list of validations to modify + :param index: The index of the series that these validations will now consider + :param override: If true, override existing index values. Otherwise keep the existing ones + :param allow_empty: Allow empty rows (NaN) to pass the validation + See :py:class:`pandas_schema.validation.IndexSeriesValidation` + """ + for valid in validations: + if override or valid.index is None: + valid.index = index - def validate(self, series: pd.Series) -> typing.List[ValidationWarning]: - """ - Creates a list of validation errors using the Validation objects contained in the Column - :param series: A pandas Series to validate - :return: An iterable of ValidationError instances generated by the validation - """ - return [error for validation in self.validations for error in validation.get_errors(series, self)] +def column_sequence( + validations: typing.Iterable['pandas_schema.core.IndexSeriesValidation'], + override: bool = False +): + """ + A utility method for setting the index data on a set of Validations. Applies a sequential position based index, so + that the first validation gets index 0, the second gets index 1 etc. Note: this will not modify any index that + already has some kind of index + :param validations: A list of validations to modify + :param override: If true, override existing index values. Otherwise keep the existing ones + """ + for i, valid in validations: + if override or valid.index is None: + valid.index = PandasIndexer(i, typ='positional') +# +# def label_column( +# validations: typing.Iterable['pandas_schema.core.IndexSeriesValidation'], +# index: typing.Union[int, str], +# ): +# """ +# A utility method for setting the label-based column for each validation +# :param validations: A list of validations to modify +# :param index: The label of the series that these validations will now consider +# """ +# return _column( +# validations, +# index, +# position=False +# ) +# +# def positional_column( +# validations: typing.Iterable['pandas_schema.core.IndexSeriesValidation'], +# index: int, +# ): +# """ +# A utility method for setting the position-based column for each validation +# :param validations: A list of validations to modify +# :param index: The index of the series that these validations will now consider +# """ +# return _column( +# validations, +# index, +# position=True + diff --git a/pandas_schema/core.py b/pandas_schema/core.py new file mode 100755 index 0000000..4435d1d --- /dev/null +++ b/pandas_schema/core.py @@ -0,0 +1,289 @@ +import abc +import math +import datetime +from itertools import chain +import pandas as pd +import numpy as np +import typing +import operator +import re +from dataclasses import dataclass + +from . import column +from .errors import PanSchArgumentError, PanSchNoIndexError +from pandas_schema.validation_warning import ValidationWarning +from pandas_schema.index import PandasIndexer, IndexValue, IndexType +from pandas.api.types import is_categorical_dtype, is_numeric_dtype + + +class BaseValidation(abc.ABC): + """ + A validation is, broadly, just a function that maps a data frame to a list of errors + """ + + @abc.abstractmethod + def validate(self, df: pd.DataFrame) -> typing.Iterable[ValidationWarning]: + """ + Validates a data frame + :param df: Data frame to validate + :return: All validation failures detected by this validation + """ + + @abc.abstractmethod + def message(self, warning: ValidationWarning) -> str: + pass + + +class IndexValidation(BaseValidation, metaclass=abc.ABCMeta): + """ + Abstract class that builds on BaseValidation to give it access to an index for selecting a Series out of the + DataFrame + """ + + def __init__(self, index: typing.Union[PandasIndexer, IndexValue], message: str = None, **kwargs): + """ + Creates a new IndexSeriesValidation + :param index: An index with which to select the series + Otherwise it's a label (ie, index=0) indicates the column with the label of 0 + """ + super().__init__(**kwargs) + if isinstance(index, PandasIndexer): + self.index = index + else: + # If it isn't already an indexer object, convert it to one + self.index = PandasIndexer(index=index) + self.custom_message = message + + def message(self, warning: ValidationWarning) -> str: + prefix = self.prefix(warning) + + if self.custom_message: + suffix = self.custom_message + else: + suffix = self.default_message(warning) + + return "{} {}".format(prefix, suffix) + + @property + def readable_name(self, **kwargs): + """ + A readable name for this validation, to be shown in validation warnings + """ + return type(self).__name__ + + def default_message(self, warnings: ValidationWarning) -> str: + return 'failed the {}'.format(self.readable_name) + + def select_series(self, df: pd.DataFrame) -> pd.Series: + """ + Select a series using the data stored in this validation + """ + if self.index is None: + raise PanSchNoIndexError() + + return self.index(df) + + def prefix(self, warning: ValidationWarning): + """ + Return a string that can be used to prefix a message that relates to this index + + This method is safe to override + """ + if self.index is None: + return "" + + if self.index.type == IndexType.POSITION: + return 'Column {}'.format(self.index.index) + else: + return 'Column "{}"'.format(self.index.index) + + +# +# class SeriesValidation(BaseValidation): +# """ +# A SeriesValidation validates a DataFrame by selecting a single series from it, and +# applying some validation to it +# """ +# +# @abc.abstractmethod +# def select_series(self, df: pd.DataFrame) -> pd.Series: +# """ +# Selects a series from the DataFrame that will be validated +# """ +# +# @abc.abstractmethod +# def validate_series(self, series: pd.Series) -> typing.Iterable[ValidationWarning]: +# """ +# Validate a single series +# """ +# +# def validate(self, df: pd.DataFrame) -> typing.Iterable[ValidationWarning]: +# series = self.select_series(df) +# return self.validate_series(series) + + +class SeriesValidation(IndexValidation): + """ + A SeriesValidation validates a DataFrame by selecting a single series from it, and + applying some validation to it + """ + + def validate(self, df: pd.DataFrame) -> typing.Iterable[ValidationWarning]: + series = self.index(df) + return self.validate_series(series) + + @abc.abstractmethod + def validate_series(self, series: pd.Series) -> typing.Iterable[ValidationWarning]: + pass + + +class WarningSeriesGenerator(BaseValidation, abc.ABC): + """ + Mixin class that indicates that this Validation can produce a "warning series", which is a pandas Series with one + or more warnings in each cell, corresponding to warnings detected in the DataFrame at the same index + """ + + @abc.abstractmethod + def get_warning_series(self, df: pd.DataFrame) -> pd.Series: + """ + Return a series of ValidationWarnings, not an iterable of ValidationWarnings like the normal validate() method + """ + + @staticmethod + def flatten_warning_series(warnings: pd.Series): + """ + Converts a warning series into an iterable of warnings + """ + return warnings[warnings.astype(bool)].explode().tolist() + + def validate(self, df: pd.DataFrame, flatten=True) -> typing.Union[ + typing.Iterable[ValidationWarning], + pd.Series + ]: + warnings = self.get_warning_series(df) + if flatten: + return self.flatten_warning_series(warnings) + else: + return warnings + + def __or__(self, other: 'WarningSeriesGenerator'): + if not isinstance(other, WarningSeriesGenerator): + raise PanSchArgumentError('The "|" operator can only be used between two' + 'Validations that subclass {}'.format(self.__class__)) + + return CombinedValidation(self, other, operator='or') + + + +class BooleanSeriesValidation(IndexValidation, WarningSeriesGenerator): + """ + Validation is defined by the function :py:meth:~select_cells that returns a boolean series. + Each cell that has False has failed the validation. + + Child classes need not create their own :py:class:~pandas_schema.core.BooleanSeriesValidation.Warning subclass, + because the data is in the same form for each cell. You need only define a :py:meth~default_message. + """ + + def __init__(self, *args, negated=False, **kwargs): + super().__init__(*args, **kwargs) + self.negated = negated + + @abc.abstractmethod + def select_cells(self, series: pd.Series) -> pd.Series: + """ + A BooleanSeriesValidation must return a boolean series. Each cell that has False has failed the + validation + :param series: The series to validate + """ + pass + + def validate_series(self, series, flatten=True) -> typing.Union[ + typing.Iterable[ValidationWarning], + pd.Series + ]: + """ + Validates a single series selected from the DataFrame + """ + selection = self.select_cells(series) + + if self.negated: + # If self.negated (which is not the default), then we don't need to flip the booleans + failed = selection + else: + # In the normal case we do need to flip the booleans, since select_cells returns True for cells that pass + # the validation, and we want cells that failed it + failed = ~selection + + # Slice out the failed items, then map each into a list of validation warnings at each respective index + warnings = series[failed].to_frame().apply(lambda row: [ValidationWarning(self, { + 'row': row.name, + 'value': row[0] + })], axis='columns', result_type='reduce') + # warnings = warnings.iloc[:, 0] + + # If flatten, return a list of ValidationWarning, otherwise return a series of lists of Validation Warnings + if flatten: + return self.flatten_warning_series(warnings) + else: + return warnings + + def get_warning_series(self, df: pd.DataFrame) -> pd.Series: + """ + Validates a series and returns a series of warnings. + """ + series = self.select_series(df) + return self.validate_series(series, flatten=False) + + def prefix(self, warning: ValidationWarning): + parent = super().prefix(warning) + # Only in this subclass do we know the contents of the warning props, since we defined them in the + # validate_series method. Thus, we can now add row index information + + return parent + ', Row {row}: "{value}"'.format(**warning.props) + + def __invert__(self) -> 'BooleanSeriesValidation': + """ + If a BooleanSeriesValidation is negated, it has the opposite result + """ + self.negated = not self.negated + return self + + +class CombinedValidation(WarningSeriesGenerator): + """ + Validates if one and/or the other validation is true for an element + """ + + def message(self, warning: ValidationWarning) -> str: + pass + + def __init__(self, validation_a: WarningSeriesGenerator, validation_b: WarningSeriesGenerator, operator: str): + super().__init__() + self.operator = operator + self.left = validation_a + self.right = validation_b + + def get_warning_series(self, df: pd.DataFrame) -> pd.Series: + # Let both validations separately select and filter a column + left_errors = self.left.validate(df, flatten=False) + right_errors = self.right.validate(df, flatten=False) + + if self.operator == 'and': + # If it's an "and" validation, left, right, or both failing means an error, so we can simply concatenate + # the lists of errors + combined = left_errors.combine(right_errors, func=operator.add, fill_value=[]) + elif self.operator == 'or': + # [error] and [] = [] + # [error_1] and [error_2] = [error_2] + # [] and [] = [] + # Thus, we can use the and operator to implement "or" validations + combined = left_errors.combine(right_errors, func=lambda l, r: l + r if l and r else [], fill_value=[]) + # func=lambda a, b: [] if len(a) == 0 or len(b) == 0 else a + b) + else: + raise Exception('Operator must be "and" or "or"') + + return combined + + @property + def default_message(self, warnings: ValidationWarning) -> str: + return '({}) {} ({})'.format(self.v_a.message, self.operator, self.v_b.message) diff --git a/pandas_schema/errors.py b/pandas_schema/errors.py old mode 100644 new mode 100755 index a9176bf..cdc3132 --- a/pandas_schema/errors.py +++ b/pandas_schema/errors.py @@ -1,8 +1,20 @@ -class PanSchError(BaseException): +class PanSchError(Exception): """ Base class for all pandas_schema exceptions """ + def __init__(self, message=None): + super().__init__(message) + + +class PanSchIndexError(PanSchError): + """ + Some issue with creating a PandasIndexer + """ + + def __init__(self, message): + super().__init__(message=message) + class PanSchInvalidSchemaError(PanSchError): """ @@ -10,6 +22,12 @@ class PanSchInvalidSchemaError(PanSchError): """ +class PanSchNoIndexError(PanSchInvalidSchemaError): + """ + A validation was provided that has not specified an index + """ + + class PanSchArgumentError(PanSchError): """ An argument passed to a function has an invalid type or value diff --git a/pandas_schema/index.py b/pandas_schema/index.py new file mode 100755 index 0000000..51f1172 --- /dev/null +++ b/pandas_schema/index.py @@ -0,0 +1,79 @@ +from pandas_schema.errors import PanSchIndexError +from dataclasses import dataclass +from typing import Union +import numpy +import pandas +from enum import Enum + +IndexValue = Union[numpy.string_, numpy.int_, str, int] +""" +A pandas index can either be an integer or string, or an array of either. This typing is a bit sketchy because really +a lot of things are accepted here +""" + + +class IndexType(Enum): + POSITION = 0 + LABEL = 1 + + +class PandasIndexer: + """ + An index into a particular axis of a DataFrame. Attempts to recreate the behaviour of `df.ix[some_index]` + """ + + # valid_types = {'position', 'label'} + index: IndexValue + """ + The index to use, either an integer for position-based indexing, or a string for label-based indexing + """ + type: IndexType + """ + The type of indexing to use, either 'position' or 'label' + """ + + axis: int + """ + The axis for the indexer + """ + + def __init__(self, index: IndexValue, typ: IndexType = None, axis: int = 1): + self.index = index + self.axis = axis + + if typ is not None: + # If the type is provided, validate it + if typ not in self.valid_types: + raise PanSchIndexError('The index type was not one of {}'.format(' or '.join(self.valid_types))) + else: + self.type = typ + else: + # If the type isn't provided, guess it based on the datatype of the index + if numpy.issubdtype(type(index), numpy.character): + self.type = IndexType.LABEL + elif numpy.issubdtype(type(index), numpy.int_): + self.type = IndexType.POSITION + else: + raise PanSchIndexError('The index value was not either an integer or string, or an array of either of ' + 'these') + + def __call__(self, df: pandas.DataFrame): + """ + Apply this index + :param df: The DataFrame to index + :param axis: The axis to index along. axis=0 will select a row, and axis=1 will select a column + """ + if self.type == IndexType.LABEL: + return df.loc(axis=self.axis)[self.index] + elif self.type == IndexType.POSITION: + return df.iloc(axis=self.axis)[self.index] + + +class RowIndexer(PandasIndexer): + def __init__(self, index: IndexValue, typ: IndexType = None): + super().__init__(index=index, typ=typ, axis=0) + + +class ColumnIndexer(PandasIndexer): + def __init__(self, index: IndexValue, typ: IndexType = None): + super().__init__(index=index, typ=typ, axis=1) diff --git a/pandas_schema/schema.py b/pandas_schema/schema.py old mode 100644 new mode 100755 index 5c0442e..83ad9c5 --- a/pandas_schema/schema.py +++ b/pandas_schema/schema.py @@ -1,9 +1,10 @@ import pandas as pd import typing -from .errors import PanSchInvalidSchemaError, PanSchArgumentError -from .validation_warning import ValidationWarning -from .column import Column +from pandas_schema.core import BaseValidation +from pandas_schema.errors import PanSchArgumentError, PanSchInvalidSchemaError +from pandas_schema.validation_warning import ValidationWarning +from pandas_schema.index import PandasIndexer class Schema: @@ -11,83 +12,32 @@ class Schema: A schema that defines the columns required in the target DataFrame """ - def __init__(self, columns: typing.Iterable[Column], ordered: bool = False): + def __init__(self, validations: typing.Iterable[BaseValidation]): """ - :param columns: A list of column objects - :param ordered: True if the Schema should associate its Columns with DataFrame columns by position only, ignoring - the header names. False if the columns should be associated by column header names only. Defaults to False + :param validations: A list of validations that will be applied to the DataFrame upon validation """ - if not columns: - raise PanSchInvalidSchemaError('An instance of the schema class must have a columns list') + if not validations: + raise PanSchInvalidSchemaError('An instance of the schema class must have a validations list') - if not isinstance(columns, typing.List): - raise PanSchInvalidSchemaError('The columns field must be a list of Column objects') + if not isinstance(validations, typing.Iterable): + raise PanSchInvalidSchemaError('The columns field must be an iterable of Validation objects') - if not isinstance(ordered, bool): - raise PanSchInvalidSchemaError('The ordered field must be a boolean') + self.validations = list(validations) - self.columns = list(columns) - self.ordered = ordered - - def validate(self, df: pd.DataFrame, columns: typing.List[str] = None) -> typing.List[ValidationWarning]: + def validate(self, df: pd.DataFrame, subset: PandasIndexer = None) -> typing.List[ValidationWarning]: """ Runs a full validation of the target DataFrame using the internal columns list :param df: A pandas DataFrame to validate - :param columns: A list of columns indicating a subset of the schema that we want to validate + :param subset: A list of columns indicating a subset of the schema that we want to validate. Can be any :return: A list of ValidationWarning objects that list the ways in which the DataFrame was invalid """ - errors = [] - df_cols = len(df.columns) - - # If no columns are passed, validate against every column in the schema. This is the default behaviour - if columns is None: - schema_cols = len(self.columns) - columns_to_pair = self.columns - if df_cols != schema_cols: - errors.append( - ValidationWarning( - 'Invalid number of columns. The schema specifies {}, but the data frame has {}'.format( - schema_cols, - df_cols) - ) - ) - return errors - - # If we did pass in columns, check that they are part of the current schema - else: - if set(columns).issubset(self.get_column_names()): - columns_to_pair = [column for column in self.columns if column.name in columns] - else: - raise PanSchArgumentError( - 'Columns {} passed in are not part of the schema'.format(set(columns).difference(self.columns)) - ) - - # We associate the column objects in the schema with data frame series either by name or by position, depending - # on the value of self.ordered - if self.ordered: - series = [x[1] for x in df.iteritems()] - column_pairs = zip(series, self.columns) - else: - column_pairs = [] - for column in columns_to_pair: - - # Throw an error if the schema column isn't in the data frame - if column.name not in df: - errors.append(ValidationWarning( - 'The column {} exists in the schema but not in the data frame'.format(column.name))) - return errors + # Apply the subset if we have one + if subset is not None: + df = subset(df) - column_pairs.append((df[column.name], column)) - - # Iterate over each pair of schema columns and data frame series and run validations - for series, column in column_pairs: - errors += column.validate(series) - - return sorted(errors, key=lambda e: e.row) - - def get_column_names(self): - """ - Returns the column names contained in the schema - """ - return [column.name for column in self.columns] + # Build the list of errors + errors = [] + for validation in self.validations: + errors.extend(validation.validate(df)) + return errors diff --git a/pandas_schema/validation_warning.py b/pandas_schema/validation_warning.py old mode 100644 new mode 100755 index 320be65..e6e3ddd --- a/pandas_schema/validation_warning.py +++ b/pandas_schema/validation_warning.py @@ -1,22 +1,31 @@ +from dataclasses import dataclass, field + + +@dataclass class ValidationWarning: """ - Represents a difference between the schema and data frame, found during the validation of the data frame + Represents a difference between the schema and data frame, found during the validation + of the data frame + """ + validation: 'pandas_schema.core.BaseValidation' + """ + The validation that spawned this warning """ - def __init__(self, message: str, value: str = None, row: int = -1, column: str = None): - self.message = message - self.value = value - """The value of the failing cell in the DataFrame""" - self.row = row - """The row index (usually an integer starting from 0) of the cell that failed the validation""" - self.column = column - """The column name of the cell that failed the validation""" + props: dict = field(default_factory=dict) + """ + List of data about this warning in addition to that provided by the validation, for + example, if a cell in the DataFrame didn't match the validation, the props might + include a `value` key, for storing what the actual value was + """ - def __str__(self) -> str: + @property + def message(self): """ - The entire warning message as a string + Return this validation as a string """ - if self.row is not None and self.column is not None and self.value is not None: - return '{{row: {}, column: "{}"}}: "{}" {}'.format(self.row, self.column, self.value, self.message) - else: - return self.message + # Internally, this actually asks the validator class to formulate a message + return self.validation.message(self) + + def __str__(self): + return self.message diff --git a/pandas_schema/validation.py b/pandas_schema/validations.py old mode 100644 new mode 100755 similarity index 55% rename from pandas_schema/validation.py rename to pandas_schema/validations.py index 2a3f2f8..2e803df --- a/pandas_schema/validation.py +++ b/pandas_schema/validations.py @@ -7,152 +7,19 @@ import operator from . import column +from .core import SeriesValidation, BooleanSeriesValidation, IndexValidation from .validation_warning import ValidationWarning from .errors import PanSchArgumentError from pandas.api.types import is_categorical_dtype, is_numeric_dtype -class _BaseValidation: - """ - The validation base class that defines any object that can create a list of errors from a Series - """ - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def get_errors(self, series: pd.Series, column: 'column.Column') -> typing.Iterable[ValidationWarning]: - """ - Return a list of errors in the given series - :param series: - :param column: - :return: - """ - - -class _SeriesValidation(_BaseValidation): - """ - Implements the _BaseValidation interface by returning a Boolean series for each element that either passes or - fails the validation - """ - __metaclass__ = abc.ABCMeta - - def __init__(self, **kwargs): - self._custom_message = kwargs.get('message') - - @property - def message(self): - return self._custom_message or self.default_message - - @abc.abstractproperty - def default_message(self) -> str: - """ - Create a message to be displayed whenever this validation fails - This should be a generic message for the validation type, but can be overwritten if the user provides a - message kwarg - """ - - @abc.abstractmethod - def validate(self, series: pd.Series) -> pd.Series: - """ - Returns a Boolean series, where each value of False is an element in the Series that has failed the validation - :param series: - :return: - """ - - def __invert__(self): - """ - Returns a negated version of this validation - """ - return _InverseValidation(self) - - def __or__(self, other: '_SeriesValidation'): - """ - Returns a validation which is true if either this or the other validation is true - """ - return _CombinedValidation(self, other, operator.or_) - - def __and__(self, other: '_SeriesValidation'): - """ - Returns a validation which is true if either this or the other validation is true - """ - return _CombinedValidation(self, other, operator.and_) - - def get_errors(self, series: pd.Series, column: 'column.Column'): - - errors = [] - - # Calculate which columns are valid using the child class's validate function, skipping empty entries if the - # column specifies to do so - simple_validation = ~self.validate(series) - if column.allow_empty: - # Failing results are those that are not empty, and fail the validation - # explicitly check to make sure the series isn't a category because issubdtype will FAIL if it is - if is_categorical_dtype(series) or is_numeric_dtype(series): - validated = ~series.isnull() & simple_validation - else: - validated = (series.str.len() > 0) & simple_validation - - else: - validated = simple_validation - - # Cut down the original series to only ones that failed the validation - indices = series.index[validated] - - # Use these indices to find the failing items. Also print the index which is probably a row number - for i in indices: - element = series[i] - errors.append(ValidationWarning( - message=self.message, - value=element, - row=i, - column=series.name - )) - - return errors - - -class _InverseValidation(_SeriesValidation): - """ - Negates an ElementValidation - """ - - def __init__(self, validation: _SeriesValidation): - self.negated = validation - super().__init__() - - def validate(self, series: pd.Series): - return ~ self.negated.validate(series) - - @property - def default_message(self): - return self.negated.message + ' ' - - -class _CombinedValidation(_SeriesValidation): - """ - Validates if one and/or the other validation is true for an element - """ - - def __init__(self, validation_a: _SeriesValidation, validation_b: _SeriesValidation, operator): - self.operator = operator - self.v_a = validation_a - self.v_b = validation_b - super().__init__() - - def validate(self, series: pd.Series): - return self.operator(self.v_a.validate(series), self.v_b.validate(series)) - - @property - def default_message(self): - return '({}) {} ({})'.format(self.v_a.message, self.operator, self.v_b.message) - - -class CustomSeriesValidation(_SeriesValidation): +class CustomSeriesValidation(BooleanSeriesValidation): """ Validates using a user-provided function that operates on an entire series (for example by using one of the pandas Series methods: http://pandas.pydata.org/pandas-docs/stable/api.html#series) """ - def __init__(self, validation: typing.Callable[[pd.Series], pd.Series], message: str): + def __init__(self, validation: typing.Callable[[pd.Series], pd.Series], *args, **kwargs): """ :param message: The error message to provide to the user if this validation fails. The row and column and failing value will automatically be prepended to this message, so you only have to provide a message that @@ -162,19 +29,20 @@ def __init__(self, validation: typing.Callable[[pd.Series], pd.Series], message: :param validation: A function that takes a pandas Series and returns a boolean Series, where each cell is equal to True if the object passed validation, and False if it failed """ + super().__init__(*args, **kwargs) self._validation = validation - super().__init__(message=message) - def validate(self, series: pd.Series) -> pd.Series: + + def select_cells(self, series: pd.Series) -> pd.Series: return self._validation(series) -class CustomElementValidation(_SeriesValidation): +class CustomElementValidation(BooleanSeriesValidation): """ Validates using a user-provided function that operates on each element """ - def __init__(self, validation: typing.Callable[[typing.Any], typing.Any], message: str): + def __init__(self, validation: typing.Callable[[typing.Any], typing.Any], *args, **kwargs): """ :param message: The error message to provide to the user if this validation fails. The row and column and failing value will automatically be prepended to this message, so you only have to provide a message that @@ -185,13 +53,13 @@ def __init__(self, validation: typing.Callable[[typing.Any], typing.Any], messag the validation, and false if it doesn't """ self._validation = validation - super().__init__(message=message) + super().__init__(*args, **kwargs) - def validate(self, series: pd.Series) -> pd.Series: + def select_cells(self, series: pd.Series) -> pd.Series: return series.apply(self._validation) -class InRangeValidation(_SeriesValidation): +class InRangeValidation(BooleanSeriesValidation): """ Checks that each element in the series is within a given numerical range """ @@ -205,16 +73,15 @@ def __init__(self, min: float = -math.inf, max: float = math.inf, **kwargs): self.max = max super().__init__(**kwargs) - @property - def default_message(self): + def default_message(self, warning: ValidationWarning): return 'was not in the range [{}, {})'.format(self.min, self.max) - def validate(self, series: pd.Series) -> pd.Series: + def select_cells(self, series: pd.Series) -> pd.Series: series = pd.to_numeric(series) return (series >= self.min) & (series < self.max) -class IsDtypeValidation(_BaseValidation): +class IsDtypeValidation(SeriesValidation): """ Checks that a series has a certain numpy dtype """ @@ -223,21 +90,24 @@ def __init__(self, dtype: np.dtype, **kwargs): """ :param dtype: The numpy dtype to check the column against """ - self.dtype = dtype super().__init__(**kwargs) + self.dtype = dtype + + def default_message(self, warning: ValidationWarning) -> str: + return 'has a dtype of {} which is not a subclass of the required type {}'.format( + self.dtype, warning.props['dtype']) - def get_errors(self, series: pd.Series, column: 'column.Column' = None): + def validate_series(self, series: pd.Series) -> typing.Iterable[ValidationWarning]: if not np.issubdtype(series.dtype, self.dtype): return [ValidationWarning( - 'The column {} has a dtype of {} which is not a subclass of the required type {}'.format( - column.name if column else '', series.dtype, self.dtype - ) + self, + {'dtype': series.dtype} )] else: return [] -class CanCallValidation(_SeriesValidation): +class CanCallValidation(BooleanSeriesValidation): """ Validates if a given function can be called on each element in a column without raising an exception """ @@ -250,12 +120,14 @@ def __init__(self, func: typing.Callable, **kwargs): if callable(type): self.callable = func else: - raise PanSchArgumentError('The object "{}" passed to CanCallValidation is not callable!'.format(type)) + raise PanSchArgumentError( + 'The object "{}" passed to CanCallValidation is not callable!'.format( + type)) super().__init__(**kwargs) - @property - def default_message(self): - return 'raised an exception when the callable {} was called on it'.format(self.callable) + def default_message(self, warning: ValidationWarning): + return 'raised an exception when the callable {} was called on it'.format( + self.callable) def can_call(self, var): try: @@ -264,7 +136,7 @@ def can_call(self, var): except: return False - def validate(self, series: pd.Series) -> pd.Series: + def select_cells(self, series: pd.Series) -> pd.Series: return series.apply(self.can_call) @@ -288,12 +160,11 @@ def __init__(self, _type: type, **kwargs): else: raise PanSchArgumentError('{} is not a valid type'.format(_type)) - @property - def default_message(self): + def default_message(self, warning: ValidationWarning): return 'cannot be converted to type {}'.format(self.callable) -class MatchesPatternValidation(_SeriesValidation): +class MatchesPatternValidation(BooleanSeriesValidation): """ Validates that a string or regular expression can match somewhere in each element in this column """ @@ -308,15 +179,14 @@ def __init__(self, pattern, options={}, **kwargs): self.options = options super().__init__(**kwargs) - @property - def default_message(self): - return 'does not match the pattern "{}"'.format(self.pattern) + def default_message(self, warning: ValidationWarning): + return 'does not match the pattern "{}"'.format(self.pattern.pattern) - def validate(self, series: pd.Series) -> pd.Series: + def select_cells(self, series: pd.Series) -> pd.Series: return series.astype(str).str.contains(self.pattern, **self.options) -class TrailingWhitespaceValidation(_SeriesValidation): +class TrailingWhitespaceValidation(BooleanSeriesValidation): """ Checks that there is no trailing whitespace in this column """ @@ -324,15 +194,14 @@ class TrailingWhitespaceValidation(_SeriesValidation): def __init__(self, **kwargs): super().__init__(**kwargs) - @property - def default_message(self): + def default_message(self, warning: ValidationWarning): return 'contains trailing whitespace' - def validate(self, series: pd.Series) -> pd.Series: + def select_cells(self, series: pd.Series) -> pd.Series: return ~series.astype(str).str.contains('\s+$') -class LeadingWhitespaceValidation(_SeriesValidation): +class LeadingWhitespaceValidation(BooleanSeriesValidation): """ Checks that there is no leading whitespace in this column """ @@ -340,15 +209,14 @@ class LeadingWhitespaceValidation(_SeriesValidation): def __init__(self, **kwargs): super().__init__(**kwargs) - @property - def default_message(self): + def default_message(self, warning: ValidationWarning): return 'contains leading whitespace' - def validate(self, series: pd.Series) -> pd.Series: + def select_cells(self, series: pd.Series) -> pd.Series: return ~series.astype(str).str.contains('^\s+') -class IsDistinctValidation(_SeriesValidation): +class IsDistinctValidation(BooleanSeriesValidation): """ Checks that every element of this column is different from each other element """ @@ -356,15 +224,14 @@ class IsDistinctValidation(_SeriesValidation): def __init__(self, **kwargs): super().__init__(**kwargs) - @property - def default_message(self): + def default_message(self, warning: ValidationWarning): return 'contains values that are not unique' - def validate(self, series: pd.Series) -> pd.Series: + def select_cells(self, series: pd.Series) -> pd.Series: return ~series.duplicated(keep='first') -class InListValidation(_SeriesValidation): +class InListValidation(BooleanSeriesValidation): """ Checks that each element in this column is contained within a list of possibilities """ @@ -378,19 +245,18 @@ def __init__(self, options: typing.Iterable, case_sensitive: bool = True, **kwar self.options = options super().__init__(**kwargs) - @property - def default_message(self): + def default_message(self, warning: ValidationWarning): values = ', '.join(str(v) for v in self.options) return 'is not in the list of legal options ({})'.format(values) - def validate(self, series: pd.Series) -> pd.Series: + def select_cells(self, series: pd.Series) -> pd.Series: if self.case_sensitive: return series.isin(self.options) else: return series.str.lower().isin([s.lower() for s in self.options]) -class DateFormatValidation(_SeriesValidation): +class DateFormatValidation(BooleanSeriesValidation): """ Checks that each element in this column is a valid date according to a provided format string """ @@ -404,8 +270,7 @@ def __init__(self, date_format: str, **kwargs): self.date_format = date_format super().__init__(**kwargs) - @property - def default_message(self): + def default_message(self, warning: ValidationWarning): return 'does not match the date format string "{}"'.format(self.date_format) def valid_date(self, val): @@ -415,5 +280,5 @@ def valid_date(self, val): except: return False - def validate(self, series: pd.Series) -> pd.Series: + def select_cells(self, series: pd.Series) -> pd.Series: return series.astype(str).apply(self.valid_date) diff --git a/pandas_schema/version.py b/pandas_schema/version.py old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 diff --git a/setup.py b/setup.py index 8d8d0fd..ff6d9a4 100755 --- a/setup.py +++ b/setup.py @@ -81,7 +81,11 @@ def run(self): ], keywords='pandas csv verification schema', packages=find_packages(include=['pandas_schema']), - install_requires=['numpy', 'pandas>=0.19'], + install_requires=[ + 'numpy', + 'pandas>=0.23', + 'dataclasses' + ], cmdclass={ 'build_readme': BuildReadme, 'build_site': BuildHtmlDocs diff --git a/test/__init__.py b/test/__init__.py old mode 100644 new mode 100755 diff --git a/test/test_column.py b/test/test_column.py deleted file mode 100644 index 38e61f0..0000000 --- a/test/test_column.py +++ /dev/null @@ -1,68 +0,0 @@ -import unittest -import pandas as pd - -from pandas_schema import Column -from pandas_schema.validation import CanConvertValidation, LeadingWhitespaceValidation, TrailingWhitespaceValidation - - -class SingleValidationColumn(unittest.TestCase): - """ - Test a column with one single validation - """ - NAME = 'col1' - - col = Column(NAME, [CanConvertValidation(int)], allow_empty=False) - ser = pd.Series([ - 'a', - 'b', - 'c' - ]) - - def test_name(self): - self.assertEqual(self.col.name, self.NAME, 'A Column does not store its name correctly') - - def test_outputs(self): - results = self.col.validate(self.ser) - - self.assertEqual(len(results), len(self.ser), 'A Column produces the wrong number of errors') - for i in range(2): - self.assertTrue(any([r.row == i for r in results]), 'A Column does not report errors for every row') - - -class DoubleValidationColumn(unittest.TestCase): - """ - Test a column with two different validations - """ - NAME = 'col1' - - col = Column(NAME, [TrailingWhitespaceValidation(), LeadingWhitespaceValidation()], allow_empty=False) - ser = pd.Series([ - ' a ', - ' b ', - ' c ' - ]) - - def test_outputs(self): - results = self.col.validate(self.ser) - - # There should be 6 errors, 2 for each row - self.assertEqual(len(results), 2 * len(self.ser), 'A Column produces the wrong number of errors') - for i in range(2): - in_row = [r for r in results if r.row == i] - self.assertEqual(len(in_row), 2, 'A Column does not report both errors for every row') - - -class AllowEmptyColumn(unittest.TestCase): - """ - Test a column with one single validation that allows empty columns - """ - NAME = 'col1' - - col = Column(NAME, [CanConvertValidation(int)], allow_empty=True) - ser = pd.Series([ - '', - ]) - - def test_outputs(self): - results = self.col.validate(self.ser) - self.assertEqual(len(results), 0, 'allow_empty is not allowing empty columns') diff --git a/test/test_example.py b/test/test_example.py old mode 100644 new mode 100755 diff --git a/test/test_metadata.py b/test/test_metadata.py old mode 100644 new mode 100755 diff --git a/test/test_schema.py b/test/test_schema.py old mode 100644 new mode 100755 diff --git a/test/test_validation.py b/test/test_validation.py old mode 100644 new mode 100755 index 7914025..2351434 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -1,43 +1,54 @@ +""" +Tests for pandas_schema.validations +""" import json import unittest import re from numpy import nan, dtype - -from pandas_schema import Column, Schema -from pandas_schema.validation import _BaseValidation -from pandas_schema.validation import * +import numpy as np +import pandas as pd + +from pandas_schema.validations import * +from pandas_schema.core import BooleanSeriesValidation, CombinedValidation, \ + BaseValidation +from pandas_schema.index import ColumnIndexer as ci +from pandas_schema.schema import Schema +from pandas_schema.column import column, column_sequence from pandas_schema import ValidationWarning -class ValidationTestBase(unittest.TestCase): - def seriesEquality(self, s1: pd.Series, s2: pd.Series, msg: str = None): - if not s1.equals(s2): - raise self.failureException(msg) - - def validate_and_compare(self, series: list, expected_result: bool, msg: str = None, series_dtype: object = None): - """ - Checks that every element in the provided series is equal to `expected_result` after validation - :param series_dtype: Explicity specifies the dtype for the generated Series - :param series: The series to check - :param expected_result: Whether the elements in this series should pass the validation - :param msg: The message to display if this test fails - """ +def get_warnings(validator: BaseValidation, series: list) -> typing.Collection[ + ValidationWarning]: + """ + Tests a validator by asserting that it generates the amount of warnings + :param series_dtype: Explicitly specifies the dtype for the generated Series + :param series: The series to check + :param expected_result: Whether the elements in this series should pass the validation + :param msg: The message to display if this test fails + """ - # Check that self.validator is correct - if not self.validator or not isinstance(self.validator, _BaseValidation): - raise ValueError('The class must have the validator field set to an instance of a Validation subclass') + # # Check that self.validator is correct + # if not self.validator or not isinstance(self.validator, BooleanSeriesValidation, index=0): + # raise ValueError('The class must have the validator field set to an instance of a Validation subclass') + # + # # Ensure we're comparing series correctly + # self.addTypeEqualityFunc(pd.Series, self.seriesEquality) - # Ensure we're comparing series correctly - self.addTypeEqualityFunc(pd.Series, self.seriesEquality) + df = pd.Series(series).to_frame() + warnings = validator.validate(df) + return list(warnings) + # + # # Now find any items where their validation does not correspond to the expected_result + # for item, result in zip(series, results): + # with self.subTest(value=item): + # self.assertEqual(result, expected_result, msg) - # Convert the input list to a series and validate it - results = self.validator.validate(pd.Series(series, dtype=series_dtype)) - # Now find any items where their validation does not correspond to the expected_result - for item, result in zip(series, results): - with self.subTest(value=item): - self.assertEqual(result, expected_result, msg) +class ValidationTestBase(unittest.TestCase): + def seriesEquality(self, s1: pd.Series, s2: pd.Series, msg: str = None): + if not s1.equals(s2): + raise self.failureException(msg) class CustomSeries(ValidationTestBase): @@ -46,13 +57,19 @@ class CustomSeries(ValidationTestBase): """ def setUp(self): - self.validator = CustomSeriesValidation(lambda s: ~s.str.contains('fail'), 'contained the word fail') + self.validator = CustomSeriesValidation( + lambda s: ~s.str.contains('fail'), + message='contained the word fail', + index=0 + ) def test_valid_inputs(self): - self.validate_and_compare(['good', 'success'], True, 'did not accept valid inputs') + assert len(get_warnings(self.validator, ['good', + 'success'])) == 0, 'did not accept valid inputs' def test_invalid_inputs(self): - self.validate_and_compare(['fail', 'failure'], False, 'accepted invalid inputs') + assert len(get_warnings(self.validator, + ['fail', 'failure'])) == 2, 'accepted invalid inputs' class CustomElement(ValidationTestBase): @@ -61,13 +78,20 @@ class CustomElement(ValidationTestBase): """ def setUp(self): - self.validator = CustomElementValidation(lambda s: s.startswith('_start_'), "Didn't begin with '_start_'") + self.validator = CustomElementValidation( + lambda s: s.startswith('_start_'), + message="Didn't begin with '_start_'", + index=0 + ) def test_valid_inputs(self): - self.validate_and_compare(['_start_sdiyhsd', '_start_234fpwunxc\n'], True, 'did not accept valid inputs') + assert len( + get_warnings(self.validator, ['_start_sdiyhsd', + '_start_234fpwunxc\n'])) == 0, 'did not accept valid inputs' def test_invalid_inputs(self): - self.validate_and_compare(['fail', '324wfp9ni'], False, 'accepted invalid inputs') + assert len(get_warnings(self.validator, + ['fail', '324wfp9ni'])) == 2, 'accepted invalid inputs' class LeadingWhitespace(ValidationTestBase): @@ -76,43 +100,31 @@ class LeadingWhitespace(ValidationTestBase): """ def setUp(self): - self.validator = LeadingWhitespaceValidation() + self.validator = LeadingWhitespaceValidation(index=0) def test_validate_trailing_whitespace(self): - self.validate_and_compare( - [ - 'trailing space ', - 'trailing tabs ', - '''trailing newline - ''' - ], - True, - 'is incorrectly failing on trailing whitespace' - ) + assert len(get_warnings(self.validator, [ + 'trailing space ', + 'trailing tabs ', + '''trailing newline + ''' + ])) == 0, 'is incorrectly failing on trailing whitespace' def test_validate_leading_whitespace(self): - self.validate_and_compare( - [ - ' leading spaces', - ' leading tabs', - ''' - leading newline''', - ], - False, - 'does not detect leading whitespace' - ) + assert len(get_warnings(self.validator, [ + ' leading spaces', + ' leading tabs', + ''' + leading newline''', + ])) == 3, 'does not detect leading whitespace' def test_validate_middle_whitespace(self): - self.validate_and_compare( - [ - 'middle spaces', - 'middle tabs', - '''middle - newline''', - ], - True, - 'is incorrectly failing on central whitespace' - ) + assert len(get_warnings(self.validator, [ + 'middle spaces', + 'middle tabs', + '''middle + newline''', + ])) == 0, 'is incorrectly failing on central whitespace' class TrailingWhitespace(ValidationTestBase): @@ -121,44 +133,32 @@ class TrailingWhitespace(ValidationTestBase): """ def setUp(self): - self.validator = TrailingWhitespaceValidation() + self.validator = TrailingWhitespaceValidation(index=0) super().setUp() def test_validate_trailing_whitespace(self): - self.validate_and_compare( - [ - 'trailing space ', - 'trailing tabs ', - '''trailing newline - ''' - ], - False, - 'is not detecting trailing whitespace' - ) + assert len(get_warnings(self.validator, [ + 'trailing space ', + 'trailing tabs ', + '''trailing newline + ''' + ])) == 3, 'is not detecting trailing whitespace' def test_validate_leading_whitespace(self): - self.validate_and_compare( - [ - ' leading spaces', - ' leading tabs', - ''' - leading newline''', - ], - True, - 'is incorrectly failing on leading whitespace' - ) + assert len(get_warnings(self.validator, [ + ' leading spaces', + ' leading tabs', + ''' + leading newline''', + ])) == 0, 'is incorrectly failing on leading whitespace' def test_validate_middle_whitespace(self): - self.validate_and_compare( - [ - 'middle spaces', - 'middle tabs', - '''middle - newline''', - ], - True, - 'is incorrectly failing on central whitespace' - ) + assert len(get_warnings(self.validator, [ + 'middle spaces', + 'middle tabs', + '''middle + newline''', + ])) == 0, 'is incorrectly failing on central whitespace' class CanCallJson(ValidationTestBase): @@ -167,29 +167,21 @@ class CanCallJson(ValidationTestBase): """ def setUp(self): - self.validator = CanCallValidation(json.loads) + self.validator = CanCallValidation(json.loads, index=0) def test_validate_valid_json(self): - self.validate_and_compare( - [ - '[1, 2, 3]', - '{"a": 1.1, "b": 2.2, "c": 3.3}', - '"string"' - ], - True, - 'is incorrectly failing on valid JSON' - ) + assert len(get_warnings(self.validator, [ + '[1, 2, 3]', + '{"a": 1.1, "b": 2.2, "c": 3.3}', + '"string"' + ])) == 0, 'is incorrectly failing on valid JSON' def test_validate_invalid_json(self): - self.validate_and_compare( - [ - '[1, 2, 3', - '{a: 1.1, b: 2.2, c: 3.3}', - 'string' - ], - False, - 'is not detecting invalid JSON' - ) + assert len(get_warnings(self.validator, [ + '[1, 2, 3', + '{a: 1.1, b: 2.2, c: 3.3}', + 'string' + ])) == 3, 'is not detecting invalid JSON' class CanCallLambda(ValidationTestBase): @@ -199,29 +191,22 @@ class CanCallLambda(ValidationTestBase): def setUp(self): # Succeed if it's divisible by 2, otherwise cause an error - self.validator = CanCallValidation(lambda x: False if x % 2 == 0 else 1 / 0) + self.validator = CanCallValidation(lambda x: False if x % 2 == 0 else 1 / 0, + index=0) def test_validate_noerror(self): - self.validate_and_compare( - [ - 2, - 4, - 6 - ], - True, - 'is incorrectly failing on even numbers' - ) + assert len(get_warnings(self.validator, [ + 2, + 4, + 6 + ])) == 0, 'is incorrectly failing on even numbers' def test_validate_error(self): - self.validate_and_compare( - [ - 1, - 3, - 5 - ], - False, - 'should fail on odd numbers' - ) + assert len(get_warnings(self.validator, [ + 1, + 3, + 5 + ])) == 3, 'should fail on odd numbers' class CanConvertInt(ValidationTestBase): @@ -230,164 +215,121 @@ class CanConvertInt(ValidationTestBase): """ def setUp(self): - self.validator = CanConvertValidation(int) + self.validator = CanConvertValidation(int, index=0) def test_valid_int(self): - self.validate_and_compare( - [ - '1', - '10', - '999', - '99999' - ], - True, - 'does not accept valid integers' - ) + assert len(get_warnings(self.validator, [ + '1', + '10', + '999', + '99999' + ])) == 0, 'does not accept valid integers' def test_invalid_int(self): - self.validate_and_compare( - [ - '1.0', - '9.5', - 'abc', - '1e-6' - ], - False, - 'accepts invalid integers' - ) + assert len(get_warnings(self.validator, [ + '1.0', + '9.5', + 'abc', + '1e-6' + ])) == 4, 'accepts invalid integers' class InListCaseSensitive(ValidationTestBase): def setUp(self): - self.validator = InListValidation(['a', 'b', 'c']) + self.validator = InListValidation(['a', 'b', 'c'], index=0) def test_valid_elements(self): - self.validate_and_compare( - [ - 'a', - 'b', - 'c' - ], - True, - 'does not accept elements that are in the validation list' - ) + assert len(get_warnings(self.validator, [ + 'a', + 'b', + 'c' + ])) == 0, 'does not accept elements that are in the validation list' def test_invalid_elements(self): - self.validate_and_compare( - [ - 'aa', - 'bb', - 'd', - 'A', - 'B', - 'C' - ], - False, - 'accepts elements that are not in the validation list' - ) + assert len(get_warnings(self.validator, [ + 'aa', + 'bb', + 'd', + 'A', + 'B', + 'C' + ])) == 6, 'accepts elements that are not in the validation list' class InListCaseInsensitive(ValidationTestBase): def setUp(self): - self.validator = InListValidation(['a', 'b', 'c'], case_sensitive=False) + self.validator = InListValidation(['a', 'b', 'c'], case_sensitive=False, + index=0) def test_valid_elements(self): - self.validate_and_compare( - [ - 'a', - 'b', - 'c', - 'A', - 'B', - 'C' - ], - True, - 'does not accept elements that are in the validation list' - ) + assert len(get_warnings(self.validator, [ + 'a', + 'b', + 'c', + 'A', + 'B', + 'C' + ])) == 0, 'does not accept elements that are in the validation list' def test_invalid_elements(self): - self.validate_and_compare( - [ - 'aa', - 'bb', - 'd', - ], - False, - 'accepts elements that are not in the validation list' - ) + assert len(get_warnings(self.validator, [ + 'aa', + 'bb', + 'd', + ])) == 3, 'accepts elements that are not in the validation list' class DateFormat(ValidationTestBase): def setUp(self): - self.validator = DateFormatValidation('%Y%m%d') + self.validator = DateFormatValidation('%Y%m%d', index=0) def test_valid_dates(self): - self.validate_and_compare( - [ - '20160404', - '00011212' - ], - True, - 'does not accept valid dates' - ) + assert len(get_warnings(self.validator, [ + '20160404', + '00011212' + ])) == 0, 'does not accept valid dates' def test_invalid_dates(self): - self.validate_and_compare( - [ - '1/2/3456', - 'yyyymmdd', - '11112233' - ], - False, - 'accepts invalid dates' - ) + assert len(get_warnings(self.validator, [ + '1/2/3456', + 'yyyymmdd', + '11112233' + ])) == 3, 'accepts invalid dates' class StringRegexMatch(ValidationTestBase): def setUp(self): - self.validator = MatchesPatternValidation('^.+\.txt$') + self.validator = MatchesPatternValidation(r'^.+\.txt$', index=0) def test_valid_strings(self): - self.validate_and_compare( - [ - 'pass.txt', - 'a.txt', - 'lots of words.txt' - ], - True, - 'does not accept strings matching the regex' - ) + assert len(get_warnings(self.validator, [ + 'pass.txt', + 'a.txt', + 'lots of words.txt' + ])) == 0, 'does not accept strings matching the regex' def test_invalid_strings(self): - self.validate_and_compare( - [ - 'pass.TXT', - '.txt', - 'lots of words.tx' - ], - False, - 'accepts strings that do not match the regex' - ) + assert len(get_warnings(self.validator, [ + 'pass.TXT', + '.txt', + 'lots of words.tx' + ])) == 3, 'accepts strings that do not match the regex' class IsDistinct(ValidationTestBase): def setUp(self): - self.validator = IsDistinctValidation() + self.validator = IsDistinctValidation(index=0) def test_valid_strings(self): - self.validate_and_compare( - [ - '1', - '2', - '3', - '4' - ], - True, - 'does not accept unique strings' - ) + assert len(get_warnings(self.validator, [ + '1', + '2', + '3', + '4' + ])) == 0, 'does not accept unique strings' def test_invalid_strings(self): - validation = self.validator.validate(pd.Series([ + validation = self.validator.select_cells(pd.Series([ '1', '1', '3', @@ -408,29 +350,33 @@ class CompiledRegexMatch(ValidationTestBase): """ def setUp(self): - self.validator = MatchesPatternValidation(re.compile('^.+\.txt$', re.IGNORECASE)) + self.validator = MatchesPatternValidation( + re.compile('^.+\.txt$', re.IGNORECASE), index=0) def test_valid_strings(self): - self.validate_and_compare( - [ - 'pass.txt', - 'a.TXT', - 'lots of words.tXt' - ], - True, - 'does not accept strings matching the regex' - ) + assert len(get_warnings(self.validator, [ + 'pass.txt', + 'a.TXT', + 'lots of words.tXt' + ])) == 0, 'does not accept strings matching the regex' def test_invalid_strings(self): - self.validate_and_compare( - [ - 'pass.txtt', - '.txt', - 'lots of words.tx' - ], - False, - 'accepts strings that do not match the regex' - ) + test_data = [ + 'pass.txtt', + '.txt', + 'lots of words.tx' + ] + warnings = get_warnings(self.validator, test_data) + + # Check that every piece of data failed + assert len(warnings) == 3, 'accepts strings that do not match the regex' + + # Also test the messages + for i, (warning, data) in enumerate(zip(warnings, test_data)): + assert 'Row {}'.format(i) in warning.message + assert 'Column 0' in warning.message + assert data in warning.message + assert self.validator.pattern.pattern in warning.message class InRange(ValidationTestBase): @@ -439,29 +385,21 @@ class InRange(ValidationTestBase): """ def setUp(self): - self.validator = InRangeValidation(7, 9) + self.validator = InRangeValidation(7, 9, index=0) def test_valid_items(self): - self.validate_and_compare( - [ - 7, - 8, - 7 - ], - True, - 'does not accept integers in the correct range' - ) + assert len(get_warnings(self.validator, [ + 7, + 8, + 7 + ])) == 0, 'does not accept integers in the correct range' def test_invalid_items(self): - self.validate_and_compare( - [ - 1, - 2, - 3 - ], - False, - 'Incorrectly accepts integers outside of the range' - ) + assert len(get_warnings(self.validator, [ + 1, + 2, + 3 + ])) == 3, 'Incorrectly accepts integers outside of the range' class Dtype(ValidationTestBase): @@ -470,10 +408,10 @@ class Dtype(ValidationTestBase): """ def setUp(self): - self.validator = IsDtypeValidation(np.number) + self.validator = IsDtypeValidation(np.number, index=0) def test_valid_items(self): - errors = self.validator.get_errors(pd.Series( + errors = self.validator.validate_series(pd.Series( [ 1, 2, @@ -483,7 +421,7 @@ def test_valid_items(self): self.assertEqual(len(errors), 0) def test_invalid_items(self): - errors = self.validator.get_errors(pd.Series( + errors = self.validator.validate_series(pd.Series( [ 'a', '', @@ -493,7 +431,6 @@ def test_invalid_items(self): self.assertEqual(len(errors), 1) self.assertEqual(type(errors[0]), ValidationWarning) - def test_schema(self): """ Test this validation inside a schema, to ensure we get helpful error messages. @@ -506,53 +443,44 @@ def test_schema(self): }) schema = Schema([ - Column('wrong_dtype1', [IsDtypeValidation(dtype('int64'))]), - Column('wrong_dtype2', [IsDtypeValidation(dtype('float64'))]), - Column('wrong_dtype3', [IsDtypeValidation(dtype('int64'))]), + IsDtypeValidation(dtype('int64'), index=ci('wrong_dtype1')), + IsDtypeValidation(dtype('float64'), index=ci('wrong_dtype2')), + IsDtypeValidation(dtype('int64'), index=ci('wrong_dtype3')), ]) errors = schema.validate(df) self.assertEqual( - sorted([str(x) for x in errors]), - sorted([ - 'The column wrong_dtype1 has a dtype of object which is not a subclass of the required type int64', - 'The column wrong_dtype2 has a dtype of int64 which is not a subclass of the required type float64', - 'The column wrong_dtype3 has a dtype of float64 which is not a subclass of the required type int64' - ]) + [x.props for x in errors], + [ + {'dtype': np.object}, + {'dtype': np.int64}, + {'dtype': np.float64}, + ] ) - class Negate(ValidationTestBase): """ Tests the ~ operator on a MatchesPatternValidation """ def setUp(self): - self.validator = ~MatchesPatternValidation('fail') + self.validator = ~MatchesPatternValidation('fail', index=0) def test_valid_items(self): - self.validate_and_compare( - [ - 'Pass', - '1', - 'True' - ], - True, - 'Rejects values that should pass' - ) + assert len(get_warnings(self.validator, [ + 'Pass', + '1', + 'True' + ])) == 0, 'Rejects values that should pass' def test_invalid_items(self): - self.validate_and_compare( - [ - 'fail', - 'thisfails', - 'failure' - ], - False, - 'Accepts values that should pass' - ) + assert len(get_warnings(self.validator, [ + 'fail', + 'thisfails', + 'failure' + ])) == 3, 'Accepts values that should pass' class Or(ValidationTestBase): @@ -561,30 +489,26 @@ class Or(ValidationTestBase): """ def setUp(self): - self.validator = MatchesPatternValidation('yes') | MatchesPatternValidation('pass') + self.validator = MatchesPatternValidation( + 'yes', index=0 + ) | MatchesPatternValidation( + 'pass', index=0 + ) def test_valid_items(self): - self.validate_and_compare( - [ - 'pass', - 'yes', - 'passyes', - '345yes345' - ], - True, - 'Rejects values that should pass' - ) + assert len(get_warnings(self.validator, [ + 'pass', + 'yes', + 'passyes', + '345yes345' + ])) == 0, 'rejects values that should pass' def test_invalid_items(self): - self.validate_and_compare( - [ - 'fail', - 'YES', - 'YPESS' - ], - False, - 'Accepts values that should pass' - ) + assert len(get_warnings(self.validator, [ + 'fail', + 'YES', + 'YPESS' + ])) == 6, 'accepts values that should pass' class CustomMessage(ValidationTestBase): @@ -596,28 +520,31 @@ def setUp(self): self.message = "UNUSUAL MESSAGE THAT WOULDN'T BE IN A NORMAL ERROR" def test_default_message(self): - validator = InRangeValidation(min=4) - for error in validator.get_errors(pd.Series( + validator = InRangeValidation(min=4, index=0) + for error in validator.validate_series(pd.Series( [ 1, 2, 3 ] - ), Column('')): - self.assertNotRegex(error.message, self.message, 'Validator not using the default warning message!') + ), flatten=True): + self.assertNotRegex(error.message, self.message, + 'Validator not using the default warning message!') def test_custom_message(self): - validator = InRangeValidation(min=4, message=self.message) - for error in validator.get_errors(pd.Series( + validator = InRangeValidation(min=4, message=self.message, index=0) + for error in validator.validate_series(pd.Series( [ 1, 2, 3 ] - ), Column('')): - self.assertRegex(error.message, self.message, 'Validator not using the custom warning message!') + ), flatten=True): + self.assertRegex(error.message, self.message, + 'Validator not using the custom warning message!') +@unittest.skip('allow_empty no longer exists') class GetErrorTests(ValidationTestBase): """ Tests for float valued columns where allow_empty=True @@ -627,18 +554,18 @@ def setUp(self): self.vals = [1.0, None, 3] def test_in_range_allow_empty_with_error(self): - validator = InRangeValidation(min=4) - errors = validator.get_errors(pd.Series(self.vals), Column('', allow_empty=True)) + validator = InRangeValidation(min=4, index=0) + errors = list(validator.validate_series(pd.Series(self.vals))) self.assertEqual(len(errors), sum(v is not None for v in self.vals)) def test_in_range_allow_empty_with_no_error(self): - validator = InRangeValidation(min=0) - errors = validator.get_errors(pd.Series(self.vals), Column('', allow_empty=True)) + validator = InRangeValidation(min=0, index=0) + errors = list(validator.validate_series(pd.Series(self.vals))) self.assertEqual(len(errors), 0) def test_in_range_allow_empty_false_with_error(self): - validator = InRangeValidation(min=4) - errors = validator.get_errors(pd.Series(self.vals), Column('', allow_empty=False)) + validator = InRangeValidation(min=4, index=0) + errors = list(validator.validate_series(pd.Series(self.vals))) self.assertEqual(len(errors), len(self.vals)) @@ -648,24 +575,25 @@ class PandasDtypeTests(ValidationTestBase): """ def setUp(self): - self.validator = InListValidation(['a', 'b', 'c'], case_sensitive=False) + self.validator = InListValidation(['a', 'b', 'c'], case_sensitive=False, + index=0) def test_valid_elements(self): - errors = self.validator.get_errors(pd.Series(['a', 'b', 'c', None, 'A', 'B', 'C'], dtype='category'), - Column('', allow_empty=True)) - self.assertEqual(len(errors), 0) + errors = self.validator.validate_series( + pd.Series(['a', 'b', 'c', 'A', 'B', 'C'], dtype='category')) + assert len(list(errors)) == 0 def test_invalid_empty_elements(self): - errors = self.validator.get_errors(pd.Series(['aa', 'bb', 'd', None], dtype='category'), - Column('', allow_empty=False)) - self.assertEqual(len(errors), 4) + errors = self.validator.validate_series( + pd.Series(['aa', 'bb', 'd', None], dtype='category')) + assert len(list(errors)) == 4 def test_invalid_and_empty_elements(self): - errors = self.validator.get_errors(pd.Series(['a', None], dtype='category'), - Column('', allow_empty=False)) - self.assertEqual(len(errors), 1) + errors = self.validator.validate_series( + pd.Series(['a', None], dtype='category')) + assert len(list(errors)) == 1 def test_invalid_elements(self): - errors = self.validator.get_errors(pd.Series(['aa', 'bb', 'd'], dtype='category'), - Column('', allow_empty=True)) - self.assertEqual(len(errors), 3) + errors = self.validator.validate_series( + pd.Series(['aa', 'bb', 'd'], dtype='category')) + assert len(list(errors)) == 3 diff --git a/test/test_validation_warning.py b/test/test_validation_warning.py old mode 100644 new mode 100755