Value Validation

ColumnValuesToBeBetween

Check if the values in a column are between a range.

If the min_value or max_value is not provided then other will be used as the threshold.

If neither min_value nor max_value is provided, then the validation will result in failure.

Parameters:

Name	Type	Description	Default
column	str	Column to validate.	required
min_value	float | None	Minimum value for a column entry length.	None
max_value	float | None	Maximum value for a column entry length.	None
threshold	float	Threshold for validation. Defaults to 0.0.	required
impact	Literal['low', 'medium', 'high']	Impact level of validation. Defaults to "low".	required
kwargs	object	KwargsType (dict): Additional keyword arguments.	{}

Source code in validoopsie/validation_catalogue/ValuesValidation/column_values_to_be_between.py

@base_validation_wrapper
class ColumnValuesToBeBetween(BaseValidationParameters):
    """Check if the values in a column are between a range.

    If the `min_value` or `max_value` is not provided then other will be used as the
    threshold.

    If neither `min_value` nor `max_value` is provided, then the validation will result
    in failure.


    Parameters:
        column (str): Column to validate.
        min_value (float | None): Minimum value for a column entry length.
        max_value (float | None): Maximum value for a column entry length.
        threshold (float, optional): Threshold for validation. Defaults to 0.0.
        impact (Literal["low", "medium", "high"], optional): Impact level of validation.
            Defaults to "low".
        kwargs: KwargsType (dict): Additional keyword arguments.

    """

    def __init__(
        self,
        column: str,
        min_value: float | None = None,
        max_value: float | None = None,
        *args,
        **kwargs: object,
    ) -> None:
        min_max_arg_check(min_value, max_value)

        super().__init__(column, *args, **kwargs)
        self.min_value = min_value
        self.max_value = max_value

    @property
    def fail_message(self) -> str:
        """Return the fail message, that will be used in the report."""
        return (
            f"The column '{self.column}' has values that are not "
            f"between {self.min_value} and {self.max_value}."
        )

    def __call__(self, frame: FrameT) -> FrameT:
        """Check if the values in a column are between a range."""
        return (
            min_max_filter(
                frame,
                f"{self.column}",
                self.min_value,
                self.max_value,
            )
            .group_by(self.column)
            .agg(
                nw.col(self.column).count().alias(f"{self.column}-count"),
            )
        )

ColumnsSumToBeBetween

Check if the sum of columns is greater than or equal to max_sum.

If the min_value or max_value is not provided then other will be used as the threshold.

If neither min_value nor max_value is provided, then the validation will result in failure.

Parameters:

Name	Type	Description	Default
columns_list	list[str]	List of columns to sum.	required
max_sum_value	float | None	Minimum sum value that columns should be greater than or equal to.	None
min_sum_value	float | None	Maximum sum value that columns should be less than or equal to.	None
threshold	float	Threshold for validation. Defaults to 0.0.	required
impact	Literal['low', 'medium', 'high']	Impact level of validation. Defaults to "low".	required
kwargs	object	KwargsType (dict): Additional keyword arguments.	{}

Source code in validoopsie/validation_catalogue/ValuesValidation/columns_sum_to_be_between.py

@base_validation_wrapper
class ColumnsSumToBeBetween(BaseValidationParameters):
    """Check if the sum of columns is greater than or equal to `max_sum`.

    If the `min_value` or `max_value` is not provided then other will be used as the
    threshold.

    If neither `min_value` nor `max_value` is provided, then the validation will result
    in failure.

    Parameters:
        columns_list (list[str]): List of columns to sum.
        max_sum_value (float | None): Minimum sum value that columns should be greater
            than or equal to.
        min_sum_value (float | None): Maximum sum value that columns should be less than
            or equal to.
        threshold (float, optional): Threshold for validation. Defaults to 0.0.
        impact (Literal["low", "medium", "high"], optional): Impact level of validation.
            Defaults to "low".
        kwargs: KwargsType (dict): Additional keyword arguments.

    """

    def __init__(
        self,
        columns_list: list[str],
        min_sum_value: float | None = None,
        max_sum_value: float | None = None,
        *args,
        **kwargs: object,
    ) -> None:
        min_max_arg_check(min_sum_value, max_sum_value)

        self.columns_list = columns_list
        self.max_sum_value = max_sum_value
        self.min_sum_value = min_sum_value
        self.column = "-".join(self.columns_list) + "-combined"
        super().__init__(self.column, *args, **kwargs)

    @property
    def fail_message(self) -> str:
        """Return the fail message, that will be used in the report."""
        return (
            f"The columns {self.columns_list} are not between {self.min_sum_value} and "
            f"{self.max_sum_value}."
        )

    def __call__(self, frame: FrameT) -> FrameT:
        """Check if the sum of columns is greater than or equal to `max_sum`."""
        # This is just in case if there is some weird column name, such as "sum"
        col_name = "-".join(self.columns_list) + "-sum"
        summed_frame = frame.select(self.columns_list).with_columns(
            nw.sum_horizontal(self.columns_list).alias(col_name),
        )

        return (
            min_max_filter(
                summed_frame,
                col_name,
                self.min_sum_value,
                self.max_sum_value,
            )
            .with_columns(
                nw.concat_str(
                    [nw.col(column) for column in self.columns_list],
                    separator=" - ",
                ).alias(
                    self.column,
                ),
            )
            .group_by(
                self.column,
            )
            .agg(
                nw.col(self.column).count().alias(f"{self.column}-count"),
            )
        )

ColumnsSumToBeEqualTo

Check if the sum of the columns is equal to a specific value.

Parameters:

Name	Type	Description	Default
columns_list	list[str]	List of columns to sum.	required
sum_value	float	Value that the columns should sum to.	required
threshold	float	Threshold for validation. Defaults to 0.0.	required
impact	Literal['low', 'medium', 'high']	Impact level of validation. Defaults to "low".	required
kwargs	object	KwargsType (dict): Additional keyword arguments.	{}

Source code in validoopsie/validation_catalogue/ValuesValidation/columns_sum_to_be_equal_to.py

@base_validation_wrapper
class ColumnsSumToBeEqualTo(BaseValidationParameters):
    """Check if the sum of the columns is equal to a specific value.

    Parameters:
        columns_list (list[str]): List of columns to sum.
        sum_value (float): Value that the columns should sum to.
        threshold (float, optional): Threshold for validation. Defaults to 0.0.
        impact (Literal["low", "medium", "high"], optional): Impact level of validation.
            Defaults to "low".
        kwargs: KwargsType (dict): Additional keyword arguments.

    """

    def __init__(
        self,
        columns_list: list[str],
        sum_value: float,
        *args,
        **kwargs: object,
    ) -> None:
        self.columns_list = columns_list
        self.sum_value = sum_value
        self.column = "-".join(self.columns_list) + "-combined"
        super().__init__(self.column, *args, **kwargs)

    @property
    def fail_message(self) -> str:
        """Return the fail message, that will be used in the report."""
        return f"The columns {self.columns_list} do not sum to {self.sum_value}."

    def __call__(self, frame: FrameT) -> FrameT:
        """Check if the sum of the columns is equal to a specific value."""
        # This is just in case if there is some weird column name, such as "sum"
        col_name = "-".join(self.columns_list) + "-sum"
        return (
            frame.select(self.columns_list)
            .with_columns(
                nw.sum_horizontal(self.columns_list).alias(col_name),
            )
            .filter(
                nw.col(col_name) != self.sum_value,
            )
            .with_columns(
                nw.concat_str(
                    [nw.col(column) for column in self.columns_list],
                    separator=" - ",
                ).alias(
                    self.column,
                ),
            )
            .group_by(
                self.column,
            )
            .agg(
                nw.col(self.column).count().alias(f"{self.column}-count"),
            )
        )