Backend registration happens at schema initialization (#1548)

* make backend registration more robust

Signed-off-by: cosmicBboy <niels.bantilan@gmail.com>

* register backends on schema init

Signed-off-by: cosmicBboy <niels.bantilan@gmail.com>

* add joblib to dev environment

Signed-off-by: cosmicBboy <niels.bantilan@gmail.com>

* fix pandas unit test

Signed-off-by: cosmicBboy <niels.bantilan@gmail.com>

* update backend registration for pyspark

Signed-off-by: cosmicBboy <niels.bantilan@gmail.com>

---------

Signed-off-by: cosmicBboy <niels.bantilan@gmail.com>

ci/requirements-py3.10-pandas1.5.3-pydantic1.10.11.txt

@@ -192,6 +192,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.10-pandas1.5.3-pydantic2.3.0.txt

@@ -194,6 +194,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.10-pandas2.0.3-pydantic1.10.11.txt

@@ -192,6 +192,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.10-pandas2.0.3-pydantic2.3.0.txt

@@ -194,6 +194,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.10-pandas2.2.0-pydantic1.10.11.txt

@@ -190,6 +190,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.17
     # via
     #   asv

ci/requirements-py3.10-pandas2.2.0-pydantic2.3.0.txt

@@ -192,6 +192,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.17
     # via
     #   asv

ci/requirements-py3.11-pandas1.5.3-pydantic1.10.11.txt

@@ -186,6 +186,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.11-pandas1.5.3-pydantic2.3.0.txt

@@ -188,6 +188,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.11-pandas2.0.3-pydantic1.10.11.txt

@@ -186,6 +186,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.11-pandas2.0.3-pydantic2.3.0.txt

@@ -188,6 +188,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.11-pandas2.2.0-pydantic1.10.11.txt

@@ -184,6 +184,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.17
     # via
     #   asv

ci/requirements-py3.11-pandas2.2.0-pydantic2.3.0.txt

@@ -186,6 +186,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.17
     # via
     #   asv

ci/requirements-py3.8-pandas1.5.3-pydantic1.10.11.txt

@@ -207,6 +207,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.8-pandas1.5.3-pydantic2.3.0.txt

@@ -209,6 +209,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.8-pandas2.0.3-pydantic1.10.11.txt

@@ -207,6 +207,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.8-pandas2.0.3-pydantic2.3.0.txt

@@ -209,6 +209,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.9-pandas1.5.3-pydantic1.10.11.txt

@@ -199,6 +199,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.9-pandas1.5.3-pydantic2.3.0.txt

@@ -201,6 +201,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.9-pandas2.0.3-pydantic1.10.11.txt

@@ -199,6 +199,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.9-pandas2.0.3-pydantic2.3.0.txt

@@ -201,6 +201,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

ci/requirements-py3.9-pandas2.2.0-pydantic1.10.11.txt

@@ -197,6 +197,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.17
     # via
     #   asv

ci/requirements-py3.9-pandas2.2.0-pydantic2.3.0.txt

@@ -199,6 +199,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.17
     # via
     #   asv

dev/requirements-3.10.txt

@@ -192,6 +192,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

dev/requirements-3.11.txt

@@ -186,6 +186,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

dev/requirements-3.8.txt

@@ -207,6 +207,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

dev/requirements-3.9.txt

@@ -199,6 +199,7 @@ jinja2==3.1.3
     #   myst-parser
     #   nbconvert
     #   sphinx
+joblib==1.3.2
 json5==0.9.14
     # via
     #   asv

environment.yml

@@ -50,6 +50,7 @@ dependencies:
 
   # testing
   - isort >= 5.7.0
+  - joblib
   - mypy = 0.982
   - pylint <= 2.17.3
   - pytest

pandera/api/base/checks.py

@@ -183,7 +183,8 @@ def from_builtin_check_name(
     @classmethod
     def register_backend(cls, type_: Type, backend: Type[BaseCheckBackend]):
         """Register a backend for the specified type."""
-        cls.BACKEND_REGISTRY[(cls, type_)] = backend
+        if (cls, type_) not in cls.BACKEND_REGISTRY:
+            cls.BACKEND_REGISTRY[(cls, type_)] = backend
 
     @classmethod
     def get_backend(cls, check_obj: Any) -> Type[BaseCheckBackend]:

pandera/api/base/schema.py

@@ -45,6 +45,7 @@ def __init__(
         self.description = description
         self.metadata = metadata
         self.drop_invalid_rows = drop_invalid_rows
+        self._register_default_backends()
 
     def validate(
         self,
@@ -94,7 +95,8 @@ def properties(self):
     @classmethod
     def register_backend(cls, type_: Type, backend: Type[BaseSchemaBackend]):
         """Register a schema backend for this class."""
-        cls.BACKEND_REGISTRY[(cls, type_)] = backend
+        if (cls, type_) not in cls.BACKEND_REGISTRY:
+            cls.BACKEND_REGISTRY[(cls, type_)] = backend
 
     @classmethod
     def get_backend(
@@ -122,6 +124,14 @@ def get_backend(
             f"Looked up the following base classes: {classes}"
         )
 
+    def _register_default_backends(self):
+        """Register default backends.
+
+        This method is invoked in the `__init__` method for subclasses that
+        implement the API for a specific dataframe object, and should be
+        overridden in those subclasses.
+        """
+
 
 def inferred_schema_guard(method):
     """

pandera/api/pandas/array.py

@@ -12,6 +12,7 @@
 from pandera.api.checks import Check
 from pandera.api.hypotheses import Hypothesis
 from pandera.api.pandas.types import PandasDtypeInputTypes, is_field
+from pandera.backends.pandas.register import register_pandas_backends
 from pandera.config import get_config_context
 from pandera.dtypes import DataType, UniqueSettings
 from pandera.engines import pandas_engine, PYDANTIC_V2
@@ -111,6 +112,9 @@ def __init__(
                 "DataFrameSchema dtype."
             )
 
+    def _register_default_backends(self):
+        register_pandas_backends()
+
     # the _is_inferred getter and setter methods are not public
     @property
     def _is_inferred(self):

pandera/api/pandas/container.py

@@ -18,6 +18,7 @@
 from pandera.api.checks import Check
 from pandera.api.hypotheses import Hypothesis
 from pandera.api.pandas.types import PandasDtypeInputTypes
+from pandera.backends.pandas.register import register_pandas_backends
 from pandera.dtypes import DataType, UniqueSettings
 from pandera.engines import pandas_engine, PYDANTIC_V2
 
@@ -171,6 +172,9 @@ def _validate_attributes(self):
                 "or `'filter'`."
             )
 
+    def _register_default_backends(self):
+        register_pandas_backends()
+
     @property
     def coerce(self) -> bool:
         """Whether to coerce series to specified type."""

pandera/api/polars/components.py

@@ -8,6 +8,7 @@
 from pandera.api.base.types import CheckList
 from pandera.api.pandas.components import Column as _Column
 from pandera.api.polars.types import PolarsDtypeInputTypes, PolarsCheckObjects
+from pandera.backends.polars.register import register_polars_backends
 from pandera.config import config_context, get_config_context
 from pandera.engines import polars_engine
 from pandera.utils import is_regex
@@ -99,6 +100,9 @@ def __init__(
         )
         self.set_regex()
 
+    def _register_default_backends(self):
+        register_polars_backends()
+
     def validate(
         self,
         check_obj: PolarsCheckObjects,

pandera/api/polars/container.py

@@ -8,6 +8,7 @@
 from pandera.api.pandas.container import DataFrameSchema as _DataFrameSchema
 from pandera.api.polars.types import PolarsCheckObjects
 from pandera.api.polars.utils import get_validation_depth
+from pandera.backends.polars.register import register_polars_backends
 from pandera.config import config_context
 from pandera.dtypes import DataType
 from pandera.engines import polars_engine
@@ -33,6 +34,9 @@ def _validate_attributes(self):
                 "polars backend, all duplicate values will be reported."
             )
 
+    def _register_default_backends(self):
+        register_polars_backends()
+
     def validate(
         self,
         check_obj: PolarsCheckObjects,

pandera/api/pyspark/column_schema.py

@@ -9,6 +9,7 @@
 from pandera.api.checks import Check
 from pandera.api.base.error_handler import ErrorHandler
 from pandera.api.pyspark.types import CheckList, PySparkDtypeInputTypes
+from pandera.backends.pyspark.register import register_pyspark_backends
 from pandera.dtypes import DataType
 from pandera.engines import pyspark_engine
 
@@ -69,6 +70,9 @@ def __init__(
         self.description = description
         self.metadata = metadata
 
+    def _register_default_backends(self):
+        register_pyspark_backends()
+
     @property
     def dtype(self) -> DataType:
         """Get the pyspark dtype"""

pandera/api/pyspark/container.py

@@ -11,12 +11,13 @@
 from pyspark.sql import DataFrame
 
 from pandera import errors
-from pandera.config import get_config_context
 from pandera.api.base.schema import BaseSchema
 from pandera.api.base.types import StrictType
 from pandera.api.checks import Check
 from pandera.api.base.error_handler import ErrorHandler
 from pandera.api.pyspark.types import CheckList, PySparkDtypeInputTypes
+from pandera.backends.pyspark.register import register_pyspark_backends
+from pandera.config import get_config_context
 from pandera.dtypes import DataType, UniqueSettings
 from pandera.engines import pyspark_engine
 
@@ -153,6 +154,9 @@ def __init__(
         self._IS_INFERRED = False
         self.metadata = metadata
 
+    def _register_default_backends(self):
+        register_pyspark_backends()
+
     @property
     def coerce(self) -> bool:
         """Whether to coerce series to specified type."""