Create a common DataFrameSchema class, update mypy used in pre-commit (…

…#1609) * define common dataframe api for DataFrameSchema Signed-off-by: cosmicBboy <niels.bantilan@gmail.com> * use mypy 1.10.0 for pre-commit Signed-off-by: cosmicBboy <niels.bantilan@gmail.com> * regenerate requirements files Signed-off-by: cosmicBboy <niels.bantilan@gmail.com> * fix lint Signed-off-by: cosmicBboy <niels.bantilan@gmail.com> * update pre-commit Signed-off-by: cosmicBboy <niels.bantilan@gmail.com> * run updated black, isort Signed-off-by: cosmicBboy <niels.bantilan@gmail.com> * update isort Signed-off-by: cosmicBboy <niels.bantilan@gmail.com> * add isort config to pyproject.toml Signed-off-by: cosmicBboy <niels.bantilan@gmail.com> * revert isort, black Signed-off-by: cosmicBboy <niels.bantilan@gmail.com> * revert black, isort Signed-off-by: cosmicBboy <niels.bantilan@gmail.com> --------- Signed-off-by: cosmicBboy <niels.bantilan@gmail.com>
unionai-oss · May 1, 2024 · d058f71 · d058f71
1 parent 1305d93
commit d058f71
Show file tree

Hide file tree

Showing 144 changed files with 1,731 additions and 1,715 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -45,7 +45,7 @@ repos:
         exclude: (^docs/|^scripts)
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.982
+    rev: v1.10.0
     hooks:
       - id: mypy
         additional_dependencies:

diff --git a/.pylintrc b/.pylintrc
@@ -54,4 +54,5 @@ disable=
     fixme,
     too-many-locals,
     redefined-outer-name,
-    logging-fstring-interpolation
+    logging-fstring-interpolation,
+    multiple-statements
diff --git a/ci/requirements-py3.10-pandas1.5.3-pydantic1.10.11.txt b/ci/requirements-py3.10-pandas1.5.3-pydantic1.10.11.txt
@@ -287,7 +287,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.10-pandas1.5.3-pydantic2.3.0.txt b/ci/requirements-py3.10-pandas1.5.3-pydantic2.3.0.txt
@@ -289,7 +289,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.10-pandas2.0.3-pydantic1.10.11.txt b/ci/requirements-py3.10-pandas2.0.3-pydantic1.10.11.txt
@@ -291,7 +291,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.10-pandas2.0.3-pydantic2.3.0.txt b/ci/requirements-py3.10-pandas2.0.3-pydantic2.3.0.txt
@@ -293,7 +293,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.10-pandas2.2.0-pydantic1.10.11.txt b/ci/requirements-py3.10-pandas2.2.0-pydantic1.10.11.txt
@@ -291,7 +291,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.10-pandas2.2.0-pydantic2.3.0.txt b/ci/requirements-py3.10-pandas2.2.0-pydantic2.3.0.txt
@@ -293,7 +293,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.11-pandas1.5.3-pydantic1.10.11.txt b/ci/requirements-py3.11-pandas1.5.3-pydantic1.10.11.txt
@@ -281,7 +281,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.11-pandas1.5.3-pydantic2.3.0.txt b/ci/requirements-py3.11-pandas1.5.3-pydantic2.3.0.txt
@@ -283,7 +283,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.11-pandas2.0.3-pydantic1.10.11.txt b/ci/requirements-py3.11-pandas2.0.3-pydantic1.10.11.txt
@@ -285,7 +285,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.11-pandas2.0.3-pydantic2.3.0.txt b/ci/requirements-py3.11-pandas2.0.3-pydantic2.3.0.txt
@@ -287,7 +287,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.11-pandas2.2.0-pydantic1.10.11.txt b/ci/requirements-py3.11-pandas2.2.0-pydantic1.10.11.txt
@@ -285,7 +285,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.11-pandas2.2.0-pydantic2.3.0.txt b/ci/requirements-py3.11-pandas2.2.0-pydantic2.3.0.txt
@@ -287,7 +287,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.8-pandas1.5.3-pydantic1.10.11.txt b/ci/requirements-py3.8-pandas1.5.3-pydantic1.10.11.txt
@@ -303,7 +303,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.8-pandas1.5.3-pydantic2.3.0.txt b/ci/requirements-py3.8-pandas1.5.3-pydantic2.3.0.txt
@@ -305,7 +305,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.8-pandas2.0.3-pydantic1.10.11.txt b/ci/requirements-py3.8-pandas2.0.3-pydantic1.10.11.txt
@@ -303,7 +303,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.8-pandas2.0.3-pydantic2.3.0.txt b/ci/requirements-py3.8-pandas2.0.3-pydantic2.3.0.txt
@@ -305,7 +305,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.9-pandas1.5.3-pydantic1.10.11.txt b/ci/requirements-py3.9-pandas1.5.3-pydantic1.10.11.txt
@@ -295,7 +295,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.9-pandas1.5.3-pydantic2.3.0.txt b/ci/requirements-py3.9-pandas1.5.3-pydantic2.3.0.txt
@@ -297,7 +297,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.9-pandas2.0.3-pydantic1.10.11.txt b/ci/requirements-py3.9-pandas2.0.3-pydantic1.10.11.txt
@@ -299,7 +299,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.9-pandas2.0.3-pydantic2.3.0.txt b/ci/requirements-py3.9-pandas2.0.3-pydantic2.3.0.txt
@@ -301,7 +301,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.9-pandas2.2.0-pydantic1.10.11.txt b/ci/requirements-py3.9-pandas2.2.0-pydantic1.10.11.txt
@@ -299,7 +299,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/ci/requirements-py3.9-pandas2.2.0-pydantic2.3.0.txt b/ci/requirements-py3.9-pandas2.2.0-pydantic2.3.0.txt
@@ -301,7 +301,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/dev/requirements-3.10.txt b/dev/requirements-3.10.txt
@@ -293,7 +293,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/dev/requirements-3.11.txt b/dev/requirements-3.11.txt
@@ -287,7 +287,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/dev/requirements-3.8.txt b/dev/requirements-3.8.txt
@@ -305,7 +305,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/dev/requirements-3.9.txt b/dev/requirements-3.9.txt
@@ -301,7 +301,7 @@ msgpack==1.0.8
     #   distributed
     #   ray
 multimethod==1.10
-mypy==0.982
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via
     #   black

diff --git a/environment.yml b/environment.yml
@@ -47,7 +47,7 @@ dependencies:
   # testing
   - isort >= 5.7.0
   - joblib
-  - mypy = 0.982
+  - mypy = 1.10.0
   - pylint <= 2.17.3
   - pytest
   - pytest-cov

diff --git a/noxfile.py b/noxfile.py
@@ -1,4 +1,5 @@
 """Nox sessions."""
+
 # isort: skip_file
 import os
 import re

diff --git a/pandera/__init__.py b/pandera/__init__.py
@@ -3,23 +3,27 @@
 import platform
 
 import pandera.backends
+import pandera.backends.base.builtin_checks
+import pandera.backends.base.builtin_hypotheses
+import pandera.backends.pandas
 from pandera import errors, external_config, typing
 from pandera.accessors import pandas_accessor
 from pandera.api import extensions
 from pandera.api.checks import Check
-from pandera.api.parsers import Parser
 from pandera.api.dataframe.model_components import (
+    Field,
     check,
     dataframe_check,
-    parser,
     dataframe_parser,
+    parser,
 )
 from pandera.api.hypotheses import Hypothesis
-from pandera.api.dataframe.model_components import Field
 from pandera.api.pandas.array import SeriesSchema
-from pandera.api.pandas.container import DataFrameSchema
 from pandera.api.pandas.components import Column, Index, MultiIndex
+from pandera.api.pandas.container import DataFrameSchema
 from pandera.api.pandas.model import DataFrameModel, SchemaModel
+from pandera.api.parsers import Parser
+from pandera.backends.pandas.register import register_pandas_backends
 from pandera.decorators import check_input, check_io, check_output, check_types
 from pandera.dtypes import (
     Bool,
@@ -65,16 +69,9 @@
     UINT64,
     pandas_version,
 )
-
-import pandera.backends.base.builtin_checks
-import pandera.backends.base.builtin_hypotheses
-import pandera.backends.pandas
-from pandera.backends.pandas.register import register_pandas_backends
-
 from pandera.schema_inference.pandas import infer_schema
 from pandera.version import __version__
 
-
 if platform.system() != "Windows":
     # pylint: disable=ungrouped-imports
     from pandera.dtypes import Complex256, Float128

diff --git a/pandera/accessors/pyspark_sql_accessor.py b/pandera/accessors/pyspark_sql_accessor.py
@@ -4,8 +4,8 @@
 import warnings
 from typing import Optional
 
-from pandera.api.pyspark.container import DataFrameSchema
 from pandera.api.base.error_handler import ErrorHandler
+from pandera.api.pyspark.container import DataFrameSchema
 
 Schemas = DataFrameSchema  # type: ignore
 Errors = ErrorHandler  # type: ignore

diff --git a/pandera/api/base/checks.py b/pandera/api/base/checks.py
@@ -162,7 +162,7 @@ def from_builtin_check_name(
         name: str,
         init_kwargs,
         error: Union[str, Callable],
-        statistics: Dict[str, Any] = None,
+        statistics: Optional[Dict[str, Any]] = None,
         **check_kwargs,
     ):
         """Create a Check object from a built-in check's name."""

diff --git a/pandera/api/base/error_handler.py b/pandera/api/base/error_handler.py
@@ -5,9 +5,9 @@
 from typing import Any, Dict, List, Union
 
 from pandera.api.checks import Check
-from pandera.config import get_config_context, ValidationDepth
-from pandera.validation_depth import ValidationScope, validation_type
+from pandera.config import ValidationDepth, get_config_context
 from pandera.errors import SchemaError, SchemaErrorReason
+from pandera.validation_depth import ValidationScope, validation_type
 
 
 class ErrorCategory(Enum):

diff --git a/pandera/api/base/parsers.py b/pandera/api/base/parsers.py
@@ -1,14 +1,7 @@
 """Data validation base parse."""
 
 import inspect
-from typing import (
-    Any,
-    Dict,
-    NamedTuple,
-    Optional,
-    Tuple,
-    Type,
-)
+from typing import Any, Dict, NamedTuple, Optional, Tuple, Type
 
 from pandera.backends.base import BaseParserBackend
 

diff --git a/pandera/api/base/schema.py b/pandera/api/base/schema.py
@@ -9,11 +9,11 @@
 import os
 from abc import ABC
 from functools import wraps
-from typing import Any, Dict, Tuple, Type, Optional, Union
+from typing import Any, Dict, Optional, Tuple, Type, Union
 
 from pandera.backends.base import BaseSchemaBackend
-from pandera.errors import BackendNotFoundError
 from pandera.dtypes import DataType
+from pandera.errors import BackendNotFoundError
 
 DtypeInputTypes = Union[str, type, DataType, Type]
 

diff --git a/pandera/api/base/types.py b/pandera/api/base/types.py
@@ -1,15 +1,16 @@
 """Base type definitions for pandera."""
 
 from typing import List, Union
+
 from pandera.api.checks import Check
-from pandera.api.parsers import Parser
 from pandera.api.hypotheses import Hypothesis
+from pandera.api.parsers import Parser
 
 try:
     # python 3.8+
     from typing import Literal  # type: ignore[attr-defined]
 except ImportError:  # pragma: no cover
-    from typing_extensions import Literal  # type: ignore[misc]
+    from typing_extensions import Literal  # type: ignore[assignment]
 
 
 StrictType = Union[bool, Literal["filter"]]