From 5bbb832a83ebb66db4b5ee740cdfc53f4df8430b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 3 Nov 2021 14:09:53 -0500 Subject: [PATCH] feat: support Python 3.10 (#1043) * feat: support Python 3.10 * fix pandas deps to match reality * run system tests with 3.10 * avoid geopandas on Python 3.10 * install google-cloud-bigquery from source * update kokoro configs * remove Python 2.7 config --- .kokoro/presubmit/prerelease-deps-3.8.cfg | 4 ++-- .kokoro/presubmit/snippets-3.10.cfg | 7 +++++++ .../{snippets-2.7.cfg => system-3.10.cfg} | 2 +- noxfile.py | 17 +++++++++++++---- samples/geography/noxfile_config.py | 7 ++++++- samples/geography/requirements.txt | 8 +++----- samples/magics/requirements.txt | 9 ++++----- samples/snippets/requirements.txt | 7 +++---- setup.py | 5 +++-- 9 files changed, 42 insertions(+), 24 deletions(-) create mode 100644 .kokoro/presubmit/snippets-3.10.cfg rename .kokoro/presubmit/{snippets-2.7.cfg => system-3.10.cfg} (82%) diff --git a/.kokoro/presubmit/prerelease-deps-3.8.cfg b/.kokoro/presubmit/prerelease-deps-3.8.cfg index f06806baf..fabe3e347 100644 --- a/.kokoro/presubmit/prerelease-deps-3.8.cfg +++ b/.kokoro/presubmit/prerelease-deps-3.8.cfg @@ -3,5 +3,5 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "prerelease_deps" -} \ No newline at end of file + value: "prerelease_deps-3.8" +} diff --git a/.kokoro/presubmit/snippets-3.10.cfg b/.kokoro/presubmit/snippets-3.10.cfg new file mode 100644 index 000000000..dde182fb9 --- /dev/null +++ b/.kokoro/presubmit/snippets-3.10.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "snippets-3.10" +} diff --git a/.kokoro/presubmit/snippets-2.7.cfg b/.kokoro/presubmit/system-3.10.cfg similarity index 82% rename from .kokoro/presubmit/snippets-2.7.cfg rename to .kokoro/presubmit/system-3.10.cfg index 3bd6134d2..30956a3ab 100644 --- a/.kokoro/presubmit/snippets-2.7.cfg +++ b/.kokoro/presubmit/system-3.10.cfg @@ -3,5 +3,5 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "snippets-2.7" + value: "system-3.10" } diff --git a/noxfile.py b/noxfile.py index 64eacaff5..6f04940c9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -27,8 +27,8 @@ BLACK_PATHS = ("docs", "google", "samples", "tests", "noxfile.py", "setup.py") DEFAULT_PYTHON_VERSION = "3.8" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] -UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.10"] +UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() # 'docfx' is excluded since it only needs to run in 'docs-presubmit' @@ -69,7 +69,12 @@ def default(session, install_extras=True): constraints_path, ) - install_target = ".[all]" if install_extras else "." + if install_extras and session.python == "3.10": + install_target = ".[bqstorage,pandas,tqdm,opentelemetry]" + elif install_extras: + install_target = ".[all]" + else: + install_target = "." session.install("-e", install_target, "-c", constraints_path) session.install("ipython", "-c", constraints_path) @@ -153,7 +158,11 @@ def system(session): # Data Catalog needed for the column ACL test with a real Policy Tag. session.install("google-cloud-datacatalog", "-c", constraints_path) - session.install("-e", ".[all]", "-c", constraints_path) + if session.python == "3.10": + extras = "[bqstorage,pandas,tqdm,opentelemetry]" + else: + extras = "[all]" + session.install("-e", f".{extras}", "-c", constraints_path) session.install("ipython", "-c", constraints_path) # Run py.test against the system tests. diff --git a/samples/geography/noxfile_config.py b/samples/geography/noxfile_config.py index 7d2e02346..315bd5be8 100644 --- a/samples/geography/noxfile_config.py +++ b/samples/geography/noxfile_config.py @@ -22,7 +22,12 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7"], + "ignored_versions": [ + "2.7", + # TODO: Enable 3.10 once there is a geopandas/fiona release. + # https://github.com/Toblerity/Fiona/issues/1043 + "3.10", + ], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index ecd428ab9..e2de86673 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -24,14 +24,12 @@ importlib-metadata==4.8.1 libcst==0.3.21 munch==2.5.0 mypy-extensions==0.4.3 -numpy==1.19.5; python_version < "3.7" -numpy==1.21.2; python_version > "3.6" packaging==21.0 pandas==1.1.5; python_version < '3.7' -pandas==1.3.2; python_version >= '3.7' +pandas==1.3.4; python_version >= '3.7' proto-plus==1.19.2 protobuf==3.18.0 -pyarrow==5.0.0 +pyarrow==6.0.0 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.20 @@ -43,7 +41,7 @@ pytz==2021.1 PyYAML==5.4.1 requests==2.26.0 rsa==4.7.2 -Shapely==1.7.1 +Shapely==1.8.0 six==1.16.0 typing-extensions==3.10.0.2 typing-inspect==0.7.1 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index f9b9d023c..5cc7ec33f 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,12 +1,11 @@ -google-cloud-bigquery==2.27.1 google-cloud-bigquery-storage==2.9.0 google-auth-oauthlib==0.4.6 grpcio==1.41.0 ipython==7.16.1; python_version < '3.7' -ipython==7.17.0; python_version >= '3.7' +ipython==7.29.0; python_version >= '3.7' matplotlib==3.3.4; python_version < '3.7' -matplotlib==3.4.1; python_version >= '3.7' +matplotlib==3.5.0rc1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.2; python_version >= '3.7' -pyarrow==5.0.0 +pandas==1.3.4; python_version >= '3.7' +pyarrow==6.0.0 pytz==2021.1 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index f9b9d023c..f79552392 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,12 +1,11 @@ -google-cloud-bigquery==2.27.1 google-cloud-bigquery-storage==2.9.0 google-auth-oauthlib==0.4.6 grpcio==1.41.0 ipython==7.16.1; python_version < '3.7' -ipython==7.17.0; python_version >= '3.7' +ipython==7.29.0; python_version >= '3.7' matplotlib==3.3.4; python_version < '3.7' matplotlib==3.4.1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.2; python_version >= '3.7' -pyarrow==5.0.0 +pandas==1.3.4; python_version >= '3.7' +pyarrow==6.0.0 pytz==2021.1 diff --git a/setup.py b/setup.py index 95dad190a..db69c45b1 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ ] + pyarrow_dep, "geopandas": ["geopandas>=0.9.0, <1.0dev", "Shapely>=1.6.0, <2.0dev"], - "pandas": ["pandas>=0.23.0"] + pyarrow_dep, + "pandas": ["pandas>=0.24.2"] + pyarrow_dep, "bignumeric_type": pyarrow_dep, "tqdm": ["tqdm >= 4.7.4, <5.0.0dev"], "opentelemetry": [ @@ -127,6 +127,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Operating System :: OS Independent", "Topic :: Internet", ], @@ -135,7 +136,7 @@ namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, - python_requires=">=3.6, <3.10", + python_requires=">=3.6, <3.11", include_package_data=True, zip_safe=False, )