From 2e92035bafa2d9864c9d44978b8613fa6ba61028 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 12 Apr 2024 11:40:02 -0500 Subject: [PATCH 01/13] Build wheels with numpy 2.0rc1 --- .github/workflows/deploy.yaml | 3 --- pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index e1b4de9..1bf04c3 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -64,9 +64,6 @@ jobs: CIBW_SKIP: "cp36-* cp37-* cp38-* pp* *-manylinux_i686 *-musllinux_i686 *-musllinux_aarch64 *-win32" CIBW_ARCHS: "${{ matrix.cibw_archs }}" CIBW_TEST_SKIP: "*_arm64 *_universal2:arm64" - # below only for building against unstable numpy - CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation" - CIBW_BEFORE_BUILD: "pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy cython setuptools versioneer" - uses: actions/upload-artifact@v4 with: diff --git a/pyproject.toml b/pyproject.toml index 3deb236..57e52e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools", "wheel", "oldest-supported-numpy", "Cython>=3", "versioneer"] +requires = ["setuptools", "wheel", "numpy>=2.0.0rc1,<3", "Cython>=3", "versioneer"] build-backend = "setuptools.build_meta" [tool.coverage.run] From 3ff7702c09f608cf78172382797e1d66ca710794 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 12 Apr 2024 13:04:19 -0500 Subject: [PATCH 02/13] Fix interpolator tests with scipy 1.13.0 --- geotiepoints/tests/test_interpolator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/geotiepoints/tests/test_interpolator.py b/geotiepoints/tests/test_interpolator.py index c342db0..567e686 100644 --- a/geotiepoints/tests/test_interpolator.py +++ b/geotiepoints/tests/test_interpolator.py @@ -349,12 +349,12 @@ def test_interpolate(self, grid_interpolator): fine_x = np.arange(16) fine_y = np.arange(32) - res = grid_interpolator.interpolate((fine_y, fine_x), method="cubic") + res = grid_interpolator.interpolate((fine_y, fine_x), method="cubic_legacy") np.testing.assert_allclose(res, self.expected, atol=2e-9) def test_interpolate_slices(self, grid_interpolator): """Test that interpolation from slices is working.""" - res = grid_interpolator.interpolate_slices((slice(0, 32), slice(0, 16)), method="cubic") + res = grid_interpolator.interpolate_slices((slice(0, 32), slice(0, 16)), method="cubic_legacy") np.testing.assert_allclose(res, self.expected, atol=2e-9) @pytest.mark.parametrize("chunks, expected_chunks", [(10, (10, 10)), @@ -369,7 +369,7 @@ def test_interpolate_dask(self, grid_interpolator, chunks, expected_chunks): with mock.patch.object(grid_interpolator, "interpolate_numpy", wraps=grid_interpolator.interpolate_numpy) as interpolate: - res = grid_interpolator.interpolate((fine_y, fine_x), method="cubic", chunks=chunks) + res = grid_interpolator.interpolate((fine_y, fine_x), method="cubic_legacy", chunks=chunks) assert not interpolate.called assert isinstance(res, da.Array) @@ -395,5 +395,5 @@ def test_interpolate_preserves_dtype(self): fine_x = np.arange(16) fine_y = np.arange(32) - res = grid_interpolator.interpolate((fine_y, fine_x), method="cubic") + res = grid_interpolator.interpolate((fine_y, fine_x), method="cubic_legacy") assert res.dtype == data.dtype From 50bb958600e289c0a0b0389c70922b81758f657d Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 12 Apr 2024 13:11:14 -0500 Subject: [PATCH 03/13] Add no build isolation to unstable CI --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ae44e3f..67920ed 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,6 +42,7 @@ jobs: if: matrix.experimental == true shell: bash -l {0} run: | + python -m pip install versioneer; \ python -m pip install \ -f https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ --no-deps --pre --upgrade \ @@ -50,7 +51,7 @@ jobs: pandas \ scipy; \ python -m pip install \ - --no-deps --upgrade \ + --no-deps --upgrade --pre --no-build-isolation \ git+https://github.com/dask/dask \ git+https://github.com/dask/distributed \ git+https://github.com/zarr-developers/zarr \ From 94619a6c808c764a9396f25b7284aeec1a918827 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 12 Apr 2024 13:16:08 -0500 Subject: [PATCH 04/13] Add Anaconda PyPI to unstable CI --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 67920ed..a216eb3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -45,6 +45,7 @@ jobs: python -m pip install versioneer; \ python -m pip install \ -f https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ + --trusted-host pypi.anaconda.org \ --no-deps --pre --upgrade \ matplotlib \ numpy \ From 08fa045466704a37a848aa379205bd221e484d01 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 12 Apr 2024 13:43:38 -0500 Subject: [PATCH 05/13] Remove scipy conda package before pip installing in unstable CI --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a216eb3..801237a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,6 +43,7 @@ jobs: shell: bash -l {0} run: | python -m pip install versioneer; \ + conda uninstall --force-remove -y scipy; \ python -m pip install \ -f https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ --trusted-host pypi.anaconda.org \ From e5537784a8a2934c01db380aca78c52090fbe605 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 12 Apr 2024 16:56:45 -0500 Subject: [PATCH 06/13] Add h5py to unstable dependencies --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 801237a..a0e8550 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,7 +43,7 @@ jobs: shell: bash -l {0} run: | python -m pip install versioneer; \ - conda uninstall --force-remove -y scipy; \ + conda uninstall --force-remove -y scipy h5py; \ python -m pip install \ -f https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ --trusted-host pypi.anaconda.org \ @@ -56,6 +56,7 @@ jobs: --no-deps --upgrade --pre --no-build-isolation \ git+https://github.com/dask/dask \ git+https://github.com/dask/distributed \ + git+https://github.com/h5py/h5py \ git+https://github.com/zarr-developers/zarr \ git+https://github.com/pydata/bottleneck \ git+https://github.com/pydata/xarray; From f457b4f660662fe184d0f1d8e7d7b4c52d26f995 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 12 Apr 2024 17:08:41 -0500 Subject: [PATCH 07/13] Add missing pkgconfig to CI --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a0e8550..206023f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: if: matrix.experimental == true shell: bash -l {0} run: | - python -m pip install versioneer; \ + python -m pip install versioneer pkgconfig; \ conda uninstall --force-remove -y scipy h5py; \ python -m pip install \ -f https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ From 7f3c1132078179a1accbe494ef19804c3788bf00 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 12 Apr 2024 17:19:57 -0500 Subject: [PATCH 08/13] Install pykdtree from source for unstable CI Needs to be built with numpy 2 --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 206023f..087a1a6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,7 +43,7 @@ jobs: shell: bash -l {0} run: | python -m pip install versioneer pkgconfig; \ - conda uninstall --force-remove -y scipy h5py; \ + conda uninstall --force-remove -y scipy h5py pykdtree; \ python -m pip install \ -f https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ --trusted-host pypi.anaconda.org \ @@ -57,6 +57,7 @@ jobs: git+https://github.com/dask/dask \ git+https://github.com/dask/distributed \ git+https://github.com/h5py/h5py \ + git+https://github.com/storpipfugl/pykdtree \ git+https://github.com/zarr-developers/zarr \ git+https://github.com/pydata/bottleneck \ git+https://github.com/pydata/xarray; From 7565e92fbf495fee0f5ae30c8b0b2275f4e993d3 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 15 Apr 2024 14:08:55 -0500 Subject: [PATCH 09/13] Add pyresample and setuptools-scm to unstable CI --- .github/workflows/ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 087a1a6..1d045f6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,8 +42,8 @@ jobs: if: matrix.experimental == true shell: bash -l {0} run: | - python -m pip install versioneer pkgconfig; \ - conda uninstall --force-remove -y scipy h5py pykdtree; \ + python -m pip install versioneer pkgconfig setuptools-scm; \ + conda uninstall --force-remove -y scipy h5py pyresample pykdtree; \ python -m pip install \ -f https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ --trusted-host pypi.anaconda.org \ @@ -58,6 +58,7 @@ jobs: git+https://github.com/dask/distributed \ git+https://github.com/h5py/h5py \ git+https://github.com/storpipfugl/pykdtree \ + git+https://github.com/pytroll/pyresample \ git+https://github.com/zarr-developers/zarr \ git+https://github.com/pydata/bottleneck \ git+https://github.com/pydata/xarray; From 934596ce1cdbe67bd58f6c4ffd6171b876abc218 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 15 Apr 2024 14:17:15 -0500 Subject: [PATCH 10/13] Force pandas uninstall before unstable reinstall --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1d045f6..9f17944 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,7 +43,7 @@ jobs: shell: bash -l {0} run: | python -m pip install versioneer pkgconfig setuptools-scm; \ - conda uninstall --force-remove -y scipy h5py pyresample pykdtree; \ + conda uninstall --force-remove -y scipy h5py pyresample pykdtree pandas xarray; \ python -m pip install \ -f https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ --trusted-host pypi.anaconda.org \ From 703fa7c6742a41651c38b9fb65e8c53cbbb083a0 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 15 Apr 2024 14:43:42 -0500 Subject: [PATCH 11/13] Replace deprecated array with asarray usage See https://numpy.org/devdocs/numpy_2_0_migration_guide.html#adapting-to-changes-in-the-copy-keyword --- geotiepoints/multilinear.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geotiepoints/multilinear.py b/geotiepoints/multilinear.py index 740421d..0bdc85a 100644 --- a/geotiepoints/multilinear.py +++ b/geotiepoints/multilinear.py @@ -59,9 +59,9 @@ class MultilinearInterpolator: __grid__ = None def __init__(self, smin, smax, orders, values=None, dtype=np.float64): - self.smin = np.array(smin, dtype=dtype, copy=False) - self.smax = np.array(smax, dtype=dtype, copy=False) - self.orders = np.array(orders, dtype=np.int_, copy=False) + self.smin = np.asarray(smin, dtype=dtype) + self.smax = np.asarray(smax, dtype=dtype) + self.orders = np.asarray(orders, dtype=np.int_) self.d = len(orders) self.dtype = dtype if values is not None: From 3c656d9b569c39820369abe6562a7c0605c7a800 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 15 Apr 2024 14:54:22 -0500 Subject: [PATCH 12/13] Replace deprecated row_stack usage with vstack --- geotiepoints/multilinear.py | 6 +++--- geotiepoints/tests/test_multilinear.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/geotiepoints/multilinear.py b/geotiepoints/multilinear.py index 0bdc85a..e6cf5f7 100644 --- a/geotiepoints/multilinear.py +++ b/geotiepoints/multilinear.py @@ -13,7 +13,7 @@ def mlinspace(smin, smax, orders): else: meshes = np.meshgrid( *[np.linspace(smin[i], smax[i], orders[i]) for i in range(len(orders))], indexing='ij') - return np.row_stack([l.flatten() for l in meshes]) + return np.vstack([l.flatten() for l in meshes]) class MultilinearInterpolator: @@ -44,8 +44,8 @@ class MultilinearInterpolator: smax = [1,1] orders = [5,5] - f = lambda x: np.row_stack([np.sqrt( x[0,:]**2 + x[1,:]**2 ), - np.power( x[0,:]**3 + x[1,:]**3, 1.0/3.0 )]) + f = lambda x: np.vstack([np.sqrt( x[0,:]**2 + x[1,:]**2 ), + np.power( x[0,:]**3 + x[1,:]**3, 1.0/3.0 )]) interp = MultilinearInterpolator(smin,smax,orders) interp.set_values( f(interp.grid) ) diff --git a/geotiepoints/tests/test_multilinear.py b/geotiepoints/tests/test_multilinear.py index 58300ee..177d418 100644 --- a/geotiepoints/tests/test_multilinear.py +++ b/geotiepoints/tests/test_multilinear.py @@ -70,7 +70,7 @@ def test_multilinear_interp(self): smax = [1, 1] orders = [5, 5] - f = lambda x: np.row_stack([ + f = lambda x: np.vstack([ np.sqrt(x[0, :]**2 + x[1, :]**2), np.power(x[0, :]**3 + x[1, :]**3, 1.0 / 3.0) ]) From c248ff434e93d08ad1a433eb77ce1bd295289a8e Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 15 Apr 2024 15:22:29 -0500 Subject: [PATCH 13/13] Fix usage of multiprocessing in tests --- geotiepoints/tests/test_modis.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/geotiepoints/tests/test_modis.py b/geotiepoints/tests/test_modis.py index c039683..b1532ff 100644 --- a/geotiepoints/tests/test_modis.py +++ b/geotiepoints/tests/test_modis.py @@ -20,6 +20,8 @@ import h5py import os +import pytest + FILENAME_250M_RESULT = os.path.join( os.path.dirname(__file__), '../../testdata/250m_lonlat_section_result.h5') FILENAME_250M_INPUT = os.path.join( @@ -68,8 +70,12 @@ def test_5_to_1(self): np.testing.assert_allclose(tlons, glons, atol=0.05) np.testing.assert_allclose(tlats, glats, atol=0.05) - def test_1000m_to_250m(self): + @pytest.mark.parametrize("ncores", [None, 4]) + def test_1000m_to_250m(self, ncores): """Test the 1 km to 250 meter interpolation facility.""" + if ncores: + import multiprocessing as mp + mp.set_start_method("spawn", force=True) with h5py.File(FILENAME_250M_RESULT) as h5f: glons = h5f['longitude'][:] / 1000. @@ -79,10 +85,7 @@ def test_1000m_to_250m(self): lons = h5f['longitude'][:] / 1000. lats = h5f['latitude'][:] / 1000. - tlons, tlats = modis1kmto250m(lons, lats) - np.testing.assert_allclose(tlons, glons, atol=0.05) - np.testing.assert_allclose(tlats, glats, atol=0.05) - - tlons, tlats = modis1kmto250m(lons, lats, cores=4) + kwargs = {"cores": ncores} if ncores is not None else {} + tlons, tlats = modis1kmto250m(lons, lats, **kwargs) np.testing.assert_allclose(tlons, glons, atol=0.05) np.testing.assert_allclose(tlats, glats, atol=0.05)