From 06f1da901ea7aced903dd557b1aadab376fb116b Mon Sep 17 00:00:00 2001 From: Mathias Louboutin Date: Mon, 24 Jan 2022 13:14:18 -0500 Subject: [PATCH 1/4] mpi: fix mask ordering for sparse gather --- devito/types/sparse.py | 1 - tests/test_mpi.py | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/devito/types/sparse.py b/devito/types/sparse.py index c743364304..7144f74e6a 100644 --- a/devito/types/sparse.py +++ b/devito/types/sparse.py @@ -169,7 +169,6 @@ def _dist_gather_mask(self, dmap=None): ret = list(self._dist_scatter_mask(dmap=dmap)) mask = ret[self._sparse_position] inds = np.unique(mask, return_index=True)[1] - inds.sort() ret[self._sparse_position] = inds.tolist() return tuple(ret) diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 2cf6867df3..dbfe5338b4 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -478,6 +478,22 @@ def test_sparse_coords(self): coords_loc += sf.coordinates.data[i, 0] assert sf.data[i] == coords_loc + @pytest.mark.parallel(mode=4) + def test_sparse_coords_issue1823(self): + grid = Grid((101, 101, 101), extent=(1000, 1000, 1000)) + coords = np.array([[1000., 0., 900.], [1000., 300., 700.], + [1000., 500., 500.], [1000., 700., 300.], + [1000., 900., 0.], [1000., 0., 850.]]) + rec = SparseTimeFunction(name="s", grid=grid, coordinates=coords, + nt=10, npoint=6) + ref = SparseTimeFunction(name="s1", grid=grid, coordinates=coords, + nt=10, npoint=6) + u = TimeFunction(name="u", grid=grid, space_order=1) + + Operator([Eq(u, u+1)]+rec.interpolate(u))() + + assert np.allclose(rec.coordinates.data[:], ref.coordinates.data) + class TestOperatorSimple(object): From 871061d68e33de220214abb23c2f2af6324766ac Mon Sep 17 00:00:00 2001 From: Mathias Louboutin Date: Wed, 26 Jan 2022 06:17:01 -0500 Subject: [PATCH 2/4] CI: mpi, clear devito cache before run --- .github/workflows/pytest-core-mpi.yml | 1 + devito/types/sparse.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest-core-mpi.yml b/.github/workflows/pytest-core-mpi.yml index 01ef13a065..1a0d977bbe 100644 --- a/.github/workflows/pytest-core-mpi.yml +++ b/.github/workflows/pytest-core-mpi.yml @@ -41,6 +41,7 @@ jobs: - name: Test with pytest run: | + python3 scripts/clear_devito_cache.py python3 -m pytest --cov --cov-config=.coveragerc --cov-report=xml -m parallel tests/ - name: Upload coverage to Codecov diff --git a/devito/types/sparse.py b/devito/types/sparse.py index 7144f74e6a..121678f674 100644 --- a/devito/types/sparse.py +++ b/devito/types/sparse.py @@ -169,7 +169,8 @@ def _dist_gather_mask(self, dmap=None): ret = list(self._dist_scatter_mask(dmap=dmap)) mask = ret[self._sparse_position] inds = np.unique(mask, return_index=True)[1] - ret[self._sparse_position] = inds.tolist() + inds.sort() + ret[self._sparse_position] = [mask[i] for i in inds] return tuple(ret) From f49a856b30f030e1f861c1d118d60733f64be682 Mon Sep 17 00:00:00 2001 From: mloubout Date: Wed, 2 Feb 2022 07:27:28 -0500 Subject: [PATCH 3/4] mpi: reuse scatter mask for gather --- devito/types/sparse.py | 49 ++++++------------------------------------ 1 file changed, 6 insertions(+), 43 deletions(-) diff --git a/devito/types/sparse.py b/devito/types/sparse.py index 121678f674..8a94c91204 100644 --- a/devito/types/sparse.py +++ b/devito/types/sparse.py @@ -151,37 +151,6 @@ def _dist_scatter_mask(self, dmap=None): ret[self._sparse_position] = mask return tuple(ret) - def _dist_subfunc_scatter_mask(self, dmap=None): - """ - This method is analogous to :meth:`_dist_scatter_mask`, although - the mask is now suitable to index into self's SubFunctions, rather - than into ``self.data``. - """ - return self._dist_scatter_mask(dmap=dmap)[self._sparse_position] - - def _dist_gather_mask(self, dmap=None): - """ - A mask to index into the ``data`` received upon returning from - ``self._dist_alltoall``. This mask creates a new data array in which - duplicate sparse data values have been discarded. The resulting data - array can thus be used to populate ``self.data``. - """ - ret = list(self._dist_scatter_mask(dmap=dmap)) - mask = ret[self._sparse_position] - inds = np.unique(mask, return_index=True)[1] - inds.sort() - ret[self._sparse_position] = [mask[i] for i in inds] - - return tuple(ret) - - def _dist_subfunc_gather_mask(self, dmap=None): - """ - This method is analogous to :meth:`_dist_subfunc_scatter_mask`, although - the mask is now suitable to index into self's SubFunctions, rather - than into ``self.data``. - """ - return self._dist_gather_mask(dmap=dmap)[self._sparse_position] - def _dist_count(self, dmap=None): """ A 2-tuple of comm-sized iterables, which tells how many sparse points @@ -261,13 +230,6 @@ def _dist_scatter(self): """ raise NotImplementedError - def _dist_gather(self, data): - """ - A ``numpy.ndarray`` containing up-to-date data and coordinate values - suitable for insertion into ``self.data``. - """ - raise NotImplementedError - def _arg_defaults(self, alias=None): key = alias or self mapper = {self: key} @@ -698,9 +660,10 @@ def _dist_scatter(self, data=None): # Compute dist map only once dmap = self._dist_datamap + mask = self._dist_scatter_mask(dmap=dmap) # Pack sparse data values so that they can be sent out via an Alltoallv - data = data[self._dist_scatter_mask(dmap=dmap)] + data = data[mask] data = np.ascontiguousarray(np.transpose(data, self._dist_reorder_mask)) # Send out the sparse point values @@ -714,7 +677,7 @@ def _dist_scatter(self, data=None): data = np.ascontiguousarray(np.transpose(data, self._dist_reorder_mask)) # Pack (reordered) coordinates so that they can be sent out via an Alltoallv - coords = self.coordinates.data._local[self._dist_subfunc_scatter_mask(dmap=dmap)] + coords = self.coordinates.data._local[mask[self._sparse_position]] # Send out the sparse point coordinates _, scount, sdisp, rshape, rcount, rdisp = self._dist_subfunc_alltoall(dmap=dmap) @@ -739,6 +702,7 @@ def _dist_gather(self, data, coords): # Compute dist map only once dmap = self._dist_datamap + mask = self._dist_scatter_mask(dmap=dmap) # Pack sparse data values so that they can be sent out via an Alltoallv data = np.ascontiguousarray(np.transpose(data, self._dist_reorder_mask)) @@ -750,7 +714,7 @@ def _dist_gather(self, data, coords): [gathered, scount, sdisp, mpitype]) # Unpack data values so that they follow the expected storage layout gathered = np.ascontiguousarray(np.transpose(gathered, self._dist_reorder_mask)) - self._data[:] = gathered[self._dist_gather_mask(dmap=dmap)] + self._data[mask] = gathered[:] if coords is not None: # Pack (reordered) coordinates so that they can be sent out via an Alltoallv @@ -762,8 +726,7 @@ def _dist_gather(self, data, coords): mpitype = MPI._typedict[np.dtype(self.coordinates.dtype).char] comm.Alltoallv([coords, rcount, rdisp, mpitype], [gathered, scount, sdisp, mpitype]) - self._coordinates.data._local[:] = \ - gathered[self._dist_subfunc_gather_mask(dmap=dmap)] + self._coordinates.data._local[mask[self._sparse_position]] = gathered[:] # Note: this method "mirrors" `_dist_scatter`: a sparse point that is sent # in `_dist_scatter` is here received; a sparse point that is received in From 7bb6ff5ad3e22d1ed0c65da77934f7450f3afae9 Mon Sep 17 00:00:00 2001 From: Fabio Luporini Date: Mon, 7 Feb 2022 16:50:37 +0100 Subject: [PATCH 4/4] tests: Fix test_get_gpu_info --- tests/test_gpu_common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_gpu_common.py b/tests/test_gpu_common.py index 41b637cc57..1e3e9f6e62 100644 --- a/tests/test_gpu_common.py +++ b/tests/test_gpu_common.py @@ -20,8 +20,9 @@ class TestGPUInfo(object): def test_get_gpu_info(self): info = get_gpu_info() + known = ['nvidia', 'tesla', 'geforce', 'unspecified'] try: - assert info['architecture'].lower() in ['tesla', 'geforce', 'unspecified'] + assert info['architecture'].lower() in known except KeyError: # There might be than one GPUs, but for now we don't care # as we're not really exploiting this info yet...