Merge pull request #1570 from CEED/jrwrigh/fix_turbspanstats_comm

fix(fluids): Make local Vecs use PETSC_COMM_SELF in turbstats
CEED · Apr 29, 2024 · 8452936 · 8452936
2 parents 8e16760 + 2788647
commit 8452936
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 6 deletions.
diff --git a/examples/fluids/src/petsc_ops.c b/examples/fluids/src/petsc_ops.c
@@ -176,9 +176,12 @@ VecType DMReturnVecType(DM dm) {
 PetscErrorCode CeedOperatorCreateLocalVecs(CeedOperator op, VecType vec_type, MPI_Comm comm, Vec *input, Vec *output) {
   CeedSize input_size, output_size;
   Ceed     ceed;
+  int      comm_size;
 
   PetscFunctionBeginUser;
   PetscCall(CeedOperatorGetCeed(op, &ceed));
+  PetscCallMPI(MPI_Comm_size(comm, &comm_size));
+  PetscCheck(comm_size == 1, PETSC_COMM_WORLD, PETSC_ERR_ARG_SIZ, "MPI_Comm must be of size 1, recieved comm of size %d", comm_size);
   PetscCallCeed(ceed, CeedOperatorGetActiveVectorLengths(op, &input_size, &output_size));
   if (input) {
     PetscCall(VecCreate(comm, input));

diff --git a/examples/fluids/src/turb_spanstats.c b/examples/fluids/src/turb_spanstats.c
@@ -169,7 +169,7 @@ PetscErrorCode GetQuadratureCoords(Ceed ceed, DM dm, CeedElemRestriction elem_re
   PetscCallCeed(ceed, CeedOperatorSetField(op_quad_coords, "input", elem_restr_x, basis_x, x_coords));
   PetscCallCeed(ceed, CeedOperatorSetField(op_quad_coords, "output", elem_restr_qx, CEED_BASIS_NONE, CEED_VECTOR_ACTIVE));
 
-  PetscCall(CeedOperatorCreateLocalVecs(op_quad_coords, DMReturnVecType(dm), PetscObjectComm((PetscObject)dm), NULL, Qx_coords));
+  PetscCall(CeedOperatorCreateLocalVecs(op_quad_coords, DMReturnVecType(dm), PETSC_COMM_SELF, NULL, Qx_coords));
   PetscCall(OperatorApplyContextCreate(NULL, NULL, ceed, op_quad_coords, CEED_VECTOR_NONE, NULL, NULL, NULL, &op_quad_coords_ctx));
 
   PetscCall(ApplyCeedOperatorLocalToLocal(NULL, *Qx_coords, op_quad_coords_ctx));
@@ -301,7 +301,6 @@ PetscErrorCode SetupL2ProjectionStats(Ceed ceed, User user, CeedData ceed_data,
   CeedOperator  op_mass, op_setup_sur, op_proj_rhs;
   CeedQFunction qf_mass, qf_stats_proj;
   CeedInt       q_data_size, num_comp_stats = user->spanstats.num_comp_stats;
-  MPI_Comm      comm = PetscObjectComm((PetscObject)user->spanstats.dm);
 
   PetscFunctionBeginUser;
   // -- Create Operator for RHS of L^2 projection of statistics
@@ -314,7 +313,7 @@ PetscErrorCode SetupL2ProjectionStats(Ceed ceed, User user, CeedData ceed_data,
   PetscCallCeed(ceed, CeedOperatorSetField(op_proj_rhs, "output", stats_data->elem_restr_parent_stats, stats_data->basis_stats, CEED_VECTOR_ACTIVE));
 
   PetscCall(OperatorApplyContextCreate(NULL, user->spanstats.dm, ceed, op_proj_rhs, NULL, NULL, NULL, NULL, &user->spanstats.op_proj_rhs_ctx));
-  PetscCall(CeedOperatorCreateLocalVecs(op_proj_rhs, DMReturnVecType(user->spanstats.dm), comm, &user->spanstats.Parent_Stats_loc, NULL));
+  PetscCall(CeedOperatorCreateLocalVecs(op_proj_rhs, DMReturnVecType(user->spanstats.dm), PETSC_COMM_SELF, &user->spanstats.Parent_Stats_loc, NULL));
 
   // -- Setup LHS of L^2 projection
   // Get q_data for mass matrix operator
@@ -340,7 +339,7 @@ PetscErrorCode SetupL2ProjectionStats(Ceed ceed, User user, CeedData ceed_data,
 
     PetscCall(MatCeedCreate(user->spanstats.dm, user->spanstats.dm, op_mass, NULL, &mat_mass));
 
-    PetscCall(KSPCreate(comm, &ksp));
+    PetscCall(KSPCreate(PetscObjectComm((PetscObject)user->spanstats.dm), &ksp));
     PetscCall(KSPSetOptionsPrefix(ksp, "turbulence_spanstats_"));
     {
       PC pc;
@@ -431,8 +430,8 @@ PetscErrorCode CreateStatisticCollectionOperator(Ceed ceed, User user, CeedData
   PetscCall(OperatorApplyContextCreate(user->dm, user->spanstats.dm, user->ceed, op_stats_collect, user->q_ceed, NULL, NULL, NULL,
                                        &user->spanstats.op_stats_collect_ctx));
 
-  PetscCall(CeedOperatorCreateLocalVecs(op_stats_collect, DMReturnVecType(user->spanstats.dm), PetscObjectComm((PetscObject)user->spanstats.dm), NULL,
-                                        &user->spanstats.Child_Stats_loc));
+  PetscCall(
+      CeedOperatorCreateLocalVecs(op_stats_collect, DMReturnVecType(user->spanstats.dm), PETSC_COMM_SELF, NULL, &user->spanstats.Child_Stats_loc));
   PetscCall(VecZeroEntries(user->spanstats.Child_Stats_loc));
 
   PetscCallCeed(ceed, CeedQFunctionDestroy(&qf_stats_collect));