Skip to content

Commit

Permalink
wait for pmodels#6972 - avoid using global variables
Browse files Browse the repository at this point in the history
  • Loading branch information
@wkliao
wkliao committed Apr 9, 2024
1 parent aae73ac commit 5d4b8cb
Show file tree
Hide file tree
Showing 11 changed files with 101 additions and 96 deletions.
2 changes: 1 addition & 1 deletion src/mpi/romio/adio/ad_gpfs/ad_gpfs_hints.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ void ADIOI_GPFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
*/
if (!fd->hints->initialized) {

ad_get_env_vars();
ad_get_env_vars(fd);
ad_gpfs_get_env_vars();
did_anything = 1;

Expand Down
16 changes: 8 additions & 8 deletions src/mpi/romio/adio/ad_gpfs/ad_gpfs_rdcoll.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,14 +171,14 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, MPI_Aint count,
* because the difference in starting and ending offsets for 1 byte is
* 0 the same as 0 bytes so it cannot be distinguished.
*/
if ((romio_read_aggmethod == 1) || (romio_read_aggmethod == 2)) {
if ((fd->romio_read_aggmethod == 1) || (fd->romio_read_aggmethod == 2)) {
count_sizes = (ADIO_Offset *) ADIOI_Malloc(nprocs * sizeof(ADIO_Offset));
MPI_Count buftype_size;
MPI_Type_size_x(datatype, &buftype_size);
my_count_size = (ADIO_Offset) count *(ADIO_Offset) buftype_size;
}
if (romio_tunegather) {
if ((romio_read_aggmethod == 1) || (romio_read_aggmethod == 2)) {
if (fd->romio_tunegather) {
if ((fd->romio_read_aggmethod == 1) || (fd->romio_read_aggmethod == 2)) {
gpfs_offsets0 = (ADIO_Offset *) ADIOI_Malloc(6 * nprocs * sizeof(ADIO_Offset));
gpfs_offsets = gpfs_offsets0 + 3 * nprocs;
for (ii = 0; ii < nprocs; ii++) {
Expand Down Expand Up @@ -218,7 +218,7 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, MPI_Aint count,
} else {
MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1, ADIO_OFFSET, fd->comm);
MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1, ADIO_OFFSET, fd->comm);
if ((romio_read_aggmethod == 1) || (romio_read_aggmethod == 2)) {
if ((fd->romio_read_aggmethod == 1) || (fd->romio_read_aggmethod == 2)) {
MPI_Allgather(&count_sizes, 1, ADIO_OFFSET, count_sizes, 1, ADIO_OFFSET, fd->comm);
}
}
Expand Down Expand Up @@ -278,7 +278,7 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, MPI_Aint count,
*
*/
int currentNonZeroDataIndex = 0;
if ((romio_read_aggmethod == 1) || (romio_read_aggmethod == 2)) {
if ((fd->romio_read_aggmethod == 1) || (fd->romio_read_aggmethod == 2)) {
/* Take out the 0-data offsets by shifting the indexes with data to the
* front and keeping track of the non-zero data index for use as the
* length. By doing this we will optimally use all available aggs
Expand All @@ -295,7 +295,7 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, MPI_Aint count,
}
}
if (gpfsmpio_tuneblocking) {
if ((romio_read_aggmethod == 1) || (romio_read_aggmethod == 2)) {
if ((fd->romio_read_aggmethod == 1) || (fd->romio_read_aggmethod == 2)) {
ADIOI_GPFS_Calc_file_domains(fd, st_offsets, end_offsets, currentNonZeroDataIndex,
nprocs_for_coll, &min_st_offset,
&fd_start, &fd_end, &fd_size, fd->fs_ptr);
Expand All @@ -305,7 +305,7 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, MPI_Aint count,
&fd_start, &fd_end, &fd_size, fd->fs_ptr);
}
} else {
if ((romio_read_aggmethod == 1) || (romio_read_aggmethod == 2)) {
if ((fd->romio_read_aggmethod == 1) || (fd->romio_read_aggmethod == 2)) {
ADIOI_Calc_file_domains(st_offsets, end_offsets, currentNonZeroDataIndex,
nprocs_for_coll, &min_st_offset,
&fd_start, &fd_end,
Expand All @@ -321,7 +321,7 @@ void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, MPI_Aint count,
}

GPFSMPIO_T_CIO_SET_GET(r, 1, 1, GPFSMPIO_CIO_T_MYREQ, GPFSMPIO_CIO_T_FD_PART);
if ((romio_read_aggmethod == 1) || (romio_read_aggmethod == 2)) {
if ((fd->romio_read_aggmethod == 1) || (fd->romio_read_aggmethod == 2)) {
/* If the user has specified to use a one-sided aggregation method then
* do that at this point instead of the two-phase I/O.
*/
Expand Down
28 changes: 14 additions & 14 deletions src/mpi/romio/adio/ad_gpfs/ad_gpfs_wrcoll.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,14 +161,14 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,
* because the difference in starting and ending offsets for 1 byte is
* 0 the same as 0 bytes so it cannot be distinguished.
*/
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {
count_sizes = (ADIO_Offset *) ADIOI_Malloc(nprocs * sizeof(ADIO_Offset));
MPI_Count buftype_size;
MPI_Type_size_x(datatype, &buftype_size);
my_count_size = (ADIO_Offset) count *(ADIO_Offset) buftype_size;
}
if (romio_tunegather) {
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if (fd->romio_tunegather) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {
gpfs_offsets0 = (ADIO_Offset *) ADIOI_Malloc(6 * nprocs * sizeof(ADIO_Offset));
gpfs_offsets = gpfs_offsets0 + 3 * nprocs;
for (ii = 0; ii < nprocs; ii++) {
Expand Down Expand Up @@ -208,7 +208,7 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,
} else {
MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1, ADIO_OFFSET, fd->comm);
MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1, ADIO_OFFSET, fd->comm);
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {
MPI_Allgather(&my_count_size, 1, ADIO_OFFSET, count_sizes, 1,
ADIO_OFFSET, fd->comm);
}
Expand Down Expand Up @@ -257,7 +257,7 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,
process may directly access only its own file domain. */
ADIO_Offset lastFileOffset = 0, firstFileOffset = -1;
int currentValidDataIndex = 0;
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {
/* Take out the 0-data offsets by shifting the indexes with data to the front
* and keeping track of the valid data index for use as the length.
*/
Expand All @@ -277,7 +277,7 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,
}

if (gpfsmpio_tuneblocking) {
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {
ADIOI_GPFS_Calc_file_domains(fd, st_offsets, end_offsets,
currentValidDataIndex,
nprocs_for_coll, &min_st_offset,
Expand All @@ -288,7 +288,7 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,
&fd_start, &fd_end, &fd_size, fd->fs_ptr);
}
} else {
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {
ADIOI_Calc_file_domains(st_offsets, end_offsets, currentValidDataIndex,
nprocs_for_coll, &min_st_offset,
&fd_start, &fd_end,
Expand All @@ -305,7 +305,7 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,

GPFSMPIO_T_CIO_SET_GET(w, 1, 1, GPFSMPIO_CIO_T_MYREQ, GPFSMPIO_CIO_T_FD_PART);

if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {
/* If the user has specified to use a one-sided aggregation method then do that at
* this point instead of the two-phase I/O.
*/
Expand Down Expand Up @@ -335,7 +335,7 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,
currentValidDataIndex, fd_start, fd_end, &holeFound,
&noStripeParms);
int anyHolesFound = 0;
if (!romio_onesided_no_rmw)
if (!fd->romio_onesided_no_rmw)
MPI_Allreduce(&holeFound, &anyHolesFound, 1, MPI_INT, MPI_MAX, fd->comm);
if (anyHolesFound == 0) {
GPFSMPIO_T_CIO_REPORT(1, fd, myrank, nprocs);
Expand All @@ -352,18 +352,18 @@ void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,
* and behavior.
*/

if (romio_onesided_inform_rmw && (myrank == 0))
if (fd->romio_onesided_inform_rmw && (myrank == 0))
FPRINTF(stderr, "Information: Holes found during one-sided "
"write aggregation algorithm --- re-running one-sided "
"write aggregation with ROMIO_ONESIDED_ALWAYS_RMW set to 1.\n");
romio_onesided_always_rmw = 1;
int prev_romio_onesided_no_rmw = romio_onesided_no_rmw;
romio_onesided_no_rmw = 1;
fd->romio_onesided_always_rmw = 1;
int prev_romio_onesided_no_rmw = fd->romio_onesided_no_rmw;
fd->romio_onesided_no_rmw = 1;
ADIOI_OneSidedWriteAggregation(fd, offset_list, len_list, contig_access_count, buf,
datatype, error_code, firstFileOffset, lastFileOffset,
currentValidDataIndex, fd_start, fd_end, &holeFound,
&noStripeParms);
romio_onesided_no_rmw = prev_romio_onesided_no_rmw;
fd->romio_onesided_no_rmw = prev_romio_onesided_no_rmw;
GPFSMPIO_T_CIO_REPORT(1, fd, myrank, nprocs);
ADIOI_Free(offset_list);
ADIOI_Free(st_offsets);
Expand Down
28 changes: 14 additions & 14 deletions src/mpi/romio/adio/ad_lustre/ad_lustre_wrcoll.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,14 +142,14 @@ void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count
* because the difference in starting and ending offsets for 1 byte is
* 0 the same as 0 bytes so it cannot be distinguished.
*/
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {
count_sizes = (ADIO_Offset *) ADIOI_Malloc(nprocs * sizeof(ADIO_Offset));
MPI_Count buftype_size;
MPI_Type_size_x(datatype, &buftype_size);
my_count_size = (ADIO_Offset) count *(ADIO_Offset) buftype_size;
}
if (romio_tunegather) {
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if (fd->romio_tunegather) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {
lustre_offsets0 = (ADIO_Offset *) ADIOI_Malloc(6 * nprocs * sizeof(ADIO_Offset));
lustre_offsets = lustre_offsets0 + 3 * nprocs;
for (i = 0; i < nprocs; i++) {
Expand Down Expand Up @@ -189,7 +189,7 @@ void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count
} else {
MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1, ADIO_OFFSET, fd->comm);
MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1, ADIO_OFFSET, fd->comm);
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {
MPI_Allgather(&my_count_size, 1, ADIO_OFFSET, count_sizes, 1,
ADIO_OFFSET, fd->comm);
}
Expand Down Expand Up @@ -221,7 +221,7 @@ void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count
if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {
ADIOI_Free(offset_list);
ADIOI_Free(st_offsets);
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2))
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2))
ADIOI_Free(count_sizes);
}

Expand All @@ -242,7 +242,7 @@ void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count

ADIO_Offset lastFileOffset = 0, firstFileOffset = -1;
int currentValidDataIndex = 0;
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {
/* Take out the 0-data offsets by shifting the indexes with data to the front
* and keeping track of the valid data index for use as the length.
*/
Expand All @@ -267,7 +267,7 @@ void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count
/* If the user has specified to use a one-sided aggregation method then do
* that at this point instead of the two-phase I/O.
*/
if ((romio_write_aggmethod == 1) || (romio_write_aggmethod == 2)) {
if ((fd->romio_write_aggmethod == 1) || (fd->romio_write_aggmethod == 2)) {

ADIOI_LUSTRE_IterateOneSided(fd, buf, striping_info, offset_list, len_list,
contig_access_count, currentValidDataIndex, count,
Expand Down Expand Up @@ -1148,7 +1148,7 @@ static void ADIOI_LUSTRE_IterateOneSided(ADIO_File fd, const void *buf, int *str
/* Remember romio_onesided_no_rmw setting if we have to re-do
* the aggregation if holes are found.
*/
int prev_romio_onesided_no_rmw = romio_onesided_no_rmw;
int prev_romio_onesided_no_rmw = fd->romio_onesided_no_rmw;

while (doAggregation) {

Expand Down Expand Up @@ -1355,7 +1355,7 @@ static void ADIOI_LUSTRE_IterateOneSided(ADIO_File fd, const void *buf, int *str
* write completes that we told it do with the stripeParms.flushCB
* flag then we need to do a barrier here.
*/
if (!romio_onesided_always_rmw && stripeParms.flushCB) {
if (!fd->romio_onesided_always_rmw && stripeParms.flushCB) {
if (fileSegmentIter < (numSegments - 1)) {
MPI_Barrier(fd->comm);
}
Expand All @@ -1374,7 +1374,7 @@ static void ADIOI_LUSTRE_IterateOneSided(ADIO_File fd, const void *buf, int *str
/* Check for holes in the data unless romio_onesided_no_rmw is set.
* If a hole is found redo the entire aggregation and write.
*/
if (!romio_onesided_no_rmw) {
if (!fd->romio_onesided_no_rmw) {
int anyHolesFound = 0;
MPI_Allreduce(&holeFound, &anyHolesFound, 1, MPI_INT, MPI_MAX, fd->comm);

Expand All @@ -1386,16 +1386,16 @@ static void ADIOI_LUSTRE_IterateOneSided(ADIO_File fd, const void *buf, int *str

currentSegementOffset =
(ADIO_Offset) startingStripeWithData *(ADIO_Offset) (striping_info[0]);
romio_onesided_always_rmw = 1;
romio_onesided_no_rmw = 1;
fd->romio_onesided_always_rmw = 1;
fd->romio_onesided_no_rmw = 1;

/* Holes are found in the data and the user has not set
* romio_onesided_no_rmw --- set romio_onesided_always_rmw to 1
* and redo the entire aggregation and write and if the user has
* romio_onesided_inform_rmw set then inform him of this condition
* and behavior.
*/
if (romio_onesided_inform_rmw && (myrank == 0)) {
if (fd->romio_onesided_inform_rmw && (myrank == 0)) {
FPRINTF(stderr, "Information: Holes found during one-sided "
"write aggregation algorithm --- re-running one-sided "
"write aggregation with ROMIO_ONESIDED_ALWAYS_RMW set to 1.\n");
Expand All @@ -1405,7 +1405,7 @@ static void ADIOI_LUSTRE_IterateOneSided(ADIO_File fd, const void *buf, int *str
} else
doAggregation = 0;
} // while doAggregation
romio_onesided_no_rmw = prev_romio_onesided_no_rmw;
fd->romio_onesided_no_rmw = prev_romio_onesided_no_rmw;

ADIOI_Free(segment_stripe_start);
ADIOI_Free(segment_stripe_end);
Expand Down
2 changes: 1 addition & 1 deletion src/mpi/romio/adio/common/ad_hints.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ void ADIOI_GEN_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
*error_code = MPI_SUCCESS;
return;
}
ad_get_env_vars();
ad_get_env_vars(fd);

/* Interpreting MPI-4.0 to mean ROMIO should only return hints it knows
* about when user calls MPI_File_get_info */
Expand Down
17 changes: 8 additions & 9 deletions src/mpi/romio/adio/common/ad_opencoll.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,6 @@
* on MPI-IO scalability").
*/

enum {
BLOCKSIZE = 0,
STRIPE_SIZE,
STRIPE_FACTOR,
START_IODEVICE,
STAT_ITEMS
} file_stats;


/* generate an MPI datatype describing the members of the ADIO_File struct that
* we want to ensure all processes have. In deferred open, aggregators will
* open the file and possibly read layout and other information.
Expand All @@ -36,6 +27,14 @@ enum {

static MPI_Datatype make_stats_type(ADIO_File fd)
{
enum file_stats {
BLOCKSIZE = 0,
STRIPE_SIZE,
STRIPE_FACTOR,
START_IODEVICE,
STAT_ITEMS
};

int lens[STAT_ITEMS];
MPI_Aint offsets[STAT_ITEMS];
MPI_Datatype types[STAT_ITEMS];
Expand Down
37 changes: 15 additions & 22 deletions src/mpi/romio/adio/common/ad_tuning.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,6 @@

#include "ad_tuning.h"

int romio_write_aggmethod;
int romio_read_aggmethod;
int romio_onesided_no_rmw;
int romio_onesided_always_rmw;
int romio_onesided_inform_rmw;
int romio_tunegather;

/* set internal variables for tuning environment variables */
/** \page mpiio_vars MPIIO Configuration
\section env_sec Environment Variables
Expand Down Expand Up @@ -76,39 +69,39 @@ int romio_tunegather;
*
*/

void ad_get_env_vars(void)
void ad_get_env_vars(ADIO_File fd)
{
char *x;

romio_write_aggmethod = 0;
fd->romio_write_aggmethod = 0;
x = getenv("ROMIO_WRITE_AGGMETHOD");
if (x)
romio_write_aggmethod = atoi(x);
fd->romio_write_aggmethod = atoi(x);

romio_read_aggmethod = 0;
fd->romio_read_aggmethod = 0;
x = getenv("ROMIO_READ_AGGMETHOD");
if (x)
romio_read_aggmethod = atoi(x);
fd->romio_read_aggmethod = atoi(x);

romio_onesided_no_rmw = 0;
fd->romio_onesided_no_rmw = 0;
x = getenv("ROMIO_ONESIDED_NO_RMW");
if (x)
romio_onesided_no_rmw = atoi(x);
fd->romio_onesided_no_rmw = atoi(x);

romio_onesided_always_rmw = 0;
fd->romio_onesided_always_rmw = 0;
x = getenv("ROMIO_ONESIDED_ALWAYS_RMW");
if (x)
romio_onesided_always_rmw = atoi(x);
if (romio_onesided_always_rmw)
romio_onesided_no_rmw = 1;
fd->romio_onesided_always_rmw = atoi(x);
if (fd->romio_onesided_always_rmw)
fd->romio_onesided_no_rmw = 1;

romio_onesided_inform_rmw = 0;
fd->romio_onesided_inform_rmw = 0;
x = getenv("ROMIO_ONESIDED_INFORM_RMW");
if (x)
romio_onesided_inform_rmw = atoi(x);
fd->romio_onesided_inform_rmw = atoi(x);

romio_tunegather = 1;
fd->romio_tunegather = 1;
x = getenv("ROMIO_TUNEGATHER");
if (x)
romio_tunegather = atoi(x);
fd->romio_tunegather = atoi(x);
}

0 comments on commit 5d4b8cb

Please sign in to comment.