collectives all gather hangs if rank 0 is not involved

[John98Zakaria]

Expected Behavior

When creating a communicator that doesn't communicate with all ranks.
I expect the communicator to be able to send data to anyone with the number of,hpx::collectives::num_sites_arg(2)

Rank0 received its data 
Rank1 received its data 
Rank2 received its data 
Rank3 received its data 

Actual Behavior

Only 2 ranks are able to communicate, in which one of them is rank 0.

Steps to Reproduce the Problem

... Please be as specific as possible while describing how to reproduce your problem.

#include <hpx/iostream.hpp>
#include <hpx/barrier.hpp>
#include <hpx/wrap_main.hpp>
#include <hpx/modules/collectives.hpp>
#include <hpx/collectives/spmd_block.hpp>

void smpd_func(hpx::lcos::spmd_block block,hpx::collectives::communicator comm1){
    const auto my_rank = hpx::get_locality_id();
    bool row1 = my_rank == 0 | my_rank == 1;
    hpx::collectives::communicator communicator = comm1;
    if (my_rank==2){
        communicator = hpx::collectives::create_communicator("comm2",hpx::collectives::num_sites_arg(2),hpx::collectives::this_site_arg(2),hpx::collectives::generation_arg(0),hpx::collectives::root_site_arg(2));
        communicator.register_as("comm2");
        hpx::cout<<"Comm on 2 was registered \n" << hpx::flush;
    }
    if (my_rank==3){
        communicator = hpx::collectives::communicator();
        communicator.connect_to("comm2");
        hpx::cout << "Rank3 connected \n" << hpx::flush;
    }

    auto res = hpx::collectives::all_gather(communicator,my_rank);
    auto value = res.get();
    hpx::cout << "Rank " << my_rank << "received its data \n" <<hpx::flush;

    block.sync_all();
}

HPX_PLAIN_ACTION(smpd_func);

int main(int argc, char* argv[]){
    auto localities = hpx::find_all_localities();
    smpd_func_action act;
    auto comm = hpx::collectives::create_communicator("comm",hpx::collectives::num_sites_arg(2));

    comm.register_as("comm1");

    hpx::lcos::define_spmd_block(
            "some_name", 1,act,comm).get();

    hpx::cout << "DONE";
}

Specifications

... Please describe your environment

• HPX Version: 1.8.0 and 1.7.1
• Platform (compiler, OS): gcc-11, linux mint
• Malloc: jemalloc

[hkaiser]

Would it work if you rewrote this example as:

#include <hpx/iostream.hpp>
#include <hpx/barrier.hpp>
#include <hpx/modules/collectives.hpp>

void func()
{
    const auto my_rank = hpx::get_locality_id();

    hpx::collectives::communicator communicator;
    if (my_rank == 2){
        // note: the communicator is 'created' if root_site == this_site
        communicator = hpx::collectives::create_communicator(
            "comm2",
            hpx::collectives::num_sites_arg(2),
            hpx::collectives::this_site_arg(0),
            hpx::collectives::generation_arg(0),
            hpx::collectives::root_site_arg(0));
        hpx::cout<<"Comm on 2 was registered \n" << hpx::flush;
    }
    if (my_rank == 3){
        // note: the communicator is 'connected' if root_site != this_site
        communicator = hpx::collectives::create_communicator(
            "comm2",
            hpx::collectives::num_sites_arg(2),
            hpx::collectives::this_site_arg(1),
            hpx::collectives::generation_arg(0),
            hpx::collectives::root_site_arg(0));
        hpx::cout << "Rank3 connected \n" << hpx::flush;
    }

    if (my_rank == 2 || my_rank == 3) {
        int value = my_rank;
        auto res = hpx::collectives::all_gather(communicator, my_rank - 2, value);
        auto gathered_value = res.get();    // should receive a vector<int>{2, 3};
        hpx::cout << "Rank " << my_rank << "received its data \n" <<hpx::flush;
    }
}

Also, please never directly #include <hpx/wrap_main.hpp>. Please see https://hpx-docs.stellar-group.org/latest/html/manual/starting_the_hpx_runtime.html for more details on how you may start the runtime.

[John98Zakaria]

Worked perfectly
I just had to swap the argument order you provided for all gather.

For future readers :
That is how it looks at the end.

#include <hpx/iostream.hpp>
#include <hpx/hpx_main.hpp>
#include <hpx/modules/collectives.hpp>
#include <hpx/future.hpp>


void func()
{
    const auto my_rank = hpx::get_locality_id();

    hpx::collectives::communicator communicator;
    if (my_rank == 2){
        // note: the communicator is 'created' if root_site == this_site
        communicator = hpx::collectives::create_communicator(
                "comm2",
                hpx::collectives::num_sites_arg(2),
                hpx::collectives::this_site_arg(0),
                hpx::collectives::generation_arg(0),
                hpx::collectives::root_site_arg(0));
        hpx::cout<<"Comm on 2 was registered \n" << hpx::flush;
    }
    if (my_rank == 3){
        // note: the communicator is 'connected' if root_site != this_site
        communicator = hpx::collectives::create_communicator(
                "comm2",
                hpx::collectives::num_sites_arg(2),
                hpx::collectives::this_site_arg(1),
                hpx::collectives::generation_arg(0),
                hpx::collectives::root_site_arg(0));
        hpx::cout << "Rank3 connected \n" << hpx::flush;
    }

    if (my_rank == 2 || my_rank == 3) {
        int value = my_rank;
        auto res = hpx::collectives::all_gather(communicator,value,hpx::collectives::this_site_arg(my_rank - 2));
        auto gathered_value = res.get();    // should receive a vector<int>{2, 3};
        hpx::cout << "Rank " << my_rank << "received its data \n" <<hpx::flush;
    }
}

HPX_PLAIN_ACTION(func,func_action);

int main(int argc, char* argv[]){
    auto localities = hpx::find_all_localities();
    func_action act;
    std::vector<hpx::future<void>> data;
    for (const auto &locality: localities) {
        data.emplace_back(hpx::async(act,locality));
    }

    hpx::wait_all(data);


    hpx::cout << "DONE";
}