Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Created test cases for the scenarios mentioned in the GitHub issue and the fixes. Elected leader leaving before the voting thread has finished. This causes a liveness issue on both ELECTION and ELECTION2. We can fix this by stopping the voting thread before starting it again. Scenarios the coordinator leaves before finishing the election process and a majority still exists. ELECTION algorithm has a liveness issue in this case, since it does not take into account changes in the view coordinator. We handle this case by calculating if the coordinator has changed between views, there is a majority, it is currently the coordinator, and there is no leader elected. Close #259.
- Loading branch information
Showing
6 changed files
with
362 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
119 changes: 119 additions & 0 deletions
119
tests/junit-functional/org/jgroups/tests/election/NetworkPartitionElectionTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| package org.jgroups.tests.election; | ||
|
|
||
| import org.jgroups.Address; | ||
| import org.jgroups.Global; | ||
| import org.jgroups.Header; | ||
| import org.jgroups.View; | ||
| import org.jgroups.protocols.raft.RAFT; | ||
| import org.jgroups.protocols.raft.election.LeaderElected; | ||
| import org.jgroups.raft.testfwk.BlockingMessageInterceptor; | ||
| import org.jgroups.raft.testfwk.PartitionedRaftCluster; | ||
| import org.jgroups.raft.testfwk.RaftTestUtils; | ||
| import org.jgroups.tests.harness.BaseRaftElectionTest; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
| import java.util.concurrent.TimeUnit; | ||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
|
|
||
| import org.testng.annotations.Test; | ||
|
|
||
| import static org.assertj.core.api.Assertions.assertThat; | ||
| import static org.jgroups.tests.harness.BaseRaftElectionTest.ALL_ELECTION_CLASSES_PROVIDER; | ||
|
|
||
| @Test(groups = Global.FUNCTIONAL, singleThreaded = true, dataProvider = ALL_ELECTION_CLASSES_PROVIDER) | ||
| public class NetworkPartitionElectionTest extends BaseRaftElectionTest.ClusterBased<PartitionedRaftCluster> { | ||
|
|
||
| { | ||
| clusterSize = 5; | ||
|
|
||
| // Since it uses a data provider, it needs to execute per method to inject the values. | ||
| recreatePerMethod = true; | ||
| } | ||
|
|
||
| public void testNetworkPartitionDuringElection(Class<?> ignore) throws Exception { | ||
| withClusterSize(5); | ||
| createCluster(); | ||
| long id = 0; | ||
|
|
||
| View view = createView(id++, 0, 1, 2, 3, 4); | ||
|
|
||
| // We intercept the first `LeaderElected` message. | ||
| AtomicBoolean onlyOnce = new AtomicBoolean(true); | ||
| BlockingMessageInterceptor interceptor = cluster.addCommandInterceptor(m -> { | ||
| for (Map.Entry<Short, Header> h : m.getHeaders().entrySet()) { | ||
| if (h.getValue() instanceof LeaderElected && onlyOnce.getAndSet(false)) { | ||
| // Assert that node A was elected | ||
| LeaderElected le = (LeaderElected) h.getValue(); | ||
| assertThat(le.leader()).isEqualTo(address(0)); | ||
| return true; | ||
| } | ||
| } | ||
| return false; | ||
| }); | ||
|
|
||
| cluster.handleView(view); | ||
|
|
||
| System.out.println("-- wait command intercept"); | ||
| assertThat(RaftTestUtils.eventually(() -> interceptor.numberOfBlockedMessages() > 0, 10, TimeUnit.SECONDS)).isTrue(); | ||
|
|
||
| // While the message is in-flight, the cluster splits. | ||
| // The previous coordinator does not have the majority to proceed. | ||
| cluster.handleView(createView(id++, 0, 1)); | ||
| cluster.handleView(createView(id++, 2, 3, 4)); | ||
|
|
||
| // We can release the elected message. | ||
| interceptor.releaseNext(); | ||
| interceptor.assertNoBlockedMessages(); | ||
|
|
||
| // Check in all instances that a new leader is elected. | ||
| System.out.println("-- waiting for leader in majority partition"); | ||
| BaseRaftElectionTest.waitUntilLeaderElected(rafts(), 10_000); | ||
|
|
||
| // Assert that A and B does not have a leader. | ||
| assertThat(raft(0).leader()).isNull(); | ||
| assertThat(raft(1).leader()).isNull(); | ||
|
|
||
| System.out.printf("-- elected during the split\n%s%n", dumpLeaderAndTerms()); | ||
| // Store who's the leader before merging. | ||
| assertThat(leaders()).hasSize(1); | ||
| RAFT leader = raft(leaders().get(0)); | ||
| long leaderTermBefore = leader.currentTerm(); | ||
|
|
||
| System.out.printf("-- merge partition, leader=%s%n", leader); | ||
| // Join the partitions. | ||
| // Note that the coordinator is different. | ||
| cluster.handleView(createView(id++, 0, 1, 2, 3, 4)); | ||
|
|
||
| // Wait until A and B receive the leader information. | ||
| BaseRaftElectionTest.waitUntilAllHaveLeaderElected(rafts(), 10_000); | ||
| System.out.printf("-- state after merge\n%s%n", dumpLeaderAndTerms()); | ||
|
|
||
| // We assert the merge did not disrupt the cluster. | ||
| // Same leader and term. | ||
| assertThat(Arrays.stream(rafts()) | ||
| .allMatch(r -> Objects.equals(leader.getAddress(), r.leader()) && r.currentTerm() == leaderTermBefore)) | ||
| .withFailMessage(this::dumpLeaderAndTerms) | ||
| .isTrue(); | ||
| } | ||
|
|
||
| private RAFT raft(Address address) { | ||
| for (RAFT raft : rafts()) { | ||
| if (Objects.equals(address, raft.getAddress())) | ||
| return raft; | ||
| } | ||
|
|
||
| throw new IllegalArgumentException(String.format("Node with address '%s' not present", address)); | ||
| } | ||
|
|
||
| @Override | ||
| protected PartitionedRaftCluster createNewMockCluster() { | ||
| return new PartitionedRaftCluster(); | ||
| } | ||
|
|
||
| @Override | ||
| protected void amendRAFTConfiguration(RAFT raft) { | ||
| raft.synchronous(true); | ||
| } | ||
| } |
Oops, something went wrong.