NRG: Ignore vote requests if leader heard more recently than election…

… timeout

This is actually a safeguard that the Raft paper prescribes as a way of avoiding
network partitioned nodes from coming back up with a high term number and causing
the existing leader to step down unnecessarily.

Signed-off-by: Neil Twigg <neil@nats.io>

server/raft.go

@@ -3875,6 +3875,20 @@ func (n *raft) processVoteRequest(vr *voteRequest) error {
 	}
 
 	n.Lock()
+
+	if n.leader != "" {
+		// A node that comes up after state reset or being in its own network partition
+		// for a long time might come back with a very high term number but potentially
+		// be behind in the log. The Raft paper addresses this in section 6 by suggesting
+		// that we should ignore vote requests if we think there's a valid leader still
+		// around so that it doesn't get forced to step down in that case.
+		if ps, ok := n.peers[n.leader]; ok && time.Since(time.Unix(0, ps.ts)) < minElectionTimeout {
+			// If we've heard from our leader recently then we should ignore a vote request.
+			n.Unlock()
+			return nil
+		}
+	}
+
 	n.resetElectionTimeout()
 
 	vresp := &voteResponse{n.term, n.id, false}

server/raft_test.go

@@ -256,7 +256,7 @@ func TestNRGSimpleElection(t *testing.T) {
 	require_NoError(t, rg.leader().node().StepDown())
 
 	// Wait for a vote request to come in.
-	msg := require_ChanRead(t, voteReqs, time.Second)
+	msg := require_ChanRead(t, voteReqs, time.Second*3)
 	vr := decodeVoteRequest(msg.Data, msg.Reply)
 	require_True(t, vr != nil)
 	require_NotEqual(t, vr.candidate, "")