Skip to content

Commit

Permalink
CBG-2706: [2.8.4] Perform additional compaction when safeSeq doesn't …
Browse files Browse the repository at this point in the history 
…change (#6084)

Co-authored-by: Adam Fraser <adam.fraser@couchbase.com>
  • Loading branch information
@gregns1 @adamcfraser
gregns1 and adamcfraser committed Feb 7, 2023
1 parent 71f014e commit 5e45e54
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 14 deletions.
27 changes: 13 additions & 14 deletions db/active_replicator_checkpointer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,22 +266,21 @@ func (c *Checkpointer) _updateCheckpointLists() (safeSeq *SequenceID) {
c.stats.ProcessedSequenceLen = len(c.processedSeqs)

maxI := c._calculateSafeExpectedSeqsIdx()
if maxI == -1 {
// nothing to do
return nil
}

seq := c.expectedSeqs[maxI]
if maxI > -1 {
seq := c.expectedSeqs[maxI]

// removes to-be checkpointed sequences from processedSeqs list
for i := 0; i <= maxI; i++ {
removeSeq := c.expectedSeqs[i]
delete(c.processedSeqs, removeSeq)
base.TracefCtx(c.ctx, base.KeyReplicate, "checkpointer: _updateCheckpointLists removed seq %v from processedSeqs map", removeSeq)
}
// removes to-be checkpointed sequences from processedSeqs list
for i := 0; i <= maxI; i++ {
removeSeq := c.expectedSeqs[i]
delete(c.processedSeqs, removeSeq)
base.TracefCtx(c.ctx, base.KeyReplicate, "checkpointer: _updateCheckpointLists removed seq %v from processedSeqs map", removeSeq)
}

// trim expectedSeqs list from beginning up to first unprocessed seq
c.expectedSeqs = c.expectedSeqs[maxI+1:]
// trim expectedSeqs list from beginning up to first unprocessed seq
c.expectedSeqs = c.expectedSeqs[maxI+1:]
safeSeq = &seq
}

// if we have many remaining expectedSeqs, see if we can shrink the lists even more
// compact contiguous blocks of sequences by keeping only the last processed sequence in both lists
Expand All @@ -303,7 +302,7 @@ func (c *Checkpointer) _updateCheckpointLists() (safeSeq *SequenceID) {
c.stats.ExpectedSequenceLenPostCleanup = len(c.expectedSeqs)
c.stats.ProcessedSequenceLenPostCleanup = len(c.processedSeqs)

return &seq
return safeSeq
}

// _calculateSafeExpectedSeqsIdx returns an index into expectedSeqs which is safe to checkpoint.
Expand Down
44 changes: 44 additions & 0 deletions db/active_replicator_checkpointer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,50 @@ func TestCheckpointerSafeSeq(t *testing.T) {
expectedExpectedSeqs: genExpectedForTest(t, "3", "5", "6", "9", "10", "13", "14", "17", "18", "19", "20"),
expectedProcessedSeqs: genProcessedForTest(t, "5", "9", "13", "17", "19"),
},
{
name: "compaction when safeSeq unchanged",
c: &Checkpointer{
expectedSeqs: genExpectedForTest(t, "47358299", "47358300", "47358301", "47358302", "47358303", "47358304", "47358305", "47358306", "47358307", "47358308",
"47358309", "47358310", "47358311", "47358312", "47358313", "47358314", "47358315", "47358316", "47358317", "47358318", "47358319", "47358320", "47358321", "47358322",
"47358323", "47358324", "47358325", "47358326", "47358327", "47358328", "47358329", "47358330", "47358331", "47358332", "47358333", "47358334", "47358335", "47358336",
"47358337", "47358338", "47358339", "47358340", "47358341", "47358342", "47358343", "47358344", "47358345", "47358346", "47358347", "47358348", "47358349", "47358350",
"47358351", "47358352", "47358353", "47358354", "47358355", "47358356", "47358357", "47358358", "47358359", "47358360", "47358361", "47358362", "47358363", "47358364",
"47358365", "47358366", "47358367", "47358368", "47358369", "47358370", "47358371", "47358372", "47358373", "47358374", "47358375", "47358376", "47358377", "47358378",
"47358379", "47358380", "47358381", "47358382", "47358383", "47358384", "47358385", "47358386", "47358387", "47358388", "47358389", "47358390", "47358391", "47358392",
"47358393", "47358394", "47358395", "47358396", "47358397", "47358398", "47358399", "47358400", "47358401", "47358402", "47358403", "47358404", "47358405", "47358406",
"47358407", "47358408", "47358409", "47358410", "47358411", "47358412", "47358413", "47358414", "47358415", "47358416", "47358417", "47358418", "47358419", "47358420",
"47358421", "47358422", "47358423", "47358424", "47358425", "47358426", "47358427", "47358428", "47358429", "47358430", "47358431", "47358432", "47358433", "47358434",
"47358435", "47358436", "47358437", "47358438", "47358439", "47358440", "47358441", "47358442", "47358443", "47358444", "47358445", "47358446", "47358447", "47358448",
"47358449", "47358450", "47358451"),
processedSeqs: genProcessedForTest(t, "47358364", "47358365", "47358366", "47358367", "47358368", "47358369", "47358370", "47358371", "47358372", "47358373",
"47358374", "47358375", "47358376", "47358377", "47358378", "47358379", "47358380", "47358381", "47358382", "47358383", "47358384", "47358385", "47358386", "47358387",
"47358388", "47358389", "47358390", "47358391", "47358392", "47358393", "47358394", "47358395", "47358396", "47358397", "47358398", "47358399", "47358400", "47358401",
"47358402", "47358403", "47358404", "47358405", "47358406", "47358407", "47358409", "47358410", "47358411", "47358412", "47358413", "47358414", "47358415", "47358416",
"47358417", "47358418", "47358419", "47358420", "47358422", "47358423", "47358424", "47358425", "47358426", "47358427", "47358428", "47358429", "47358430", "47358431",
"47358432", "47358433", "47358434", "47358435", "47358436", "47358437", "47358438", "47358439", "47358440", "47358442", "47358443", "47358444", "47358448", "47358449",
"47358450", "47358451", "47358452", "47358453", "47358454", "47358455", "47358456", "47358457", "47358458", "47358459", "47358460", "47358461", "47358462", "47358463",
"47358464", "47358465", "47358466", "47358467", "47358468", "47358469", "47358470", "47358471", "47358472", "47358473", "47358474", "47358475", "47358477", "47358478",
"47358479", "47358481", "47358482", "47358484", "47358486", "47358487", "47358489", "47358490", "47358491", "47358492", "47358493", "47358495", "47358496", "47358497",
"47358498", "47358499", "47358500", "47358501", "47358502", "47358503", "47358504", "47358505", "47358507", "47358508", "47358509", "47358510", "47358511", "47358512",
"47358513", "47358514", "47358515", "47358516", "47358517", "47358518", "47358519", "47358520", "47358521", "47358522", "47358523", "47358524", "47358525", "47358526",
"47358527", "47358528", "47358529", "47358530", "47358531", "47358532"),
expectedSeqCompactionThreshold: 5, // this many expected seqs to trigger compaction
},
expectedSafeSeq: nil,
expectedExpectedSeqsIdx: -1,
expectedExpectedSeqs: genExpectedForTest(t, "47358299", "47358300", "47358301", "47358302", "47358303", "47358304", "47358305", "47358306", "47358307", "47358308",
"47358309", "47358310", "47358311", "47358312", "47358313", "47358314", "47358315", "47358316", "47358317", "47358318", "47358319", "47358320", "47358321", "47358322",
"47358323", "47358324", "47358325", "47358326", "47358327", "47358328", "47358329", "47358330", "47358331", "47358332", "47358333", "47358334", "47358335", "47358336",
"47358337", "47358338", "47358339", "47358340", "47358341", "47358342", "47358343", "47358344", "47358345", "47358346", "47358347", "47358348", "47358349", "47358350",
"47358351", "47358352", "47358353", "47358354", "47358355", "47358356", "47358357", "47358358", "47358359", "47358360", "47358361", "47358362", "47358363", "47358407",
"47358408", "47358420", "47358421", "47358440", "47358441", "47358444", "47358445", "47358446", "47358447", "47358451"),
expectedProcessedSeqs: genProcessedForTest(t, "47358407", "47358420", "47358440", "47358444", "47358451", "47358452", "47358453", "47358454", "47358455", "47358456",
"47358457", "47358458", "47358459", "47358460", "47358461", "47358462", "47358463", "47358464", "47358465", "47358466", "47358467", "47358468", "47358469", "47358470",
"47358471", "47358472", "47358473", "47358474", "47358475", "47358477", "47358478", "47358479", "47358481", "47358482", "47358484", "47358486", "47358487", "47358489",
"47358490", "47358491", "47358492", "47358493", "47358495", "47358496", "47358497", "47358498", "47358499", "47358500", "47358501", "47358502", "47358503", "47358504",
"47358505", "47358507", "47358508", "47358509", "47358510", "47358511", "47358512", "47358513", "47358514", "47358515", "47358516", "47358517", "47358518", "47358519",
"47358520", "47358521", "47358522", "47358523", "47358524", "47358525", "47358526", "47358527", "47358528", "47358529", "47358530", "47358531", "47358532"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down

0 comments on commit 5e45e54

Please sign in to comment.