Reduce FST block size for BlockTreeTermsWriter (#12604)

apache · Oct 4, 2023 · 0e6c29e · 0e6c29e
1 parent 43739e2
commit 0e6c29e
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 1 deletion.
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
@@ -26,6 +26,11 @@ Bug Fixes
 * GITHUB#12352: [Tessellator] Improve the checks that validate the diagonal between two polygon nodes so
   the resulting polygons are valid counter clockwise polygons. (Ignacio Vera)
 
+Optimizations
+---------------------
+* GITHUB#12604: Estimate the block size of FST BytesStore in BlockTreeTermsWriter
+  to reduce GC load during indexing. (Guo Feng)
+
 ======================= Lucene 8.11.2 =======================
 
 Bug Fixes

diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java
@@ -55,6 +55,7 @@
 import org.apache.lucene.util.fst.BytesRefFSTEnum;
 import org.apache.lucene.util.fst.FST;
 import org.apache.lucene.util.fst.Util;
+import org.apache.lucene.util.packed.PackedInts;
 
 /*
   TODO:
@@ -427,10 +428,22 @@ public void compileIndex(List<PendingBlock> blocks, RAMOutputStream scratchBytes
         }
       }
 
+      long estimateSize = prefix.length;
+      for (PendingBlock block : blocks) {
+        if (block.subIndices != null) {
+          for (FST<BytesRef> subIndex : block.subIndices) {
+            estimateSize += subIndex.numBytes();
+          }
+        }
+      }
+      int estimateBitsRequired = PackedInts.bitsRequired(estimateSize);
+      int pageBits = Math.min(15, Math.max(6, estimateBitsRequired));
+
+
       final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
       final Builder<BytesRef> indexBuilder = new Builder<>(FST.INPUT_TYPE.BYTE1,
                                                            0, 0, true, false, Integer.MAX_VALUE,
-                                                           outputs, true, 15);
+                                                           outputs, true, pageBits);
       //if (DEBUG) {
       //  System.out.println("  compile index for prefix=" + prefix);
       //}

diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
@@ -489,6 +489,10 @@ void finish(long newStartNode) throws IOException {
     startNode = newStartNode;
     bytes.finish();
   }
+
+  public long numBytes() {
+    return bytes.getPosition();
+  }
 
   public T getEmptyOutput() {
     return emptyOutput;