fix: text splitter with tiny documents (fixes #193) (#196)

Azure-Samples · Mar 22, 2024 · 2e808c7 · 2e808c7
1 parent 6efe240
commit 2e808c7
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/packages/indexer/src/lib/document-processor.ts b/packages/indexer/src/lib/document-processor.ts
@@ -75,6 +75,10 @@ export class DocumentProcessor {
     let start = 0;
     let end = length;
 
+    if (end <= MAX_SECTION_LENGTH) {
+      return [{ content: allText, page: findPage(0) }];
+    }
+
     while (start + SECTION_OVERLAP < length) {
       let lastWord = -1;
       end = start + MAX_SECTION_LENGTH;