Unit tests that replicate the issue Appendium#60 and a fix proposal f…

…or one of the found defects.

flatpack/src/main/java/net/sf/flatpack/util/ParserUtils.java

@@ -182,6 +182,12 @@ public static List<String> splitLine(final String line, final char delimiter, fi
                     } else {
                         newBlock[sizeSelected++] = currentChar;
                     }
+                } else if (insideQualifier && currentChar == qualifier && i + 1 < size && trimmedLine.charAt(i + 1) == qualifier) {
+                    newBlock[sizeSelected++] = qualifier;
+                    newBlock[sizeSelected++] = qualifier;
+                    i += 1;
+                    previousChar = qualifier;
+                    continue;
                 } else {
                     if (i + 1 < size && delimiter != ' ') {
                         // this is used to allow unescaped qualifiers to be contained within the element

flatpack/src/test/java/net/sf/flatpack/delim/csv/CsvParserTest.java

@@ -2,10 +2,13 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
+import java.io.ByteArrayInputStream;
 import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
 
 import junit.framework.TestCase;
 import net.sf.flatpack.DataSet;
+import net.sf.flatpack.DelimiterParser;
 import net.sf.flatpack.Parser;
 import net.sf.flatpack.brparse.BuffReaderParseFactory;
 import net.sf.flatpack.util.FPConstants;
@@ -61,4 +64,40 @@ public void testCsvWithWrongPzMap() {
 
     }
 
+    /**
+     * Fails with the error "Odd number of qualifiers"
+     */
+    public void testCsvDocumentWithMultilineString() {
+        final String testCsv =
+            "Bob,Smith,bsmiht@test.com,\"This is a long fragment of text" + System.lineSeparator() +
+            "that should be processed as a single field\", 1988, 111-222-33,\"another field with new line character" + System.lineSeparator() +
+            "that should be considered as a field of the same data row\"";
+
+        final String[] expectedResult = {
+            "Bob",
+            "Smith",
+            "bsmiht@test.com",
+            "This is a long fragment of text" + System.lineSeparator() + "that should be processed as a single field",
+            " 1988",
+            " 111-222-33",
+            "another field with new line character" + System.lineSeparator() + "that should be considered as a field of the same data row"
+        };
+
+        final ByteArrayInputStream bis = new ByteArrayInputStream(testCsv.getBytes(StandardCharsets.UTF_8));
+        final DelimiterParser parser = new DelimiterParser(bis, ',', '"', false);
+        final DataSet result =  parser.parse();
+
+        assertThat(result.getErrorCount()).isEqualTo(0);
+        assertThat(result.getColumns().length).isEqualTo(expectedResult.length);
+        assertThat(result.getRowCount()).isEqualTo(1);
+
+        result.next();
+        String[] row = result.getColumns();
+
+        for (int i = 0; i < expectedResult.length; ++i) {
+            assertThat(expectedResult[i]).isEqualTo(row[i]);
+        }
+
+    }
+
 }

flatpack/src/test/java/net/sf/flatpack/parserutils/ParserUtilsSplitLineTest.java

@@ -236,6 +236,18 @@ public void testMultilineExtreme() {
         assertThat(results4.get(2)).isEqualTo("\r\n");
     }
 
+    public void testLineWithQualifiersAsText() {
+        final String testLine = "Bob, Smith,\"\"\"Test\"\" , 2, Some string, still string, also part of the string.\",11111111";
+        final List<String> result = ParserUtils.splitLine(testLine, ',', '"', 10, true, true);
+
+        assertThat(result.size()).isEqualTo(4);
+        assertThat(result.get(0)).isEqualTo("Bob");
+        assertThat(result.get(1)).isEqualTo(" Smith");
+        assertThat(result.get(2)).isEqualTo("\"Test\" , 2, Some string, still string, also part of the string.");
+        assertThat(result.get(3)).isEqualTo("11111111");
+
+    }
+
     public static void main(final String[] args) {
         junit.textui.TestRunner.run(ParserUtilsSplitLineTest.class);
     }