Merge pull request #79 from ckrahlisch/master

Adds strict delimitation option
TinyCsvParser · Jun 17, 2021 · 199b5b2 · 199b5b2
2 parents 508872d + ff5c098
commit 199b5b2
Show file tree

Hide file tree

Showing 6 changed files with 68 additions and 18 deletions.
diff --git a/TinyCsvParser/NuGet/nupkg/TinyCsvParser.2.6.1.nupkg b/TinyCsvParser/NuGet/nupkg/TinyCsvParser.2.6.1.nupkg
diff --git a/TinyCsvParser/NuGet/nupkg/TinyCsvParser.2.6.1.symbols.nupkg b/TinyCsvParser/NuGet/nupkg/TinyCsvParser.2.6.1.symbols.nupkg
diff --git a/TinyCsvParser/TinyCsvParser.Test/Tokenizer/Rfc4180TokenizerTest.cs b/TinyCsvParser/TinyCsvParser.Test/Tokenizer/Rfc4180TokenizerTest.cs
@@ -96,6 +96,46 @@ public void Rfc4180_QuotedString_Double_Quoted_Data_Test()
             Assert.AreEqual("Also goes by \"Mike\", among friends that is", tokens[2]);
         }
 
+        [Test]
+        public void RFC4180_CsvParser_StrictDelimitation_Test()
+        {
+            // Use a " as Quote Character, a \\ as Escape Character and a , as Delimiter.
+            var options = new Options('"', '\\', ';', true);
+
+            // Initialize the Rfc4180 Tokenizer:
+            var tokenizer = new RFC4180Tokenizer(options);
+
+            // Now Build the Parser:
+            CsvParserOptions csvParserOptions = new CsvParserOptions(true, tokenizer);
+            SampleEntityMapping csvMapper = new SampleEntityMapping();
+            CsvParser<SampleEntity> csvParser = new CsvParser<SampleEntity>(csvParserOptions, csvMapper);
+
+
+            var stringBuilder = new StringBuilder()
+                .AppendLine("Name; Age; Description")
+                .AppendLine("\"Michael, Chester\";24;\"Also goes by \"Mike\", among friends that is\"")
+                .AppendLine("\"Robert, Willliamson\"; ;\"All-around nice guy who always says hi\"");
+
+            // Define the NewLine Character to split at:
+            CsvReaderOptions csvReaderOptions = new CsvReaderOptions(new[] { Environment.NewLine });
+
+            var result = csvParser
+                .ReadFromString(csvReaderOptions, stringBuilder.ToString())
+                .ToList();
+
+            Assert.AreEqual(2, result.Count);
+
+            Assert.AreEqual(true, result.All(x => x.IsValid));
+
+            Assert.AreEqual("Michael, Chester", result[0].Result.Name);
+            Assert.AreEqual(24, result[0].Result.Age);
+            Assert.AreEqual("Also goes by \"Mike\", among friends that is", result[0].Result.Description);
+
+            Assert.AreEqual("Robert, Willliamson", result[1].Result.Name);
+            Assert.AreEqual(false, result[1].Result.Age.HasValue);
+            Assert.AreEqual("All-around nice guy who always says hi", result[1].Result.Description);
+        }
+
         [Test]
         public void Rfc4180_Issue3_Empty_Column_Test()
         {

diff --git a/TinyCsvParser/TinyCsvParser/TinyCsvParser.csproj b/TinyCsvParser/TinyCsvParser/TinyCsvParser.csproj
@@ -12,10 +12,10 @@
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <Copyright>Copyright © 2019 Philipp Wagner</Copyright>
     <GeneratePackageOnBuild>True</GeneratePackageOnBuild>
-    <AssemblyVersion>2.6.0.0</AssemblyVersion>
-    <FileVersion>2.6.0.0</FileVersion>
+    <AssemblyVersion>2.6.1.0</AssemblyVersion>
+    <FileVersion>2.6.1.0</FileVersion>
     <RepositoryType>git</RepositoryType>
-    <Version>2.6.0</Version>
+    <Version>2.6.1</Version>
     <IncludeSource>True</IncludeSource>
     <IncludeSymbols>True</IncludeSymbols>
     <SignAssembly>true</SignAssembly>

diff --git a/TinyCsvParser/TinyCsvParser/Tokenizer/RFC4180/Options.cs b/TinyCsvParser/TinyCsvParser/Tokenizer/RFC4180/Options.cs
@@ -7,17 +7,19 @@ public class Options
         public readonly char QuoteCharacter;
         public readonly char EscapeCharacter;
         public readonly char DelimiterCharacter;
+        public readonly bool StrictDelimitation;
 
-        public Options(char quoteCharacter, char escapeCharacter, char delimiterCharacter)
+        public Options(char quoteCharacter, char escapeCharacter, char delimiterCharacter, bool strictDelimitation = false)
         {
             QuoteCharacter = quoteCharacter;
             EscapeCharacter = escapeCharacter;
             DelimiterCharacter = delimiterCharacter;
+            StrictDelimitation = strictDelimitation;
         }
 
         public override string ToString()
         {
-            return $"Options (QuoteCharacter = {QuoteCharacter}, EscapeCharacter = {EscapeCharacter}, DelimiterCharacter = {DelimiterCharacter})";
+            return $"Options (QuoteCharacter = {QuoteCharacter}, EscapeCharacter = {EscapeCharacter}, DelimiterCharacter = {DelimiterCharacter}, StrictDelimitation = {StrictDelimitation})";
         }
     }
 }
diff --git a/TinyCsvParser/TinyCsvParser/Tokenizer/RFC4180/Reader.cs b/TinyCsvParser/TinyCsvParser/Tokenizer/RFC4180/Reader.cs
@@ -72,25 +72,28 @@ private Token NextToken(StringReader reader)
             }
             else
             {
-                if (IsQuoteCharacter(c))
+                if (!options.StrictDelimitation)
                 {
-                    result = ReadQuoted(reader);
+                    if (IsQuoteCharacter(c))
+                    {
+                        result = ReadQuoted(reader);
 
-                    Skip(reader);
+                        Skip(reader);
 
-                    if (IsEndOfStream(reader.Peek()))
-                    {
-                        return new Token(TokenType.EndOfRecord, result);
-                    }
+                        if (IsEndOfStream(reader.Peek()))
+                        {
+                            return new Token(TokenType.EndOfRecord, result);
+                        }
 
-                    if (IsDelimiter(reader.Peek()))
-                    {
-                        reader.Read();
-                    }
+                        if (IsDelimiter(reader.Peek()))
+                        {
+                            reader.Read();
+                        }
 
-                    return new Token(TokenType.Token, result);
+                        return new Token(TokenType.Token, result);
+                    }
                 }
-                                
+
                 if (IsEndOfStream(c)) 
                 {
                     return new Token(TokenType.EndOfRecord);
@@ -99,6 +102,11 @@ private Token NextToken(StringReader reader)
                 {
                     result = reader.ReadTo(options.DelimiterCharacter).Trim();
 
+                    if (options.StrictDelimitation)
+                    {
+                        result = result.TrimStart(options.QuoteCharacter).TrimEnd(options.QuoteCharacter);
+                    }
+
                     Skip(reader);
 
                     if (IsEndOfStream(reader.Peek()))