Merge pull request #4 from DrewMcArthur/fix/2-decode-ignored-any

improve support for ignoring values when decoding
haydnv · Apr 2, 2024 · 8fbf971 · 8fbf971
2 parents 4186d53 + 342c7a8
commit 8fbf971
Show file tree

Hide file tree

Showing 4 changed files with 581 additions and 13 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "destream_json"
-version = "0.12.1"
+version = "0.12.2"
 authors = ["code@tinychain.net"]
 edition = "2021"
 license = "Apache-2.0"
@@ -18,6 +18,7 @@ value = ["number-general/stream"]
 all = ["tokio-io", "value"]
 
 [dependencies]
+async-recursion = "1.1"
 async-trait = "0.1"
 base64 = "0.22"
 bytes = "1.5"
@@ -33,3 +34,4 @@ number-general = { version = "0.11", features=["stream"] }
 tokio = { version = "1.35", features = ["fs", "macros"] }
 tokio-util = { version = "0.7", features = ["io"] }
 tokio-test = "0.4"
+test-case = "3.3.1"
diff --git a/src/constants.rs b/src/constants.rs
@@ -14,3 +14,56 @@ pub const NUMERIC: [u8; 15] = [
     b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0', b'-', b'e', b'E', b'.',
 ];
 pub const QUOTE: &[u8] = b"\"";
+
+// Lookup table of bytes that must be escaped. A value of true at
+// index i means that byte i requires an escape sequence in the input.
+// taken from
+// https://github.com/serde-rs/json/blob/4a0be88b5ac6cda971a52df9f027b551fe566347/src/read.rs#L789
+pub static ESCAPE_CHARS: [bool; 256] = {
+    const CT: bool = true; // control character \x00..=\x1F
+    const QU: bool = true; // quote \x22
+    const BS: bool = true; // backslash \x5C
+    const __: bool = false; // allow unescaped
+    [
+        //   1   2   3   4   5   6   7   8   9   A   B   C   D   E   F
+        CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, // 0
+        CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, CT, // 1
+        __, __, QU, __, __, __, __, __, __, __, __, __, __, __, __, __, // 2
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 3
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 4
+        __, __, __, __, __, __, __, __, __, __, __, __, BS, __, __, __, // 5
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 6
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 7
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 8
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 9
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // A
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // B
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // C
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // D
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // E
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // F
+    ]
+};
+
+pub static HEX: [u8; 256] = {
+    const __: u8 = 255; // not a hex digit
+    [
+        //   1   2   3   4   5   6   7   8   9   A   B   C   D   E   F
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 0
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 1
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 2
+        0, 1, 2, 3, 4, 5, 6, 7, 8, 9, __, __, __, __, __, __, // 3
+        __, 10, 11, 12, 13, 14, 15, __, __, __, __, __, __, __, __, __, // 4
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 5
+        __, 10, 11, 12, 13, 14, 15, __, __, __, __, __, __, __, __, __, // 6
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 7
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 8
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 9
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // A
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // B
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // C
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // D
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // E
+        __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // F
+    ]
+};