Add tests comparing to original JS library

Author: eelmafia

.github/workflows/test.yml

@@ -19,5 +19,5 @@ jobs:
           rebar3-version: "3"
           # elixir-version: "1.15.4"
       - run: gleam deps download
-      - run: gleam test
+      - run: generate_testcases.sh && gleam test
       - run: gleam format --check src test

generate_testcases.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if ! command -v node &> /dev/null
+then
+    exit 1
+fi
+
+if ! command -v npm &> /dev/null
+then
+    exit 1
+fi
+
+if ! grep -q '"lz-string"' package.json &> /dev/null; then
+    npm install lz-string --save
+else
+    npm update lz-string
+fi
+
+# Run the JS file with Node.js
+node lz_string_generate_test_cases.js

gleam.toml

@@ -12,6 +12,7 @@ links = [{ title = "Gleam", href = "https://gleam.run" }]
 [dependencies]
 gleam_stdlib = ">= 0.34.0 and < 2.0.0"
 gleam_erlang = ">= 0.26.0 and < 1.0.0"
+file_streams = ">= 1.1.1 and < 2.0.0"
 
 [dev-dependencies]
 gleeunit = ">= 1.0.0 and < 2.0.0"

lz_string_generate_test_cases.js

@@ -0,0 +1,99 @@
+const LZString = require('lz-string');
+const fs = require('node:fs');
+
+function writeInt32LE(buffer, value, offset) {
+    buffer.writeInt32LE(value, offset);
+}
+
+function generate_random_string(length){
+    let string = ""
+    for (let i = 0; i < length; i++ ){
+        string += String.fromCodePoint(random_int_in_range())
+    }
+    return string
+}
+
+function random_int_in_range(){
+    let j = Math.floor(Math.random() * 65_535)
+    while (!(j < 55295 || j >= 57344)){
+        j = Math.floor(Math.random() * 65_535)
+    }
+    return j
+}
+
+function compressed_sentence(str, type){
+    if (type == "UINT8") {
+        return LZString.compressToUint8Array(str)
+    } else if (type == "Base64"){
+        return LZString.compressToBase64(str)
+    } else if (type == "URI"){
+        return LZString.compressToEncodedURIComponent(str)
+    }
+}
+
+function write_random_strings(amount, type, chars){
+    for (let i = 0; i <= amount; i++){
+        let str = generate_random_string(chars)
+        write_test_case_to_file({
+            input:  str,
+            output: compressed_sentence(str, type)
+        })
+    }
+}
+function every_UTF8_char(){
+    let codePoints = [];
+
+    // Add code points from 0 to 55295
+    for (let i = 0; i <= 55295; i++) {
+        codePoints.push(i);
+    }
+
+    // Add code points from 57344 to 65535
+    for (let i = 57344; i <= 65535; i++) {
+        codePoints.push(i);
+    }
+
+    // Convert code points to a string
+    return String.fromCodePoint(...codePoints);
+}
+
+const file = fs.openSync('output.bin', 'w');
+
+
+function create_cases(){
+    //2 known cases as a sanity test
+    write_test_case_to_file({input: "hello, i am a 猫", output: compressed_sentence("hello, i am a 猫", "UINT8")})
+    write_test_case_to_file({input: "今日は 今日は 今日は 今日は 今日は 今日は", output: compressed_sentence("今日は 今日は 今日は 今日は 今日は 今日は", "UINT8")})
+
+    //Generate 1000 random strings and compress them
+    write_random_strings(1000, "UINT8", 1000)
+    write_random_strings(1000, "Base64", 1000)
+    write_random_strings(1000, "URI", 1000)
+
+    //Generatea  really long string and compress it
+    write_random_strings(1, "UINT8", 1000000)
+    write_random_strings(1, "Base64", 1000000)
+    write_random_strings(1, "URI", 1000000)
+
+    let allUTF8Characters = every_UTF8_char()
+    write_test_case_to_file({input: "", output: compressed_sentence(allUTF8Characters, "UINT8")})
+    write_test_case_to_file({input: "", output: compressed_sentence(allUTF8Characters, "Base64")})
+    write_test_case_to_file({input: "", output: compressed_sentence(allUTF8Characters, "URI")})
+}
+
+// First 4 bytes are the size of the input string followed by the input string
+// and the same for the output string
+function write_test_case_to_file(testCase){
+    const inputBuffer = Buffer.from(testCase.input, 'utf-8');
+    const outputBuffer = Buffer.from(testCase.output);
+
+    const buffer = Buffer.alloc(8 + inputBuffer.length + outputBuffer.length);
+    writeInt32LE(buffer, inputBuffer.length, 0);
+    inputBuffer.copy(buffer, 4);
+    writeInt32LE(buffer, outputBuffer.length, 4 + inputBuffer.length);
+    outputBuffer.copy(buffer, 8 + inputBuffer.length);
+    fs.writeSync(file, buffer);
+}
+
+create_cases()
+fs.closeSync(file);

manifest.toml

@@ -2,12 +2,14 @@
 # You typically do not need to edit this file
 
 packages = [
+  { name = "file_streams", version = "1.1.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "file_streams", source = "hex", outer_checksum = "73FC5AD6CA7016E521EFCD7CCB14CA8D186C7C2B0669A53EF7EC18C57A20DBBA" },
   { name = "gleam_erlang", version = "0.26.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "3DF72F95F4716883FA51396FB0C550ED3D55195B541568CAF09745984FD37AD1" },
   { name = "gleam_stdlib", version = "0.40.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "86606B75A600BBD05E539EB59FABC6E307EEEA7B1E5865AFB6D980A93BCB2181" },
   { name = "gleeunit", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "F7A7228925D3EE7D0813C922E062BFD6D7E9310F0BEE585D3A42F3307E3CFD13" },
 ]
 
 [requirements]
-gleam_erlang = { version = ">= 0.26.0 and < 1.0.0"}
+file_streams = { version = ">= 1.1.1 and < 2.0.0"}
+gleam_erlang = { version = ">= 0.26.0 and < 1.0.0" }
 gleam_stdlib = { version = ">= 0.34.0 and < 2.0.0" }
 gleeunit = { version = ">= 1.0.0 and < 2.0.0" }

src/internal_lib/lib.gleam

@@ -23,11 +23,7 @@ pub fn decode_base64(
     [char, ..rest] -> {
       case dict.get(key_dict, char) {
         Ok(num) -> {
-          decode_base64(
-            rest,
-            key_dict,
-            bit_array.append(bitstring, <<num:size(6)>>),
-          )
+          decode_base64(rest, key_dict, <<bitstring:bits, <<num:size(6)>>:bits>>)
         }
         _ -> Error(EInvalidInput)
       }

test/gleamlz_string_test.gleam

@@ -1,14 +1,24 @@
+import file_streams/file_stream.{type FileStream}
 import gleam/bit_array
 import gleam/erlang/atom
 import gleam/list
+import gleam/result
 import gleam/string
 import gleamlz_string
 import gleeunit
 import gleeunit/should
 import helpers/test_helpers
 
+type Mode {
+  Uint8
+  Base64
+  URI
+}
+
 const known_string = "hello, i am a 猫"
 
+const filename = "output.bin"
+
 pub fn main() {
   gleeunit.main()
 }
@@ -138,7 +148,7 @@ pub fn high_entropy_string_test_() {
 
 pub fn large_low_entropy_string_test_() {
   let assert Ok(timeout) = atom.from_string("timeout")
-  #(timeout, 60.0, [
+  #(timeout, 300.0, [
     fn() {
       let str =
         bit_array.base16_encode(test_helpers.generate_random_bytes(1_000_000))
@@ -157,3 +167,168 @@ pub fn large_low_entropy_string_test_() {
     },
   ])
 }
+
+pub fn invalid_input_test() {
+  gleamlz_string.decompress_from_uint8(<<5>>)
+  |> should.be_error
+
+  gleamlz_string.decompress_from_base64(known_string)
+  |> should.be_error
+
+  gleamlz_string.decompress_from_encoded_uri(known_string)
+  |> should.be_error
+}
+
+//Tests with the OG javascript library output
+
+pub fn js_lib_test_() {
+  let assert Ok(timeout) = atom.from_string("timeout")
+  #(timeout, 300.0, [
+    fn() {
+      let assert Ok(stream) = file_stream.open_read(filename)
+      //Test 2 known strings
+      js_test_known(stream)
+      |> should.be_ok
+
+      //1000 strings of 1_000 chars each
+      js_test_random_uint8(stream, 1000)
+      |> should.be_ok
+
+      js_test_random_base64(stream, 1000)
+      |> should.be_ok
+
+      js_test_random_uri(stream, 1000)
+      |> should.be_ok
+
+      //1 string 1_000_000 chars
+      js_test_random_uint8(stream, 1)
+      |> should.be_ok
+
+      js_test_random_base64(stream, 1)
+      |> should.be_ok
+
+      js_test_random_uri(stream, 1)
+      |> should.be_ok
+
+      //every single utf8 character
+      js_test_all_utf8_uint8(stream)
+      |> should.be_ok
+
+      js_test_all_utf8_base64(stream)
+      |> should.be_ok
+      |> should.be_ok
+
+      js_test_all_utf8_uri(stream)
+      |> should.be_ok
+      |> should.be_ok
+
+      let assert Ok(Nil) = file_stream.close(stream)
+    },
+  ])
+}
+
+fn js_test_known(stream: FileStream) {
+  [
+    "hello, i am a 猫",
+    "今日は 今日は 今日は 今日は 今日は 今日は",
+  ]
+  |> list.try_each(fn(string) { js_compress_vs_known(stream, string) })
+}
+
+fn js_test_random_uint8(fstream: FileStream, n: Int) {
+  list.range(0, n)
+  |> list.try_each(fn(_x) { js_decompress_vs_unknown(fstream, Uint8) })
+}
+
+fn js_test_random_base64(fstream: FileStream, n: Int) {
+  list.range(0, n)
+  |> list.try_each(fn(_x) { js_decompress_vs_unknown(fstream, Base64) })
+}
+
+pub fn js_test_random_uri(fstream: FileStream, n: Int) {
+  list.range(0, n)
+  |> list.try_each(fn(_x) { js_decompress_vs_unknown(fstream, URI) })
+}
+
+fn js_test_all_utf8_uint8(fstream: FileStream) {
+  let allutf8chars = test_helpers.all_utf8_chars()
+
+  use #(_input_str, output_str) <- result.map(read_js_input_output(fstream))
+
+  output_str
+  |> gleamlz_string.decompress_from_uint8()
+  |> should.equal(Ok(allutf8chars))
+}
+
+fn js_test_all_utf8_base64(fstream: FileStream) {
+  let allutf8chars = test_helpers.all_utf8_chars()
+  js_decompress_vs_known(fstream, allutf8chars, Base64)
+}
+
+fn js_test_all_utf8_uri(fstream: FileStream) {
+  let allutf8chars = test_helpers.all_utf8_chars()
+  js_decompress_vs_known(fstream, allutf8chars, URI)
+}
+
+//Compress a known string and match the JS output for the same
+fn js_compress_vs_known(fstream: FileStream, known_string: String) {
+  result.map(read_js_input_output(fstream), fn(result) {
+    let compressed = gleamlz_string.compress_to_uint8(known_string)
+    compressed
+    |> should.equal(result.1)
+
+    compressed
+    |> should.not_equal(<<>>)
+  })
+}
+
+fn js_decompress_vs_known(fstream: FileStream, known_string: String, mode: Mode) {
+  use result <- result.map(read_js_input_output(fstream))
+  use output_str <- result.map(bit_array.to_string(result.1))
+  case mode {
+    Uint8 -> {
+      gleamlz_string.decompress_from_uint8(result.1)
+      |> should.equal(Ok(known_string))
+    }
+    Base64 -> {
+      gleamlz_string.decompress_from_base64(output_str)
+      |> should.equal(Ok(known_string))
+    }
+    URI -> {
+      gleamlz_string.decompress_from_encoded_uri(output_str)
+      |> should.equal(Ok(known_string))
+    }
+  }
+}
+
+fn js_decompress_vs_unknown(fstream: FileStream, mode: Mode) {
+  use result <- result.map(read_js_input_output(fstream))
+  use input_str <- result.map(bit_array.to_string(result.0))
+  use output_str <- result.map(bit_array.to_string(result.1))
+
+  case mode {
+    Uint8 -> {
+      gleamlz_string.decompress_from_uint8(result.1)
+      |> should.equal(Ok(input_str))
+    }
+    Base64 -> {
+      gleamlz_string.decompress_from_base64(output_str)
+      |> should.equal(Ok(input_str))
+    }
+    URI -> {
+      gleamlz_string.decompress_from_encoded_uri(output_str)
+      |> should.equal(Ok(input_str))
+    }
+  }
+}
+
+fn read_js_input_output(stream: FileStream) {
+  use input_size <- result.try(file_stream.read_uint32_le(stream))
+  use str <- result.try(file_stream.read_bytes(stream, input_size))
+  use output_size <- result.try(file_stream.read_uint32_le(stream))
+  use js_compressed_string <- result.map(file_stream.read_bytes(
+    stream,
+    output_size,
+  ))
+  #(str, js_compressed_string)
+}