Add tokenizer test for other languages

botsquad · Mar 7, 2024 · 8b466ab · 8b466ab
1 parent 7b92644
commit 8b466ab
Showing 1 changed file with 9 additions and 0 deletions.
diff --git a/test/bubble_match/sentence/tokenizer_test.exs b/test/bubble_match/sentence/tokenizer_test.exs
@@ -34,4 +34,13 @@ defmodule BubbleMatch.Sentence.TokenizerTest do
     assert [] = Tokenizer.tokenize("   ")
     assert [] = Tokenizer.tokenize("")
   end
+
+  test "other languages" do
+    # polish
+    assert [_, _] = Tokenizer.tokenize("dzień dobry")
+    # arabic
+    assert [_, _] = Tokenizer.tokenize("صباح الخير")
+    # ukrainan
+    assert [_, _] = Tokenizer.tokenize("Добрий ранок")
+  end
 end