Merge pull request #54 from kellyjonbrazil/dev

Dev v1.5.5

CHANGELOG

@@ -1,5 +1,9 @@
 jello changelog
 
+20230114 v1.5.5
+- Fix schema output to ensure invalid variable names are enclosed in bracket notation
+- Fix to allow blank lines when slurping JSON Lines objects
+
 20220730 v1.5.4
 - Add `__main__.py` to package for `python -m jello` use cases
 

jello/__init__.py

@@ -1,8 +1,8 @@
 """jello - query JSON at the command line with python syntax"""
 
 
-__version__ = '1.5.4'
+__version__ = '1.5.5'
 AUTHOR = 'Kelly Brazil'
 WEBSITE = 'https://github.com/kellyjonbrazil/jello'
-COPYRIGHT = '© 2020-2022 Kelly Brazil'
+COPYRIGHT = '© 2020-2023 Kelly Brazil'
 LICENSE = 'MIT License'

jello/lib.py

@@ -5,6 +5,7 @@
 import ast
 import json
 import shutil
+from keyword import iskeyword
 from textwrap import TextWrapper
 from jello.dotmap import DotMap
 
@@ -22,6 +23,21 @@
     PYGMENTS_INSTALLED = False
 
 
+def is_valid_variable_name(name: str) -> bool:
+    dict_methods = [
+        '__class__', '__class_getitem__', '__contains__', '__delattr__',
+        '__delitem__', '__dir__', '__eq__', '__format__', '__ge__',
+        '__getattribute__', '__getitem__', '__getstate__', '__gt__',
+        '__init__', '__init_subclass__', '__ior__', '__iter__', '__le__',
+        '__len__', '__lt__', '__ne__', '__new__', '__or__', '__reduce__',
+        '__reduce_ex__', '__repr__', '__reversed__', '__ror__', '__setattr__',
+        '__setitem__', '__sizeof__', '__str__', '__subclasshook__', 'clear',
+        'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem',
+        'setdefault', 'update', 'values'
+    ]
+    return name.isidentifier() and not iskeyword(name) and name not in dict_methods
+
+
 class opts:
     initialize = None
     version_info = None
@@ -177,11 +193,11 @@ def _schema_gen(self, src, path='_'):
             self._schema_list.append(f'{path} = {val};{padding}{val_type}')
 
             for k, v in src.items():
-                # encapsulate key in brackets if it includes spaces
-                if ' ' in k:
-                    k = f'["{k}"]'
-                else:
+                # encapsulate key in brackets if it is not a valid variable name
+                if is_valid_variable_name(k):
                     k = f'.{k}'
+                else:
+                    k = f'["{k}"]'
 
                 if isinstance(v, list):
                     # print empty brackets as first list definition
@@ -335,7 +351,7 @@ def load_json(data):
     except Exception as e:
         try:
             # if json.loads fails, try loading as json lines
-            json_dict = [json.loads(i) for i in data.splitlines()]
+            json_dict = [json.loads(i) for i in data.splitlines() if i.strip()]
         except Exception:
             # raise original JSON exception instead of JSON Lines exception
             raise e

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name='jello',
-    version='1.5.4',
+    version='1.5.5',
     author='Kelly Brazil',
     author_email='kellyjonbrazil@gmail.com',
     description='Filter JSON and JSON Lines data with Python syntax.',

tests/test_create_schema.py

@@ -139,6 +139,16 @@ def setUp(self):
 
         self.deep_nest_sample = [[[[{"foo":[[[[1,2,3]]]]}]]]]
 
+        self.reserved_or_invalid_keynames_sample = {
+            "True": 1,
+            "get": 1,
+            "fromkeys": 1,
+            "aquaero-hid-3-1cb1": 1,
+            "if": 1,
+            "return": 1,
+            "__class__": 1
+        }
+
     # ------------ Tests ------------
 
     #
@@ -534,5 +544,18 @@ def test_deep_nest_m(self):
         expected = '_ = [];\n_[0] = [];\n_[0][0] = [];\n_[0][0][0] = [];\n_[0][0][0][0] = {};\n_[0][0][0][0].foo = [];\n_[0][0][0][0].foo[0] = [];\n_[0][0][0][0].foo[0][0] = [];\n_[0][0][0][0].foo[0][0][0] = [];\n_[0][0][0][0].foo[0][0][0][0] = 1;\n_[0][0][0][0].foo[0][0][0][1] = 2;\n_[0][0][0][0].foo[0][0][0][2] = 3;'
         self.assertEqual(self.schema.create_schema(data_in), expected)
 
+    #
+    # Handle invalid or reserved key names
+    #
+
+    def test_dict_reserved_or_invalid_keynames(self):
+        """
+        Test self.reserved_or_invalid_keynames_sample
+        """
+        data_in = self.reserved_or_invalid_keynames_sample
+        expected = '_ = {};\n_["True"] = 1;\n_["get"] = 1;\n_["fromkeys"] = 1;\n_["aquaero-hid-3-1cb1"] = 1;\n_["if"] = 1;\n_["return"] = 1;\n_["__class__"] = 1;'
+        output = self.schema.create_schema(data_in)
+        self.assertEqual(output, expected)
+
 if __name__ == '__main__':
     unittest.main()

tests/test_load_json.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+import unittest
+from jello.lib import load_json
+
+
+class MyTests(unittest.TestCase):
+    pretty_json =  '''\
+{
+    "foo": 1,
+    "bar": 2,
+    "baz": 3
+}'''
+
+    compact_json = '''{"foo": 1,"bar": 2,"baz": 3}'''
+
+    json_lines = '''\
+{"foo": 1,"bar": 2,"baz": 3}
+{"foo": 4,"bar": 5,"baz": 6}
+{"foo": 7,"bar": 8,"baz": 9}'''
+
+    json_lines_extra_spaces = '''\
+     
+{"foo": 1,"bar": 2,"baz": 3}
+
+{"foo": 4,"bar": 5,"baz": 6}
+          
+{"foo": 7,"bar": 8,"baz": 9}
+     
+    '''
+
+    def test_load_pretty_json(self):
+        """
+        Test with pretty JSON
+        """
+        expected = {'foo': 1, 'bar': 2, 'baz': 3}
+        self.assertEqual(load_json(self.pretty_json), expected)
+
+    def test_load_compact_json(self):
+        """
+        Test with compact JSON
+        """
+        expected = {'foo': 1, 'bar': 2, 'baz': 3}
+        self.assertEqual(load_json(self.compact_json), expected)
+
+    def test_load_json_lines(self):
+        """
+        Test with JSON Lines
+        """
+        expected = [{'foo': 1, 'bar': 2, 'baz': 3}, {'foo': 4, 'bar': 5, 'baz': 6}, {'foo': 7, 'bar': 8, 'baz': 9}]
+        self.assertEqual(load_json(self.json_lines), expected)
+
+    def test_load_json_lines_with_extra_whitespace(self):
+        """
+        Test with JSON Lines with extra blank lines and whitespace
+        """
+        expected = [{'foo': 1, 'bar': 2, 'baz': 3}, {'foo': 4, 'bar': 5, 'baz': 6}, {'foo': 7, 'bar': 8, 'baz': 9}]
+        self.assertEqual(load_json(self.json_lines_extra_spaces), expected)
+
+
+if __name__ == '__main__':
+    unittest.main()