Fix #229, #293: Modify parser.py to re-insert quotes correctly

fire/parser.py

@@ -73,7 +73,6 @@ def DefaultParseValue(value):
     # If _LiteralEval can't parse the value, treat it as a string.
     return value
 
-
 def _LiteralEval(value):
   """Parse value as a Python literal, or container of containers and literals.
 
@@ -93,38 +92,128 @@ def _LiteralEval(value):
       literals.
     SyntaxError: If the value string has a syntax error.
   """
+
+  ptr = 0   # Represents the current position in 'value'
+  brace_stack = []   # Keeps track of curly braces
+
+  # Set of characters that will help distinguish between two container elements
+  delimiter = {',', ']', '}', ')', ' '}
+
+  while ptr < len(value):
+    if value[ptr] in {'{', '[', '('}:
+      # Beginning of container encountered
+      if value[ptr] == '{':
+        if not brace_stack:
+          # Potential dictionary encountered, add colon as delimiter
+          delimiter.add(':')
+        brace_stack.append('{')
+
+      ptr += 1
+    elif value[ptr] in {"'", '"'}:
+      # Beginning of string encountered
+      if value[ptr: ptr + 3] in {"'''", '"""'}:
+        # Triple quotes for strings
+        left = ptr
+        right = ptr + 3
+
+        # Search for the ending triple quotes
+        while (right < len(value) and
+               value[right: right + 3] != value[left: left + 3]):
+          right += 1
+
+        ptr = right + 3
+      else:
+        # Single quotes for strings
+        left = ptr
+        right = ptr + 1
+
+        # Search for the ending single quote
+        while (right < len(value) and
+               value[right] != value[left]):
+          right += 1
+
+        ptr = right + 1
+
+      # Skip all delimiters occuring after this
+      while ptr < len(value) and value[ptr] in delimiter:
+        if value[ptr] == '}' and brace_stack:
+          brace_stack.pop()
+          if not brace_stack:
+            # All potential dictionaries exited,
+            # remove colon (:) from delimiter
+            delimiter.remove(':')
+
+        ptr += 1
+    else:
+      # Literal encountered
+      left = ptr
+      right = ptr + 1
+
+      delimiter.remove(' ') # Don't consider space as a delimiter in strings
+
+      while right < len(value) and value[right] not in delimiter:
+        right += 1
+
+      delimiter.add(' ')
+
+      element = value[left: right]  # Single element identified
+
+      # If the element is of type string or ellipsis,
+      # put quotes around it
+      if isinstance(_SingleLiteralEval(element), str) or element == "...":
+        value = _InsertQuotes(value, left, right)
+        right += 2
+
+      ptr = right
+
+      # Skip all delimiters occuring after this
+      while ptr < len(value) and value[ptr] in delimiter:
+        if value[ptr] == '}' and brace_stack:
+          brace_stack.pop()
+          if not brace_stack:
+            # All potential dictionaries exited,
+            # remove colon (:) from delimiter
+            delimiter.remove(':')
+
+        ptr += 1
+
   root = ast.parse(value, mode='eval')
-  if isinstance(root.body, ast.BinOp):  # pytype: disable=attribute-error
-    raise ValueError(value)
-
-  for node in ast.walk(root):
-    for field, child in ast.iter_fields(node):
-      if isinstance(child, list):
-        for index, subchild in enumerate(child):
-          if isinstance(subchild, ast.Name):
-            child[index] = _Replacement(subchild)
-
-      elif isinstance(child, ast.Name):
-        replacement = _Replacement(child)
-        node.__setattr__(field, replacement)
-
-  # ast.literal_eval supports the following types:
-  # strings, bytes, numbers, tuples, lists, dicts, sets, booleans, and None
-  # (bytes and set literals only starting with Python 3.2)
+
   return ast.literal_eval(root)
 
 
-def _Replacement(node):
-  """Returns a node to use in place of the supplied node in the AST.
+def _SingleLiteralEval(value):
+  """Parse value as a Python literal only.
+
+  Args:
+    value: A string to be parsed as a literal only.
+  Returns:
+    The Python value representing the value arg.
+  Raises:
+    ValueError: If the value is not an expression with only containers and
+      literals.
+    SyntaxError: If the value string has a syntax error.
+  """
+
+  try:
+    return ast.literal_eval(value)
+  except (SyntaxError, ValueError):
+    # If _SingleLiteralEval can't parse the value, treat it as a string.
+    return value
+
+
+def _InsertQuotes(string, left_index, right_index):
+  """Insert quotes in the passed string.
 
   Args:
-    node: A node of type Name. Could be a variable, or builtin constant.
+    string: A string in which quotes will be inserted.
+    left_index: An integer representing the index where
+      starting quote will be inserted.
+    right_index: An integer representing the index where
+      closing quote will be inserted.
   Returns:
-    A node to use in place of the supplied Node. Either the same node, or a
-    String node whose value matches the Name node's id.
+    The modified string with quotes inserted at left_index and right_index.
   """
-  value = node.id
-  # These are the only builtin constants supported by literal_eval.
-  if value in ('True', 'False', 'None'):
-    return node
-  return ast.Str(value)
+  return (string[:left_index] + "'" +
+          string[left_index: right_index] + "'" +
+          string[right_index:])

fire/parser_test.py

@@ -50,6 +50,10 @@ def testDefaultParseValueStrings(self):
     self.assertEqual(parser.DefaultParseValue('path/file.jpg'), 'path/file.jpg')
     self.assertEqual(parser.DefaultParseValue('hello world'), 'hello world')
     self.assertEqual(parser.DefaultParseValue('--flag'), '--flag')
+    self.assertEqual(
+        parser.DefaultParseValue(
+            '{"[all (delimiters)], are present {in} this:string!"}'),
+        {'[all (delimiters)], are present {in} this:string!'})
 
   def testDefaultParseValueQuotedStrings(self):
     self.assertEqual(parser.DefaultParseValue("'hello'"), 'hello')
@@ -119,6 +123,24 @@ def testDefaultParseValueNestedContainers(self):
     self.assertEqual(
         parser.DefaultParseValue('[(A, 2, "3"), 5, {alph: 10.2, beta: "cat"}]'),
         [('A', 2, '3'), 5, {'alph': 10.2, 'beta': 'cat'}])
+    self.assertEqual(
+        parser.DefaultParseValue(
+            '[1234, "test-item", [nested-list!, "with spaces", 1e5]]'),
+        [1234, 'test-item', ['nested-list!', 'with spaces', 1e5]])
+    self.assertEqual(
+        parser.DefaultParseValue(
+            '{12.34, "test-item", (2+3j, "with spaces", True)}'),
+        {12.34, "test-item", (2+3j, "with spaces", True)})
+    self.assertEqual(
+        parser.DefaultParseValue(
+            '[{"abc": (3+4j), "colon:here": ("spaces, and commas", 123.45), '\
+            '100000.0: {"abc": True}}]'),
+        [{'abc': (3+4j), 'colon:here': ('spaces, and commas', 123.45),
+          100000.0: {'abc': True}}])
+    self.assertEqual(
+        parser.DefaultParseValue(
+            '{12.34, "test-item", (2+3j, "with spaces", True)}'),
+        {12.34, "test-item", (2+3j, "with spaces", True)})
 
   def testDefaultParseValueComments(self):
     self.assertEqual(parser.DefaultParseValue('"0#comments"'), '0#comments')