Merge pull request #1617 from google/google_sync

Google sync

pytype/io.py

@@ -286,14 +286,17 @@ def process_one_file(options):
       write_pickle(ret.ast, options, ret.context.loader)
 
   if options.unused_imports_info_files:
-    # Sort the paths to make the output stable.
-    cwd = os.getcwd()
-    unused_paths = sorted(ret.context.loader.get_unused_imports_map_paths())
-    with options.open_function(
-        options.unused_imports_info_files, "wt", encoding="utf-8"
-    ) as f:
-      for unused_path in unused_paths:
-        f.write(f"{os.path.relpath(unused_path, cwd)}\n")
+    if options.use_rewrite:
+      pass  # not implemented yet
+    else:
+      # Sort the paths to make the output stable.
+      cwd = os.getcwd()
+      unused_paths = sorted(ret.context.loader.get_unused_imports_map_paths())
+      with options.open_function(
+          options.unused_imports_info_files, "wt", encoding="utf-8"
+      ) as f:
+        for unused_path in unused_paths:
+          f.write(f"{os.path.relpath(unused_path, cwd)}\n")
   exit_status = handle_errors(ret.context.errorlog, options)
 
   # Touch output file upon success.

pytype/overlays/enum_overlay.py

@@ -338,13 +338,34 @@ def _make_new(self, node, member_type, cls):
     # to `unsolvable` if the enum has no members. Technically, `__new__` should
     # not accept any arguments, because it will always fail if the enum has no
     # members. But `unsolvable` is much simpler to implement and use.
+    value_types = [member_type, cls]
+    # If this enum class defines the _missing_ classmethod, then widen the value
+    # type to include the type of the value parameter in _missing_.
+    if "_missing_" in cls:
+      if isinstance(cls, abstract.PyTDClass):
+        missing_sigs = []
+        with self.ctx.allow_recursive_convert():
+          missing_var = cls.load_lazy_attribute("_missing_")
+        for missing in missing_var.data:
+          if isinstance(missing, abstract.PyTDFunction):
+            missing_sigs.extend(sig.signature for sig in missing.signatures)
+      else:
+        missing_sigs = []
+        for val in cls.members["_missing_"].data:
+          if isinstance(val, special_builtins.ClassMethodInstance):
+            for missing in val.func.data:
+              if isinstance(missing, abstract.SignedFunction):
+                missing_sigs.append(missing.signature)
+      for missing_sig in missing_sigs:
+        value_type = missing_sig.annotations.get(
+            "value", self.ctx.convert.unsolvable)
+        value_types.append(value_type)
     return overlay_utils.make_method(
         ctx=self.ctx,
         node=node,
         name="__new__",
         params=[
-            overlay_utils.Param("value",
-                                abstract.Union([member_type, cls], self.ctx))
+            overlay_utils.Param("value", abstract.Union(value_types, self.ctx))
         ],
         return_type=cls)
 

pytype/rewrite/abstract/CMakeLists.txt

@@ -48,6 +48,7 @@ py_library(
   DEPS
     .base
     .functions
+    pytype.types.types
 )
 
 py_test(
@@ -80,6 +81,8 @@ py_test(
   DEPS
     .base
     .containers
+    .internal
+    pytype.rewrite.flow.flow
     pytype.rewrite.tests.test_utils
 )
 

pytype/rewrite/abstract/abstract.py

@@ -42,3 +42,4 @@
 
 get_atomic_constant = _utils.get_atomic_constant
 join_values = _utils.join_values
+is_any = _utils.is_any

pytype/rewrite/abstract/classes.py

@@ -9,6 +9,7 @@
 import immutabledict
 from pytype.rewrite.abstract import base
 from pytype.rewrite.abstract import functions as functions_lib
+from pytype.types import types
 
 log = logging.getLogger(__name__)
 
@@ -211,7 +212,7 @@ def set_attribute(self, name: str, value: base.BaseValue) -> None:
     log.info('Ignoring attribute set on %r: %s -> %r', self, name, value)
 
 
-class Module(BaseInstance):
+class Module(BaseInstance, types.Module):
   """A module."""
 
   def __init__(self, ctx: base.ContextType, name: str):

pytype/rewrite/abstract/containers.py

@@ -24,21 +24,21 @@ def __init__(self, ctx: base.ContextType, constant: _List[_Variable]):
   def __repr__(self):
     return f'List({self.constant!r})'
 
-  def append(self, val: _Variable):
-    self.constant.append(val)
+  def append(self, var: _Variable) -> 'List':
+    return List(self._ctx, self.constant + [var])
 
-  def extend(self, val: _Variable):
+  def extend(self, var: _Variable) -> base.BaseValue:
     try:
-      const = utils.get_atomic_constant(val)
-      if not isinstance(const, list):
-        const = None
+      val = var.get_atomic_value()
     except ValueError:
-      const = None
-
-    if const:
-      self.constant.extend(const)
+      # This list has multiple possible values, so it is no longer a constant.
+      return self._ctx.abstract_loader.load_raw_type(list).instantiate()
+    if isinstance(val, List):
+      new_constant = self.constant + val.constant
     else:
-      self.constant.append(internal.Splat(self._ctx, val).to_variable())
+      splat = internal.Splat(self._ctx, val)
+      new_constant = self.constant + [splat.to_variable()]
+    return List(self._ctx, new_constant)
 
 
 class Dict(base.PythonConstant[_Dict[_Variable, _Variable]]):
@@ -54,21 +54,16 @@ def __init__(
   def __repr__(self):
     return f'Dict({self.constant!r})'
 
-  def setitem(self, key, val):
-    self.constant[key] = val
+  def setitem(self, key: _Variable, val: _Variable) -> 'Dict':
+    return Dict(self._ctx, {**self.constant, key: val})
 
-  def update(self, val: _Variable):
+  def update(self, var: _Variable) -> base.BaseValue:
     try:
-      const = utils.get_atomic_constant(val)
-      if not isinstance(const, dict):
-        const = None
+      val = utils.get_atomic_constant(var, dict)
     except ValueError:
-      const = None
-
-    if const:
-      self.constant.update(const)
-    else:
-      self.indefinite = True
+      # This dict has multiple possible values, so it is no longer a constant.
+      return self._ctx.abstract_loader.load_raw_type(dict).instantiate()
+    return Dict(self._ctx, {**self.constant, **val})
 
 
 class Set(base.PythonConstant[_Set[_Variable]]):
@@ -81,8 +76,8 @@ def __init__(self, ctx: base.ContextType, constant: _Set[_Variable]):
   def __repr__(self):
     return f'Set({self.constant!r})'
 
-  def add(self, val: _Variable):
-    self.constant.add(val)
+  def add(self, val: _Variable) -> 'Set':
+    return Set(self._ctx, self.constant | {val})
 
 
 class Tuple(base.PythonConstant[_Tuple[_Variable, ...]]):

pytype/rewrite/abstract/containers_test.py

@@ -2,6 +2,8 @@
 
 from pytype.rewrite.abstract import base
 from pytype.rewrite.abstract import containers
+from pytype.rewrite.abstract import internal
+from pytype.rewrite.flow import variables
 from pytype.rewrite.tests import test_utils
 from typing_extensions import assert_type
 
@@ -25,6 +27,36 @@ def test_constant_type(self):
     c = containers.List(self.ctx, [a])
     assert_type(c.constant, List[_AbstractVariable])
 
+  def test_append(self):
+    l1 = containers.List(self.ctx, [self.const_var("a")])
+    l2 = l1.append(self.const_var("b"))
+    self.assertEqual(l2.constant, [self.const_var("a"), self.const_var("b")])
+
+  def test_extend(self):
+    l1 = containers.List(self.ctx, [self.const_var("a")])
+    l2 = containers.List(self.ctx, [self.const_var("b")])
+    l3 = l1.extend(l2.to_variable())
+    self.assertIsInstance(l3, containers.List)
+    self.assertEqual(l3.constant, [self.const_var("a"), self.const_var("b")])
+
+  def test_extend_splat(self):
+    l1 = containers.List(self.ctx, [self.const_var("a")])
+    l2 = self.ctx.abstract_loader.load_raw_type(list).instantiate()
+    l3 = l1.extend(l2.to_variable())
+    self.assertIsInstance(l3, containers.List)
+    self.assertEqual(
+        l3.constant,
+        [self.const_var("a"), internal.Splat(self.ctx, l2).to_variable()])
+
+  def test_extend_multiple_bindings(self):
+    l1 = containers.List(self.ctx, [self.const_var("a")])
+    l2 = containers.List(self.ctx, [self.const_var("b")])
+    l3 = containers.List(self.ctx, [self.const_var("c")])
+    var = variables.Variable((variables.Binding(l2), variables.Binding(l3)))
+    l4 = l1.extend(var)
+    self.assertEqual(
+        l4, self.ctx.abstract_loader.load_raw_type(list).instantiate())
+
 
 class DictTest(BaseTest):
 
@@ -34,6 +66,33 @@ def test_constant_type(self):
     c = containers.Dict(self.ctx, {a: b})
     assert_type(c.constant, Dict[_AbstractVariable, _AbstractVariable])
 
+  def test_setitem(self):
+    d1 = containers.Dict(self.ctx, {})
+    d2 = d1.setitem(self.const_var("a"), self.const_var("b"))
+    self.assertEqual(d2.constant, {self.const_var("a"): self.const_var("b")})
+
+  def test_update(self):
+    d1 = containers.Dict(self.ctx, {})
+    d2 = containers.Dict(self.ctx, {self.const_var("a"): self.const_var("b")})
+    d3 = d1.update(d2.to_variable())
+    self.assertIsInstance(d3, containers.Dict)
+    self.assertEqual(d3.constant, {self.const_var("a"): self.const_var("b")})
+
+  def test_update_indefinite(self):
+    d1 = containers.Dict(self.ctx, {})
+    indef = self.ctx.abstract_loader.load_raw_type(dict).instantiate()
+    d2 = d1.update(indef.to_variable())
+    self.assertEqual(d2, indef)
+
+  def test_update_multiple_bindings(self):
+    d1 = containers.Dict(self.ctx, {})
+    d2 = containers.Dict(self.ctx, {self.const_var("a"): self.const_var("b")})
+    d3 = containers.Dict(self.ctx, {self.const_var("c"): self.const_var("d")})
+    var = variables.Variable((variables.Binding(d2), variables.Binding(d3)))
+    d4 = d1.update(var)
+    self.assertEqual(
+        d4, self.ctx.abstract_loader.load_raw_type(dict).instantiate())
+
 
 class SetTest(BaseTest):
 
@@ -42,6 +101,11 @@ def test_constant_type(self):
     c = containers.Set(self.ctx, {a})
     assert_type(c.constant, Set[_AbstractVariable])
 
+  def test_add(self):
+    c1 = containers.Set(self.ctx, set())
+    c2 = c1.add(self.const_var("a"))
+    self.assertEqual(c2.constant, {self.const_var("a")})
+
 
 class TupleTest(BaseTest):
 

pytype/rewrite/abstract/functions.py

@@ -236,6 +236,11 @@ def map_args(self, args: Args[_FrameT]) -> MappedArgs[_FrameT]:
     # TODO(b/241479600): Implement this properly, with error detection.
     argdict = dict(zip(self.param_names, args.posargs))
     argdict.update(args.kwargs)
+    def add_arg(k, v):
+      if k:
+        argdict[k] = v or self._ctx.consts.Any.to_variable()
+    add_arg(self.varargs_name, args.starargs)
+    add_arg(self.kwargs_name, args.starstarargs)
     return MappedArgs(signature=self, argdict=argdict, frame=args.frame)
 
   def make_fake_args(self) -> MappedArgs[FrameType]:

pytype/rewrite/abstract/internal.py

@@ -54,7 +54,7 @@ class Splat(base.BaseValue):
   (x, *ys, z) in starargs) and let the function arg matcher unpack them.
   """
 
-  def __init__(self, ctx: base.ContextType, iterable: _Variable):
+  def __init__(self, ctx: base.ContextType, iterable: base.BaseValue):
     super().__init__(ctx)
     self.iterable = iterable
 

pytype/rewrite/abstract/internal_test.py

@@ -22,7 +22,7 @@ class SplatTest(test_utils.ContextfulTestBase):
   def test_basic(self):
     # Basic smoke test, remove when we have some real functionality to test.
     cls = self.ctx.abstract_loader.load_raw_type(tuple)
-    seq = cls.instantiate().to_variable()
+    seq = cls.instantiate()
     x = internal.Splat(self.ctx, seq)
     self.assertEqual(x.iterable, seq)
 

pytype/rewrite/abstract/utils.py

@@ -35,3 +35,7 @@ def join_values(
     return values[0]
   else:
     return ctx.consts.Any
+
+
+def is_any(value: base.BaseValue):
+  return isinstance(value, base.Singleton) and value.name == 'Any'

pytype/rewrite/flow/variables.py

@@ -22,6 +22,11 @@ def __repr__(self):
       return f'Bind[{self.value}]'
     return f'Bind[{self.value} if {self.condition}]'
 
+  @property
+  def data(self):
+    # Temporary alias for 'value' for compatibility with current pytype.
+    return self.value
+
 
 @_frozen_dataclass
 class Variable(Generic[_T]):
@@ -39,6 +44,11 @@ def from_value(
   def values(self) -> Tuple[_T, ...]:
     return tuple(b.value for b in self.bindings)
 
+  @property
+  def data(self):
+    # Temporary alias for 'values' for compatibility with current pytype.
+    return self.values
+
   def display_name(self) -> str:
     return f'variable {self.name}' if self.name else 'anonymous variable'
 
@@ -77,6 +87,11 @@ def with_condition(self, condition: conditions.Condition) -> 'Variable[_T]':
   def with_name(self, name: Optional[str]) -> 'Variable[_T]':
     return dataclasses.replace(self, name=name)
 
+  def with_value(self, value: _T2) -> 'Variable[_T2]':
+    assert len(self.bindings) == 1
+    new_binding = dataclasses.replace(self.bindings[0], value=value)
+    return dataclasses.replace(self, bindings=(new_binding,))
+
   def __repr__(self):
     bindings = ' | '.join(repr(b) for b in self.bindings)
     if self.name:

pytype/rewrite/flow/variables_test.py

@@ -126,6 +126,12 @@ def test_empty_does_not_have_atomic_value(self):
     x = variables.Variable(bindings=())
     self.assertFalse(x.has_atomic_value(5))
 
+  def test_with_value(self):
+    x = variables.Variable.from_value(5)
+    y = x.with_value('test')
+    assert_type(y, variables.Variable[str])
+    self.assertEqual(y.get_atomic_value(), 'test')
+
 
 if __name__ == '__main__':
   unittest.main()