Merge pull request #1613 from google/google_sync

Google sync

pytype/abstract/_interpreter_function.py

@@ -186,9 +186,6 @@ def __init__(self, name, def_opcode, code, f_locals, f_globals, defaults,
     self.last_frame = None  # for BuildClass
     self._store_call_records = False
     self.is_class_builder = False  # Will be set by BuildClass.
-    if name.endswith(".__init_subclass__"):
-      # __init_subclass__ is automatically promoted to a classmethod
-      self.is_classmethod = True
     # Whether to cache the return value irrespective of call args
     self.cache_return = False
 

pytype/overlays/special_builtins.py

@@ -776,6 +776,8 @@ def call(self, node, func, args, alias_map=None):
     arg = args.posargs[0]
     if not _check_method_decorator_arg(arg, "classmethod", self.ctx):
       return node, self.ctx.new_unsolvable(node)
+    if any(isinstance(v, ClassMethodInstance) for v in arg.data):
+      return node, arg
     for d in arg.data:
       d.is_classmethod = True
       d.is_attribute_of_class = True

pytype/rewrite/CMakeLists.txt

@@ -116,6 +116,7 @@ py_test(
     load_abstract_test.py
   DEPS
     pytype.rewrite.abstract.abstract
+    pytype.rewrite.flow.flow
     pytype.rewrite.tests.test_utils
 )
 

pytype/rewrite/abstract/CMakeLists.txt

@@ -8,8 +8,8 @@ py_library(
   DEPS
     .base
     .classes
+    .containers
     .functions
-    .instances
     .internal
     .utils
 )
@@ -63,43 +63,43 @@ py_test(
 
 py_library(
   NAME
-    functions
+    containers
   SRCS
-    functions.py
+    containers.py
   DEPS
     .base
-    pytype.blocks.blocks
-    pytype.pytd.pytd
 )
 
 py_test(
   NAME
-    functions_test
+    containers_test
   SRCS
-    functions_test.py
+    containers_test.py
   DEPS
-    .classes
-    .functions
+    .base
+    .containers
     pytype.rewrite.tests.test_utils
 )
 
 py_library(
   NAME
-    instances
+    functions
   SRCS
-    instances.py
+    functions.py
   DEPS
     .base
+    pytype.blocks.blocks
+    pytype.pytd.pytd
 )
 
 py_test(
   NAME
-    instances_test
+    functions_test
   SRCS
-    instances_test.py
+    functions_test.py
   DEPS
-    .base
-    .instances
+    .classes
+    .functions
     pytype.rewrite.tests.test_utils
 )
 

pytype/rewrite/abstract/abstract.py

@@ -2,8 +2,8 @@
 
 from pytype.rewrite.abstract import base as _base
 from pytype.rewrite.abstract import classes as _classes
+from pytype.rewrite.abstract import containers as _containers
 from pytype.rewrite.abstract import functions as _functions
-from pytype.rewrite.abstract import instances as _instances
 from pytype.rewrite.abstract import internal as _internal
 from pytype.rewrite.abstract import utils as _utils
 
@@ -17,6 +17,7 @@
 BaseInstance = _classes.BaseInstance
 FrozenInstance = _classes.FrozenInstance
 InterpreterClass = _classes.InterpreterClass
+Module = _classes.Module
 MutableInstance = _classes.MutableInstance
 
 Args = _functions.Args
@@ -30,12 +31,13 @@
 SimpleFunction = _functions.SimpleFunction
 SimpleReturn = _functions.SimpleReturn
 
-Dict = _instances.Dict
-List = _instances.List
-Set = _instances.Set
-Tuple = _instances.Tuple
+Dict = _containers.Dict
+List = _containers.List
+Set = _containers.Set
+Tuple = _containers.Tuple
 
 ConstKeyDict = _internal.ConstKeyDict
+FunctionArgTuple = _internal.FunctionArgTuple
 Splat = _internal.Splat
 
 get_atomic_constant = _utils.get_atomic_constant

pytype/rewrite/abstract/classes.py

@@ -80,13 +80,13 @@ def get_attribute(self, name: str) -> Optional[base.BaseValue]:
 
   def set_attribute(self, name: str, value: base.BaseValue) -> None:
     # SimpleClass is used to model imported classes, which we treat as frozen.
-    log.info('Ignoring attribute set on %r: %s -> %r',
-             self, name, value)
+    log.info('Ignoring attribute set on %r: %s -> %r', self, name, value)
 
   def instantiate(self) -> 'FrozenInstance':
     """Creates an instance of this class."""
     if self._canonical_instance:
-      log.info('Getting cached instance of class %s', self.full_name)
+      if self.module not in ('builtins', 'typing'):
+        log.info('Getting cached instance of class %s', self.full_name)
       return self._canonical_instance
     log.info('Instantiating class %s', self.full_name)
     for setup_method_name in self.setup_methods:
@@ -208,5 +208,30 @@ def _attrs(self):
   def set_attribute(self, name: str, value: base.BaseValue) -> None:
     # The VM may try to set an attribute on a frozen instance in the process of
     # analyzing a class's methods. This is fine; we just ignore it.
-    log.info('Ignoring attribute set on %r: %s -> %r',
-             self, name, value)
+    log.info('Ignoring attribute set on %r: %s -> %r', self, name, value)
+
+
+class Module(BaseInstance):
+  """A module."""
+
+  def __init__(self, ctx: base.ContextType, name: str):
+    cls = ctx.abstract_loader.load_builtin('module')
+    super().__init__(ctx, cls, members={})
+    self.name = name
+
+  def __repr__(self):
+    return f'Module({self.name})'
+
+  @property
+  def _attrs(self):
+    return (self.name,)
+
+  def set_attribute(self, name: str, value: base.BaseValue) -> None:
+    # We don't allow modifying imported modules.
+    log.info('Ignoring attribute set on %r: %s -> %r', self, name, value)
+
+  def get_attribute(self, name: str) -> Optional[base.BaseValue]:
+    try:
+      return self._ctx.abstract_loader.load_value(self.name, name)
+    except KeyError:
+      return super().get_attribute(name)

pytype/rewrite/abstract/instances.py → pytype/rewrite/abstract/containers.py

@@ -1,4 +1,4 @@
-"""Abstract representations of class instances."""
+"""Abstract representations of builtin containers."""
 
 import logging
 

pytype/rewrite/abstract/instances_test.py → pytype/rewrite/abstract/containers_test.py

@@ -1,7 +1,7 @@
 from typing import Dict, List, Set, Tuple
 
 from pytype.rewrite.abstract import base
-from pytype.rewrite.abstract import instances
+from pytype.rewrite.abstract import containers
 from pytype.rewrite.tests import test_utils
 from typing_extensions import assert_type
 
@@ -22,7 +22,7 @@ class ListTest(BaseTest):
 
   def test_constant_type(self):
     a = self.const_var("a")
-    c = instances.List(self.ctx, [a])
+    c = containers.List(self.ctx, [a])
     assert_type(c.constant, List[_AbstractVariable])
 
 
@@ -31,15 +31,15 @@ class DictTest(BaseTest):
   def test_constant_type(self):
     a = self.const_var("a")
     b = self.const_var("b")
-    c = instances.Dict(self.ctx, {a: b})
+    c = containers.Dict(self.ctx, {a: b})
     assert_type(c.constant, Dict[_AbstractVariable, _AbstractVariable])
 
 
 class SetTest(BaseTest):
 
   def test_constant_type(self):
     a = self.const_var("a")
-    c = instances.Set(self.ctx, {a})
+    c = containers.Set(self.ctx, {a})
     assert_type(c.constant, Set[_AbstractVariable])
 
 
@@ -48,7 +48,7 @@ class TupleTest(BaseTest):
   def test_constant_type(self):
     a = self.const_var("a")
     b = self.const_var("b")
-    c = instances.Tuple(self.ctx, (a, b))
+    c = containers.Tuple(self.ctx, (a, b))
     assert_type(c.constant, Tuple[_AbstractVariable, ...])
 
 

pytype/rewrite/abstract/internal.py

@@ -1,6 +1,6 @@
 """Abstract types used internally by pytype."""
 
-from typing import Any, Dict, Sequence
+from typing import Dict, Sequence, Tuple
 
 import immutabledict
 
@@ -17,7 +17,7 @@ class ConstKeyDict(base.BaseValue):
   Used by the python interpreter to construct function args.
   """
 
-  def __init__(self, ctx: base.ContextType, constant: Dict[Any, _Variable]):
+  def __init__(self, ctx: base.ContextType, constant: Dict[str, _Variable]):
     super().__init__(ctx)
     assert isinstance(constant, dict), constant
     self.constant = constant
@@ -30,6 +30,22 @@ def _attrs(self):
     return (immutabledict.immutabledict(self.constant),)
 
 
+class FunctionArgTuple(base.BaseValue):
+  """Representation of a function arg tuple."""
+
+  def __init__(self, ctx: base.ContextType, constant: Tuple[_Variable, ...]):
+    super().__init__(ctx)
+    assert isinstance(constant, tuple), constant
+    self.constant = constant
+
+  def __repr__(self):
+    return f"FunctionArgTuple({self.constant!r})"
+
+  @property
+  def _attrs(self):
+    return (self.constant,)
+
+
 class Splat(base.BaseValue):
   """Representation of unpacked iterables.
 

pytype/rewrite/frame.py

@@ -221,7 +221,7 @@ def load_global(self, name) -> _AbstractVariable:
       return self.load_builtin(name)
 
   def load_builtin(self, name) -> _AbstractVariable:
-    builtin = self._ctx.abstract_loader.load_builtin_by_name(name)
+    builtin = self._ctx.abstract_loader.load_builtin(name)
     return builtin.to_variable(name)
 
   def load_deref(self, name) -> _AbstractVariable:
@@ -392,8 +392,15 @@ def byte_RESUME(self, opcode):
   # Load and store operations
 
   def byte_LOAD_CONST(self, opcode):
-    constant = self._ctx.consts[self._code.consts[opcode.arg]]
-    self._stack.push(constant.to_variable())
+    const = self._code.consts[opcode.arg]
+    if isinstance(const, tuple):
+      # Tuple literals with all primitive elements are stored as a single raw
+      # constant; we need to wrap each element in a variable for consistency
+      # with tuples created via BUILD_TUPLE
+      val = self._ctx.abstract_loader.build_tuple(const)
+    else:
+      val = self._ctx.consts[const]
+    self._stack.push(val.to_variable())
 
   def byte_RETURN_VALUE(self, opcode):
     self._returns.append(self._stack.pop())
@@ -524,6 +531,12 @@ def byte_LOAD_METHOD(self, opcode):
     self._stack.push(self._ctx.consts.singles['NULL'].to_variable())
     self._stack.push(self._load_attr(instance_var, method_name))
 
+  def byte_IMPORT_NAME(self, opcode):
+    full_name = opcode.argval
+    unused_level_var, unused_fromlist = self._stack.popn(2)
+    module = abstract.Module(self._ctx, full_name)
+    return self._stack.push(module.to_variable())
+
   # ---------------------------------------------------------------
   # Function and method calls
 
@@ -558,6 +571,40 @@ def byte_CALL_FUNCTION(self, opcode):
     callargs = abstract.Args(posargs=tuple(args), frame=self)
     self._call_function(func, callargs)
 
+  def _unpack_starargs(self, starargs):
+    # TODO(b/331853896): This follows vm_utils.ensure_unpacked_starargs, but
+    # does not yet handle indefinite iterables.
+    posargs = starargs.get_atomic_value()
+    if isinstance(posargs, abstract.FunctionArgTuple):
+      # This has already been converted
+      pass
+    elif isinstance(posargs, abstract.Tuple):
+      posargs = abstract.FunctionArgTuple(self._ctx, posargs.constant)
+    elif isinstance(posargs, tuple):
+      posargs = abstract.FunctionArgTuple(self._ctx, posargs)
+    else:
+      assert False, f'unexpected posargs type: {posargs}: {type(posargs)}'
+    return posargs
+
+  def _unpack_starstarargs(self, starstarargs):
+    kwargs = abstract.get_atomic_constant(starstarargs, dict)
+    return {abstract.get_atomic_constant(k, str): v
+            for k, v in kwargs.items()}
+
+  def byte_CALL_FUNCTION_EX(self, opcode):
+    if opcode.arg & _Flags.CALL_FUNCTION_EX_HAS_KWARGS:
+      starstarargs = self._stack.pop()
+      kwargs = self._unpack_starstarargs(starstarargs)
+    else:
+      kwargs = _EMPTY_MAP
+    starargs = self._stack.pop()
+    posargs = self._unpack_starargs(starargs).constant
+    func = self._stack.pop()
+    if self._code.python_version >= (3, 11):
+      self._stack.pop_and_discard()
+    callargs = abstract.Args(posargs=posargs, kwargs=kwargs, frame=self)
+    self._call_function(func, callargs)
+
   def byte_CALL_METHOD(self, opcode):
     args = self._stack.popn(opcode.arg)
     func = self._stack.pop()
@@ -615,6 +662,8 @@ def byte_BUILD_CONST_KEY_MAP(self, opcode):
     # to abstract objects because they are used internally to construct function
     # call args.
     keys = abstract.get_atomic_constant(keys, tuple)
+    # Unpack the keys into raw strings.
+    keys = [abstract.get_atomic_constant(k, str) for k in keys]
     assert len(keys) == n_elts
     vals = self._stack.popn(n_elts)
     ret = dict(zip(keys, vals))

pytype/rewrite/frame_test.py

@@ -635,6 +635,16 @@ def f(x, *, y):
     self.assertConstantVar(callargs.posargs[0], 1)
     self.assertConstantVar(callargs.kwargs['y'], 2)
 
+  @test_utils.skipBeforePy((3, 11), 'Relies on 3.11+ bytecode')
+  def test_call_function_ex_no_crash(self):
+    frame = self._make_frame("""
+      def f(x, y, z):
+        pass
+      a = (1, 2)
+      f(*a, z=3)
+    """)
+    frame.run()
+
 
 if __name__ == '__main__':
   unittest.main()