Merge pull request #1620 from google/google_sync

Google sync

.github/workflows/build.yml

@@ -55,7 +55,7 @@ jobs:
     # used to build Python there.  It is the latter that determines
     # the wheel's platform tag.
     # https://github.com/actions/virtual-environments/issues/696
-    runs-on: macos-latest
+    runs-on: macos-13
     strategy:
       matrix:
         python_version: ['3.8', '3.9', '3.10', '3.11']

pytype/abstract/_classes.py

@@ -42,8 +42,6 @@ def call(self, node, func, args, alias_map=None):
     args = args.simplify(node, self.ctx)
     funcvar, name = args.posargs[0:2]
     kwargs = args.namedargs
-    # TODO(mdemello): Check if there are any changes between python2 and
-    # python3 in the final metaclass computation.
     # TODO(b/123450483): Any remaining kwargs need to be passed to the
     # metaclass.
     metaclass = kwargs.get("metaclass", None)

pytype/datatypes.py

@@ -5,9 +5,15 @@
 import itertools
 from typing import Dict, Optional, TypeVar
 
+import immutabledict
+
 _K = TypeVar("_K")
 _V = TypeVar("_V")
 
+# Public alias for immutabledict to save users the extra import.
+immutabledict = immutabledict.immutabledict
+EMPTY_MAP = immutabledict()
+
 
 class UnionFind:
   r"""A disjoint-set data structure for `AliasingDict`.

pytype/rewrite/CMakeLists.txt

@@ -58,6 +58,7 @@ py_library(
   DEPS
     pytype.pytd.pytd
     pytype.rewrite.abstract.abstract
+    pytype.rewrite.overlays.overlays
 )
 
 py_test(
@@ -71,14 +72,39 @@ py_test(
     pytype.rewrite.tests.test_utils
 )
 
+py_library(
+  NAME
+    function_call_helper
+  SRCS
+    function_call_helper.py
+  DEPS
+    .context
+    pytype.utils
+    pytype.rewrite.abstract.abstract
+    pytype.rewrite.flow.flow
+)
+
+py_test(
+  NAME
+    function_call_helper_test
+  SRCS
+    function_call_helper_test.py
+  DEPS
+    .frame
+    pytype.rewrite.abstract.abstract
+    pytype.rewrite.tests.test_utils
+)
+
 py_library(
   NAME
     frame
   SRCS
     frame.py
   DEPS
     .context
+    .function_call_helper
     .stack
+    pytype.utils
     pytype.blocks.blocks
     pytype.rewrite.abstract.abstract
     pytype.rewrite.flow.flow
@@ -106,7 +132,6 @@ py_library(
     pytype.load_pytd
     pytype.pytd.pytd
     pytype.rewrite.abstract.abstract
-    pytype.rewrite.overlays.overlays
 )
 
 py_test(

pytype/rewrite/abstract/CMakeLists.txt

@@ -48,6 +48,8 @@ py_library(
   DEPS
     .base
     .functions
+    pytype.utils
+    pytype.pytd.pytd
     pytype.types.types
 )
 
@@ -81,8 +83,6 @@ py_test(
   DEPS
     .base
     .containers
-    .internal
-    pytype.rewrite.flow.flow
     pytype.rewrite.tests.test_utils
 )
 
@@ -93,6 +93,10 @@ py_library(
     functions.py
   DEPS
     .base
+    .containers
+    .internal
+    .utils
+    pytype.utils
     pytype.blocks.blocks
     pytype.pytd.pytd
 )

pytype/rewrite/abstract/abstract.py

@@ -36,7 +36,7 @@
 Set = _containers.Set
 Tuple = _containers.Tuple
 
-ConstKeyDict = _internal.ConstKeyDict
+FunctionArgDict = _internal.FunctionArgDict
 FunctionArgTuple = _internal.FunctionArgTuple
 Splat = _internal.Splat
 

pytype/rewrite/abstract/base.py

@@ -24,6 +24,7 @@ class ContextType(Protocol):
   abstract_loader: Any
   pytd_converter: Any
   consts: Any
+  types: Any
 
 
 class BaseValue(types.BaseValue, abc.ABC):
@@ -136,6 +137,9 @@ def _attrs(self):
   def instantiate(self) -> 'Singleton':
     return self
 
+  def get_attribute(self, name: str) -> 'Singleton':
+    return self
+
 
 class Union(BaseValue):
   """Union of values."""
@@ -161,4 +165,5 @@ def _attrs(self):
   def instantiate(self):
     return Union(self._ctx, tuple(o.instantiate() for o in self.options))
 
+
 AbstractVariableType = variables.Variable[BaseValue]

pytype/rewrite/abstract/classes.py

@@ -6,7 +6,8 @@
 
 from typing import Dict, List, Mapping, Optional, Protocol, Sequence
 
-import immutabledict
+from pytype import datatypes
+from pytype.pytd import mro as mro_lib
 from pytype.rewrite.abstract import base
 from pytype.rewrite.abstract import functions as functions_lib
 from pytype.types import types
@@ -36,13 +37,18 @@ def __init__(
       ctx: base.ContextType,
       name: str,
       members: Dict[str, base.BaseValue],
+      bases: Sequence['SimpleClass'] = (),
+      keywords: Mapping[str, base.BaseValue] = datatypes.EMPTY_MAP,
       module: Optional[str] = None,
   ):
     super().__init__(ctx)
     self.name = name
     self.members = members
+    self.bases = bases
+    self.keywords = keywords
     self.module = module
     self._canonical_instance: Optional['FrozenInstance'] = None
+    self._mro: Optional[Sequence['SimpleClass']] = None
 
     if isinstance((init := members.get('__init__')),
                   functions_lib.SimpleFunction):
@@ -76,8 +82,17 @@ def full_name(self):
     else:
       return self.name
 
+  @property
+  def metaclass(self) -> Optional[base.BaseValue]:
+    return self.keywords.get('metaclass')
+
   def get_attribute(self, name: str) -> Optional[base.BaseValue]:
-    return self.members.get(name)
+    if name in self.members:
+      return self.members[name]
+    mro = self.mro()
+    if len(mro) > 1:
+      return mro[1].get_attribute(name)
+    return None
 
   def set_attribute(self, name: str, value: base.BaseValue) -> None:
     # SimpleClass is used to model imported classes, which we treat as frozen.
@@ -95,7 +110,7 @@ def instantiate(self) -> 'FrozenInstance':
       if isinstance(setup_method, functions_lib.InterpreterFunction):
         _ = setup_method.bind_to(self).analyze()
     constructor = self.get_attribute(self.constructor)
-    if constructor:
+    if constructor and constructor.full_name != 'builtins.object.__new__':
       log.error('Custom __new__ not yet implemented')
     instance = MutableInstance(self._ctx, self)
     for initializer_name in self.initializers:
@@ -116,16 +131,40 @@ def call(self, args: functions_lib.Args) -> ClassCallReturn:
         _ = initializer.bind_to(instance).call(args)
     return ClassCallReturn(instance)
 
+  def mro(self) -> Sequence['SimpleClass']:
+    if self._mro:
+      return self._mro
+    if self.full_name == 'builtins.object':
+      self._mro = mro = [self]
+      return mro
+    bases = list(self.bases)
+    obj_type = self._ctx.types[object]
+    if not bases or bases[-1] != obj_type:
+      bases.append(obj_type)
+    mro_bases = [[self]] + [list(base.mro()) for base in bases] + [bases]
+    self._mro = mro = mro_lib.MROMerge(mro_bases)
+    return mro
+
+  def set_type_parameters(self, params):
+    # A dummy implementation to let type annotations with parameters not crash.
+    del params  # not implemented yet
+    # We eventually want to return a new class with the type parameters set
+    return self
+
 
 class InterpreterClass(SimpleClass):
   """Class defined in the current module."""
 
   def __init__(
-      self, ctx: base.ContextType, name: str,
+      self,
+      ctx: base.ContextType,
+      name: str,
       members: Dict[str, base.BaseValue],
+      bases: Sequence[SimpleClass],
+      keywords: Mapping[str, base.BaseValue],
       functions: Sequence[functions_lib.InterpreterFunction],
       classes: Sequence['InterpreterClass']):
-    super().__init__(ctx, name, members)
+    super().__init__(ctx, name, members, bases, keywords)
     # Functions and classes defined in this class's body. Unlike 'members',
     # ignores the effects of post-definition transformations like decorators.
     self.functions = functions
@@ -136,7 +175,7 @@ def __repr__(self):
 
   @property
   def _attrs(self):
-    return (self.name, immutabledict.immutabledict(self.members))
+    return (self.name, datatypes.immutabledict(self.members))
 
 
 class BaseInstance(base.BaseValue):
@@ -174,7 +213,7 @@ def __repr__(self):
 
   @property
   def _attrs(self):
-    return (self.cls, immutabledict.immutabledict(self.members))
+    return (self.cls, datatypes.immutabledict(self.members))
 
   def set_attribute(self, name: str, value: base.BaseValue) -> None:
     if name in self.members:
@@ -195,7 +234,7 @@ class FrozenInstance(BaseInstance):
 
   def __init__(self, ctx: base.ContextType, instance: MutableInstance):
     super().__init__(
-        ctx, instance.cls, immutabledict.immutabledict(instance.members))
+        ctx, instance.cls, datatypes.immutabledict(instance.members))
 
   def __repr__(self):
     return f'FrozenInstance({self.cls.name})'

pytype/rewrite/abstract/classes_test.py

@@ -16,6 +16,12 @@ def test_get_nonexistent_attribute(self):
     cls = classes.SimpleClass(self.ctx, 'X', {})
     self.assertIsNone(cls.get_attribute('x'))
 
+  def test_get_parent_attribute(self):
+    x = self.ctx.consts[5]
+    parent = classes.SimpleClass(self.ctx, 'Parent', {'x': x})
+    child = classes.SimpleClass(self.ctx, 'Child', {}, bases=[parent])
+    self.assertEqual(child.get_attribute('x'), x)
+
   def test_instantiate(self):
     cls = classes.SimpleClass(self.ctx, 'X', {})
     instance = cls.instantiate()
@@ -26,6 +32,11 @@ def test_call(self):
     instance = cls.call(functions.Args()).get_return_value()
     self.assertEqual(instance.cls, cls)
 
+  def test_mro(self):
+    parent = classes.SimpleClass(self.ctx, 'Parent', {})
+    child = classes.SimpleClass(self.ctx, 'Child', {}, bases=[parent])
+    self.assertEqual(child.mro(), [child, parent, self.ctx.types[object]])
+
 
 class MutableInstanceTest(test_utils.ContextfulTestBase):
 

pytype/rewrite/abstract/containers.py

@@ -27,43 +27,46 @@ def __repr__(self):
   def append(self, var: _Variable) -> 'List':
     return List(self._ctx, self.constant + [var])
 
-  def extend(self, var: _Variable) -> base.BaseValue:
-    try:
-      val = var.get_atomic_value()
-    except ValueError:
-      # This list has multiple possible values, so it is no longer a constant.
-      return self._ctx.abstract_loader.load_raw_type(list).instantiate()
-    if isinstance(val, List):
-      new_constant = self.constant + val.constant
-    else:
-      splat = internal.Splat(self._ctx, val)
-      new_constant = self.constant + [splat.to_variable()]
+  def extend(self, val: 'List') -> 'List':
+    new_constant = self.constant + val.constant
     return List(self._ctx, new_constant)
 
 
 class Dict(base.PythonConstant[_Dict[_Variable, _Variable]]):
   """Representation of a Python dict."""
 
   def __init__(
-      self, ctx: base.ContextType, constant: _Dict[_Variable, _Variable]
+      self, ctx: base.ContextType, constant: _Dict[_Variable, _Variable],
   ):
     assert isinstance(constant, dict), constant
     super().__init__(ctx, constant)
-    self.indefinite = False
 
   def __repr__(self):
     return f'Dict({self.constant!r})'
 
+  @classmethod
+  def from_function_arg_dict(
+      cls, ctx: base.ContextType, val: internal.FunctionArgDict
+  ) -> 'Dict':
+    assert not val.indefinite
+    new_constant = {
+        ctx.consts[k].to_variable(): v
+        for k, v in val.constant.items()
+    }
+    return cls(ctx, new_constant)
+
   def setitem(self, key: _Variable, val: _Variable) -> 'Dict':
     return Dict(self._ctx, {**self.constant, key: val})
 
-  def update(self, var: _Variable) -> base.BaseValue:
-    try:
-      val = utils.get_atomic_constant(var, dict)
-    except ValueError:
-      # This dict has multiple possible values, so it is no longer a constant.
-      return self._ctx.abstract_loader.load_raw_type(dict).instantiate()
-    return Dict(self._ctx, {**self.constant, **val})
+  def update(self, val: 'Dict') -> base.BaseValue:
+    return Dict(self._ctx, {**self.constant, **val.constant})
+
+  def to_function_arg_dict(self) -> internal.FunctionArgDict:
+    new_const = {
+        utils.get_atomic_constant(k, str): v
+        for k, v in self.constant.items()
+    }
+    return internal.FunctionArgDict(self._ctx, new_const)
 
 
 class Set(base.PythonConstant[_Set[_Variable]]):