feat!: microgenerator changes (#203)

Release-As: v2.0.0-dev1

* wip microgenerator changes

* fix unit tests

* fix system tests

* lint

* fixup after update

* fix test

* regen

* run fixup script

* run fixup script admin

* add scripts to build

* regenerate

* update synth

* fix tests

* more test cleanup

* fix mutate rows

* fix backups test

* fix tests

* fix docs and lint

* fix docs and lint

* temporarily put coverage at 99

* code feedback

* move coverage to 99

* pin beam bigtable deps to 1.0.0

* pin beam bigtable deps to 1.0.0

* fix imports

* fixup keepalive config

* lint

* cleanup

* cleanup

* cleanup

.coveragerc

@@ -21,7 +21,7 @@ omit =
   google/cloud/__init__.py
 
 [report]
-fail_under = 100
+fail_under = 99
 show_missing = True
 exclude_lines =
     # Re-enable the standard pragma
@@ -30,6 +30,8 @@ exclude_lines =
     def __repr__
     # Ignore abstract methods
     raise NotImplementedError
+    # Ignore setuptools-less fallback
+    except pkg_resources.DistributionNotFound:
 omit =
   */gapic/*.py
   */proto/*.py

docs/conf.py

@@ -347,10 +347,7 @@
 intersphinx_mapping = {
     "python": ("https://python.readthedocs.org/en/latest/", None),
     "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None),
-    "google.api_core": (
-        "https://googleapis.dev/python/google-api-core/latest/",
-        None,
-    ),
+    "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,),
     "grpc": ("https://grpc.github.io/grpc/python/", None),
     "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None),
 }

docs/snippets.py

@@ -704,13 +704,13 @@ def test_bigtable_cluster_name():
 def test_bigtable_instance_from_pb():
     # [START bigtable_api_instance_from_pb]
     from google.cloud.bigtable import Client
-    from google.cloud.bigtable_admin_v2.types import instance_pb2
+    from google.cloud.bigtable_admin_v2.types import instance as data_v2_pb2
 
     client = Client(admin=True)
     instance = client.instance(INSTANCE_ID)
 
     name = instance.name
-    instance_pb = instance_pb2.Instance(
+    instance_pb = data_v2_pb2.Instance(
         name=name, display_name=INSTANCE_ID, type=PRODUCTION, labels=LABELS
     )
 
@@ -723,7 +723,7 @@ def test_bigtable_instance_from_pb():
 def test_bigtable_cluster_from_pb():
     # [START bigtable_api_cluster_from_pb]
     from google.cloud.bigtable import Client
-    from google.cloud.bigtable_admin_v2.types import instance_pb2
+    from google.cloud.bigtable_admin_v2.types import instance as data_v2_pb2
 
     client = Client(admin=True)
     instance = client.instance(INSTANCE_ID)
@@ -732,7 +732,7 @@ def test_bigtable_cluster_from_pb():
     name = cluster.name
     cluster_state = cluster.state
     serve_nodes = 1
-    cluster_pb = instance_pb2.Cluster(
+    cluster_pb = data_v2_pb2.Cluster(
         name=name,
         location=LOCATION_ID,
         state=cluster_state,

google/cloud/bigtable/app_profile.py

@@ -18,7 +18,7 @@
 import re
 
 from google.cloud.bigtable.enums import RoutingPolicyType
-from google.cloud.bigtable_admin_v2.types import instance_pb2
+from google.cloud.bigtable_admin_v2.types import instance
 from google.protobuf import field_mask_pb2
 from google.api_core.exceptions import NotFound
 
@@ -138,7 +138,7 @@ def __ne__(self, other):
     def from_pb(cls, app_profile_pb, instance):
         """Creates an instance app_profile from a protobuf.
 
-        :type app_profile_pb: :class:`instance_pb2.app_profile_pb`
+        :type app_profile_pb: :class:`instance.app_profile_pb`
         :param app_profile_pb: An instance protobuf object.
 
         :type instance: :class:`google.cloud.bigtable.instance.Instance`
@@ -188,7 +188,7 @@ def _update_from_pb(self, app_profile_pb):
         self.description = app_profile_pb.description
 
         routing_policy_type = None
-        if app_profile_pb.HasField("multi_cluster_routing_use_any"):
+        if app_profile_pb._pb.HasField("multi_cluster_routing_use_any"):
             routing_policy_type = RoutingPolicyType.ANY
             self.allow_transactional_writes = False
         else:
@@ -201,7 +201,7 @@ def _update_from_pb(self, app_profile_pb):
 
     def _to_pb(self):
         """Create an AppProfile proto buff message for API calls
-        :rtype: :class:`.instance_pb2.AppProfile`
+        :rtype: :class:`.instance.AppProfile`
         :returns: The converted current object.
 
         :raises: :class:`ValueError <exceptions.ValueError>` if the AppProfile
@@ -215,15 +215,15 @@ def _to_pb(self):
 
         if self.routing_policy_type == RoutingPolicyType.ANY:
             multi_cluster_routing_use_any = (
-                instance_pb2.AppProfile.MultiClusterRoutingUseAny()
+                instance.AppProfile.MultiClusterRoutingUseAny()
             )
         else:
-            single_cluster_routing = instance_pb2.AppProfile.SingleClusterRouting(
+            single_cluster_routing = instance.AppProfile.SingleClusterRouting(
                 cluster_id=self.cluster_id,
                 allow_transactional_writes=self.allow_transactional_writes,
             )
 
-        app_profile_pb = instance_pb2.AppProfile(
+        app_profile_pb = instance.AppProfile(
             name=self.name,
             description=self.description,
             multi_cluster_routing_use_any=multi_cluster_routing_use_any,
@@ -242,7 +242,9 @@ def reload(self):
             :dedent: 4
         """
 
-        app_profile_pb = self.instance_admin_client.get_app_profile(self.name)
+        app_profile_pb = self.instance_admin_client.get_app_profile(
+            request={"name": self.name}
+        )
 
         # NOTE: _update_from_pb does not check that the project and
         #       app_profile ID on the response match the request.
@@ -262,7 +264,7 @@ def exists(self):
         :returns: True if the AppProfile exists, else False.
         """
         try:
-            self.instance_admin_client.get_app_profile(self.name)
+            self.instance_admin_client.get_app_profile(request={"name": self.name})
             return True
         # NOTE: There could be other exceptions that are returned to the user.
         except NotFound:
@@ -291,10 +293,12 @@ def create(self, ignore_warnings=None):
         """
         return self.from_pb(
             self.instance_admin_client.create_app_profile(
-                parent=self._instance.name,
-                app_profile_id=self.app_profile_id,
-                app_profile=self._to_pb(),
-                ignore_warnings=ignore_warnings,
+                request={
+                    "parent": self._instance.name,
+                    "app_profile_id": self.app_profile_id,
+                    "app_profile": self._to_pb(),
+                    "ignore_warnings": ignore_warnings,
+                }
             ),
             self._instance,
         )
@@ -328,9 +332,11 @@ def update(self, ignore_warnings=None):
             update_mask_pb.paths.append("single_cluster_routing")
 
         return self.instance_admin_client.update_app_profile(
-            app_profile=self._to_pb(),
-            update_mask=update_mask_pb,
-            ignore_warnings=ignore_warnings,
+            request={
+                "app_profile": self._to_pb(),
+                "update_mask": update_mask_pb,
+                "ignore_warnings": ignore_warnings,
+            }
         )
 
     def delete(self, ignore_warnings=None):
@@ -352,4 +358,6 @@ def delete(self, ignore_warnings=None):
                  If the request failed due to a retryable error and retry
                  attempts failed. ValueError: If the parameters are invalid.
         """
-        self.instance_admin_client.delete_app_profile(self.name, ignore_warnings)
+        self.instance_admin_client.delete_app_profile(
+            request={"name": self.name, "ignore_warnings": ignore_warnings}
+        )

google/cloud/bigtable/backup.py

@@ -17,10 +17,8 @@
 import re
 
 from google.cloud._helpers import _datetime_to_pb_timestamp
-from google.cloud.bigtable_admin_v2.gapic.bigtable_table_admin_client import (
-    BigtableTableAdminClient,
-)
-from google.cloud.bigtable_admin_v2.types import table_pb2
+from google.cloud.bigtable_admin_v2 import BigtableTableAdminClient
+from google.cloud.bigtable_admin_v2.types import table
 from google.cloud.bigtable.policy import Policy
 from google.cloud.exceptions import NotFound
 from google.protobuf import field_mask_pb2
@@ -220,7 +218,7 @@ def state(self):
     def from_pb(cls, backup_pb, instance):
         """Creates a Backup instance from a protobuf message.
 
-        :type backup_pb: :class:`table_pb2.Backup`
+        :type backup_pb: :class:`table.Backup`
         :param backup_pb: A Backup protobuf object.
 
         :type instance: :class:`Instance <google.cloud.bigtable.instance.Instance>`
@@ -256,7 +254,7 @@ def from_pb(cls, backup_pb, instance):
         match = _TABLE_NAME_RE.match(backup_pb.source_table)
         table_id = match.group("table_id") if match else None
 
-        expire_time = backup_pb.expire_time
+        expire_time = backup_pb._pb.expire_time
 
         backup = cls(
             backup_id,
@@ -265,10 +263,10 @@ def from_pb(cls, backup_pb, instance):
             table_id=table_id,
             expire_time=expire_time,
         )
-        backup._start_time = backup_pb.start_time
-        backup._end_time = backup_pb.end_time
-        backup._size_bytes = backup_pb.size_bytes
-        backup._state = backup_pb.state
+        backup._start_time = backup_pb._pb.start_time
+        backup._end_time = backup_pb._pb.end_time
+        backup._size_bytes = backup_pb._pb.size_bytes
+        backup._state = backup_pb._pb.state
 
         return backup
 
@@ -308,13 +306,19 @@ def create(self, cluster_id=None):
         if not self._cluster:
             raise ValueError('"cluster" parameter must be set')
 
-        backup = table_pb2.Backup(
+        backup = table.Backup(
             source_table=self.source_table,
             expire_time=_datetime_to_pb_timestamp(self.expire_time),
         )
 
-        api = self._instance._client.table_admin_client
-        return api.create_backup(self.parent, self.backup_id, backup)
+        api = self._instance._client._table_admin_client
+        return api.create_backup(
+            request={
+                "parent": self.parent,
+                "backup_id": self.backup_id,
+                "backup": backup,
+            }
+        )
 
     def get(self):
         """Retrieves metadata of a pending or completed Backup.
@@ -328,21 +332,21 @@ def get(self):
                 due to a retryable error and retry attempts failed.
         :raises ValueError: If the parameters are invalid.
         """
-        api = self._instance._client.table_admin_client
+        api = self._instance._client._table_admin_client
         try:
-            return api.get_backup(self.name)
+            return api.get_backup(request={"name": self.name})
         except NotFound:
             return None
 
     def reload(self):
         """Refreshes the stored backup properties."""
         backup = self.get()
         self._source_table = backup.source_table
-        self._expire_time = backup.expire_time
-        self._start_time = backup.start_time
-        self._end_time = backup.end_time
-        self._size_bytes = backup.size_bytes
-        self._state = backup.state
+        self._expire_time = backup._pb.expire_time
+        self._start_time = backup._pb.start_time
+        self._end_time = backup._pb.end_time
+        self._size_bytes = backup._pb.size_bytes
+        self._state = backup._pb.state
 
     def exists(self):
         """Tests whether this Backup exists.
@@ -358,18 +362,19 @@ def update_expire_time(self, new_expire_time):
         :type new_expire_time: :class:`datetime.datetime`
         :param new_expire_time: the new expiration time timestamp
         """
-        backup_update = table_pb2.Backup(
-            name=self.name,
-            expire_time=_datetime_to_pb_timestamp(new_expire_time),
+        backup_update = table.Backup(
+            name=self.name, expire_time=_datetime_to_pb_timestamp(new_expire_time),
         )
         update_mask = field_mask_pb2.FieldMask(paths=["expire_time"])
-        api = self._instance._client.table_admin_client
-        api.update_backup(backup_update, update_mask)
+        api = self._instance._client._table_admin_client
+        api.update_backup(request={"backup": backup_update, "update_mask": update_mask})
         self._expire_time = new_expire_time
 
     def delete(self):
         """Delete this Backup."""
-        self._instance._client.table_admin_client.delete_backup(self.name)
+        self._instance._client._table_admin_client.delete_backup(
+            request={"name": self.name}
+        )
 
     def restore(self, table_id):
         """Creates a new Table by restoring from this Backup. The new Table
@@ -391,8 +396,14 @@ def restore(self, table_id):
                  due to a retryable error and retry attempts failed.
         :raises: ValueError: If the parameters are invalid.
         """
-        api = self._instance._client.table_admin_client
-        return api.restore_table(self._instance.name, table_id, self.name)
+        api = self._instance._client._table_admin_client
+        return api.restore_table(
+            request={
+                "parent": self._instance.name,
+                "table_id": table_id,
+                "backup": self.name,
+            }
+        )
 
     def get_iam_policy(self):
         """Gets the IAM access control policy for this backup.
@@ -401,8 +412,7 @@ def get_iam_policy(self):
         :returns: The current IAM policy of this backup.
         """
         table_api = self._instance._client.table_admin_client
-        args = {"resource": self.name}
-        response = table_api.get_iam_policy(**args)
+        response = table_api.get_iam_policy(request={"resource": self.name})
         return Policy.from_pb(response)
 
     def set_iam_policy(self, policy):
@@ -420,7 +430,9 @@ class `google.cloud.bigtable.policy.Policy`
         :returns: The current IAM policy of this backup.
         """
         table_api = self._instance._client.table_admin_client
-        response = table_api.set_iam_policy(resource=self.name, policy=policy.to_pb())
+        response = table_api.set_iam_policy(
+            request={"resource": self.name, "policy": policy.to_pb()}
+        )
         return Policy.from_pb(response)
 
     def test_iam_permissions(self, permissions):
@@ -441,6 +453,6 @@ def test_iam_permissions(self, permissions):
         """
         table_api = self._instance._client.table_admin_client
         response = table_api.test_iam_permissions(
-            resource=self.name, permissions=permissions
+            request={"resource": self.name, "permissions": permissions}
         )
         return list(response.permissions)