Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat!: microgenerator changes #203

Merged
merged 35 commits into from Feb 11, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
9e41c07
wip microgenerator changes
kolea2 Oct 27, 2020
8f6f90a
fix unit tests
kolea2 Jan 21, 2021
2239cd0
fix system tests
kolea2 Jan 22, 2021
136a0e8
lint
kolea2 Jan 25, 2021
e52f324
Merge branch 'master' of github.com:kolea2/python-bigtable into micro…
kolea2 Jan 25, 2021
8e9d5c9
fixup after update
kolea2 Jan 25, 2021
ce4a2ea
fix test
kolea2 Jan 25, 2021
ab2d728
regen
kolea2 Feb 1, 2021
c6e75b1
run fixup script
kolea2 Feb 2, 2021
32fa3b2
run fixup script admin
kolea2 Feb 2, 2021
48520c2
add scripts to build
kolea2 Feb 2, 2021
947babb
regenerate
kolea2 Feb 2, 2021
1f9d175
update synth
kolea2 Feb 2, 2021
bf2a7d0
fix tests
kolea2 Feb 2, 2021
293909d
more test cleanup
kolea2 Feb 3, 2021
48f8a10
fix mutate rows
kolea2 Feb 3, 2021
71bef98
Merge branch 'master' of github.com:googleapis/python-bigtable into m…
kolea2 Feb 3, 2021
0a0604a
fix backups test
kolea2 Feb 3, 2021
5e920a2
Merge branch 'master' of github.com:googleapis/python-bigtable into m…
kolea2 Feb 3, 2021
3f34761
fix tests
kolea2 Feb 4, 2021
5469fe2
fix docs and lint
kolea2 Feb 4, 2021
19b1883
fix docs and lint
kolea2 Feb 4, 2021
11f292b
temporarily put coverage at 99
kolea2 Feb 4, 2021
226a760
code feedback
kolea2 Feb 4, 2021
d89876f
Merge branch 'master' of github.com:googleapis/python-bigtable into m…
kolea2 Feb 8, 2021
ad82e4a
move coverage to 99
kolea2 Feb 8, 2021
a5150da
pin beam bigtable deps to 1.0.0
kolea2 Feb 8, 2021
0353c11
pin beam bigtable deps to 1.0.0
kolea2 Feb 8, 2021
90a5d2c
Merge branch 'master' of github.com:googleapis/python-bigtable into m…
kolea2 Feb 10, 2021
d0e40a2
fix imports
kolea2 Feb 10, 2021
51f18f0
fixup keepalive config
kolea2 Feb 10, 2021
9163ec3
lint
kolea2 Feb 10, 2021
b7ef779
cleanup
kolea2 Feb 10, 2021
27f9283
cleanup
kolea2 Feb 11, 2021
92a3fad
cleanup
kolea2 Feb 11, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 4 additions & 4 deletions docs/snippets.py
Expand Up @@ -704,13 +704,13 @@ def test_bigtable_cluster_name():
def test_bigtable_instance_from_pb():
# [START bigtable_api_instance_from_pb]
from google.cloud.bigtable import Client
from google.cloud.bigtable_admin_v2.types import instance
from google.cloud.bigtable_admin_v2.types import instance as data_v2_pb2

client = Client(admin=True)
instance = client.instance(INSTANCE_ID)

name = instance.name
instance_pb = instance.Instance(
instance_pb = data_v2_pb2.Instance(
name=name, display_name=INSTANCE_ID, type=PRODUCTION, labels=LABELS
)

Expand All @@ -723,7 +723,7 @@ def test_bigtable_instance_from_pb():
def test_bigtable_cluster_from_pb():
# [START bigtable_api_cluster_from_pb]
from google.cloud.bigtable import Client
from google.cloud.bigtable_admin_v2.types import instance
from google.cloud.bigtable_admin_v2.types import instance as data_v2_pb2

client = Client(admin=True)
instance = client.instance(INSTANCE_ID)
Expand All @@ -732,7 +732,7 @@ def test_bigtable_cluster_from_pb():
name = cluster.name
cluster_state = cluster.state
serve_nodes = 1
cluster_pb = instance.Cluster(
cluster_pb = data_v2_pb2.Cluster(
name=name,
location=LOCATION_ID,
state=cluster_state,
Expand Down
2 changes: 1 addition & 1 deletion google/cloud/bigtable/client.py
Expand Up @@ -217,7 +217,7 @@ def project_path(self):
:rtype: str
:returns: Return a fully-qualified project string.
"""
return "projects/{project}".format(project=self.project)
return self.instance_admin_client.common_project_path(self.project)

@property
def table_data_client(self):
Expand Down
2 changes: 1 addition & 1 deletion google/cloud/bigtable/table.py
Expand Up @@ -1094,7 +1094,7 @@ def _do_mutate_retryable_rows(self):
entries=entries,
app_profile_id=self.app_profile_id,
retry=None,
# **kwargs
**kwargs
)
except (ServiceUnavailable, DeadlineExceeded, Aborted):
# If an exception, considered retryable by `RETRY_CODES`, is
Expand Down
18 changes: 10 additions & 8 deletions tests/unit/test_app_profile.py
Expand Up @@ -562,15 +562,16 @@ def test_update_app_profile_routing_any(self):

instance_api.update_app_profile.return_value = response_pb
app_profile._instance._client._instance_admin_client = instance_api
app_profile.update(ignore_warnings=ignore_warnings)
result = app_profile.update(ignore_warnings=ignore_warnings)
actual_request = client._instance_admin_client.update_app_profile.call_args_list[
0
].kwargs

self.assertEqual(actual_request, expected_request)
# self.assertIsInstance(result, operation.Operation)
# self.assertEqual(result.operation.name, self.OP_NAME)
# self.assertIsInstance(result.metadata, messages_v2_pb2.UpdateAppProfileMetadata)
self.assertEqual(
result.metadata.type_url,
"type.googleapis.com/google.bigtable.admin.v2.UpdateAppProfileMetadata",
)

def test_update_app_profile_routing_single(self):
from google.longrunning import operations_pb2
Expand Down Expand Up @@ -626,14 +627,15 @@ def test_update_app_profile_routing_single(self):
}
}

app_profile.update(ignore_warnings=ignore_warnings)
result = app_profile.update(ignore_warnings=ignore_warnings)
actual_request = client._instance_admin_client.update_app_profile.call_args_list[
0
].kwargs
self.assertEqual(actual_request, expected_request)
# self.assertIsInstance(result, operation.Operation)
# self.assertEqual(result.operation.name, self.OP_NAME)
# self.assertIsInstance(result.metadata, messages_v2_pb2.UpdateAppProfileMetadata)
self.assertEqual(
result.metadata.type_url,
"type.googleapis.com/google.bigtable.admin.v2.UpdateAppProfileMetadata",
)

def test_update_app_profile_with_wrong_routing_policy(self):
credentials = _make_credentials()
Expand Down
6 changes: 0 additions & 6 deletions tests/unit/test_cluster.py
Expand Up @@ -391,9 +391,6 @@ def test_create(self):
0
].kwargs
self.assertEqual(actual_request, expected_request)
# self.assertIsInstance(result, operation.Operation)
# self.assertEqual(result.operation.name, self.OP_NAME)
# self.assertIsInstance(result.metadata, messages_v2_pb2.CreateClusterMetadata)

def test_update(self):
import datetime
Expand Down Expand Up @@ -456,9 +453,6 @@ def test_update(self):
].kwargs

self.assertEqual(actual_request, expected_request)
# self.assertIsInstance(result, operation.Operation)
# self.assertEqual(result.operation.name, self.OP_NAME)
# self.assertIsInstance(result.metadata, messages_v2_pb2.UpdateClusterMetadata)

def test_delete(self):
from google.protobuf import empty_pb2
Expand Down
2 changes: 2 additions & 0 deletions tests/unit/test_instance.py
Expand Up @@ -316,6 +316,7 @@ def test_create(self):
self.LABELS,
)
instance_api, response = self._instance_api_response_for_create()
instance_api.common_project_path.return_value = "projects/project"
client._instance_admin_client = instance_api
serve_nodes = 3

Expand Down Expand Up @@ -367,6 +368,7 @@ def test_create_w_clusters(self):
self.LABELS,
)
instance_api, response = self._instance_api_response_for_create()
instance_api.common_project_path.return_value = "projects/project"
client._instance_admin_client = instance_api

# Perform the method and check the result.
Expand Down
155 changes: 12 additions & 143 deletions tests/unit/test_table.py
Expand Up @@ -143,7 +143,7 @@ class TestTable(unittest.TestCase):
ROW_KEY_1 = b"row-key-1"
ROW_KEY_2 = b"row-key-2"
ROW_KEY_3 = b"row-key-3"
FAMILY_NAME = u"family"
FAMILY_NAME = "family"
QUALIFIER = b"qualifier"
TIMESTAMP_MICROS = 100
VALUE = b"value"
Expand Down Expand Up @@ -525,9 +525,9 @@ def test_get_cluster_states(self):

# build expected result
expected_result = {
u"cluster-id1": ClusterState(INITIALIZING),
u"cluster-id2": ClusterState(PLANNED_MAINTENANCE),
u"cluster-id3": ClusterState(READY),
"cluster-id1": ClusterState(INITIALIZING),
"cluster-id2": ClusterState(PLANNED_MAINTENANCE),
"cluster-id3": ClusterState(READY),
}

# Perform the method and check the result.
Expand Down Expand Up @@ -858,22 +858,7 @@ def test_read_retry_rows(self):
response_iterator = _MockReadRowsIterator(response_2)

# Patch the stub used by the API method.
data_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)
table_api.table_path.return_value = (
kolea2 marked this conversation as resolved.
Show resolved Hide resolved
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)
data_api.table_path.return_value = f"projects/{self.PROJECT_ID}/instances/{self.INSTANCE_ID}/tables/{self.TABLE_ID}"

client._table_data_client.read_rows = mock.Mock(
side_effect=[
Expand Down Expand Up @@ -1547,9 +1532,6 @@ def test_callable_empty_rows(self):

data_api = mock.create_autospec(BigtableClient)
table_api = mock.create_autospec(bigtable_table_admin.BigtableTableAdminClient)
table_api.table_path.return_value = (
"projects/self.PROJECT_ID/instances/self.INSTANCE_ID/tables/self.TABLE_ID"
)
credentials = _make_credentials()
client = self._make_client(
project="project-id", credentials=credentials, admin=True
Expand Down Expand Up @@ -1604,22 +1586,6 @@ def test_callable_no_retry_strategy(self):
response = self._make_responses(response_codes)
data_api.mutate_rows = mock.MagicMock(return_value=[response])

data_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)
table_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)
table._instance._client._table_data_client = data_api
table._instance._client._table_admin_client = table_api

Expand Down Expand Up @@ -1654,22 +1620,7 @@ def test_callable_retry(self):

data_api = mock.create_autospec(BigtableClient)
table_api = mock.create_autospec(bigtable_table_admin.BigtableTableAdminClient)
data_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)
table_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)

credentials = _make_credentials()
client = self._make_client(
project="project-id", credentials=credentials, admin=True
Expand Down Expand Up @@ -1740,22 +1691,7 @@ def test_do_mutate_retryable_rows(self):

data_api = mock.create_autospec(BigtableClient)
table_api = mock.create_autospec(bigtable_table_admin.BigtableTableAdminClient)
data_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)
table_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)

credentials = _make_credentials()
client = self._make_client(
project="project-id", credentials=credentials, admin=True
Expand Down Expand Up @@ -1826,22 +1762,7 @@ def test_do_mutate_retryable_rows_retry(self):

# Patch the stub used by the API method.
client._table_data_client.mutate_rows.side_effect = [[response]]
data_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)
table_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)

table._instance._client._table_data_client = data_api
table._instance._client._table_admin_client = table_api

Expand Down Expand Up @@ -1902,22 +1823,7 @@ def test_do_mutate_retryable_rows_second_retry(self):

# Patch the stub used by the API method.
client._table_data_client.mutate_rows.side_effect = [[response]]
data_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)
table_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)

table._instance._client._table_data_client = data_api
table._instance._client._table_admin_client = table_api

Expand Down Expand Up @@ -1981,22 +1887,7 @@ def test_do_mutate_retryable_rows_second_try(self):

# Patch the stub used by the API method.
client._table_data_client.mutate_rows.side_effect = [[response]]
data_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)
table_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)

table._instance._client._table_data_client = data_api
table._instance._client._table_admin_client = table_api

Expand Down Expand Up @@ -2049,14 +1940,7 @@ def test_do_mutate_retryable_rows_second_try_no_retryable(self):
worker.responses_statuses = self._make_responses_statuses(
[self.SUCCESS, self.NON_RETRYABLE]
)
table_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)

table._instance._client._table_admin_client = table_api

statuses = worker._do_mutate_retryable_rows()
Expand Down Expand Up @@ -2093,22 +1977,7 @@ def test_do_mutate_retryable_rows_mismatch_num_responses(self):

# Patch the stub used by the API method.
client._table_data_client.mutate_rows.side_effect = [[response]]
data_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)
table_api.table_path.return_value = (
"projects/"
+ self.PROJECT_ID
+ "/instances/"
+ self.INSTANCE_ID
+ "/tables/"
+ self.TABLE_ID
)

table._instance._client._table_data_client = data_api
table._instance._client._table_admin_client = table_api

Expand Down