diff --git a/docs/snippets.py b/docs/snippets.py index 8c106e63d..3f9b9a88c 100644 --- a/docs/snippets.py +++ b/docs/snippets.py @@ -478,132 +478,6 @@ def test_update_table_cmek(client, to_delete): # [END bigquery_update_table_cmek] -@pytest.mark.skip( - reason=( - "update_table() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" - ) -) -def test_manage_views(client, to_delete): - project = client.project - source_dataset_id = "source_dataset_{}".format(_millis()) - source_dataset_ref = bigquery.DatasetReference(project, source_dataset_id) - source_dataset = bigquery.Dataset(source_dataset_ref) - source_dataset = client.create_dataset(source_dataset) - to_delete.append(source_dataset) - - job_config = bigquery.LoadJobConfig() - job_config.schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - job_config.skip_leading_rows = 1 - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv" - source_table_id = "us_states" - load_job = client.load_table_from_uri( - uri, source_dataset.table(source_table_id), job_config=job_config - ) - load_job.result() - - shared_dataset_id = "shared_dataset_{}".format(_millis()) - shared_dataset_ref = bigquery.DatasetReference(project, shared_dataset_id) - shared_dataset = bigquery.Dataset(shared_dataset_ref) - shared_dataset = client.create_dataset(shared_dataset) - to_delete.append(shared_dataset) - - # [START bigquery_create_view] - # from google.cloud import bigquery - # client = bigquery.Client() - # project = 'my-project' - # source_dataset_id = 'my_source_dataset' - # source_table_id = 'us_states' - # shared_dataset_ref = bigquery.DatasetReference(project, 'my_shared_dataset') - - # This example shows how to create a shared view of a source table of - # US States. The source table contains all 50 states, while the view will - # contain only states with names starting with 'W'. - view_ref = shared_dataset_ref.table("my_shared_view") - view = bigquery.Table(view_ref) - sql_template = 'SELECT name, post_abbr FROM `{}.{}.{}` WHERE name LIKE "W%"' - view.view_query = sql_template.format(project, source_dataset_id, source_table_id) - view = client.create_table(view) # API request - - print("Successfully created view at {}".format(view.full_table_id)) - # [END bigquery_create_view] - - # [START bigquery_update_view_query] - # from google.cloud import bigquery - # client = bigquery.Client() - # project = 'my-project' - # source_dataset_id = 'my_source_dataset' - # source_table_id = 'us_states' - # shared_dataset_ref = bigquery.DatasetReference(project, 'my_shared_dataset') - - # This example shows how to update a shared view of a source table of - # US States. The view's query will be updated to contain only states with - # names starting with 'M'. - view_ref = shared_dataset_ref.table("my_shared_view") - view = bigquery.Table(view_ref) - sql_template = 'SELECT name, post_abbr FROM `{}.{}.{}` WHERE name LIKE "M%"' - view.view_query = sql_template.format(project, source_dataset_id, source_table_id) - view = client.update_table(view, ["view_query"]) # API request - # [END bigquery_update_view_query] - - # [START bigquery_get_view] - # from google.cloud import bigquery - # client = bigquery.Client() - # shared_dataset_id = 'my_shared_dataset' - project = client.project - shared_dataset_ref = bigquery.DatasetReference(project, shared_dataset_id) - view_ref = shared_dataset_ref.table("my_shared_view") - view = client.get_table(view_ref) # API Request - - # Display view properties - print("View at {}".format(view.full_table_id)) - print("View Query:\n{}".format(view.view_query)) - # [END bigquery_get_view] - assert view.view_query is not None - - analyst_group_email = "example-analyst-group@google.com" - # [START bigquery_grant_view_access] - # from google.cloud import bigquery - # client = bigquery.Client() - - # Assign access controls to the dataset containing the view - # shared_dataset_id = 'my_shared_dataset' - # analyst_group_email = 'data_analysts@example.com' - project = client.project - shared_dataset_ref = bigquery.DatasetReference(project, shared_dataset_id) - shared_dataset = client.get_dataset(shared_dataset_ref) # API request - access_entries = shared_dataset.access_entries - access_entries.append( - bigquery.AccessEntry("READER", "groupByEmail", analyst_group_email) - ) - shared_dataset.access_entries = access_entries - shared_dataset = client.update_dataset( - shared_dataset, ["access_entries"] - ) # API request - - # Authorize the view to access the source dataset - # project = 'my-project' - # source_dataset_id = 'my_source_dataset' - project = client.project - source_dataset_ref = bigquery.DatasetReference(project, source_dataset_id) - source_dataset = client.get_dataset(source_dataset_ref) # API request - view_reference = { - "projectId": project, - "datasetId": shared_dataset_id, - "tableId": "my_shared_view", - } - access_entries = source_dataset.access_entries - access_entries.append(bigquery.AccessEntry(None, "view", view_reference)) - source_dataset.access_entries = access_entries - source_dataset = client.update_dataset( - source_dataset, ["access_entries"] - ) # API request - # [END bigquery_grant_view_access] - - def test_load_table_add_column(client, to_delete): dataset_id = "load_table_add_column_{}".format(_millis()) project = client.project diff --git a/samples/snippets/__init__.py b/samples/snippets/__init__.py new file mode 100644 index 000000000..c6334245a --- /dev/null +++ b/samples/snippets/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/samples/snippets/conftest.py b/samples/snippets/conftest.py new file mode 100644 index 000000000..4d3f60f92 --- /dev/null +++ b/samples/snippets/conftest.py @@ -0,0 +1,28 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from google.cloud import bigquery + + +@pytest.fixture(scope="session") +def bigquery_client(): + bigquery_client = bigquery.Client() + return bigquery_client + + +@pytest.fixture(scope="session") +def project_id(bigquery_client): + return bigquery_client.project diff --git a/samples/snippets/materialized_view.py b/samples/snippets/materialized_view.py index d925ec230..429bd98b4 100644 --- a/samples/snippets/materialized_view.py +++ b/samples/snippets/materialized_view.py @@ -25,7 +25,7 @@ def create_materialized_view(override_values={}): # To facilitate testing, we replace values with alternatives # provided by the testing harness. view_id = override_values.get("view_id", view_id) - base_table_id = override_values.get("base_table_id", view_id) + base_table_id = override_values.get("base_table_id", base_table_id) # [START bigquery_create_materialized_view] view = bigquery.Table(view_id) view.mview_query = f""" diff --git a/samples/snippets/materialized_view_test.py b/samples/snippets/materialized_view_test.py index fc3db533c..8c3ae9a6c 100644 --- a/samples/snippets/materialized_view_test.py +++ b/samples/snippets/materialized_view_test.py @@ -19,17 +19,12 @@ from google.cloud import bigquery import pytest -import materialized_view +from . import materialized_view def temp_suffix(): - return str(uuid.uuid4()).replace("-", "_") - - -@pytest.fixture(scope="module") -def bigquery_client(): - bigquery_client = bigquery.Client() - return bigquery_client + now = datetime.datetime.now() + return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}" @pytest.fixture(autouse=True) @@ -37,11 +32,6 @@ def bigquery_client_patch(monkeypatch, bigquery_client): monkeypatch.setattr(bigquery, "Client", lambda: bigquery_client) -@pytest.fixture(scope="module") -def project_id(bigquery_client): - return bigquery_client.project - - @pytest.fixture(scope="module") def dataset_id(bigquery_client): dataset_id = f"mvdataset_{temp_suffix()}" diff --git a/samples/snippets/view.py b/samples/snippets/view.py new file mode 100644 index 000000000..3d8429e85 --- /dev/null +++ b/samples/snippets/view.py @@ -0,0 +1,167 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def create_view(override_values={}): + # [START bigquery_create_view] + from google.cloud import bigquery + + client = bigquery.Client() + + view_id = "my-project.my_dataset.my_view" + source_id = "my-project.my_dataset.my_table" + # [END bigquery_create_view] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_id = override_values.get("view_id", view_id) + source_id = override_values.get("source_id", source_id) + # [START bigquery_create_view] + view = bigquery.Table(view_id) + + # The source table in this example is created from a CSV file in Google + # Cloud Storage located at + # `gs://cloud-samples-data/bigquery/us-states/us-states.csv`. It contains + # 50 US states, while the view returns only those states with names + # starting with the letter 'W'. + view.view_query = ( + f"SELECT name, post_abbr FROM `{source_id}` WHERE name LIKE 'W%'" + ) + + # Make an API request to create the view. + view = client.create_table(view) + print(f"Created {view.table_type}: {str(view.reference)}") + # [END bigquery_create_view] + return view + + +def update_view(override_values={}): + # [START bigquery_update_view_query] + from google.cloud import bigquery + + client = bigquery.Client() + + view_id = "my-project.my_dataset.my_view" + source_id = "my-project.my_dataset.my_table" + # [END bigquery_update_view_query] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_id = override_values.get("view_id", view_id) + source_id = override_values.get("source_id", source_id) + # [START bigquery_update_view_query] + view = bigquery.Table(view_id) + + # The source table in this example is created from a CSV file in Google + # Cloud Storage located at + # `gs://cloud-samples-data/bigquery/us-states/us-states.csv`. It contains + # 50 US states, while the view returns only those states with names + # starting with the letter 'M'. + view.view_query = ( + f"SELECT name, post_abbr FROM `{source_id}` WHERE name LIKE 'M%'" + ) + + # Make an API request to update the query property of the view. + view = client.update_table(view, ["view_query"]) + print(f"Updated {view.table_type}: {str(view.reference)}") + # [END bigquery_update_view_query] + return view + + +def get_view(override_values={}): + # [START bigquery_get_view] + from google.cloud import bigquery + + client = bigquery.Client() + + view_id = "my-project.my_dataset.my_view" + # [END bigquery_get_view] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_id = override_values.get("view_id", view_id) + # [START bigquery_get_view] + # Make an API request to get the table resource. + view = client.get_table(view_id) + + # Display view properties + print(f"Retrieved {view.table_type}: {str(view.reference)}") + print(f"View Query:\n{view.view_query}") + # [END bigquery_get_view] + return view + + +def grant_access(override_values={}): + # [START bigquery_grant_view_access] + from google.cloud import bigquery + + client = bigquery.Client() + + # To use a view, the analyst requires ACLs to both the view and the source + # table. Create an authorized view to allow an analyst to use a view + # without direct access permissionsto the source table. + view_dataset_id = "my-project.my_view_dataset" + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_dataset_id = override_values.get("view_dataset_id", view_dataset_id) + # [START bigquery_grant_view_access] + # Make an API request to get the view dataset ACLs. + view_dataset = client.get_dataset(view_dataset_id) + + analyst_group_email = 'data_analysts@example.com' + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + analyst_group_email = override_values.get("analyst_group_email", analyst_group_email) + # [START bigquery_grant_view_access] + access_entries = view_dataset.access_entries + access_entries.append( + bigquery.AccessEntry("READER", "groupByEmail", analyst_group_email) + ) + view_dataset.access_entries = access_entries + + # Make an API request to update the ACLs property of the view dataset. + view_dataset = client.update_dataset( + view_dataset, ["access_entries"] + ) + + # Group members of "data_analysts@example.com" now have access to the view, + # but they require access to the source table to use it. To remove this + # restriction, authorize the view to access the source dataset. + source_dataset_id = "my-project.my_source_dataset" + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + source_dataset_id = override_values.get("source_dataset_id", source_dataset_id) + # [START bigquery_grant_view_access] + # Make an API request to set the source dataset ACLs. + source_dataset = client.get_dataset(source_dataset_id) + + view_reference = { + "projectId": "my-project", + "datasetId": "my_view_dataset", + "tableId": "my_authorized_view", + } + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_reference = override_values.get("view_reference", view_reference) + # [START bigquery_grant_view_access] + access_entries = source_dataset.access_entries + access_entries.append(bigquery.AccessEntry(None, "view", view_reference)) + source_dataset.access_entries = access_entries + + # Make an API request to update the ACLs property of the source dataset. + source_dataset = client.update_dataset( + source_dataset, ["access_entries"] + ) + # [END bigquery_grant_view_access] + return view_dataset, source_dataset diff --git a/samples/snippets/view_test.py b/samples/snippets/view_test.py new file mode 100644 index 000000000..8717b552b --- /dev/null +++ b/samples/snippets/view_test.py @@ -0,0 +1,87 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import uuid + +from google.cloud import bigquery +import pytest + +from . import view + + +def temp_suffix(): + now = datetime.datetime.now() + return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}" + + +@pytest.fixture(autouse=True) +def bigquery_client_patch(monkeypatch, bigquery_client): + monkeypatch.setattr(bigquery, "Client", lambda: bigquery_client) + + +@pytest.fixture(scope="module") +def view_dataset_id(bigquery_client, project_id): + dataset_id = f"{project_id}.view_{temp_suffix()}" + bigquery_client.create_dataset(dataset_id) + yield dataset_id + bigquery_client.delete_dataset(dataset_id, delete_contents=True) + + +@pytest.fixture(scope="module") +def view_id(bigquery_client, view_dataset_id): + view_id = f"{view_dataset_id}.my_view" + yield view_id + bigquery_client.delete_table(view_id, not_found_ok=True) + + +@pytest.fixture(scope="module") +def source_dataset_id(bigquery_client, project_id): + dataset_id = f"{project_id}.view_{temp_suffix()}" + bigquery_client.create_dataset(dataset_id) + yield dataset_id + bigquery_client.delete_dataset(dataset_id, delete_contents=True) + + +@pytest.fixture(scope="module") +def source_table_id(bigquery_client, source_dataset_id): + source_table_id = f"{source_dataset_id}.us_states" + job_config = bigquery.LoadJobConfig( + schema=[ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ], + skip_leading_rows=1 + ) + load_job = bigquery_client.load_table_from_uri( + "gs://cloud-samples-data/bigquery/us-states/us-states.csv", + source_table_id, + job_config=job_config + ) + load_job.result() + yield source_table_id + bigquery_client.delete_table(source_table_id, not_found_ok=True) + + +def test_view(capsys, view_id, source_table_id): + override_values = { + "view_id": view_id, + "source_id": source_table_id, + } + got = view.create_view(override_values) + assert source_table_id in got.view_query + out, _ = capsys.readouterr() + assert view_id in out + + # TODO: test other view samples