From 079b6a162f6929bf801366d92f8daeb3318426c4 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 10 Dec 2020 17:05:08 -0600 Subject: [PATCH] docs: move and refresh view samples (#420) docs: restore old view snippets remove relative imports docs: fix missing space in comment sort imports --- samples/snippets/conftest.py | 27 ++++ samples/snippets/materialized_view.py | 2 +- samples/snippets/materialized_view_test.py | 14 +- samples/snippets/view.py | 164 +++++++++++++++++++++ samples/snippets/view_test.py | 117 +++++++++++++++ 5 files changed, 311 insertions(+), 13 deletions(-) create mode 100644 samples/snippets/conftest.py create mode 100644 samples/snippets/view.py create mode 100644 samples/snippets/view_test.py diff --git a/samples/snippets/conftest.py b/samples/snippets/conftest.py new file mode 100644 index 000000000..d22a33318 --- /dev/null +++ b/samples/snippets/conftest.py @@ -0,0 +1,27 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery +import pytest + + +@pytest.fixture(scope="session") +def bigquery_client(): + bigquery_client = bigquery.Client() + return bigquery_client + + +@pytest.fixture(scope="session") +def project_id(bigquery_client): + return bigquery_client.project diff --git a/samples/snippets/materialized_view.py b/samples/snippets/materialized_view.py index d925ec230..429bd98b4 100644 --- a/samples/snippets/materialized_view.py +++ b/samples/snippets/materialized_view.py @@ -25,7 +25,7 @@ def create_materialized_view(override_values={}): # To facilitate testing, we replace values with alternatives # provided by the testing harness. view_id = override_values.get("view_id", view_id) - base_table_id = override_values.get("base_table_id", view_id) + base_table_id = override_values.get("base_table_id", base_table_id) # [START bigquery_create_materialized_view] view = bigquery.Table(view_id) view.mview_query = f""" diff --git a/samples/snippets/materialized_view_test.py b/samples/snippets/materialized_view_test.py index fc3db533c..75c6b2106 100644 --- a/samples/snippets/materialized_view_test.py +++ b/samples/snippets/materialized_view_test.py @@ -23,13 +23,8 @@ def temp_suffix(): - return str(uuid.uuid4()).replace("-", "_") - - -@pytest.fixture(scope="module") -def bigquery_client(): - bigquery_client = bigquery.Client() - return bigquery_client + now = datetime.datetime.now() + return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}" @pytest.fixture(autouse=True) @@ -37,11 +32,6 @@ def bigquery_client_patch(monkeypatch, bigquery_client): monkeypatch.setattr(bigquery, "Client", lambda: bigquery_client) -@pytest.fixture(scope="module") -def project_id(bigquery_client): - return bigquery_client.project - - @pytest.fixture(scope="module") def dataset_id(bigquery_client): dataset_id = f"mvdataset_{temp_suffix()}" diff --git a/samples/snippets/view.py b/samples/snippets/view.py new file mode 100644 index 000000000..ad3f11717 --- /dev/null +++ b/samples/snippets/view.py @@ -0,0 +1,164 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_view(override_values={}): + # [START bigquery_create_view] + from google.cloud import bigquery + + client = bigquery.Client() + + view_id = "my-project.my_dataset.my_view" + source_id = "my-project.my_dataset.my_table" + # [END bigquery_create_view] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_id = override_values.get("view_id", view_id) + source_id = override_values.get("source_id", source_id) + # [START bigquery_create_view] + view = bigquery.Table(view_id) + + # The source table in this example is created from a CSV file in Google + # Cloud Storage located at + # `gs://cloud-samples-data/bigquery/us-states/us-states.csv`. It contains + # 50 US states, while the view returns only those states with names + # starting with the letter 'W'. + view.view_query = f"SELECT name, post_abbr FROM `{source_id}` WHERE name LIKE 'W%'" + + # Make an API request to create the view. + view = client.create_table(view) + print(f"Created {view.table_type}: {str(view.reference)}") + # [END bigquery_create_view] + return view + + +def get_view(override_values={}): + # [START bigquery_get_view] + from google.cloud import bigquery + + client = bigquery.Client() + + view_id = "my-project.my_dataset.my_view" + # [END bigquery_get_view] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_id = override_values.get("view_id", view_id) + # [START bigquery_get_view] + # Make an API request to get the table resource. + view = client.get_table(view_id) + + # Display view properties + print(f"Retrieved {view.table_type}: {str(view.reference)}") + print(f"View Query:\n{view.view_query}") + # [END bigquery_get_view] + return view + + +def update_view(override_values={}): + # [START bigquery_update_view_query] + from google.cloud import bigquery + + client = bigquery.Client() + + view_id = "my-project.my_dataset.my_view" + source_id = "my-project.my_dataset.my_table" + # [END bigquery_update_view_query] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_id = override_values.get("view_id", view_id) + source_id = override_values.get("source_id", source_id) + # [START bigquery_update_view_query] + view = bigquery.Table(view_id) + + # The source table in this example is created from a CSV file in Google + # Cloud Storage located at + # `gs://cloud-samples-data/bigquery/us-states/us-states.csv`. It contains + # 50 US states, while the view returns only those states with names + # starting with the letter 'M'. + view.view_query = f"SELECT name, post_abbr FROM `{source_id}` WHERE name LIKE 'M%'" + + # Make an API request to update the query property of the view. + view = client.update_table(view, ["view_query"]) + print(f"Updated {view.table_type}: {str(view.reference)}") + # [END bigquery_update_view_query] + return view + + +def grant_access(override_values={}): + # [START bigquery_grant_view_access] + from google.cloud import bigquery + + client = bigquery.Client() + + # To use a view, the analyst requires ACLs to both the view and the source + # table. Create an authorized view to allow an analyst to use a view + # without direct access permissions to the source table. + view_dataset_id = "my-project.my_view_dataset" + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_dataset_id = override_values.get("view_dataset_id", view_dataset_id) + # [START bigquery_grant_view_access] + # Make an API request to get the view dataset ACLs. + view_dataset = client.get_dataset(view_dataset_id) + + analyst_group_email = "data_analysts@example.com" + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + analyst_group_email = override_values.get( + "analyst_group_email", analyst_group_email + ) + # [START bigquery_grant_view_access] + access_entries = view_dataset.access_entries + access_entries.append( + bigquery.AccessEntry("READER", "groupByEmail", analyst_group_email) + ) + view_dataset.access_entries = access_entries + + # Make an API request to update the ACLs property of the view dataset. + view_dataset = client.update_dataset(view_dataset, ["access_entries"]) + print(f"Access to view: {view_dataset.access_entries}") + + # Group members of "data_analysts@example.com" now have access to the view, + # but they require access to the source table to use it. To remove this + # restriction, authorize the view to access the source dataset. + source_dataset_id = "my-project.my_source_dataset" + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + source_dataset_id = override_values.get("source_dataset_id", source_dataset_id) + # [START bigquery_grant_view_access] + # Make an API request to set the source dataset ACLs. + source_dataset = client.get_dataset(source_dataset_id) + + view_reference = { + "projectId": "my-project", + "datasetId": "my_view_dataset", + "tableId": "my_authorized_view", + } + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_reference = override_values.get("view_reference", view_reference) + # [START bigquery_grant_view_access] + access_entries = source_dataset.access_entries + access_entries.append(bigquery.AccessEntry(None, "view", view_reference)) + source_dataset.access_entries = access_entries + + # Make an API request to update the ACLs property of the source dataset. + source_dataset = client.update_dataset(source_dataset, ["access_entries"]) + print(f"Access to source: {source_dataset.access_entries}") + # [END bigquery_grant_view_access] + return view_dataset, source_dataset diff --git a/samples/snippets/view_test.py b/samples/snippets/view_test.py new file mode 100644 index 000000000..77105b61a --- /dev/null +++ b/samples/snippets/view_test.py @@ -0,0 +1,117 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import uuid + +from google.cloud import bigquery +import pytest + +import view + + +def temp_suffix(): + now = datetime.datetime.now() + return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}" + + +@pytest.fixture(autouse=True) +def bigquery_client_patch(monkeypatch, bigquery_client): + monkeypatch.setattr(bigquery, "Client", lambda: bigquery_client) + + +@pytest.fixture(scope="module") +def view_dataset_id(bigquery_client, project_id): + dataset_id = f"{project_id}.view_{temp_suffix()}" + bigquery_client.create_dataset(dataset_id) + yield dataset_id + bigquery_client.delete_dataset(dataset_id, delete_contents=True) + + +@pytest.fixture(scope="module") +def view_id(bigquery_client, view_dataset_id): + view_id = f"{view_dataset_id}.my_view" + yield view_id + bigquery_client.delete_table(view_id, not_found_ok=True) + + +@pytest.fixture(scope="module") +def source_dataset_id(bigquery_client, project_id): + dataset_id = f"{project_id}.view_{temp_suffix()}" + bigquery_client.create_dataset(dataset_id) + yield dataset_id + bigquery_client.delete_dataset(dataset_id, delete_contents=True) + + +@pytest.fixture(scope="module") +def source_table_id(bigquery_client, source_dataset_id): + source_table_id = f"{source_dataset_id}.us_states" + job_config = bigquery.LoadJobConfig( + schema=[ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ], + skip_leading_rows=1, + ) + load_job = bigquery_client.load_table_from_uri( + "gs://cloud-samples-data/bigquery/us-states/us-states.csv", + source_table_id, + job_config=job_config, + ) + load_job.result() + yield source_table_id + bigquery_client.delete_table(source_table_id, not_found_ok=True) + + +def test_view(capsys, view_id, view_dataset_id, source_table_id, source_dataset_id): + override_values = { + "view_id": view_id, + "source_id": source_table_id, + } + got = view.create_view(override_values) + assert source_table_id in got.view_query + out, _ = capsys.readouterr() + assert view_id in out + + got = view.get_view(override_values) + assert source_table_id in got.view_query + assert "'W%'" in got.view_query + out, _ = capsys.readouterr() + assert view_id in out + assert source_table_id in out + assert "'W%'" in out + + got = view.update_view(override_values) + assert source_table_id in got.view_query + assert "'M%'" in got.view_query + out, _ = capsys.readouterr() + assert view_id in out + + project_id, dataset_id, table_id = view_id.split(".") + override_values = { + "analyst_group_email": "cloud-dpes-bigquery@google.com", + "view_dataset_id": view_dataset_id, + "source_dataset_id": source_dataset_id, + "view_reference": { + "projectId": project_id, + "datasetId": dataset_id, + "tableId": table_id, + }, + } + view_dataset, source_dataset = view.grant_access(override_values) + assert len(view_dataset.access_entries) != 0 + assert len(source_dataset.access_entries) != 0 + out, _ = capsys.readouterr() + assert "cloud-dpes-bigquery@google.com" in out + assert table_id in out