From f27c75589be9bdf40bd5d4cf872b7af056eb1b61 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 18 Jun 2020 14:04:55 +0300 Subject: [PATCH 1/8] feat(datastore): Add Datastore Admin API samples --- datastore/cloud-client/admin.py | 85 ++++++++++++++++++++++++++++ datastore/cloud-client/admin_test.py | 37 ++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 datastore/cloud-client/admin.py create mode 100644 datastore/cloud-client/admin_test.py diff --git a/datastore/cloud-client/admin.py b/datastore/cloud-client/admin.py new file mode 100644 index 000000000000..aeb6156e055b --- /dev/null +++ b/datastore/cloud-client/admin.py @@ -0,0 +1,85 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START datastore_admin_client_create] +from google.cloud.datastore_admin_v1.gapic import datastore_admin_client + + +def client_create(): + """Creates a new Datastore admin client.""" + client = datastore_admin_client.DatastoreAdminClient() + + print("Admin client created\n") + return client +# [END datastore_admin_client_create] + + +# [START datastore_admin_entities_export] +def export_entities(project_id, output_url_prefix): + """ + Exports a copy of all or a subset of entities from + Datastore to another storage system, such as Cloud Storage. + """ + # project_id = "project-id" + # output_url_prefix = "gs://bucket-name" + client = datastore_admin_client.DatastoreAdminClient() + + op = client.export_entities(project_id, output_url_prefix) + entities = op.result() + + print("Entities were exported\n") + return entities +# [END datastore_admin_entities_export] + + +# [START datastore_admin_entities_import] +def import_entities(project_id, input_url): + """Imports entities into Datastore.""" + # project_id := "project-id" + # input_url := "gs://bucket-name/overall-export-metadata-file" + client = datastore_admin_client.DatastoreAdminClient() + + op = client.import_entities(project_id, input_url) + op.result() + + print("Entities were imported\n") +# [END datastore_admin_entities_import] + + +# [START datastore_admin_index_get] +def get_index(project_id, index_id): + """Gets an index.""" + # project_id := "my-project-id" + # index_id := "my-index" + client = datastore_admin_client.DatastoreAdminClient() + index = client.get_index(project_id, index_id) + + print("Got index: %v\n", index.index_id) + return index +# [END datastore_admin_index_get] + + +# [START datastore_admin_index_list] +def list_indexes(project_id): + """Lists the indexes.""" + # project_id := "my-project-id" + client = datastore_admin_client.DatastoreAdminClient() + + indexes = [] + for index in client.list_indexes(project_id): + indexes.append(index) + print("Got index: %v\n", index.index_id) + + print("Got list of indexes\n") + return indexes +# [END datastore_admin_index_list] diff --git a/datastore/cloud-client/admin_test.py b/datastore/cloud-client/admin_test.py new file mode 100644 index 000000000000..2f7c777364ab --- /dev/null +++ b/datastore/cloud-client/admin_test.py @@ -0,0 +1,37 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest + +import admin + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] + + +class TestDatastoreAdminSnippets: + def test_client_create(self): + assert admin.client_create() + + def test_get_index(self): + indexes = admin.list_indexes(PROJECT) + if not indexes: + pytest.skip( + "Skipping datastore test. At least " + "one index should present in database." + ) + + assert admin.get_index(PROJECT, indexes[0].index_id) + + def test_list_index(self): + assert admin.list_indexes(PROJECT) From a1bb45e0430151ae081907467759ab05dcf741a7 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Fri, 26 Jun 2020 11:39:37 +0300 Subject: [PATCH 2/8] add test for entities import/export --- datastore/cloud-client/admin.py | 7 ++++--- datastore/cloud-client/admin_test.py | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/datastore/cloud-client/admin.py b/datastore/cloud-client/admin.py index aeb6156e055b..3db3d0c9bd4a 100644 --- a/datastore/cloud-client/admin.py +++ b/datastore/cloud-client/admin.py @@ -35,10 +35,10 @@ def export_entities(project_id, output_url_prefix): client = datastore_admin_client.DatastoreAdminClient() op = client.export_entities(project_id, output_url_prefix) - entities = op.result() + response = op.result() print("Entities were exported\n") - return entities + return response # [END datastore_admin_entities_export] @@ -50,9 +50,10 @@ def import_entities(project_id, input_url): client = datastore_admin_client.DatastoreAdminClient() op = client.import_entities(project_id, input_url) - op.result() + response = op.result() print("Entities were imported\n") + return response # [END datastore_admin_entities_import] diff --git a/datastore/cloud-client/admin_test.py b/datastore/cloud-client/admin_test.py index 2f7c777364ab..0fffd58e7baa 100644 --- a/datastore/cloud-client/admin_test.py +++ b/datastore/cloud-client/admin_test.py @@ -13,10 +13,12 @@ import os import pytest +from retrying import retry import admin PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BUCKET = os.environ["CLOUD_STORAGE_BUCKET"] class TestDatastoreAdminSnippets: @@ -35,3 +37,10 @@ def test_get_index(self): def test_list_index(self): assert admin.list_indexes(PROJECT) + + @retry(stop_max_attempt_number=3, stop_max_delay=540000) + def test_export_import_entities(self): + response = admin.export_entities(PROJECT, "gs://" + BUCKET) + assert response + + assert admin.import_entities(PROJECT, response.output_url) From bd7e915de6d4eec510472fb6b0e4f88ab881e318 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Fri, 26 Jun 2020 12:14:04 +0300 Subject: [PATCH 3/8] add requirements into the README file --- datastore/cloud-client/README.rst | 9 +++++++++ datastore/cloud-client/README.rst.in | 8 +++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/datastore/cloud-client/README.rst b/datastore/cloud-client/README.rst index 70454fdec821..42860de75989 100644 --- a/datastore/cloud-client/README.rst +++ b/datastore/cloud-client/README.rst @@ -14,6 +14,15 @@ This directory contains samples for Google Cloud Datastore. `Google Cloud Datast .. _Google Cloud Datastore: https://cloud.google.com/datastore/docs + +To run the sample, you need to have `Datastore Import Export Admin` role. + + +Set environment variables: + `GOOGLE_CLOUD_PROJECT` - Google Cloud project id + `CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name + + Setup ------------------------------------------------------------------------------- diff --git a/datastore/cloud-client/README.rst.in b/datastore/cloud-client/README.rst.in index aeda020c25ae..cff3c9ece3e7 100644 --- a/datastore/cloud-client/README.rst.in +++ b/datastore/cloud-client/README.rst.in @@ -4,10 +4,16 @@ product: name: Google Cloud Datastore short_name: Cloud Datastore url: https://cloud.google.com/datastore/docs - description: > + description: > `Google Cloud Datastore`_ is a NoSQL document database built for automatic scaling, high performance, and ease of application development. +required_role: Datastore Import Export Admin +other_required_steps: > + Set environment variables: + `GOOGLE_CLOUD_PROJECT` - Google Cloud project id + `CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name + setup: - auth - install_deps From 575a23ed7be6d1bff6ae922b09e4a1e9f68bfc74 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Mon, 29 Jun 2020 10:23:27 +0300 Subject: [PATCH 4/8] fix imports order, add Storage roles tip, add retrying into requirements --- datastore/cloud-client/README.rst | 2 ++ datastore/cloud-client/README.rst.in | 3 +++ datastore/cloud-client/admin_test.py | 3 ++- datastore/cloud-client/requirements-test.txt | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datastore/cloud-client/README.rst b/datastore/cloud-client/README.rst index 42860de75989..537dbe053860 100644 --- a/datastore/cloud-client/README.rst +++ b/datastore/cloud-client/README.rst @@ -22,6 +22,8 @@ Set environment variables: `GOOGLE_CLOUD_PROJECT` - Google Cloud project id `CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name +For entities import/export you also need a `Storage Admin`, or `Storage Owner` role set in your Service Account. + Setup ------------------------------------------------------------------------------- diff --git a/datastore/cloud-client/README.rst.in b/datastore/cloud-client/README.rst.in index cff3c9ece3e7..f8d3d157ff69 100644 --- a/datastore/cloud-client/README.rst.in +++ b/datastore/cloud-client/README.rst.in @@ -14,6 +14,9 @@ other_required_steps: > `GOOGLE_CLOUD_PROJECT` - Google Cloud project id `CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name + For entities import/export you also need a `Storage Admin`, or + `Storage Owner` role set in your Service Account. + setup: - auth - install_deps diff --git a/datastore/cloud-client/admin_test.py b/datastore/cloud-client/admin_test.py index 0fffd58e7baa..1f1de019164a 100644 --- a/datastore/cloud-client/admin_test.py +++ b/datastore/cloud-client/admin_test.py @@ -12,10 +12,11 @@ # limitations under the License. import os + +import admin import pytest from retrying import retry -import admin PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BUCKET = os.environ["CLOUD_STORAGE_BUCKET"] diff --git a/datastore/cloud-client/requirements-test.txt b/datastore/cloud-client/requirements-test.txt index 40857e816e6b..a242cc23d978 100644 --- a/datastore/cloud-client/requirements-test.txt +++ b/datastore/cloud-client/requirements-test.txt @@ -1,3 +1,4 @@ backoff==1.10.0 pytest==5.3.2 +retrying==1.3.3 flaky==3.6.1 From b447a3605ca297896bfa9e7a5db33d10615d3169 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Wed, 1 Jul 2020 14:44:29 +0300 Subject: [PATCH 5/8] use backoff instead of retrying, add App Engine tip --- datastore/cloud-client/README.rst | 5 +++-- datastore/cloud-client/README.rst.in | 6 +++--- datastore/cloud-client/admin_test.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/datastore/cloud-client/README.rst b/datastore/cloud-client/README.rst index 537dbe053860..d246f5d0362a 100644 --- a/datastore/cloud-client/README.rst +++ b/datastore/cloud-client/README.rst @@ -15,14 +15,15 @@ This directory contains samples for Google Cloud Datastore. `Google Cloud Datast .. _Google Cloud Datastore: https://cloud.google.com/datastore/docs -To run the sample, you need to have `Datastore Import Export Admin` role. Set environment variables: `GOOGLE_CLOUD_PROJECT` - Google Cloud project id `CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name -For entities import/export you also need a `Storage Admin`, or `Storage Owner` role set in your Service Account. +Roles to be set in your Service Account and App Engine default service account: + `Datastore Import Export Admin`, or `Cloud Datastore Owner`, or `Owner` + `Storage Admin`, or `Owner` Setup diff --git a/datastore/cloud-client/README.rst.in b/datastore/cloud-client/README.rst.in index f8d3d157ff69..5eef577baaac 100644 --- a/datastore/cloud-client/README.rst.in +++ b/datastore/cloud-client/README.rst.in @@ -8,14 +8,14 @@ product: `Google Cloud Datastore`_ is a NoSQL document database built for automatic scaling, high performance, and ease of application development. -required_role: Datastore Import Export Admin other_required_steps: > Set environment variables: `GOOGLE_CLOUD_PROJECT` - Google Cloud project id `CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name - For entities import/export you also need a `Storage Admin`, or - `Storage Owner` role set in your Service Account. + Roles to be set in your Service Account and App Engine default service account: + `Datastore Import Export Admin`, or `Cloud Datastore Owner`, or `Owner` + `Storage Admin`, or `Owner` setup: - auth diff --git a/datastore/cloud-client/admin_test.py b/datastore/cloud-client/admin_test.py index 1f1de019164a..851c44a5de5c 100644 --- a/datastore/cloud-client/admin_test.py +++ b/datastore/cloud-client/admin_test.py @@ -15,7 +15,7 @@ import admin import pytest -from retrying import retry +import backoff PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] @@ -39,7 +39,7 @@ def test_get_index(self): def test_list_index(self): assert admin.list_indexes(PROJECT) - @retry(stop_max_attempt_number=3, stop_max_delay=540000) + @backoff.on_exception(backoff.expo, AssertionError, max_tries=3, max_time=540000) def test_export_import_entities(self): response = admin.export_entities(PROJECT, "gs://" + BUCKET) assert response From 9ba88d0b1565f97da62c634f76502379805a5e84 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Wed, 1 Jul 2020 14:45:32 +0300 Subject: [PATCH 6/8] del retrying from requrements-test.txt --- datastore/cloud-client/requirements-test.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/datastore/cloud-client/requirements-test.txt b/datastore/cloud-client/requirements-test.txt index a242cc23d978..40857e816e6b 100644 --- a/datastore/cloud-client/requirements-test.txt +++ b/datastore/cloud-client/requirements-test.txt @@ -1,4 +1,3 @@ backoff==1.10.0 pytest==5.3.2 -retrying==1.3.3 flaky==3.6.1 From eb01543887c64ded0b417c39868bc6bcef33ae97 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Wed, 1 Jul 2020 15:17:39 +0300 Subject: [PATCH 7/8] fix imports order --- datastore/cloud-client/admin_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datastore/cloud-client/admin_test.py b/datastore/cloud-client/admin_test.py index 851c44a5de5c..8b334f8da971 100644 --- a/datastore/cloud-client/admin_test.py +++ b/datastore/cloud-client/admin_test.py @@ -13,9 +13,10 @@ import os -import admin -import pytest import backoff +import pytest + +import admin PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] From 3508a6f23f861f3629d4bb54ea55363fff351e54 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 9 Jul 2020 10:26:02 +0300 Subject: [PATCH 8/8] add timeouts --- datastore/cloud-client/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/cloud-client/admin.py b/datastore/cloud-client/admin.py index 3db3d0c9bd4a..73e6991dd97d 100644 --- a/datastore/cloud-client/admin.py +++ b/datastore/cloud-client/admin.py @@ -35,7 +35,7 @@ def export_entities(project_id, output_url_prefix): client = datastore_admin_client.DatastoreAdminClient() op = client.export_entities(project_id, output_url_prefix) - response = op.result() + response = op.result(timeout=200) print("Entities were exported\n") return response @@ -50,7 +50,7 @@ def import_entities(project_id, input_url): client = datastore_admin_client.DatastoreAdminClient() op = client.import_entities(project_id, input_url) - response = op.result() + response = op.result(timeout=200) print("Entities were imported\n") return response