diff --git a/datastore/cloud-client/README.rst b/datastore/cloud-client/README.rst index 70454fdec821..d246f5d0362a 100644 --- a/datastore/cloud-client/README.rst +++ b/datastore/cloud-client/README.rst @@ -14,6 +14,18 @@ This directory contains samples for Google Cloud Datastore. `Google Cloud Datast .. _Google Cloud Datastore: https://cloud.google.com/datastore/docs + + + +Set environment variables: + `GOOGLE_CLOUD_PROJECT` - Google Cloud project id + `CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name + +Roles to be set in your Service Account and App Engine default service account: + `Datastore Import Export Admin`, or `Cloud Datastore Owner`, or `Owner` + `Storage Admin`, or `Owner` + + Setup ------------------------------------------------------------------------------- diff --git a/datastore/cloud-client/README.rst.in b/datastore/cloud-client/README.rst.in index aeda020c25ae..5eef577baaac 100644 --- a/datastore/cloud-client/README.rst.in +++ b/datastore/cloud-client/README.rst.in @@ -4,10 +4,19 @@ product: name: Google Cloud Datastore short_name: Cloud Datastore url: https://cloud.google.com/datastore/docs - description: > + description: > `Google Cloud Datastore`_ is a NoSQL document database built for automatic scaling, high performance, and ease of application development. +other_required_steps: > + Set environment variables: + `GOOGLE_CLOUD_PROJECT` - Google Cloud project id + `CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name + + Roles to be set in your Service Account and App Engine default service account: + `Datastore Import Export Admin`, or `Cloud Datastore Owner`, or `Owner` + `Storage Admin`, or `Owner` + setup: - auth - install_deps diff --git a/datastore/cloud-client/admin.py b/datastore/cloud-client/admin.py new file mode 100644 index 000000000000..73e6991dd97d --- /dev/null +++ b/datastore/cloud-client/admin.py @@ -0,0 +1,86 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START datastore_admin_client_create] +from google.cloud.datastore_admin_v1.gapic import datastore_admin_client + + +def client_create(): + """Creates a new Datastore admin client.""" + client = datastore_admin_client.DatastoreAdminClient() + + print("Admin client created\n") + return client +# [END datastore_admin_client_create] + + +# [START datastore_admin_entities_export] +def export_entities(project_id, output_url_prefix): + """ + Exports a copy of all or a subset of entities from + Datastore to another storage system, such as Cloud Storage. + """ + # project_id = "project-id" + # output_url_prefix = "gs://bucket-name" + client = datastore_admin_client.DatastoreAdminClient() + + op = client.export_entities(project_id, output_url_prefix) + response = op.result(timeout=200) + + print("Entities were exported\n") + return response +# [END datastore_admin_entities_export] + + +# [START datastore_admin_entities_import] +def import_entities(project_id, input_url): + """Imports entities into Datastore.""" + # project_id := "project-id" + # input_url := "gs://bucket-name/overall-export-metadata-file" + client = datastore_admin_client.DatastoreAdminClient() + + op = client.import_entities(project_id, input_url) + response = op.result(timeout=200) + + print("Entities were imported\n") + return response +# [END datastore_admin_entities_import] + + +# [START datastore_admin_index_get] +def get_index(project_id, index_id): + """Gets an index.""" + # project_id := "my-project-id" + # index_id := "my-index" + client = datastore_admin_client.DatastoreAdminClient() + index = client.get_index(project_id, index_id) + + print("Got index: %v\n", index.index_id) + return index +# [END datastore_admin_index_get] + + +# [START datastore_admin_index_list] +def list_indexes(project_id): + """Lists the indexes.""" + # project_id := "my-project-id" + client = datastore_admin_client.DatastoreAdminClient() + + indexes = [] + for index in client.list_indexes(project_id): + indexes.append(index) + print("Got index: %v\n", index.index_id) + + print("Got list of indexes\n") + return indexes +# [END datastore_admin_index_list] diff --git a/datastore/cloud-client/admin_test.py b/datastore/cloud-client/admin_test.py new file mode 100644 index 000000000000..8b334f8da971 --- /dev/null +++ b/datastore/cloud-client/admin_test.py @@ -0,0 +1,48 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import backoff +import pytest + +import admin + + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BUCKET = os.environ["CLOUD_STORAGE_BUCKET"] + + +class TestDatastoreAdminSnippets: + def test_client_create(self): + assert admin.client_create() + + def test_get_index(self): + indexes = admin.list_indexes(PROJECT) + if not indexes: + pytest.skip( + "Skipping datastore test. At least " + "one index should present in database." + ) + + assert admin.get_index(PROJECT, indexes[0].index_id) + + def test_list_index(self): + assert admin.list_indexes(PROJECT) + + @backoff.on_exception(backoff.expo, AssertionError, max_tries=3, max_time=540000) + def test_export_import_entities(self): + response = admin.export_entities(PROJECT, "gs://" + BUCKET) + assert response + + assert admin.import_entities(PROJECT, response.output_url)