From d4eb02ee936e5b8b183c760f3285db00c33cc8bf Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Thu, 28 May 2020 23:58:17 +0000 Subject: [PATCH] Remove Bigtable samples --- bigtable/README.rst | 84 --- bigtable/README.rst.in | 25 - bigtable/hello/README.rst | 115 ----- bigtable/hello/README.rst.in | 23 - bigtable/hello/main.py | 130 ----- bigtable/hello/main_test.py | 39 -- bigtable/hello/requirements-test.txt | 1 - bigtable/hello/requirements.txt | 2 - bigtable/hello_happybase/README.rst | 122 ----- bigtable/hello_happybase/README.rst.in | 32 -- bigtable/hello_happybase/main.py | 118 ----- bigtable/hello_happybase/main_test.py | 41 -- .../hello_happybase/requirements-test.txt | 1 - bigtable/hello_happybase/requirements.txt | 1 - bigtable/instanceadmin/README.rst | 120 ----- bigtable/instanceadmin/README.rst.in | 23 - bigtable/instanceadmin/instanceadmin.py | 259 ---------- bigtable/instanceadmin/requirements-test.txt | 1 - bigtable/instanceadmin/requirements.txt | 1 - bigtable/metricscaler/Dockerfile | 24 - bigtable/metricscaler/README.rst | 128 ----- bigtable/metricscaler/README.rst.in | 29 -- bigtable/metricscaler/metricscaler.py | 209 -------- bigtable/metricscaler/metricscaler_test.py | 200 -------- bigtable/metricscaler/requirements-test.txt | 2 - bigtable/metricscaler/requirements.txt | 2 - bigtable/quickstart/README.rst | 108 ---- bigtable/quickstart/README.rst.in | 23 - bigtable/quickstart/main.py | 58 --- bigtable/quickstart/main_test.py | 55 -- bigtable/quickstart/requirements-test.txt | 1 - bigtable/quickstart/requirements.txt | 1 - bigtable/quickstart_happybase/README.rst | 108 ---- bigtable/quickstart_happybase/README.rst.in | 23 - bigtable/quickstart_happybase/main.py | 62 --- bigtable/quickstart_happybase/main_test.py | 55 -- .../requirements-test.txt | 1 - .../quickstart_happybase/requirements.txt | 1 - bigtable/snippets/filters/filter_snippets.py | 290 ----------- bigtable/snippets/filters/filters_test.py | 226 --------- .../snippets/filters/requirements-test.txt | 1 - bigtable/snippets/filters/requirements.txt | 2 - .../snippets/filters/snapshots/__init__.py | 0 .../filters/snapshots/snap_filters_test.py | 480 ------------------ bigtable/snippets/reads/read_snippets.py | 152 ------ bigtable/snippets/reads/reads_test.py | 121 ----- bigtable/snippets/reads/requirements-test.txt | 1 - bigtable/snippets/reads/requirements.txt | 2 - bigtable/snippets/reads/snapshots/__init__.py | 0 .../reads/snapshots/snap_reads_test.py | 142 ------ bigtable/snippets/writes/__init__.py | 0 .../snippets/writes/requirements-test.txt | 2 - bigtable/snippets/writes/requirements.txt | 1 - bigtable/snippets/writes/write_batch.py | 55 -- .../snippets/writes/write_conditionally.py | 44 -- bigtable/snippets/writes/write_increment.py | 34 -- bigtable/snippets/writes/write_simple.py | 49 -- bigtable/snippets/writes/writes_test.py | 94 ---- bigtable/tableadmin/README.rst | 115 ----- bigtable/tableadmin/README.rst.in | 23 - bigtable/tableadmin/requirements-test.txt | 1 - bigtable/tableadmin/requirements.txt | 1 - bigtable/tableadmin/tableadmin.py | 283 ----------- bigtable/tableadmin/tableadmin_test.py | 66 --- 64 files changed, 4413 deletions(-) delete mode 100644 bigtable/README.rst delete mode 100644 bigtable/README.rst.in delete mode 100644 bigtable/hello/README.rst delete mode 100644 bigtable/hello/README.rst.in delete mode 100644 bigtable/hello/main.py delete mode 100644 bigtable/hello/main_test.py delete mode 100644 bigtable/hello/requirements-test.txt delete mode 100644 bigtable/hello/requirements.txt delete mode 100644 bigtable/hello_happybase/README.rst delete mode 100644 bigtable/hello_happybase/README.rst.in delete mode 100644 bigtable/hello_happybase/main.py delete mode 100644 bigtable/hello_happybase/main_test.py delete mode 100644 bigtable/hello_happybase/requirements-test.txt delete mode 100644 bigtable/hello_happybase/requirements.txt delete mode 100644 bigtable/instanceadmin/README.rst delete mode 100644 bigtable/instanceadmin/README.rst.in delete mode 100644 bigtable/instanceadmin/instanceadmin.py delete mode 100644 bigtable/instanceadmin/requirements-test.txt delete mode 100755 bigtable/instanceadmin/requirements.txt delete mode 100644 bigtable/metricscaler/Dockerfile delete mode 100644 bigtable/metricscaler/README.rst delete mode 100644 bigtable/metricscaler/README.rst.in delete mode 100644 bigtable/metricscaler/metricscaler.py delete mode 100644 bigtable/metricscaler/metricscaler_test.py delete mode 100644 bigtable/metricscaler/requirements-test.txt delete mode 100644 bigtable/metricscaler/requirements.txt delete mode 100644 bigtable/quickstart/README.rst delete mode 100644 bigtable/quickstart/README.rst.in delete mode 100644 bigtable/quickstart/main.py delete mode 100644 bigtable/quickstart/main_test.py delete mode 100644 bigtable/quickstart/requirements-test.txt delete mode 100644 bigtable/quickstart/requirements.txt delete mode 100644 bigtable/quickstart_happybase/README.rst delete mode 100644 bigtable/quickstart_happybase/README.rst.in delete mode 100644 bigtable/quickstart_happybase/main.py delete mode 100644 bigtable/quickstart_happybase/main_test.py delete mode 100644 bigtable/quickstart_happybase/requirements-test.txt delete mode 100644 bigtable/quickstart_happybase/requirements.txt delete mode 100644 bigtable/snippets/filters/filter_snippets.py delete mode 100644 bigtable/snippets/filters/filters_test.py delete mode 100644 bigtable/snippets/filters/requirements-test.txt delete mode 100755 bigtable/snippets/filters/requirements.txt delete mode 100644 bigtable/snippets/filters/snapshots/__init__.py delete mode 100644 bigtable/snippets/filters/snapshots/snap_filters_test.py delete mode 100644 bigtable/snippets/reads/read_snippets.py delete mode 100644 bigtable/snippets/reads/reads_test.py delete mode 100644 bigtable/snippets/reads/requirements-test.txt delete mode 100755 bigtable/snippets/reads/requirements.txt delete mode 100644 bigtable/snippets/reads/snapshots/__init__.py delete mode 100644 bigtable/snippets/reads/snapshots/snap_reads_test.py delete mode 100644 bigtable/snippets/writes/__init__.py delete mode 100644 bigtable/snippets/writes/requirements-test.txt delete mode 100755 bigtable/snippets/writes/requirements.txt delete mode 100644 bigtable/snippets/writes/write_batch.py delete mode 100644 bigtable/snippets/writes/write_conditionally.py delete mode 100644 bigtable/snippets/writes/write_increment.py delete mode 100644 bigtable/snippets/writes/write_simple.py delete mode 100644 bigtable/snippets/writes/writes_test.py delete mode 100644 bigtable/tableadmin/README.rst delete mode 100644 bigtable/tableadmin/README.rst.in delete mode 100644 bigtable/tableadmin/requirements-test.txt delete mode 100755 bigtable/tableadmin/requirements.txt delete mode 100644 bigtable/tableadmin/tableadmin.py delete mode 100755 bigtable/tableadmin/tableadmin_test.py diff --git a/bigtable/README.rst b/bigtable/README.rst deleted file mode 100644 index 3f958d7e2a5e..000000000000 --- a/bigtable/README.rst +++ /dev/null @@ -1,84 +0,0 @@ -.. This file is automatically generated. Do not edit this file directly. - -Google Cloud Bigtable Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/README.rst - - -This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. - - -This directory contains samples that demonstrate using the Google Cloud Client Library for -Python as well as the `Google Cloud Client Library HappyBase package`_ to connect to and -interact with Cloud Bigtable. - -.. _Google Cloud Client Library HappyBase package: - https://github.com/GoogleCloudPlatform/google-cloud-python-happybase - - -.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs - -Setup -------------------------------------------------------------------------------- - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - - - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/README.rst.in b/bigtable/README.rst.in deleted file mode 100644 index b8f68e21bc68..000000000000 --- a/bigtable/README.rst.in +++ /dev/null @@ -1,25 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google Cloud Bigtable - short_name: Cloud Bigtable - url: https://cloud.google.com/bigtable/docs - description: > - `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's - the same database that powers many core Google services, including Search, - Analytics, Maps, and Gmail. - -description: | - This directory contains samples that demonstrate using the Google Cloud Client Library for - Python as well as the `Google Cloud Client Library HappyBase package`_ to connect to and - interact with Cloud Bigtable. - - .. _Google Cloud Client Library HappyBase package: - https://github.com/GoogleCloudPlatform/google-cloud-python-happybase - -setup: -- auth -- install_deps - - -cloud_client_library: true diff --git a/bigtable/hello/README.rst b/bigtable/hello/README.rst deleted file mode 100644 index 893932ad5e73..000000000000 --- a/bigtable/hello/README.rst +++ /dev/null @@ -1,115 +0,0 @@ -.. This file is automatically generated. Do not edit this file directly. - -Google Cloud Bigtable Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/README.rst - - -This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. - - - - -.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs - -Setup -------------------------------------------------------------------------------- - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - -Samples -------------------------------------------------------------------------------- - -Basic example -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/main.py,bigtable/hello/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python main.py - - usage: main.py [-h] [--table TABLE] project_id instance_id - - Demonstrates how to connect to Cloud Bigtable and run some basic operations. - Prerequisites: - Create a Cloud Bigtable cluster. - https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google - Application Default Credentials. - https://developers.google.com/identity/protocols/application-default- - credentials - - positional arguments: - project_id Your Cloud Platform project ID. - instance_id ID of the Cloud Bigtable instance to connect to. - - optional arguments: - -h, --help show this help message and exit - --table TABLE Table to create and destroy. (default: Hello-Bigtable) - - - - - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/hello/README.rst.in b/bigtable/hello/README.rst.in deleted file mode 100644 index ed9253c115a4..000000000000 --- a/bigtable/hello/README.rst.in +++ /dev/null @@ -1,23 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google Cloud Bigtable - short_name: Cloud Bigtable - url: https://cloud.google.com/bigtable/docs - description: > - `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's - the same database that powers many core Google services, including Search, - Analytics, Maps, and Gmail. - -setup: -- auth -- install_deps - -samples: -- name: Basic example - file: main.py - show_help: true - -cloud_client_library: true - -folder: bigtable/hello \ No newline at end of file diff --git a/bigtable/hello/main.py b/bigtable/hello/main.py deleted file mode 100644 index 073270847232..000000000000 --- a/bigtable/hello/main.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Demonstrates how to connect to Cloud Bigtable and run some basic operations. - -Prerequisites: - -- Create a Cloud Bigtable cluster. - https://cloud.google.com/bigtable/docs/creating-cluster -- Set your Google Application Default Credentials. - https://developers.google.com/identity/protocols/application-default-credentials -""" - -import argparse -# [START bigtable_hw_imports] -import datetime - -from google.cloud import bigtable -from google.cloud.bigtable import column_family -from google.cloud.bigtable import row_filters -# [END bigtable_hw_imports] - - -def main(project_id, instance_id, table_id): - # [START bigtable_hw_connect] - # The client must be created with admin=True because it will create a - # table. - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - # [END bigtable_hw_connect] - - # [START bigtable_hw_create_table] - print('Creating the {} table.'.format(table_id)) - table = instance.table(table_id) - - print('Creating column family cf1 with Max Version GC rule...') - # Create a column family with GC policy : most recent N versions - # Define the GC policy to retain only the most recent 2 versions - max_versions_rule = column_family.MaxVersionsGCRule(2) - column_family_id = 'cf1' - column_families = {column_family_id: max_versions_rule} - if not table.exists(): - table.create(column_families=column_families) - else: - print("Table {} already exists.".format(table_id)) - # [END bigtable_hw_create_table] - - # [START bigtable_hw_write_rows] - print('Writing some greetings to the table.') - greetings = ['Hello World!', 'Hello Cloud Bigtable!', 'Hello Python!'] - rows = [] - column = 'greeting'.encode() - for i, value in enumerate(greetings): - # Note: This example uses sequential numeric IDs for simplicity, - # but this can result in poor performance in a production - # application. Since rows are stored in sorted order by key, - # sequential keys can result in poor distribution of operations - # across nodes. - # - # For more information about how to design a Bigtable schema for - # the best performance, see the documentation: - # - # https://cloud.google.com/bigtable/docs/schema-design - row_key = 'greeting{}'.format(i).encode() - row = table.direct_row(row_key) - row.set_cell(column_family_id, - column, - value, - timestamp=datetime.datetime.utcnow()) - rows.append(row) - table.mutate_rows(rows) - # [END bigtable_hw_write_rows] - - # [START bigtable_hw_create_filter] - # Create a filter to only retrieve the most recent version of the cell - # for each column accross entire row. - row_filter = row_filters.CellsColumnLimitFilter(1) - # [END bigtable_hw_create_filter] - - # [START bigtable_hw_get_with_filter] - print('Getting a single greeting by row key.') - key = 'greeting0'.encode() - - row = table.read_row(key, row_filter) - cell = row.cells[column_family_id][column][0] - print(cell.value.decode('utf-8')) - # [END bigtable_hw_get_with_filter] - - # [START bigtable_hw_scan_with_filter] - print('Scanning for all greetings:') - partial_rows = table.read_rows(filter_=row_filter) - - for row in partial_rows: - cell = row.cells[column_family_id][column][0] - print(cell.value.decode('utf-8')) - # [END bigtable_hw_scan_with_filter] - - # [START bigtable_hw_delete_table] - print('Deleting the {} table.'.format(table_id)) - table.delete() - # [END bigtable_hw_delete_table] - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('project_id', help='Your Cloud Platform project ID.') - parser.add_argument( - 'instance_id', help='ID of the Cloud Bigtable instance to connect to.') - parser.add_argument( - '--table', - help='Table to create and destroy.', - default='Hello-Bigtable') - - args = parser.parse_args() - main(args.project_id, args.instance_id, args.table) diff --git a/bigtable/hello/main_test.py b/bigtable/hello/main_test.py deleted file mode 100644 index 49b8098fcd7e..000000000000 --- a/bigtable/hello/main_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import random - -from main import main - -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] -BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE'] -TABLE_NAME_FORMAT = 'hello-world-test-{}' -TABLE_NAME_RANGE = 10000 - - -def test_main(capsys): - table_name = TABLE_NAME_FORMAT.format( - random.randrange(TABLE_NAME_RANGE)) - - main(PROJECT, BIGTABLE_INSTANCE, table_name) - - out, _ = capsys.readouterr() - assert 'Creating the {} table.'.format(table_name) in out - assert 'Writing some greetings to the table.' in out - assert 'Getting a single greeting by row key.' in out - assert 'Hello World!' in out - assert 'Scanning for all greetings' in out - assert 'Hello Cloud Bigtable!' in out - assert 'Deleting the {} table.'.format(table_name) in out diff --git a/bigtable/hello/requirements-test.txt b/bigtable/hello/requirements-test.txt deleted file mode 100644 index 781d4326c947..000000000000 --- a/bigtable/hello/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==5.3.2 diff --git a/bigtable/hello/requirements.txt b/bigtable/hello/requirements.txt deleted file mode 100644 index 29ecf15a2b72..000000000000 --- a/bigtable/hello/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -google-cloud-bigtable==1.2.1 -google-cloud-core==1.3.0 diff --git a/bigtable/hello_happybase/README.rst b/bigtable/hello_happybase/README.rst deleted file mode 100644 index 82a376535373..000000000000 --- a/bigtable/hello_happybase/README.rst +++ /dev/null @@ -1,122 +0,0 @@ -.. This file is automatically generated. Do not edit this file directly. - -Google Cloud Bigtable Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello_happybase/README.rst - - -This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. - - -This sample demonstrates using the `Google Cloud Client Library HappyBase -package`_, an implementation of the `HappyBase API`_ to connect to and -interact with Cloud Bigtable. - -.. _Google Cloud Client Library HappyBase package: - https://github.com/GoogleCloudPlatform/google-cloud-python-happybase -.. _HappyBase API: http://happybase.readthedocs.io/en/stable/ - - -.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs - -Setup -------------------------------------------------------------------------------- - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - -Samples -------------------------------------------------------------------------------- - -Basic example -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello_happybase/main.py,bigtable/hello_happybase/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python main.py - - usage: main.py [-h] [--table TABLE] project_id instance_id - - Demonstrates how to connect to Cloud Bigtable and run some basic operations. - Prerequisites: - Create a Cloud Bigtable cluster. - https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google - Application Default Credentials. - https://developers.google.com/identity/protocols/application-default- - credentials - - positional arguments: - project_id Your Cloud Platform project ID. - instance_id ID of the Cloud Bigtable instance to connect to. - - optional arguments: - -h, --help show this help message and exit - --table TABLE Table to create and destroy. (default: Hello-Bigtable) - - - - - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/hello_happybase/README.rst.in b/bigtable/hello_happybase/README.rst.in deleted file mode 100644 index 8ef6a956b5e9..000000000000 --- a/bigtable/hello_happybase/README.rst.in +++ /dev/null @@ -1,32 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google Cloud Bigtable - short_name: Cloud Bigtable - url: https://cloud.google.com/bigtable/docs - description: > - `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's - the same database that powers many core Google services, including Search, - Analytics, Maps, and Gmail. - -description: | - This sample demonstrates using the `Google Cloud Client Library HappyBase - package`_, an implementation of the `HappyBase API`_ to connect to and - interact with Cloud Bigtable. - - .. _Google Cloud Client Library HappyBase package: - https://github.com/GoogleCloudPlatform/google-cloud-python-happybase - .. _HappyBase API: http://happybase.readthedocs.io/en/stable/ - -setup: -- auth -- install_deps - -samples: -- name: Basic example - file: main.py - show_help: true - -cloud_client_library: true - -folder: bigtable/hello_happybase \ No newline at end of file diff --git a/bigtable/hello_happybase/main.py b/bigtable/hello_happybase/main.py deleted file mode 100644 index ade4acbf0d84..000000000000 --- a/bigtable/hello_happybase/main.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Demonstrates how to connect to Cloud Bigtable and run some basic operations. - -Prerequisites: - -- Create a Cloud Bigtable cluster. - https://cloud.google.com/bigtable/docs/creating-cluster -- Set your Google Application Default Credentials. - https://developers.google.com/identity/protocols/application-default-credentials -""" - -import argparse - -# [START bigtable_hw_imports_happybase] -from google.cloud import bigtable -from google.cloud import happybase -# [END bigtable_hw_imports_happybase] - - -def main(project_id, instance_id, table_name): - # [START bigtable_hw_connect_happybase] - # The client must be created with admin=True because it will create a - # table. - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - connection = happybase.Connection(instance=instance) - # [END bigtable_hw_connect_happybase] - - try: - # [START bigtable_hw_create_table_happybase] - print('Creating the {} table.'.format(table_name)) - column_family_name = 'cf1' - connection.create_table( - table_name, - { - column_family_name: dict() # Use default options. - }) - # [END bigtable_hw_create_table_happybase] - - # [START bigtable_hw_write_rows_happybase] - print('Writing some greetings to the table.') - table = connection.table(table_name) - column_name = '{fam}:greeting'.format(fam=column_family_name) - greetings = [ - 'Hello World!', - 'Hello Cloud Bigtable!', - 'Hello HappyBase!', - ] - - for i, value in enumerate(greetings): - # Note: This example uses sequential numeric IDs for simplicity, - # but this can result in poor performance in a production - # application. Since rows are stored in sorted order by key, - # sequential keys can result in poor distribution of operations - # across nodes. - # - # For more information about how to design a Bigtable schema for - # the best performance, see the documentation: - # - # https://cloud.google.com/bigtable/docs/schema-design - row_key = 'greeting{}'.format(i) - table.put( - row_key, {column_name.encode('utf-8'): value.encode('utf-8')} - ) - # [END bigtable_hw_write_rows_happybase] - - # [START bigtable_hw_get_by_key_happybase] - print('Getting a single greeting by row key.') - key = 'greeting0'.encode('utf-8') - row = table.row(key) - print('\t{}: {}'.format(key, row[column_name.encode('utf-8')])) - # [END bigtable_hw_get_by_key_happybase] - - # [START bigtable_hw_scan_all_happybase] - print('Scanning for all greetings:') - - for key, row in table.scan(): - print('\t{}: {}'.format(key, row[column_name.encode('utf-8')])) - # [END bigtable_hw_scan_all_happybase] - - # [START bigtable_hw_delete_table_happybase] - print('Deleting the {} table.'.format(table_name)) - connection.delete_table(table_name) - # [END bigtable_hw_delete_table_happybase] - - finally: - connection.close() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('project_id', help='Your Cloud Platform project ID.') - parser.add_argument( - 'instance_id', help='ID of the Cloud Bigtable instance to connect to.') - parser.add_argument( - '--table', - help='Table to create and destroy.', - default='Hello-Bigtable') - - args = parser.parse_args() - main(args.project_id, args.instance_id, args.table) diff --git a/bigtable/hello_happybase/main_test.py b/bigtable/hello_happybase/main_test.py deleted file mode 100644 index f72fc0b2e52b..000000000000 --- a/bigtable/hello_happybase/main_test.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import random - -from main import main - -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] -BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE'] -TABLE_NAME_FORMAT = 'hello-world-hb-test-{}' -TABLE_NAME_RANGE = 10000 - - -def test_main(capsys): - table_name = TABLE_NAME_FORMAT.format( - random.randrange(TABLE_NAME_RANGE)) - main( - PROJECT, - BIGTABLE_INSTANCE, - table_name) - - out, _ = capsys.readouterr() - assert 'Creating the {} table.'.format(table_name) in out - assert 'Writing some greetings to the table.' in out - assert 'Getting a single greeting by row key.' in out - assert 'Hello World!' in out - assert 'Scanning for all greetings' in out - assert 'Hello Cloud Bigtable!' in out - assert 'Deleting the {} table.'.format(table_name) in out diff --git a/bigtable/hello_happybase/requirements-test.txt b/bigtable/hello_happybase/requirements-test.txt deleted file mode 100644 index 781d4326c947..000000000000 --- a/bigtable/hello_happybase/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==5.3.2 diff --git a/bigtable/hello_happybase/requirements.txt b/bigtable/hello_happybase/requirements.txt deleted file mode 100644 index a144f03e1bc5..000000000000 --- a/bigtable/hello_happybase/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-cloud-happybase==0.33.0 diff --git a/bigtable/instanceadmin/README.rst b/bigtable/instanceadmin/README.rst deleted file mode 100644 index 16f176a6099c..000000000000 --- a/bigtable/instanceadmin/README.rst +++ /dev/null @@ -1,120 +0,0 @@ -.. This file is automatically generated. Do not edit this file directly. - - -Google Cloud Bigtable table creation -=============================================================================== - -https://cloud.google.com/bigtable/docs/quickstart-cbt - -This page explains how to use the cbt command to connect to a Cloud Bigtable instance, perform basic administrative tasks, and read and write data in a table. - -Google Cloud Bigtable Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/README.rst - - -This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. - - - - -.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs - -Setup -------------------------------------------------------------------------------- - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - -Samples -------------------------------------------------------------------------------- - -Basic example -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/instanceadmin.py,bigtable/instanceadmin/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python instanceadmin.py - - usage: instanceadmin.py [-h] [run] [dev-instance] [del-instance] [add-cluster] [del-cluster] project_id instance_id cluster_id - - Demonstrates how to connect to Cloud Bigtable and run some basic operations - to create instance, create cluster, delete instance and delete cluster. - Prerequisites: - Create a Cloud Bigtable cluster. - https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google - Application Default Credentials. - https://developers.google.com/identity/protocols/application-default- - credentials - - positional arguments: - project_id Your Cloud Platform project ID. - instance_id ID of the Cloud Bigtable instance to connect to. - cluster_id ID of the Cloud Bigtable cluster to connect to. - - optional arguments: - -h, --help show this help message and exit - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/instanceadmin/README.rst.in b/bigtable/instanceadmin/README.rst.in deleted file mode 100644 index c085e40a6278..000000000000 --- a/bigtable/instanceadmin/README.rst.in +++ /dev/null @@ -1,23 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google Cloud Bigtable and run some basic operations. - short_name: Cloud Bigtable - url: https://cloud.google.com/bigtable/docs - description: > - `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's - the same database that powers many core Google services, including Search, - Analytics, Maps, and Gmail. - -setup: -- auth -- install_deps - -samples: -- name: Basic example with Bigtable Column family and GC rules. - file: instanceadmin.py - show_help: true - -cloud_client_library: true - -folder: bigtable/instanceadmin \ No newline at end of file diff --git a/bigtable/instanceadmin/instanceadmin.py b/bigtable/instanceadmin/instanceadmin.py deleted file mode 100644 index 32120eb63751..000000000000 --- a/bigtable/instanceadmin/instanceadmin.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Demonstrates how to connect to Cloud Bigtable and run some basic operations. -# http://www.apache.org/licenses/LICENSE-2.0 -Prerequisites: -- Create a Cloud Bigtable project. - https://cloud.google.com/bigtable/docs/ -- Set your Google Application Default Credentials. - https://developers.google.com/identity/protocols/application-default-credentials - -Operations performed: -- Create a Cloud Bigtable Instance. -- List Instance for a Cloud Bigtable. -- Delete a Cloud Bigtable Instance. -- Create a Cloud Bigtable Cluster. -- List Cloud Bigtable Clusters. -- Delete a Cloud Bigtable Cluster. -""" - -import argparse - -from google.cloud import bigtable -from google.cloud.bigtable import enums - - -def run_instance_operations(project_id, instance_id): - ''' Check Instance exists. - Creates a Production instance with default Cluster. - List instances in a project. - List clusters in an instance. - - :type project_id: str - :param project_id: Project id of the client. - - :type instance_id: str - :param instance_id: Instance of the client. - ''' - client = bigtable.Client(project=project_id, admin=True) - location_id = 'us-central1-f' - serve_nodes = 3 - storage_type = enums.StorageType.SSD - production = enums.Instance.Type.PRODUCTION - labels = {'prod-label': 'prod-label'} - instance = client.instance(instance_id, instance_type=production, - labels=labels) - - # [START bigtable_check_instance_exists] - if not instance.exists(): - print('Instance {} does not exists.'.format(instance_id)) - else: - print('Instance {} already exists.'.format(instance_id)) - # [END bigtable_check_instance_exists] - - # [START bigtable_create_prod_instance] - cluster = instance.cluster("ssd-cluster1", location_id=location_id, - serve_nodes=serve_nodes, - default_storage_type=storage_type) - if not instance.exists(): - print('\nCreating an Instance') - # Create instance with given options - instance.create(clusters=[cluster]) - print('\nCreated instance: {}'.format(instance_id)) - # [END bigtable_create_prod_instance] - - # [START bigtable_list_instances] - print('\nListing Instances:') - for instance_local in client.list_instances()[0]: - print(instance_local.instance_id) - # [END bigtable_list_instances] - - # [START bigtable_get_instance] - print('\nName of instance:{}\nLabels:{}'.format(instance.display_name, - instance.labels)) - # [END bigtable_get_instance] - - # [START bigtable_get_clusters] - print('\nListing Clusters...') - for cluster in instance.list_clusters()[0]: - print(cluster.cluster_id) - # [END bigtable_get_clusters] - - -def create_dev_instance(project_id, instance_id, cluster_id): - ''' Creates a Development instance with the name "hdd-instance" - location us-central1-f - Cluster nodes should not be set while creating Development - Instance - - :type project_id: str - :param project_id: Project id of the client. - - :type instance_id: str - :param instance_id: Instance of the client. - ''' - - client = bigtable.Client(project=project_id, admin=True) - - # [START bigtable_create_dev_instance] - print('\nCreating a DEVELOPMENT Instance') - # Set options to create an Instance - location_id = 'us-central1-f' - development = enums.Instance.Type.DEVELOPMENT - storage_type = enums.StorageType.HDD - labels = {'dev-label': 'dev-label'} - - # Create instance with given options - instance = client.instance(instance_id, instance_type=development, - labels=labels) - cluster = instance.cluster(cluster_id, location_id=location_id, - default_storage_type=storage_type) - - # Create development instance with given options - if not instance.exists(): - instance.create(clusters=[cluster]) - print('Created development instance: {}'.format(instance_id)) - else: - print('Instance {} already exists.'.format(instance_id)) - - # [END bigtable_create_dev_instance] - - -def delete_instance(project_id, instance_id): - ''' Delete the Instance - - :type project_id: str - :param project_id: Project id of the client. - - :type instance_id: str - :param instance_id: Instance of the client. - ''' - - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - # [START bigtable_delete_instance] - print('\nDeleting Instance') - if not instance.exists(): - print('Instance {} does not exists.'.format(instance_id)) - else: - instance.delete() - print('Deleted Instance: {}'.format(instance_id)) - # [END bigtable_delete_instance] - - -def add_cluster(project_id, instance_id, cluster_id): - ''' Add Cluster - - :type project_id: str - :param project_id: Project id of the client. - - :type instance_id: str - :param instance_id: Instance of the client. - - :type cluster_id: str - :param cluster_id: Cluster id. - ''' - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - - location_id = 'us-central1-a' - serve_nodes = 3 - storage_type = enums.StorageType.SSD - - if not instance.exists(): - print('Instance {} does not exists.'.format(instance_id)) - else: - print('\nAdding Cluster to Instance {}'.format(instance_id)) - # [START bigtable_create_cluster] - print('\nListing Clusters...') - for cluster in instance.list_clusters()[0]: - print(cluster.cluster_id) - cluster = instance.cluster(cluster_id, location_id=location_id, - serve_nodes=serve_nodes, - default_storage_type=storage_type) - if cluster.exists(): - print( - '\nCluster not created, as {} already exists.'. - format(cluster_id) - ) - else: - cluster.create() - print('\nCluster created: {}'.format(cluster_id)) - # [END bigtable_create_cluster] - - -def delete_cluster(project_id, instance_id, cluster_id): - ''' Delete the cluster - - :type project_id: str - :param project_id: Project id of the client. - - :type instance_id: str - :param instance_id: Instance of the client. - - :type cluster_id: str - :param cluster_id: Cluster id. - ''' - - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - cluster = instance.cluster(cluster_id) - - # [START bigtable_delete_cluster] - print('\nDeleting Cluster') - if cluster.exists(): - cluster.delete() - print('Cluster deleted: {}'.format(cluster_id)) - else: - print('\nCluster {} does not exist.'.format(cluster_id)) - - # [END bigtable_delete_cluster] - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('command', - help='run, dev-instance, del-instance, \ - add-cluster or del-cluster. \ - Operation to perform on Instance.') - parser.add_argument('project_id', - help='Your Cloud Platform project ID.') - parser.add_argument('instance_id', - help='ID of the Cloud Bigtable instance to \ - connect to.') - parser.add_argument('cluster_id', - help='ID of the Cloud Bigtable cluster to \ - connect to.') - - args = parser.parse_args() - - if args.command.lower() == 'run': - run_instance_operations(args.project_id, args.instance_id) - elif args.command.lower() == 'dev-instance': - create_dev_instance(args.project_id, args.instance_id, - args.cluster_id) - elif args.command.lower() == 'del-instance': - delete_instance(args.project_id, args.instance_id) - elif args.command.lower() == 'add-cluster': - add_cluster(args.project_id, args.instance_id, args.cluster_id) - elif args.command.lower() == 'del-cluster': - delete_cluster(args.project_id, args.instance_id, args.cluster_id) - else: - print('Command should be either run \n Use argument -h, \ - --help to show help and exit.') diff --git a/bigtable/instanceadmin/requirements-test.txt b/bigtable/instanceadmin/requirements-test.txt deleted file mode 100644 index 781d4326c947..000000000000 --- a/bigtable/instanceadmin/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==5.3.2 diff --git a/bigtable/instanceadmin/requirements.txt b/bigtable/instanceadmin/requirements.txt deleted file mode 100755 index 2771c2e4c4d0..000000000000 --- a/bigtable/instanceadmin/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-cloud-bigtable==1.2.1 diff --git a/bigtable/metricscaler/Dockerfile b/bigtable/metricscaler/Dockerfile deleted file mode 100644 index d8a5ec0c1a9b..000000000000 --- a/bigtable/metricscaler/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -FROM python:3 - -WORKDIR /usr/src/app - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -COPY . . - -ENTRYPOINT [ "python", "./metricscaler.py"] -CMD ["--help"] diff --git a/bigtable/metricscaler/README.rst b/bigtable/metricscaler/README.rst deleted file mode 100644 index c64bbff1d8af..000000000000 --- a/bigtable/metricscaler/README.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. This file is automatically generated. Do not edit this file directly. - -Google Cloud Bigtable Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/metricscaler/README.rst - - -This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. - - -This sample demonstrates how to use `Stackdriver Monitoring`_ -to scale Cloud Bigtable based on CPU usage. - -.. _Stackdriver Monitoring: http://cloud.google.com/monitoring/docs/ - - -.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs/ - -Setup -------------------------------------------------------------------------------- - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - -Samples -------------------------------------------------------------------------------- - -Metricscaling example -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/metricscaler/metricscaler.py,bigtable/metricscaler/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python metricscaler.py - - usage: metricscaler.py [-h] [--high_cpu_threshold HIGH_CPU_THRESHOLD] - [--low_cpu_threshold LOW_CPU_THRESHOLD] - [--short_sleep SHORT_SLEEP] [--long_sleep LONG_SLEEP] - bigtable_instance bigtable_cluster - - Scales Cloud Bigtable clusters based on CPU usage. - - positional arguments: - bigtable_instance ID of the Cloud Bigtable instance to connect to. - bigtable_cluster ID of the Cloud Bigtable cluster to connect to. - - optional arguments: - -h, --help show this help message and exit - --high_cpu_threshold HIGH_CPU_THRESHOLD - If Cloud Bigtable CPU usage is above this threshold, - scale up - --low_cpu_threshold LOW_CPU_THRESHOLD - If Cloud Bigtable CPU usage is below this threshold, - scale down - --short_sleep SHORT_SLEEP - How long to sleep in seconds between checking metrics - after no scale operation - --long_sleep LONG_SLEEP - How long to sleep in seconds between checking metrics - after a scaling operation - - - - - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/metricscaler/README.rst.in b/bigtable/metricscaler/README.rst.in deleted file mode 100644 index 44a548e4c1fb..000000000000 --- a/bigtable/metricscaler/README.rst.in +++ /dev/null @@ -1,29 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google Cloud Bigtable - short_name: Cloud Bigtable - url: https://cloud.google.com/bigtable/docs/ - description: > - `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's - the same database that powers many core Google services, including Search, - Analytics, Maps, and Gmail. - -description: | - This sample demonstrates how to use `Stackdriver Monitoring`_ - to scale Cloud Bigtable based on CPU usage. - - .. _Stackdriver Monitoring: http://cloud.google.com/monitoring/docs/ - -setup: -- auth -- install_deps - -samples: -- name: Metricscaling example - file: metricscaler.py - show_help: true - -cloud_client_library: true - -folder: bigtable/metricscaler \ No newline at end of file diff --git a/bigtable/metricscaler/metricscaler.py b/bigtable/metricscaler/metricscaler.py deleted file mode 100644 index 1957a81f1aa9..000000000000 --- a/bigtable/metricscaler/metricscaler.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Sample that demonstrates how to use Stackdriver Monitoring metrics to -programmatically scale a Google Cloud Bigtable cluster.""" - -import argparse -import os -import time - -from google.cloud import bigtable -from google.cloud import monitoring_v3 -from google.cloud.bigtable import enums -from google.cloud.monitoring_v3 import query - -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] - - -def get_cpu_load(): - """Returns the most recent Cloud Bigtable CPU load measurement. - - Returns: - float: The most recent Cloud Bigtable CPU usage metric - """ - # [START bigtable_cpu] - client = monitoring_v3.MetricServiceClient() - cpu_query = query.Query(client, - project=PROJECT, - metric_type='bigtable.googleapis.com/' - 'cluster/cpu_load', - minutes=5) - cpu = next(cpu_query.iter()) - return cpu.points[0].value.double_value - # [END bigtable_cpu] - - -def get_storage_utilization(): - """Returns the most recent Cloud Bigtable storage utilization measurement. - - Returns: - float: The most recent Cloud Bigtable storage utilization metric - """ - # [START bigtable_metric_scaler_storage_utilization] - client = monitoring_v3.MetricServiceClient() - utilization_query = query.Query(client, - project=PROJECT, - metric_type='bigtable.googleapis.com/' - 'cluster/storage_utilization', - minutes=5) - utilization = next(utilization_query.iter()) - return utilization.points[0].value.double_value - # [END bigtable_metric_scaler_storage_utilization] - - -def scale_bigtable(bigtable_instance, bigtable_cluster, scale_up): - """Scales the number of Cloud Bigtable nodes up or down. - - Edits the number of nodes in the Cloud Bigtable cluster to be increased - or decreased, depending on the `scale_up` boolean argument. Currently - the `incremental` strategy from `strategies.py` is used. - - - Args: - bigtable_instance (str): Cloud Bigtable instance ID to scale - bigtable_cluster (str): Cloud Bigtable cluster ID to scale - scale_up (bool): If true, scale up, otherwise scale down - """ - - # The minimum number of nodes to use. The default minimum is 3. If you have - # a lot of data, the rule of thumb is to not go below 2.5 TB per node for - # SSD lusters, and 8 TB for HDD. The - # "bigtable.googleapis.com/disk/bytes_used" metric is useful in figuring - # out the minimum number of nodes. - min_node_count = 3 - - # The maximum number of nodes to use. The default maximum is 30 nodes per - # zone. If you need more quota, you can request more by following the - # instructions at https://cloud.google.com/bigtable/quota. - max_node_count = 30 - - # The number of nodes to change the cluster by. - size_change_step = 3 - - # [START bigtable_scale] - bigtable_client = bigtable.Client(admin=True) - instance = bigtable_client.instance(bigtable_instance) - instance.reload() - - if instance.type_ == enums.Instance.Type.DEVELOPMENT: - raise ValueError("Development instances cannot be scaled.") - - cluster = instance.cluster(bigtable_cluster) - cluster.reload() - - current_node_count = cluster.serve_nodes - - if scale_up: - if current_node_count < max_node_count: - new_node_count = min( - current_node_count + size_change_step, max_node_count) - cluster.serve_nodes = new_node_count - cluster.update() - print('Scaled up from {} to {} nodes.'.format( - current_node_count, new_node_count)) - else: - if current_node_count > min_node_count: - new_node_count = max( - current_node_count - size_change_step, min_node_count) - cluster.serve_nodes = new_node_count - cluster.update() - print('Scaled down from {} to {} nodes.'.format( - current_node_count, new_node_count)) - # [END bigtable_scale] - - -def main( - bigtable_instance, - bigtable_cluster, - high_cpu_threshold, - low_cpu_threshold, - high_storage_threshold, - short_sleep, - long_sleep -): - """Main loop runner that autoscales Cloud Bigtable. - - Args: - bigtable_instance (str): Cloud Bigtable instance ID to autoscale - high_cpu_threshold (float): If CPU is higher than this, scale up. - low_cpu_threshold (float): If CPU is lower than this, scale down. - high_storage_threshold (float): If storage is higher than this, - scale up. - short_sleep (int): How long to sleep after no operation - long_sleep (int): How long to sleep after the number of nodes is - changed - """ - cluster_cpu = get_cpu_load() - cluster_storage = get_storage_utilization() - print('Detected cpu of {}'.format(cluster_cpu)) - print('Detected storage utilization of {}'.format(cluster_storage)) - try: - if cluster_cpu > high_cpu_threshold or cluster_storage > high_storage_threshold: - scale_bigtable(bigtable_instance, bigtable_cluster, True) - time.sleep(long_sleep) - elif cluster_cpu < low_cpu_threshold: - if cluster_storage < high_storage_threshold: - scale_bigtable(bigtable_instance, bigtable_cluster, False) - time.sleep(long_sleep) - else: - print('CPU within threshold, sleeping.') - time.sleep(short_sleep) - except Exception as e: - print("Error during scaling: %s", e) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Scales Cloud Bigtable clusters based on CPU usage.') - parser.add_argument( - 'bigtable_instance', - help='ID of the Cloud Bigtable instance to connect to.') - parser.add_argument( - 'bigtable_cluster', - help='ID of the Cloud Bigtable cluster to connect to.') - parser.add_argument( - '--high_cpu_threshold', - help='If Cloud Bigtable CPU usage is above this threshold, scale up', - default=0.6) - parser.add_argument( - '--low_cpu_threshold', - help='If Cloud Bigtable CPU usage is below this threshold, scale down', - default=0.2) - parser.add_argument( - '--high_storage_threshold', - help='If Cloud Bigtable storage utilization is above this threshold, ' - 'scale up', - default=0.6) - parser.add_argument( - '--short_sleep', - help='How long to sleep in seconds between checking metrics after no ' - 'scale operation', - default=60) - parser.add_argument( - '--long_sleep', - help='How long to sleep in seconds between checking metrics after a ' - 'scaling operation', - default=60 * 10) - args = parser.parse_args() - - while True: - main( - args.bigtable_instance, - args.bigtable_cluster, - float(args.high_cpu_threshold), - float(args.low_cpu_threshold), - float(args.high_storage_threshold), - int(args.short_sleep), - int(args.long_sleep)) diff --git a/bigtable/metricscaler/metricscaler_test.py b/bigtable/metricscaler/metricscaler_test.py deleted file mode 100644 index 1d4a3a3d11ec..000000000000 --- a/bigtable/metricscaler/metricscaler_test.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit and system tests for metricscaler.py""" - -import os -import time -import uuid - -from google.cloud import bigtable -from google.cloud.bigtable import enums -from mock import patch - -import pytest - -from metricscaler import get_cpu_load -from metricscaler import get_storage_utilization -from metricscaler import main -from metricscaler import scale_bigtable - - -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] -BIGTABLE_ZONE = os.environ['BIGTABLE_ZONE'] -SIZE_CHANGE_STEP = 3 -INSTANCE_ID_FORMAT = 'metric-scale-test-{}' -BIGTABLE_INSTANCE = INSTANCE_ID_FORMAT.format(str(uuid.uuid4())[:10]) -BIGTABLE_DEV_INSTANCE = INSTANCE_ID_FORMAT.format(str(uuid.uuid4())[:10]) - - -# System tests to verify API calls succeed - - -def test_get_cpu_load(): - assert float(get_cpu_load()) > 0.0 - - -def test_get_storage_utilization(): - assert float(get_storage_utilization()) > 0.0 - - -@pytest.fixture() -def instance(): - cluster_id = BIGTABLE_INSTANCE - - client = bigtable.Client(project=PROJECT, admin=True) - - serve_nodes = 3 - storage_type = enums.StorageType.SSD - production = enums.Instance.Type.PRODUCTION - labels = {'prod-label': 'prod-label'} - instance = client.instance(BIGTABLE_INSTANCE, instance_type=production, - labels=labels) - - if not instance.exists(): - cluster = instance.cluster(cluster_id, location_id=BIGTABLE_ZONE, - serve_nodes=serve_nodes, - default_storage_type=storage_type) - instance.create(clusters=[cluster]) - - yield - - instance.delete() - - -@pytest.fixture() -def dev_instance(): - cluster_id = BIGTABLE_DEV_INSTANCE - - client = bigtable.Client(project=PROJECT, admin=True) - - storage_type = enums.StorageType.SSD - development = enums.Instance.Type.DEVELOPMENT - labels = {'dev-label': 'dev-label'} - instance = client.instance(BIGTABLE_DEV_INSTANCE, - instance_type=development, - labels=labels) - - if not instance.exists(): - cluster = instance.cluster(cluster_id, location_id=BIGTABLE_ZONE, - default_storage_type=storage_type) - instance.create(clusters=[cluster]) - - yield - - instance.delete() - - -def test_scale_bigtable(instance): - bigtable_client = bigtable.Client(admin=True) - - instance = bigtable_client.instance(BIGTABLE_INSTANCE) - instance.reload() - - cluster = instance.cluster(BIGTABLE_INSTANCE) - cluster.reload() - original_node_count = cluster.serve_nodes - - scale_bigtable(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, True) - - for n in range(10): - time.sleep(10) - cluster.reload() - new_node_count = cluster.serve_nodes - try: - assert (new_node_count == (original_node_count + SIZE_CHANGE_STEP)) - except AssertionError: - if n == 9: - raise - - scale_bigtable(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, False) - - for n in range(10): - time.sleep(10) - cluster.reload() - final_node_count = cluster.serve_nodes - try: - assert final_node_count == original_node_count - except AssertionError: - if n == 9: - raise - - -def test_handle_dev_instance(capsys, dev_instance): - with pytest.raises(ValueError): - scale_bigtable(BIGTABLE_DEV_INSTANCE, BIGTABLE_DEV_INSTANCE, True) - - -@patch('time.sleep') -@patch('metricscaler.get_storage_utilization') -@patch('metricscaler.get_cpu_load') -@patch('metricscaler.scale_bigtable') -def test_main(scale_bigtable, get_cpu_load, get_storage_utilization, sleep): - SHORT_SLEEP = 5 - LONG_SLEEP = 10 - - # Test okay CPU, okay storage utilization - get_cpu_load.return_value = 0.5 - get_storage_utilization.return_value = 0.5 - - main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, - LONG_SLEEP) - scale_bigtable.assert_not_called() - scale_bigtable.reset_mock() - - # Test high CPU, okay storage utilization - get_cpu_load.return_value = 0.7 - get_storage_utilization.return_value = 0.5 - main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, - LONG_SLEEP) - scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, - BIGTABLE_INSTANCE, True) - scale_bigtable.reset_mock() - - # Test low CPU, okay storage utilization - get_storage_utilization.return_value = 0.5 - get_cpu_load.return_value = 0.2 - main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, - LONG_SLEEP) - scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, - BIGTABLE_INSTANCE, False) - scale_bigtable.reset_mock() - - # Test okay CPU, high storage utilization - get_cpu_load.return_value = 0.5 - get_storage_utilization.return_value = 0.7 - - main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, - LONG_SLEEP) - scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, - BIGTABLE_INSTANCE, True) - scale_bigtable.reset_mock() - - # Test high CPU, high storage utilization - get_cpu_load.return_value = 0.7 - get_storage_utilization.return_value = 0.7 - main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, - LONG_SLEEP) - scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, - BIGTABLE_INSTANCE, True) - scale_bigtable.reset_mock() - - # Test low CPU, high storage utilization - get_cpu_load.return_value = 0.2 - get_storage_utilization.return_value = 0.7 - main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, - LONG_SLEEP) - scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, - BIGTABLE_INSTANCE, True) - scale_bigtable.reset_mock() diff --git a/bigtable/metricscaler/requirements-test.txt b/bigtable/metricscaler/requirements-test.txt deleted file mode 100644 index 41c4d5110536..000000000000 --- a/bigtable/metricscaler/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest==5.3.2 -mock==3.0.5 diff --git a/bigtable/metricscaler/requirements.txt b/bigtable/metricscaler/requirements.txt deleted file mode 100644 index ab135bd21a06..000000000000 --- a/bigtable/metricscaler/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -google-cloud-bigtable==1.2.1 -google-cloud-monitoring==1.0.0 diff --git a/bigtable/quickstart/README.rst b/bigtable/quickstart/README.rst deleted file mode 100644 index c7ffabf8c3bd..000000000000 --- a/bigtable/quickstart/README.rst +++ /dev/null @@ -1,108 +0,0 @@ -.. This file is automatically generated. Do not edit this file directly. - -Google Cloud Bigtable Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/quickstart/README.rst - - -This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. - - - - -.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs - -Setup -------------------------------------------------------------------------------- - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - -Samples -------------------------------------------------------------------------------- - -Quickstart -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/quickstart/main.py,bigtable/quickstart/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python main.py - - usage: main.py [-h] [--table TABLE] project_id instance_id - - positional arguments: - project_id Your Cloud Platform project ID. - instance_id ID of the Cloud Bigtable instance to connect to. - - optional arguments: - -h, --help show this help message and exit - --table TABLE Existing table used in the quickstart. (default: my-table) - - - - - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/quickstart/README.rst.in b/bigtable/quickstart/README.rst.in deleted file mode 100644 index 94f070a7c887..000000000000 --- a/bigtable/quickstart/README.rst.in +++ /dev/null @@ -1,23 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google Cloud Bigtable - short_name: Cloud Bigtable - url: https://cloud.google.com/bigtable/docs - description: > - `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's - the same database that powers many core Google services, including Search, - Analytics, Maps, and Gmail. - -setup: -- auth -- install_deps - -samples: -- name: Quickstart - file: main.py - show_help: true - -cloud_client_library: true - -folder: bigtable/quickstart \ No newline at end of file diff --git a/bigtable/quickstart/main.py b/bigtable/quickstart/main.py deleted file mode 100644 index 3763296f1e4c..000000000000 --- a/bigtable/quickstart/main.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START bigtable_quickstart] -import argparse - -from google.cloud import bigtable - - -def main(project_id="project-id", instance_id="instance-id", - table_id="my-table"): - # Create a Cloud Bigtable client. - client = bigtable.Client(project=project_id) - - # Connect to an existing Cloud Bigtable instance. - instance = client.instance(instance_id) - - # Open an existing table. - table = instance.table(table_id) - - row_key = 'r1' - row = table.read_row(row_key.encode('utf-8')) - - column_family_id = 'cf1' - column_id = 'c1'.encode('utf-8') - value = row.cells[column_family_id][column_id][0].value.decode('utf-8') - - print('Row key: {}\nData: {}'.format(row_key, value)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('project_id', help='Your Cloud Platform project ID.') - parser.add_argument( - 'instance_id', help='ID of the Cloud Bigtable instance to connect to.') - parser.add_argument( - '--table', - help='Existing table used in the quickstart.', - default='my-table') - - args = parser.parse_args() - main(args.project_id, args.instance_id, args.table) -# [END bigtable_quickstart] diff --git a/bigtable/quickstart/main_test.py b/bigtable/quickstart/main_test.py deleted file mode 100644 index 55c06f413df9..000000000000 --- a/bigtable/quickstart/main_test.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2018 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import random - -from google.cloud import bigtable -import pytest - -from main import main - - -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] -BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE'] -TABLE_ID_FORMAT = 'quickstart-test-{}' -TABLE_ID_RANGE = 10000 - - -@pytest.fixture() -def table(): - table_id = TABLE_ID_FORMAT.format( - random.randrange(TABLE_ID_RANGE)) - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) - column_family_id = 'cf1' - column_families = {column_family_id: None} - table.create(column_families=column_families) - - row = table.direct_row("r1") - row.set_cell(column_family_id, "c1", "test-value") - row.commit() - - yield table_id - - table.delete() - - -def test_main(capsys, table): - table_id = table - main(PROJECT, BIGTABLE_INSTANCE, table_id) - - out, _ = capsys.readouterr() - assert 'Row key: r1\nData: test-value\n' in out diff --git a/bigtable/quickstart/requirements-test.txt b/bigtable/quickstart/requirements-test.txt deleted file mode 100644 index 781d4326c947..000000000000 --- a/bigtable/quickstart/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==5.3.2 diff --git a/bigtable/quickstart/requirements.txt b/bigtable/quickstart/requirements.txt deleted file mode 100644 index 2771c2e4c4d0..000000000000 --- a/bigtable/quickstart/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-cloud-bigtable==1.2.1 diff --git a/bigtable/quickstart_happybase/README.rst b/bigtable/quickstart_happybase/README.rst deleted file mode 100644 index e2d1c45a2729..000000000000 --- a/bigtable/quickstart_happybase/README.rst +++ /dev/null @@ -1,108 +0,0 @@ -.. This file is automatically generated. Do not edit this file directly. - -Google Cloud Bigtable Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/quickstart_happybase/README.rst - - -This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. - - - - -.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs - -Setup -------------------------------------------------------------------------------- - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - -Samples -------------------------------------------------------------------------------- - -Quickstart -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/quickstart_happybase/main.py,bigtable/quickstart_happybase/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python main.py - - usage: main.py [-h] [--table TABLE] project_id instance_id - - positional arguments: - project_id Your Cloud Platform project ID. - instance_id ID of the Cloud Bigtable instance to connect to. - - optional arguments: - -h, --help show this help message and exit - --table TABLE Existing table used in the quickstart. (default: my-table) - - - - - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/quickstart_happybase/README.rst.in b/bigtable/quickstart_happybase/README.rst.in deleted file mode 100644 index 811a0b868fb3..000000000000 --- a/bigtable/quickstart_happybase/README.rst.in +++ /dev/null @@ -1,23 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google Cloud Bigtable - short_name: Cloud Bigtable - url: https://cloud.google.com/bigtable/docs - description: > - `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's - the same database that powers many core Google services, including Search, - Analytics, Maps, and Gmail. - -setup: -- auth -- install_deps - -samples: -- name: Quickstart - file: main.py - show_help: true - -cloud_client_library: true - -folder: bigtable/quickstart_happybase \ No newline at end of file diff --git a/bigtable/quickstart_happybase/main.py b/bigtable/quickstart_happybase/main.py deleted file mode 100644 index 056e3666bb5b..000000000000 --- a/bigtable/quickstart_happybase/main.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# [START bigtable_quickstart_happybase] -import argparse - -from google.cloud import bigtable -from google.cloud import happybase - - -def main(project_id="project-id", instance_id="instance-id", - table_id="my-table"): - # Creates a Bigtable client - client = bigtable.Client(project=project_id) - - # Connect to an existing instance:my-bigtable-instance - instance = client.instance(instance_id) - - connection = happybase.Connection(instance=instance) - - try: - # Connect to an existing table:my-table - table = connection.table(table_id) - - key = 'r1' - row = table.row(key.encode('utf-8')) - - column = 'cf1:c1'.encode('utf-8') - value = row[column].decode('utf-8') - print('Row key: {}\nData: {}'.format(key, value)) - - finally: - connection.close() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('project_id', help='Your Cloud Platform project ID.') - parser.add_argument( - 'instance_id', help='ID of the Cloud Bigtable instance to connect to.') - parser.add_argument( - '--table', - help='Existing table used in the quickstart.', - default='my-table') - - args = parser.parse_args() - main(args.project_id, args.instance_id, args.table) -# [END bigtable_quickstart_happybase] diff --git a/bigtable/quickstart_happybase/main_test.py b/bigtable/quickstart_happybase/main_test.py deleted file mode 100644 index 5d4ae1e7acd7..000000000000 --- a/bigtable/quickstart_happybase/main_test.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2018 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import random - -from google.cloud import bigtable -import pytest - -from main import main - - -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] -BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE'] -TABLE_ID_FORMAT = 'quickstart-hb-test-{}' -TABLE_ID_RANGE = 10000 - - -@pytest.fixture() -def table(): - table_id = TABLE_ID_FORMAT.format( - random.randrange(TABLE_ID_RANGE)) - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) - column_family_id = 'cf1' - column_families = {column_family_id: None} - table.create(column_families=column_families) - - row = table.direct_row("r1") - row.set_cell(column_family_id, "c1", "test-value") - row.commit() - - yield table_id - - table.delete() - - -def test_main(capsys, table): - table_id = table - main(PROJECT, BIGTABLE_INSTANCE, table_id) - - out, _ = capsys.readouterr() - assert 'Row key: r1\nData: test-value\n' in out diff --git a/bigtable/quickstart_happybase/requirements-test.txt b/bigtable/quickstart_happybase/requirements-test.txt deleted file mode 100644 index 781d4326c947..000000000000 --- a/bigtable/quickstart_happybase/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==5.3.2 diff --git a/bigtable/quickstart_happybase/requirements.txt b/bigtable/quickstart_happybase/requirements.txt deleted file mode 100644 index a144f03e1bc5..000000000000 --- a/bigtable/quickstart_happybase/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-cloud-happybase==0.33.0 diff --git a/bigtable/snippets/filters/filter_snippets.py b/bigtable/snippets/filters/filter_snippets.py deleted file mode 100644 index c815eae99b8e..000000000000 --- a/bigtable/snippets/filters/filter_snippets.py +++ /dev/null @@ -1,290 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2020, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START bigtable_filters_print] -import datetime - -from google.cloud import bigtable -import google.cloud.bigtable.row_filters as row_filters - -# Write your code here. -# [START_EXCLUDE] - - -# [START bigtable_filters_limit_row_sample] -def filter_limit_row_sample(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows(filter_=row_filters.RowSampleFilter(.75)) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_row_sample] -# [START bigtable_filters_limit_row_regex] -def filter_limit_row_regex(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows( - filter_=row_filters.RowKeyRegexFilter(".*#20190501$".encode("utf-8"))) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_row_regex] -# [START bigtable_filters_limit_cells_per_col] -def filter_limit_cells_per_col(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows(filter_=row_filters.CellsColumnLimitFilter(2)) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_cells_per_col] -# [START bigtable_filters_limit_cells_per_row] -def filter_limit_cells_per_row(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows(filter_=row_filters.CellsRowLimitFilter(2)) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_cells_per_row] -# [START bigtable_filters_limit_cells_per_row_offset] -def filter_limit_cells_per_row_offset(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows(filter_=row_filters.CellsRowOffsetFilter(2)) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_cells_per_row_offset] -# [START bigtable_filters_limit_col_family_regex] -def filter_limit_col_family_regex(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows( - filter_=row_filters.FamilyNameRegexFilter("stats_.*$".encode("utf-8"))) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_col_family_regex] -# [START bigtable_filters_limit_col_qualifier_regex] -def filter_limit_col_qualifier_regex(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows( - filter_=row_filters.ColumnQualifierRegexFilter( - "connected_.*$".encode("utf-8"))) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_col_qualifier_regex] -# [START bigtable_filters_limit_col_range] -def filter_limit_col_range(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows( - filter_=row_filters.ColumnRangeFilter("cell_plan", - b"data_plan_01gb", - b"data_plan_10gb", - inclusive_end=False)) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_col_range] -# [START bigtable_filters_limit_value_range] -def filter_limit_value_range(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows( - filter_=row_filters.ValueRangeFilter(b"PQ2A.190405", b"PQ2A.190406")) - - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_value_range] -# [START bigtable_filters_limit_value_regex] - - -def filter_limit_value_regex(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows( - filter_=row_filters.ValueRegexFilter("PQ2A.*$".encode("utf-8"))) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_value_regex] -# [START bigtable_filters_limit_timestamp_range] -def filter_limit_timestamp_range(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - end = datetime.datetime(2019, 5, 1) - - rows = table.read_rows( - filter_=row_filters.TimestampRangeFilter( - row_filters.TimestampRange(end=end))) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_timestamp_range] -# [START bigtable_filters_limit_block_all] -def filter_limit_block_all(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows(filter_=row_filters.BlockAllFilter(True)) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_block_all] -# [START bigtable_filters_limit_pass_all] -def filter_limit_pass_all(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows(filter_=row_filters.PassAllFilter(True)) - for row in rows: - print_row(row) - - -# [END bigtable_filters_limit_pass_all] -# [START bigtable_filters_modify_strip_value] -def filter_modify_strip_value(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows( - filter_=row_filters.StripValueTransformerFilter(True)) - for row in rows: - print_row(row) - - -# [END bigtable_filters_modify_strip_value] -# [START bigtable_filters_modify_apply_label] -def filter_modify_apply_label(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows( - filter_=row_filters.ApplyLabelFilter(label="labelled")) - for row in rows: - print_row(row) - - -# [END bigtable_filters_modify_apply_label] -# [START bigtable_filters_composing_chain] -def filter_composing_chain(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows(filter_=row_filters.RowFilterChain( - filters=[row_filters.CellsColumnLimitFilter(1), - row_filters.FamilyNameRegexFilter("cell_plan")])) - for row in rows: - print_row(row) - - -# [END bigtable_filters_composing_chain] -# [START bigtable_filters_composing_interleave] -def filter_composing_interleave(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows(filter_=row_filters.RowFilterUnion( - filters=[row_filters.ValueRegexFilter("true"), - row_filters.ColumnQualifierRegexFilter("os_build")])) - for row in rows: - print_row(row) - - -# [END bigtable_filters_composing_interleave] -# [START bigtable_filters_composing_condition] -def filter_composing_condition(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows(filter_=row_filters.ConditionalRowFilter( - base_filter=row_filters.RowFilterChain(filters=[ - row_filters.ColumnQualifierRegexFilter( - "data_plan_10gb"), - row_filters.ValueRegexFilter( - "true")]), - true_filter=row_filters.ApplyLabelFilter(label="passed-filter"), - false_filter=row_filters.ApplyLabelFilter(label="filtered-out") - - )) - for row in rows: - print_row(row) - - -# [END bigtable_filters_composing_condition] -# [END_EXCLUDE] - - -def print_row(row): - print("Reading data for {}:".format(row.row_key.decode('utf-8'))) - for cf, cols in sorted(row.cells.items()): - print("Column Family {}".format(cf)) - for col, cells in sorted(cols.items()): - for cell in cells: - labels = " [{}]".format(",".join(cell.labels)) \ - if len(cell.labels) else "" - print( - "\t{}: {} @{}{}".format(col.decode('utf-8'), - cell.value.decode('utf-8'), - cell.timestamp, labels)) - print("") -# [END bigtable_filters_print] diff --git a/bigtable/snippets/filters/filters_test.py b/bigtable/snippets/filters/filters_test.py deleted file mode 100644 index f46541bffd35..000000000000 --- a/bigtable/snippets/filters/filters_test.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2020, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import datetime -import os -import uuid - -from google.cloud import bigtable -import pytest - -import filter_snippets - - -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] -BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE'] -TABLE_ID_PREFIX = 'mobile-time-series-{}' - - -@pytest.fixture(scope="module", autouse=True) -def table_id(): - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={'stats_summary': None, 'cell_plan': None}) - - timestamp = datetime.datetime(2019, 5, 1) - timestamp_minus_hr = datetime.datetime(2019, 5, 1) - datetime.timedelta( - hours=1) - - rows = [ - table.direct_row("phone#4c410523#20190501"), - table.direct_row("phone#4c410523#20190502"), - table.direct_row("phone#4c410523#20190505"), - table.direct_row("phone#5c10102#20190501"), - table.direct_row("phone#5c10102#20190502"), - ] - - rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) - rows[0].set_cell("cell_plan", "data_plan_01gb", "true", timestamp_minus_hr) - rows[0].set_cell("cell_plan", "data_plan_01gb", "false", timestamp) - rows[0].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) - rows[1].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) - rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[2].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) - rows[3].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) - rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) - rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[4].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) - - table.mutate_rows(rows) - - yield table_id - - table.delete() - - -def test_filter_limit_row_sample(capsys, snapshot, table_id): - filter_snippets.filter_limit_row_sample(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - assert 'Reading data for' in out - - -def test_filter_limit_row_regex(capsys, snapshot, table_id): - filter_snippets.filter_limit_row_regex(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_limit_cells_per_col(capsys, snapshot, table_id): - filter_snippets.filter_limit_cells_per_col(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_limit_cells_per_row(capsys, snapshot, table_id): - filter_snippets.filter_limit_cells_per_row(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_limit_cells_per_row_offset(capsys, snapshot, table_id): - filter_snippets.filter_limit_cells_per_row_offset(PROJECT, - BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_limit_col_family_regex(capsys, snapshot, table_id): - filter_snippets.filter_limit_col_family_regex(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_limit_col_qualifier_regex(capsys, snapshot, table_id): - filter_snippets.filter_limit_col_qualifier_regex(PROJECT, - BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_limit_col_range(capsys, snapshot, table_id): - filter_snippets.filter_limit_col_range(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_limit_value_range(capsys, snapshot, table_id): - filter_snippets.filter_limit_value_range(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_limit_value_regex(capsys, snapshot, table_id): - filter_snippets.filter_limit_value_regex(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_limit_timestamp_range(capsys, snapshot, table_id): - filter_snippets.filter_limit_timestamp_range(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_limit_block_all(capsys, snapshot, table_id): - filter_snippets.filter_limit_block_all(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_limit_pass_all(capsys, snapshot, table_id): - filter_snippets.filter_limit_pass_all(PROJECT, BIGTABLE_INSTANCE, table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_modify_strip_value(capsys, snapshot, table_id): - filter_snippets.filter_modify_strip_value(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_modify_apply_label(capsys, snapshot, table_id): - filter_snippets.filter_modify_apply_label(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_composing_chain(capsys, snapshot, table_id): - filter_snippets.filter_composing_chain(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_composing_interleave(capsys, snapshot, table_id): - filter_snippets.filter_composing_interleave(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_filter_composing_condition(capsys, snapshot, table_id): - filter_snippets.filter_composing_condition(PROJECT, BIGTABLE_INSTANCE, - table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) diff --git a/bigtable/snippets/filters/requirements-test.txt b/bigtable/snippets/filters/requirements-test.txt deleted file mode 100644 index 781d4326c947..000000000000 --- a/bigtable/snippets/filters/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==5.3.2 diff --git a/bigtable/snippets/filters/requirements.txt b/bigtable/snippets/filters/requirements.txt deleted file mode 100755 index a64e924f1be3..000000000000 --- a/bigtable/snippets/filters/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -google-cloud-bigtable==1.2.1 -snapshottest==0.5.1 \ No newline at end of file diff --git a/bigtable/snippets/filters/snapshots/__init__.py b/bigtable/snippets/filters/snapshots/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/bigtable/snippets/filters/snapshots/snap_filters_test.py b/bigtable/snippets/filters/snapshots/snap_filters_test.py deleted file mode 100644 index a0580f565990..000000000000 --- a/bigtable/snippets/filters/snapshots/snap_filters_test.py +++ /dev/null @@ -1,480 +0,0 @@ -# -*- coding: utf-8 -*- -# snapshottest: v1 - https://goo.gl/zC4yUc -# flake8: noqa -from __future__ import unicode_literals - -from snapshottest import Snapshot - -snapshots = Snapshot() - -snapshots['test_filter_limit_row_regex 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 -\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_limit_cells_per_col 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 -\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_limit_cells_per_row 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 -\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_limit_cells_per_row_offset 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family stats_summary -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family stats_summary -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family stats_summary -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family stats_summary -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_limit_col_family_regex 1'] = '''Reading data for phone#4c410523#20190501: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_limit_col_qualifier_regex 1'] = '''Reading data for phone#4c410523#20190501: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_limit_col_range 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 -\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_limit_value_range 1'] = '''Reading data for phone#4c410523#20190501: -Column Family stats_summary -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family stats_summary -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_limit_value_regex 1'] = '''Reading data for phone#4c410523#20190501: -Column Family stats_summary -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family stats_summary -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family stats_summary -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family stats_summary -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family stats_summary -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_limit_timestamp_range 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 - -''' - -snapshots['test_filter_limit_block_all 1'] = '' - -snapshots['test_filter_limit_pass_all 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 -\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_modify_strip_value 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_01gb: @2019-05-01 00:00:00+00:00 -\tdata_plan_01gb: @2019-04-30 23:00:00+00:00 -\tdata_plan_05gb: @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: @2019-05-01 00:00:00+00:00 -\tconnected_wifi: @2019-05-01 00:00:00+00:00 -\tos_build: @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family cell_plan -\tdata_plan_05gb: @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: @2019-05-01 00:00:00+00:00 -\tconnected_wifi: @2019-05-01 00:00:00+00:00 -\tos_build: @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family cell_plan -\tdata_plan_05gb: @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: @2019-05-01 00:00:00+00:00 -\tconnected_wifi: @2019-05-01 00:00:00+00:00 -\tos_build: @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family cell_plan -\tdata_plan_10gb: @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: @2019-05-01 00:00:00+00:00 -\tconnected_wifi: @2019-05-01 00:00:00+00:00 -\tos_build: @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family cell_plan -\tdata_plan_10gb: @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tconnected_cell: @2019-05-01 00:00:00+00:00 -\tconnected_wifi: @2019-05-01 00:00:00+00:00 -\tos_build: @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_modify_apply_label 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 [labelled] -\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 [labelled] -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [labelled] -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 [labelled] - -Reading data for phone#4c410523#20190502: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [labelled] -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 [labelled] - -Reading data for phone#4c410523#20190505: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [labelled] -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [labelled] -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [labelled] - -Reading data for phone#5c10102#20190501: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [labelled] -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 [labelled] - -Reading data for phone#5c10102#20190502: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [labelled] -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [labelled] -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [labelled] - -''' - -snapshots['test_filter_composing_chain 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_composing_interleave 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 -Column Family stats_summary -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_filter_composing_condition 1'] = '''Reading data for phone#4c410523#20190501: -Column Family cell_plan -\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 [filtered-out] -\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 [filtered-out] -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [filtered-out] -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 [filtered-out] - -Reading data for phone#4c410523#20190502: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [filtered-out] -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 [filtered-out] - -Reading data for phone#4c410523#20190505: -Column Family cell_plan -\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [filtered-out] -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [filtered-out] -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [filtered-out] - -Reading data for phone#5c10102#20190501: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [passed-filter] -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [passed-filter] -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [passed-filter] -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 [passed-filter] - -Reading data for phone#5c10102#20190502: -Column Family cell_plan -\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [passed-filter] -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [passed-filter] -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [passed-filter] -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [passed-filter] - -''' diff --git a/bigtable/snippets/reads/read_snippets.py b/bigtable/snippets/reads/read_snippets.py deleted file mode 100644 index 6936b4c64c8b..000000000000 --- a/bigtable/snippets/reads/read_snippets.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2020, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START bigtable_reads_print] -from google.cloud import bigtable -import google.cloud.bigtable.row_filters as row_filters -from google.cloud.bigtable.row_set import RowSet - -# Write your code here. -# [START_EXCLUDE] - - -# [START bigtable_reads_row] -def read_row(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - row_key = "phone#4c410523#20190501" - - row = table.read_row(row_key) - print_row(row) - - -# [END bigtable_reads_row] - -# [START bigtable_reads_row_partial] -def read_row_partial(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - row_key = "phone#4c410523#20190501" - col_filter = row_filters.ColumnQualifierRegexFilter(b'os_build') - - row = table.read_row(row_key, filter_=col_filter) - print_row(row) - - -# [END bigtable_reads_row_partial] -# [START bigtable_reads_rows] -def read_rows(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - row_set = RowSet() - row_set.add_row_key(b"phone#4c410523#20190501") - row_set.add_row_key(b"phone#4c410523#20190502") - - rows = table.read_rows(row_set=row_set) - for row in rows: - print_row(row) - - -# [END bigtable_reads_rows] -# [START bigtable_reads_row_range] -def read_row_range(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - row_set = RowSet() - row_set.add_row_range_from_keys( - start_key=b"phone#4c410523#20190501", - end_key=b"phone#4c410523#201906201") - - rows = table.read_rows(row_set=row_set) - for row in rows: - print_row(row) - - -# [END bigtable_reads_row_range] -# [START bigtable_reads_row_ranges] -def read_row_ranges(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - row_set = RowSet() - row_set.add_row_range_from_keys( - start_key=b"phone#4c410523#20190501", - end_key=b"phone#4c410523#201906201") - row_set.add_row_range_from_keys( - start_key=b"phone#5c10102#20190501", - end_key=b"phone#5c10102#201906201") - - rows = table.read_rows(row_set=row_set) - for row in rows: - print_row(row) - - -# [END bigtable_reads_row_ranges] -# [START bigtable_reads_prefix] -def read_prefix(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - prefix = "phone#" - end_key = prefix[:-1] + chr(ord(prefix[-1]) + 1) - - row_set = RowSet() - row_set.add_row_range_from_keys(prefix.encode("utf-8"), - end_key.encode("utf-8")) - - rows = table.read_rows(row_set=row_set) - for row in rows: - print_row(row) - - -# [END bigtable_reads_prefix] -# [START bigtable_reads_filter] -def read_filter(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - rows = table.read_rows(filter_=row_filters.ValueRegexFilter(b"PQ2A.*$")) - for row in rows: - print_row(row) - - -# [END bigtable_reads_filter] -# [END_EXCLUDE] - - -def print_row(row): - print("Reading data for {}:".format(row.row_key.decode('utf-8'))) - for cf, cols in sorted(row.cells.items()): - print("Column Family {}".format(cf)) - for col, cells in sorted(cols.items()): - for cell in cells: - labels = " [{}]".format(",".join(cell.labels)) \ - if len(cell.labels) else "" - print( - "\t{}: {} @{}{}".format(col.decode('utf-8'), - cell.value.decode('utf-8'), - cell.timestamp, labels)) - print("") -# [END bigtable_reads_print] diff --git a/bigtable/snippets/reads/reads_test.py b/bigtable/snippets/reads/reads_test.py deleted file mode 100644 index fc3421000229..000000000000 --- a/bigtable/snippets/reads/reads_test.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2020, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import os -import uuid - -from google.cloud import bigtable -import pytest - -import read_snippets - - -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] -BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE'] -TABLE_ID_PREFIX = 'mobile-time-series-{}' - - -@pytest.fixture(scope="module", autouse=True) -def table_id(): - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={'stats_summary': None}) - - # table = instance.table(table_id) - - timestamp = datetime.datetime(2019, 5, 1) - rows = [ - table.direct_row("phone#4c410523#20190501"), - table.direct_row("phone#4c410523#20190502"), - table.direct_row("phone#4c410523#20190505"), - table.direct_row("phone#5c10102#20190501"), - table.direct_row("phone#5c10102#20190502"), - ] - - rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) - rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) - rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) - rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) - rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) - rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - - table.mutate_rows(rows) - - yield table_id - - table.delete() - - -def test_read_row(capsys, snapshot, table_id): - read_snippets.read_row(PROJECT, BIGTABLE_INSTANCE, table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_read_row_partial(capsys, snapshot, table_id): - read_snippets.read_row_partial(PROJECT, BIGTABLE_INSTANCE, table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_read_rows(capsys, snapshot, table_id): - read_snippets.read_rows(PROJECT, BIGTABLE_INSTANCE, table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_read_row_range(capsys, snapshot, table_id): - read_snippets.read_row_range(PROJECT, BIGTABLE_INSTANCE, table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_read_row_ranges(capsys, snapshot, table_id): - read_snippets.read_row_ranges(PROJECT, BIGTABLE_INSTANCE, table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_read_prefix(capsys, snapshot, table_id): - read_snippets.read_prefix(PROJECT, BIGTABLE_INSTANCE, table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) - - -def test_read_filter(capsys, snapshot, table_id): - read_snippets.read_filter(PROJECT, BIGTABLE_INSTANCE, table_id) - - out, _ = capsys.readouterr() - snapshot.assert_match(out) diff --git a/bigtable/snippets/reads/requirements-test.txt b/bigtable/snippets/reads/requirements-test.txt deleted file mode 100644 index 781d4326c947..000000000000 --- a/bigtable/snippets/reads/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==5.3.2 diff --git a/bigtable/snippets/reads/requirements.txt b/bigtable/snippets/reads/requirements.txt deleted file mode 100755 index a64e924f1be3..000000000000 --- a/bigtable/snippets/reads/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -google-cloud-bigtable==1.2.1 -snapshottest==0.5.1 \ No newline at end of file diff --git a/bigtable/snippets/reads/snapshots/__init__.py b/bigtable/snippets/reads/snapshots/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/bigtable/snippets/reads/snapshots/snap_reads_test.py b/bigtable/snippets/reads/snapshots/snap_reads_test.py deleted file mode 100644 index f45e98f2e57c..000000000000 --- a/bigtable/snippets/reads/snapshots/snap_reads_test.py +++ /dev/null @@ -1,142 +0,0 @@ -# -*- coding: utf-8 -*- -# snapshottest: v1 - https://goo.gl/zC4yUc -from __future__ import unicode_literals - -from snapshottest import Snapshot - - -snapshots = Snapshot() - -snapshots['test_read_row_partial 1'] = '''Reading data for phone#4c410523#20190501: -Column Family stats_summary -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_read_rows 1'] = '''Reading data for phone#4c410523#20190501: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_read_row_range 1'] = '''Reading data for phone#4c410523#20190501: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_read_row_ranges 1'] = '''Reading data for phone#4c410523#20190501: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_read_prefix 1'] = '''Reading data for phone#4c410523#20190501: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_read_filter 1'] = '''Reading data for phone#4c410523#20190501: -Column Family stats_summary -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190502: -Column Family stats_summary -\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 - -Reading data for phone#4c410523#20190505: -Column Family stats_summary -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190501: -Column Family stats_summary -\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 - -Reading data for phone#5c10102#20190502: -Column Family stats_summary -\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 - -''' - -snapshots['test_read_row 1'] = '''Reading data for phone#4c410523#20190501: -Column Family stats_summary -\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 -\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 - -''' diff --git a/bigtable/snippets/writes/__init__.py b/bigtable/snippets/writes/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/bigtable/snippets/writes/requirements-test.txt b/bigtable/snippets/writes/requirements-test.txt deleted file mode 100644 index 8855f3cf1f88..000000000000 --- a/bigtable/snippets/writes/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -backoff==1.10.0 -pytest==5.3.2 diff --git a/bigtable/snippets/writes/requirements.txt b/bigtable/snippets/writes/requirements.txt deleted file mode 100755 index 618a0d90714d..000000000000 --- a/bigtable/snippets/writes/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-cloud-bigtable==1.2.1 \ No newline at end of file diff --git a/bigtable/snippets/writes/write_batch.py b/bigtable/snippets/writes/write_batch.py deleted file mode 100644 index ecc8f273b0a6..000000000000 --- a/bigtable/snippets/writes/write_batch.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# [START bigtable_writes_batch] -import datetime - -from google.cloud import bigtable - - -def write_batch(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - timestamp = datetime.datetime.utcnow() - column_family_id = "stats_summary" - - rows = [table.direct_row("tablet#a0b81f74#20190501"), - table.direct_row("tablet#a0b81f74#20190502")] - - rows[0].set_cell(column_family_id, - "connected_wifi", - 1, - timestamp) - rows[0].set_cell(column_family_id, - "os_build", - "12155.0.0-rc1", - timestamp) - rows[1].set_cell(column_family_id, - "connected_wifi", - 1, - timestamp) - rows[1].set_cell(column_family_id, - "os_build", - "12145.0.0-rc6", - timestamp) - - response = table.mutate_rows(rows) - for i, status in enumerate(response): - if status.code != 0: - print("Error writing row: {}".format(status.message)) - - print('Successfully wrote 2 rows.') -# [END bigtable_writes_batch] diff --git a/bigtable/snippets/writes/write_conditionally.py b/bigtable/snippets/writes/write_conditionally.py deleted file mode 100644 index 5f3d4d607dc8..000000000000 --- a/bigtable/snippets/writes/write_conditionally.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# [START bigtable_writes_conditional] -import datetime - -from google.cloud import bigtable -from google.cloud.bigtable import row_filters - - -def write_conditional(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - timestamp = datetime.datetime.utcnow() - column_family_id = "stats_summary" - - row_key = "phone#4c410523#20190501" - - row_filter = row_filters.RowFilterChain( - filters=[row_filters.FamilyNameRegexFilter(column_family_id), - row_filters.ColumnQualifierRegexFilter('os_build'), - row_filters.ValueRegexFilter("PQ2A\\..*")]) - row = table.conditional_row(row_key, filter_=row_filter) - row.set_cell(column_family_id, - "os_name", - "android", - timestamp) - row.commit() - - print('Successfully updated row\'s os_name.') -# [END bigtable_writes_conditional] diff --git a/bigtable/snippets/writes/write_increment.py b/bigtable/snippets/writes/write_increment.py deleted file mode 100644 index 73ce52c2f6d2..000000000000 --- a/bigtable/snippets/writes/write_increment.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# [START bigtable_writes_increment] -from google.cloud import bigtable - - -def write_increment(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - column_family_id = "stats_summary" - - row_key = "phone#4c410523#20190501" - row = table.append_row(row_key) - - # Decrement the connected_wifi value by 1. - row.increment_cell_value(column_family_id, "connected_wifi", -1) - row.commit() - - print('Successfully updated row {}.'.format(row_key)) -# [END bigtable_writes_increment] diff --git a/bigtable/snippets/writes/write_simple.py b/bigtable/snippets/writes/write_simple.py deleted file mode 100644 index b4222d234798..000000000000 --- a/bigtable/snippets/writes/write_simple.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START bigtable_writes_simple] -import datetime - -from google.cloud import bigtable - - -def write_simple(project_id, instance_id, table_id): - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - timestamp = datetime.datetime.utcnow() - column_family_id = "stats_summary" - - row_key = "phone#4c410523#20190501" - - row = table.direct_row(row_key) - row.set_cell(column_family_id, - "connected_cell", - 1, - timestamp) - row.set_cell(column_family_id, - "connected_wifi", - 1, - timestamp) - row.set_cell(column_family_id, - "os_build", - "PQ2A.190405.003", - timestamp) - - row.commit() - - print('Successfully wrote row {}.'.format(row_key)) -# [END bigtable_writes_simple] diff --git a/bigtable/snippets/writes/writes_test.py b/bigtable/snippets/writes/writes_test.py deleted file mode 100644 index abe3000959ec..000000000000 --- a/bigtable/snippets/writes/writes_test.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2018 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import uuid - -import backoff -from google.api_core.exceptions import DeadlineExceeded -from google.cloud import bigtable -import pytest - -from .write_batch import write_batch -from .write_conditionally import write_conditional -from .write_increment import write_increment -from .write_simple import write_simple - - -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] -BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE'] -TABLE_ID_PREFIX = 'mobile-time-series-{}' - - -@pytest.fixture -def bigtable_client(): - return bigtable.Client(project=PROJECT, admin=True) - - -@pytest.fixture -def bigtable_instance(bigtable_client): - return bigtable_client.instance(BIGTABLE_INSTANCE) - - -@pytest.fixture -def table_id(bigtable_instance): - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) - table = bigtable_instance.table(table_id) - if table.exists(): - table.delete() - - column_family_id = 'stats_summary' - column_families = {column_family_id: None} - table.create(column_families=column_families) - - yield table_id - - table.delete() - - -def test_writes(capsys, table_id): - - # `row.commit()` sometimes ends up with DeadlineExceeded, so now - # we put retries with a hard deadline. - @backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=60) - def _write_simple(): - write_simple(PROJECT, BIGTABLE_INSTANCE, table_id) - - _write_simple() - out, _ = capsys.readouterr() - assert 'Successfully wrote row' in out - - @backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=60) - def _write_increment(): - write_increment(PROJECT, BIGTABLE_INSTANCE, table_id) - - _write_increment() - out, _ = capsys.readouterr() - assert 'Successfully updated row' in out - - @backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=60) - def _write_conditional(): - write_conditional(PROJECT, BIGTABLE_INSTANCE, table_id) - - _write_conditional() - out, _ = capsys.readouterr() - assert 'Successfully updated row\'s os_name' in out - - @backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=60) - def _write_batch(): - write_batch(PROJECT, BIGTABLE_INSTANCE, table_id) - - _write_batch() - out, _ = capsys.readouterr() - assert 'Successfully wrote 2 rows' in out diff --git a/bigtable/tableadmin/README.rst b/bigtable/tableadmin/README.rst deleted file mode 100644 index f7f83d6d2a1b..000000000000 --- a/bigtable/tableadmin/README.rst +++ /dev/null @@ -1,115 +0,0 @@ -.. This file is automatically generated. Do not edit this file directly. - -Google Cloud Bigtable Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/README.rst - - -This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. - - - - -.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs - -Setup -------------------------------------------------------------------------------- - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - -Samples -------------------------------------------------------------------------------- - -Basic example -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/tableadmin.py,bigtable/hello/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python tableadmin.py - - usage: tableadmin.py [-h] [run] [delete] [--table TABLE] project_id instance_id - - Demonstrates how to connect to Cloud Bigtable and run some basic operations. - Prerequisites: - Create a Cloud Bigtable cluster. - https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google - Application Default Credentials. - https://developers.google.com/identity/protocols/application-default- - credentials - - positional arguments: - project_id Your Cloud Platform project ID. - instance_id ID of the Cloud Bigtable instance to connect to. - - optional arguments: - -h, --help show this help message and exit - --table TABLE Table to create and destroy. (default: Hello-Bigtable) - - - - - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/tableadmin/README.rst.in b/bigtable/tableadmin/README.rst.in deleted file mode 100644 index 7fd37641969a..000000000000 --- a/bigtable/tableadmin/README.rst.in +++ /dev/null @@ -1,23 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google Cloud Bigtable and run some basic operations. - short_name: Cloud Bigtable - url: https://cloud.google.com/bigtable/docs - description: > - `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's - the same database that powers many core Google services, including Search, - Analytics, Maps, and Gmail. - -setup: -- auth -- install_deps - -samples: -- name: Basic example with Bigtable Column family and GC rules. - file: tableadmin.py - show_help: true - -cloud_client_library: true - -folder: bigtable/tableadmin \ No newline at end of file diff --git a/bigtable/tableadmin/requirements-test.txt b/bigtable/tableadmin/requirements-test.txt deleted file mode 100644 index 781d4326c947..000000000000 --- a/bigtable/tableadmin/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==5.3.2 diff --git a/bigtable/tableadmin/requirements.txt b/bigtable/tableadmin/requirements.txt deleted file mode 100755 index 2771c2e4c4d0..000000000000 --- a/bigtable/tableadmin/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-cloud-bigtable==1.2.1 diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py deleted file mode 100644 index 29551a7f390c..000000000000 --- a/bigtable/tableadmin/tableadmin.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Demonstrates how to connect to Cloud Bigtable and run some basic operations. -# http://www.apache.org/licenses/LICENSE-2.0 -Prerequisites: -- Create a Cloud Bigtable cluster. - https://cloud.google.com/bigtable/docs/creating-cluster -- Set your Google Application Default Credentials. - https://developers.google.com/identity/protocols/application-default-credentials - -Operations performed: -- Create a Cloud Bigtable table. -- List tables for a Cloud Bigtable instance. -- Print metadata of the newly created table. -- Create Column Families with different GC rules. - - GC Rules like: MaxAge, MaxVersions, Union, Intersection and Nested. -- Delete a Bigtable table. -""" - -import argparse -import datetime - -from google.cloud import bigtable -from google.cloud.bigtable import column_family - - -def create_table(project_id, instance_id, table_id): - ''' Create a Bigtable table - - :type project_id: str - :param project_id: Project id of the client. - - :type instance_id: str - :param instance_id: Instance of the client. - - :type table_id: str - :param table_id: Table id to create table. - ''' - - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - # Check whether table exists in an instance. - # Create table if it does not exists. - print('Checking if table {} exists...'.format(table_id)) - if table.exists(): - print('Table {} already exists.'.format(table_id)) - else: - print('Creating the {} table.'.format(table_id)) - table.create() - print('Created table {}.'.format(table_id)) - - return client, instance, table - - -def run_table_operations(project_id, instance_id, table_id): - ''' Create a Bigtable table and perform basic operations on it - - :type project_id: str - :param project_id: Project id of the client. - - :type instance_id: str - :param instance_id: Instance of the client. - - :type table_id: str - :param table_id: Table id to create table. - ''' - - client, instance, table = create_table(project_id, instance_id, table_id) - - # [START bigtable_list_tables] - tables = instance.list_tables() - print('Listing tables in current project...') - if tables != []: - for tbl in tables: - print(tbl.table_id) - else: - print('No table exists in current project...') - # [END bigtable_list_tables] - - # [START bigtable_create_family_gc_max_age] - print('Creating column family cf1 with with MaxAge GC Rule...') - # Create a column family with GC policy : maximum age - # where age = current time minus cell timestamp - - # Define the GC rule to retain data with max age of 5 days - max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) - - column_family1 = table.column_family('cf1', max_age_rule) - column_family1.create() - print('Created column family cf1 with MaxAge GC Rule.') - # [END bigtable_create_family_gc_max_age] - - # [START bigtable_create_family_gc_max_versions] - print('Creating column family cf2 with max versions GC rule...') - # Create a column family with GC policy : most recent N versions - # where 1 = most recent version - - # Define the GC policy to retain only the most recent 2 versions - max_versions_rule = column_family.MaxVersionsGCRule(2) - - column_family2 = table.column_family('cf2', max_versions_rule) - column_family2.create() - print('Created column family cf2 with Max Versions GC Rule.') - # [END bigtable_create_family_gc_max_versions] - - # [START bigtable_create_family_gc_union] - print('Creating column family cf3 with union GC rule...') - # Create a column family with GC policy to drop data that matches - # at least one condition. - # Define a GC rule to drop cells older than 5 days or not the - # most recent version - union_rule = column_family.GCRuleUnion([ - column_family.MaxAgeGCRule(datetime.timedelta(days=5)), - column_family.MaxVersionsGCRule(2)]) - - column_family3 = table.column_family('cf3', union_rule) - column_family3.create() - print('Created column family cf3 with Union GC rule') - # [END bigtable_create_family_gc_union] - - # [START bigtable_create_family_gc_intersection] - print('Creating column family cf4 with Intersection GC rule...') - # Create a column family with GC policy to drop data that matches - # all conditions - # GC rule: Drop cells older than 5 days AND older than the most - # recent 2 versions - intersection_rule = column_family.GCRuleIntersection([ - column_family.MaxAgeGCRule(datetime.timedelta(days=5)), - column_family.MaxVersionsGCRule(2)]) - - column_family4 = table.column_family('cf4', intersection_rule) - column_family4.create() - print('Created column family cf4 with Intersection GC rule.') - # [END bigtable_create_family_gc_intersection] - - # [START bigtable_create_family_gc_nested] - print('Creating column family cf5 with a Nested GC rule...') - # Create a column family with nested GC policies. - # Create a nested GC rule: - # Drop cells that are either older than the 10 recent versions - # OR - # Drop cells that are older than a month AND older than the - # 2 recent versions - rule1 = column_family.MaxVersionsGCRule(10) - rule2 = column_family.GCRuleIntersection([ - column_family.MaxAgeGCRule(datetime.timedelta(days=30)), - column_family.MaxVersionsGCRule(2)]) - - nested_rule = column_family.GCRuleUnion([rule1, rule2]) - - column_family5 = table.column_family('cf5', nested_rule) - column_family5.create() - print('Created column family cf5 with a Nested GC rule.') - # [END bigtable_create_family_gc_nested] - - # [START bigtable_list_column_families] - print('Printing Column Family and GC Rule for all column families...') - column_families = table.list_column_families() - for column_family_name, gc_rule in sorted(column_families.items()): - print('Column Family:', column_family_name) - print('GC Rule:') - print(gc_rule.to_pb()) - # Sample output: - # Column Family: cf4 - # GC Rule: - # gc_rule { - # intersection { - # rules { - # max_age { - # seconds: 432000 - # } - # } - # rules { - # max_num_versions: 2 - # } - # } - # } - # [END bigtable_list_column_families] - - print('Print column family cf1 GC rule before update...') - print('Column Family: cf1') - print(column_family1.to_pb()) - - # [START bigtable_update_gc_rule] - print('Updating column family cf1 GC rule...') - # Update the column family cf1 to update the GC rule - column_family1 = table.column_family( - 'cf1', - column_family.MaxVersionsGCRule(1)) - column_family1.update() - print('Updated column family cf1 GC rule\n') - # [END bigtable_update_gc_rule] - - print('Print column family cf1 GC rule after update...') - print('Column Family: cf1') - print(column_family1.to_pb()) - - # [START bigtable_delete_family] - print('Delete a column family cf2...') - # Delete a column family - column_family2.delete() - print('Column family cf2 deleted successfully.') - # [END bigtable_delete_family] - - print('execute command "python tableadmin.py delete [project_id] \ - [instance_id] --table [tableName]" to delete the table.') - - -def delete_table(project_id, instance_id, table_id): - ''' Delete bigtable. - - :type project_id: str - :param project_id: Project id of the client. - - :type instance_id: str - :param instance_id: Instance of the client. - - :type table_id: str - :param table_id: Table id to create table. - ''' - - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - # [START bigtable_delete_table] - # Delete the entire table - - print('Checking if table {} exists...'.format(table_id)) - if table.exists(): - print('Table {} exists.'.format(table_id)) - print('Deleting {} table.'.format(table_id)) - table.delete() - print('Deleted {} table.'.format(table_id)) - else: - print('Table {} does not exists.'.format(table_id)) - # [END bigtable_delete_table] - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('command', - help='run or delete. \ - Operation to perform on table.') - parser.add_argument( - '--table', - help='Cloud Bigtable Table name.', - default='Hello-Bigtable') - - parser.add_argument('project_id', - help='Your Cloud Platform project ID.') - parser.add_argument( - 'instance_id', - help='ID of the Cloud Bigtable instance to connect to.') - - args = parser.parse_args() - - if args.command.lower() == 'run': - run_table_operations(args.project_id, args.instance_id, - args.table) - elif args.command.lower() == 'delete': - delete_table(args.project_id, args.instance_id, args.table) - else: - print('Command should be either run or delete.\n Use argument -h,\ - --help to show help and exit.') diff --git a/bigtable/tableadmin/tableadmin_test.py b/bigtable/tableadmin/tableadmin_test.py deleted file mode 100755 index 782f6b621677..000000000000 --- a/bigtable/tableadmin/tableadmin_test.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import random - -from tableadmin import create_table -from tableadmin import delete_table -from tableadmin import run_table_operations - -PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] -BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE'] -TABLE_NAME_FORMAT = 'tableadmin-test-{}' -TABLE_NAME_RANGE = 10000 - - -def test_run_table_operations(capsys): - table_name = TABLE_NAME_FORMAT.format( - random.randrange(TABLE_NAME_RANGE)) - - run_table_operations(PROJECT, BIGTABLE_INSTANCE, table_name) - out, _ = capsys.readouterr() - - assert 'Creating the ' + table_name + ' table.' in out - assert 'Listing tables in current project.' in out - assert 'Creating column family cf1 with with MaxAge GC Rule' in out - assert 'Created column family cf1 with MaxAge GC Rule.' in out - assert 'Created column family cf2 with Max Versions GC Rule.' in out - assert 'Created column family cf3 with Union GC rule' in out - assert 'Created column family cf4 with Intersection GC rule.' in out - assert 'Created column family cf5 with a Nested GC rule.' in out - assert 'Printing Column Family and GC Rule for all column families.' in out - assert 'Updating column family cf1 GC rule...' in out - assert 'Updated column family cf1 GC rule' in out - assert 'Print column family cf1 GC rule after update...' in out - assert 'Column Family: cf1' in out - assert 'max_num_versions: 1' in out - assert 'Delete a column family cf2...' in out - assert 'Column family cf2 deleted successfully.' in out - - delete_table(PROJECT, BIGTABLE_INSTANCE, table_name) - - -def test_delete_table(capsys): - table_name = TABLE_NAME_FORMAT.format( - random.randrange(TABLE_NAME_RANGE)) - create_table(PROJECT, BIGTABLE_INSTANCE, table_name) - - delete_table(PROJECT, BIGTABLE_INSTANCE, table_name) - out, _ = capsys.readouterr() - - assert 'Table ' + table_name + ' exists.' in out - assert 'Deleting ' + table_name + ' table.' in out - assert 'Deleted ' + table_name + ' table.' in out