tests: move systests into separate modules, refactor using pytest (#474)

* tests: move instance API systests to own module

Refactor to use pytest fixtures / idioms, rather than old 'Config'
setup / teardown.

Toward #472.

* tests: move database API systests to own module

Refactor to use pytest fixtures / idioms, rather than old 'Config'
setup / teardown.

Toward #472.

* tests: move table API systests to own module

Refactor to use pytest fixtures / idioms, rather than old 'Config'
setup / teardown.

Toward #472.

* tests: move backup API systests to own module [WIP]

Refactor to use pytest fixtures / idioms, rather than old 'Config'
setup / teardown.

Toward #472.

* tests: move streaming/chunnking systests to own module

Refactor to use pytest fixtures / idioms, rather than old 'Config'
setup / teardown.

Toward #472.

* tests: move session API systests to own module

Refactor to use pytest fixtures / idioms, rather than old 'Config'
setup/ teardown.

Toward #472.

* tests: move dbapi systests to owwn module

Refactor to use pytest fixtures / idioms, rather than old 'Confog'
setup / teardown.

Toward #472.

* tests: remove legacy systest setup / teardown code

Closes #472.

* tests: don't pre-create datbase before restore attempt

* tests: fix instance config fixtures under emulator

* tests: clean up alt instnce at module scope

Avoids clash with 'test_list_instances' expectatons.

* tests: work around MethodNotImplemented

Raised from 'ListBackups' API on the CI emulator, but not locally.

* chore: drop use of pytz in systests

See #479 for rationale.

* chore: fix fossil in comment

* chore: move '_check_batch_status' to only calling module

Likewise the 'FauxCall' helper class it uses.

* chore: improve testcase name

* tests: replicate dbapi systest changes from #412 into new module

tests/system/_helpers.py

@@ -0,0 +1,110 @@
+# Copyright 2021 Google LLC All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import operator
+import os
+import time
+
+from google.api_core import exceptions
+from google.cloud.spanner_v1 import instance as instance_mod
+from tests import _fixtures
+from test_utils import retry
+from test_utils import system
+
+
+CREATE_INSTANCE_ENVVAR = "GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE"
+CREATE_INSTANCE = os.getenv(CREATE_INSTANCE_ENVVAR) is not None
+
+INSTANCE_ID_ENVVAR = "GOOGLE_CLOUD_TESTS_SPANNER_INSTANCE"
+INSTANCE_ID_DEFAULT = "google-cloud-python-systest"
+INSTANCE_ID = os.environ.get(INSTANCE_ID_ENVVAR, INSTANCE_ID_DEFAULT)
+
+SKIP_BACKUP_TESTS_ENVVAR = "SKIP_BACKUP_TESTS"
+SKIP_BACKUP_TESTS = os.getenv(SKIP_BACKUP_TESTS_ENVVAR) is not None
+
+SPANNER_OPERATION_TIMEOUT_IN_SECONDS = int(
+    os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 60)
+)
+
+USE_EMULATOR_ENVVAR = "SPANNER_EMULATOR_HOST"
+USE_EMULATOR = os.getenv(USE_EMULATOR_ENVVAR) is not None
+
+EMULATOR_PROJECT_ENVVAR = "GCLOUD_PROJECT"
+EMULATOR_PROJECT_DEFAULT = "emulator-test-project"
+EMULATOR_PROJECT = os.getenv(EMULATOR_PROJECT_ENVVAR, EMULATOR_PROJECT_DEFAULT)
+
+
+DDL_STATEMENTS = (
+    _fixtures.EMULATOR_DDL_STATEMENTS if USE_EMULATOR else _fixtures.DDL_STATEMENTS
+)
+
+retry_true = retry.RetryResult(operator.truth)
+retry_false = retry.RetryResult(operator.not_)
+
+retry_503 = retry.RetryErrors(exceptions.ServiceUnavailable)
+retry_429_503 = retry.RetryErrors(
+    exceptions.TooManyRequests, exceptions.ServiceUnavailable,
+)
+retry_mabye_aborted_txn = retry.RetryErrors(exceptions.ServerError, exceptions.Aborted)
+retry_mabye_conflict = retry.RetryErrors(exceptions.ServerError, exceptions.Conflict)
+
+
+def _has_all_ddl(database):
+    # Predicate to test for EC completion.
+    return len(database.ddl_statements) == len(DDL_STATEMENTS)
+
+
+retry_has_all_dll = retry.RetryInstanceState(_has_all_ddl)
+
+
+def scrub_instance_backups(to_scrub):
+    try:
+        for backup_pb in to_scrub.list_backups():
+            bkp = instance_mod.Backup.from_pb(backup_pb, to_scrub)
+            try:
+                # Instance cannot be deleted while backups exist.
+                retry_429_503(bkp.delete)()
+            except exceptions.NotFound:  # lost the race
+                pass
+    except exceptions.MethodNotImplemented:
+        # The CI emulator raises 501:  local versions seem fine.
+        pass
+
+
+def scrub_instance_ignore_not_found(to_scrub):
+    """Helper for func:`cleanup_old_instances`"""
+    scrub_instance_backups(to_scrub)
+
+    try:
+        retry_429_503(to_scrub.delete)()
+    except exceptions.NotFound:  # lost the race
+        pass
+
+
+def cleanup_old_instances(spanner_client):
+    cutoff = int(time.time()) - 1 * 60 * 60  # one hour ago
+    instance_filter = "labels.python-spanner-systests:true"
+
+    for instance_pb in spanner_client.list_instances(filter_=instance_filter):
+        instance = instance_mod.Instance.from_pb(instance_pb, spanner_client)
+
+        if "created" in instance.labels:
+            create_time = int(instance.labels["created"])
+
+            if create_time <= cutoff:
+                scrub_instance_ignore_not_found(instance)
+
+
+def unique_id(prefix, separator="-"):
+    return f"{prefix}{system.unique_resource_id(separator)}"

tests/system/_sample_data.py

@@ -0,0 +1,87 @@
+# Copyright 2021 Google LLC All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import math
+
+from google.api_core import datetime_helpers
+from google.cloud._helpers import UTC
+from google.cloud import spanner_v1
+
+
+TABLE = "contacts"
+COLUMNS = ("contact_id", "first_name", "last_name", "email")
+ROW_DATA = (
+    (1, u"Phred", u"Phlyntstone", u"phred@example.com"),
+    (2, u"Bharney", u"Rhubble", u"bharney@example.com"),
+    (3, u"Wylma", u"Phlyntstone", u"wylma@example.com"),
+)
+ALL = spanner_v1.KeySet(all_=True)
+SQL = "SELECT * FROM contacts ORDER BY contact_id"
+
+COUNTERS_TABLE = "counters"
+COUNTERS_COLUMNS = ("name", "value")
+
+
+def _assert_timestamp(value, nano_value):
+    assert isinstance(value, datetime.datetime)
+    assert value.tzinfo is None
+    assert nano_value.tzinfo is UTC
+
+    assert value.year == nano_value.year
+    assert value.month == nano_value.month
+    assert value.day == nano_value.day
+    assert value.hour == nano_value.hour
+    assert value.minute == nano_value.minute
+    assert value.second == nano_value.second
+    assert value.microsecond == nano_value.microsecond
+
+    if isinstance(value, datetime_helpers.DatetimeWithNanoseconds):
+        assert value.nanosecond == nano_value.nanosecond
+    else:
+        assert value.microsecond * 1000 == nano_value.nanosecond
+
+
+def _check_rows_data(rows_data, expected=ROW_DATA, recurse_into_lists=True):
+    assert len(rows_data) == len(expected)
+
+    for row, expected in zip(rows_data, expected):
+        _check_row_data(row, expected, recurse_into_lists=recurse_into_lists)
+
+
+def _check_row_data(row_data, expected, recurse_into_lists=True):
+    assert len(row_data) == len(expected)
+
+    for found_cell, expected_cell in zip(row_data, expected):
+        _check_cell_data(
+            found_cell, expected_cell, recurse_into_lists=recurse_into_lists
+        )
+
+
+def _check_cell_data(found_cell, expected_cell, recurse_into_lists=True):
+
+    if isinstance(found_cell, datetime_helpers.DatetimeWithNanoseconds):
+        _assert_timestamp(expected_cell, found_cell)
+
+    elif isinstance(found_cell, float) and math.isnan(found_cell):
+        assert math.isnan(expected_cell)
+
+    elif isinstance(found_cell, list) and recurse_into_lists:
+        assert len(found_cell) == len(expected_cell)
+
+        for found_item, expected_item in zip(found_cell, expected_cell):
+            _check_cell_data(found_item, expected_item)
+
+    else:
+        assert found_cell == expected_cell

tests/system/conftest.py

@@ -0,0 +1,153 @@
+# Copyright 2021 Google LLC All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+import pytest
+
+from google.cloud import spanner_v1
+from . import _helpers
+
+
+@pytest.fixture(scope="function")
+def if_create_instance():
+    if not _helpers.CREATE_INSTANCE:
+        pytest.skip(f"{_helpers.CREATE_INSTANCE_ENVVAR} not set in environment.")
+
+
+@pytest.fixture(scope="function")
+def no_create_instance():
+    if _helpers.CREATE_INSTANCE:
+        pytest.skip(f"{_helpers.CREATE_INSTANCE_ENVVAR} set in environment.")
+
+
+@pytest.fixture(scope="function")
+def if_backup_tests():
+    if _helpers.SKIP_BACKUP_TESTS:
+        pytest.skip(f"{_helpers.SKIP_BACKUP_TESTS_ENVVAR} set in environment.")
+
+
+@pytest.fixture(scope="function")
+def not_emulator():
+    if _helpers.USE_EMULATOR:
+        pytest.skip(f"{_helpers.USE_EMULATOR_ENVVAR} set in environment.")
+
+
+@pytest.fixture(scope="session")
+def spanner_client():
+    if _helpers.USE_EMULATOR:
+        from google.auth.credentials import AnonymousCredentials
+
+        credentials = AnonymousCredentials()
+        return spanner_v1.Client(
+            project=_helpers.EMULATOR_PROJECT, credentials=credentials,
+        )
+    else:
+        return spanner_v1.Client()  # use google.auth.default credentials
+
+
+@pytest.fixture(scope="session")
+def operation_timeout():
+    return _helpers.SPANNER_OPERATION_TIMEOUT_IN_SECONDS
+
+
+@pytest.fixture(scope="session")
+def shared_instance_id():
+    if _helpers.CREATE_INSTANCE:
+        return f"{_helpers.unique_id('google-cloud')}"
+
+    return _helpers.INSTANCE_ID
+
+
+@pytest.fixture(scope="session")
+def instance_configs(spanner_client):
+    configs = list(_helpers.retry_503(spanner_client.list_instance_configs)())
+
+    if not _helpers.USE_EMULATOR:
+
+        # Defend against back-end returning configs for regions we aren't
+        # actually allowed to use.
+        configs = [config for config in configs if "-us-" in config.name]
+
+    yield configs
+
+
+@pytest.fixture(scope="session")
+def instance_config(instance_configs):
+    if not instance_configs:
+        raise ValueError("No instance configs found.")
+
+    yield instance_configs[0]
+
+
+@pytest.fixture(scope="session")
+def existing_instances(spanner_client):
+    instances = list(_helpers.retry_503(spanner_client.list_instances)())
+
+    yield instances
+
+
+@pytest.fixture(scope="session")
+def shared_instance(
+    spanner_client,
+    operation_timeout,
+    shared_instance_id,
+    instance_config,
+    existing_instances,  # evalutate before creating one
+):
+    _helpers.cleanup_old_instances(spanner_client)
+
+    if _helpers.CREATE_INSTANCE:
+        create_time = str(int(time.time()))
+        labels = {"python-spanner-systests": "true", "created": create_time}
+
+        instance = spanner_client.instance(
+            shared_instance_id, instance_config.name, labels=labels
+        )
+        created_op = _helpers.retry_429_503(instance.create)()
+        created_op.result(operation_timeout)  # block until completion
+
+    else:  # reuse existing instance
+        instance = spanner_client.instance(shared_instance_id)
+        instance.reload()
+
+    yield instance
+
+    if _helpers.CREATE_INSTANCE:
+        _helpers.retry_429_503(instance.delete)()
+
+
+@pytest.fixture(scope="session")
+def shared_database(shared_instance, operation_timeout):
+    database_name = _helpers.unique_id("test_database")
+    pool = spanner_v1.BurstyPool(labels={"testcase": "database_api"})
+    database = shared_instance.database(
+        database_name, ddl_statements=_helpers.DDL_STATEMENTS, pool=pool
+    )
+    operation = database.create()
+    operation.result(operation_timeout)  # raises on failure / timeout.
+
+    yield database
+
+    database.drop()
+
+
+@pytest.fixture(scope="function")
+def databases_to_delete():
+    to_delete = []
+
+    yield to_delete
+
+    for database in to_delete:
+        database.drop()