diff --git a/google/cloud/spanner_v1/backup.py b/google/cloud/spanner_v1/backup.py index 405a9e2be2..2277a33fce 100644 --- a/google/cloud/spanner_v1/backup.py +++ b/google/cloud/spanner_v1/backup.py @@ -51,14 +51,23 @@ class Backup(object): :param expire_time: (Optional) The expire time that will be used to create the backup. Required if the create method needs to be called. + + :type version_time: :class:`datetime.datetime` + :param version_time: (Optional) The version time that was specified for + the externally consistent copy of the database. If + not present, it is the same as the `create_time` of + the backup. """ - def __init__(self, backup_id, instance, database="", expire_time=None): + def __init__( + self, backup_id, instance, database="", expire_time=None, version_time=None + ): self.backup_id = backup_id self._instance = instance self._database = database self._expire_time = expire_time self._create_time = None + self._version_time = version_time self._size_bytes = None self._state = None self._referencing_databases = None @@ -109,6 +118,16 @@ def create_time(self): """ return self._create_time + @property + def version_time(self): + """Version time of this backup. + + :rtype: :class:`datetime.datetime` + :returns: a datetime object representing the version time of + this backup + """ + return self._version_time + @property def size_bytes(self): """Size of this backup in bytes. @@ -190,7 +209,11 @@ def create(self): raise ValueError("database not set") api = self._instance._client.database_admin_api metadata = _metadata_with_prefix(self.name) - backup = BackupPB(database=self._database, expire_time=self.expire_time,) + backup = BackupPB( + database=self._database, + expire_time=self.expire_time, + version_time=self.version_time, + ) future = api.create_backup( parent=self._instance.name, @@ -228,6 +251,7 @@ def reload(self): self._database = pb.database self._expire_time = pb.expire_time self._create_time = pb.create_time + self._version_time = pb.version_time self._size_bytes = pb.size_bytes self._state = BackupPB.State(pb.state) self._referencing_databases = pb.referencing_databases diff --git a/google/cloud/spanner_v1/database.py b/google/cloud/spanner_v1/database.py index c1c7953648..7a89ccdb3e 100644 --- a/google/cloud/spanner_v1/database.py +++ b/google/cloud/spanner_v1/database.py @@ -107,6 +107,8 @@ def __init__(self, database_id, instance, ddl_statements=(), pool=None): self._state = None self._create_time = None self._restore_info = None + self._version_retention_period = None + self._earliest_version_time = None if pool is None: pool = BurstyPool() @@ -204,6 +206,25 @@ def restore_info(self): """ return self._restore_info + @property + def version_retention_period(self): + """The period in which Cloud Spanner retains all versions of data + for the database. + + :rtype: str + :returns: a string representing the duration of the version retention period + """ + return self._version_retention_period + + @property + def earliest_version_time(self): + """The earliest time at which older versions of the data can be read. + + :rtype: :class:`datetime.datetime` + :returns: a datetime object representing the earliest version time + """ + return self._earliest_version_time + @property def ddl_statements(self): """DDL Statements used to define database schema. @@ -313,6 +334,8 @@ def reload(self): self._state = DatabasePB.State(response.state) self._create_time = response.create_time self._restore_info = response.restore_info + self._version_retention_period = response.version_retention_period + self._earliest_version_time = response.earliest_version_time def update_ddl(self, ddl_statements, operation_id=""): """Update DDL for this database. diff --git a/google/cloud/spanner_v1/instance.py b/google/cloud/spanner_v1/instance.py index b422c57afd..ffaed41c91 100644 --- a/google/cloud/spanner_v1/instance.py +++ b/google/cloud/spanner_v1/instance.py @@ -400,7 +400,7 @@ def list_databases(self, page_size=None): ) return page_iter - def backup(self, backup_id, database="", expire_time=None): + def backup(self, backup_id, database="", expire_time=None, version_time=None): """Factory to create a backup within this instance. :type backup_id: str @@ -415,13 +415,29 @@ def backup(self, backup_id, database="", expire_time=None): :param expire_time: Optional. The expire time that will be used when creating the backup. Required if the create method needs to be called. + + :type version_time: :class:`datetime.datetime` + :param version_time: + Optional. The version time that will be used to create the externally + consistent copy of the database. If not present, it is the same as + the `create_time` of the backup. """ try: return Backup( - backup_id, self, database=database.name, expire_time=expire_time + backup_id, + self, + database=database.name, + expire_time=expire_time, + version_time=version_time, ) except AttributeError: - return Backup(backup_id, self, database=database, expire_time=expire_time) + return Backup( + backup_id, + self, + database=database, + expire_time=expire_time, + version_time=version_time, + ) def list_backups(self, filter_="", page_size=None): """List backups for the instance. diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 90031a3e3a..86be97d3eb 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -355,6 +355,62 @@ def test_create_database(self): database_ids = [database.name for database in Config.INSTANCE.list_databases()] self.assertIn(temp_db.name, database_ids) + @unittest.skipIf( + USE_EMULATOR, "PITR-lite features are not supported by the emulator" + ) + def test_create_database_pitr_invalid_retention_period(self): + pool = BurstyPool(labels={"testcase": "create_database_pitr"}) + temp_db_id = "temp_db" + unique_resource_id("_") + retention_period = "0d" + ddl_statements = [ + "ALTER DATABASE {}" + " SET OPTIONS (version_retention_period = '{}')".format( + temp_db_id, retention_period + ) + ] + temp_db = Config.INSTANCE.database( + temp_db_id, pool=pool, ddl_statements=ddl_statements + ) + with self.assertRaises(exceptions.InvalidArgument): + temp_db.create() + + @unittest.skipIf( + USE_EMULATOR, "PITR-lite features are not supported by the emulator" + ) + def test_create_database_pitr_success(self): + pool = BurstyPool(labels={"testcase": "create_database_pitr"}) + temp_db_id = "temp_db" + unique_resource_id("_") + retention_period = "7d" + ddl_statements = [ + "ALTER DATABASE {}" + " SET OPTIONS (version_retention_period = '{}')".format( + temp_db_id, retention_period + ) + ] + temp_db = Config.INSTANCE.database( + temp_db_id, pool=pool, ddl_statements=ddl_statements + ) + operation = temp_db.create() + self.to_delete.append(temp_db) + + # We want to make sure the operation completes. + operation.result(30) # raises on failure / timeout. + + database_ids = [database.name for database in Config.INSTANCE.list_databases()] + self.assertIn(temp_db.name, database_ids) + + temp_db.reload() + self.assertEqual(temp_db.version_retention_period, retention_period) + + with temp_db.snapshot() as snapshot: + results = snapshot.execute_sql( + "SELECT OPTION_VALUE AS version_retention_period " + "FROM INFORMATION_SCHEMA.DATABASE_OPTIONS " + "WHERE SCHEMA_NAME = '' AND OPTION_NAME = 'version_retention_period'" + ) + for result in results: + self.assertEqual(result[0], retention_period) + def test_table_not_found(self): temp_db_id = "temp_db" + unique_resource_id("_") @@ -407,6 +463,62 @@ def test_update_database_ddl_with_operation_id(self): self.assertEqual(len(temp_db.ddl_statements), len(ddl_statements)) + @unittest.skipIf( + USE_EMULATOR, "PITR-lite features are not supported by the emulator" + ) + def test_update_database_ddl_pitr_invalid(self): + pool = BurstyPool(labels={"testcase": "update_database_ddl_pitr"}) + temp_db_id = "temp_db" + unique_resource_id("_") + retention_period = "0d" + temp_db = Config.INSTANCE.database(temp_db_id, pool=pool) + create_op = temp_db.create() + self.to_delete.append(temp_db) + + # We want to make sure the operation completes. + create_op.result(240) # raises on failure / timeout. + + self.assertIsNone(temp_db.version_retention_period) + + ddl_statements = DDL_STATEMENTS + [ + "ALTER DATABASE {}" + " SET OPTIONS (version_retention_period = '{}')".format( + temp_db_id, retention_period + ) + ] + with self.assertRaises(exceptions.InvalidArgument): + temp_db.update_ddl(ddl_statements) + + @unittest.skipIf( + USE_EMULATOR, "PITR-lite features are not supported by the emulator" + ) + def test_update_database_ddl_pitr_success(self): + pool = BurstyPool(labels={"testcase": "update_database_ddl_pitr"}) + temp_db_id = "temp_db" + unique_resource_id("_") + retention_period = "7d" + temp_db = Config.INSTANCE.database(temp_db_id, pool=pool) + create_op = temp_db.create() + self.to_delete.append(temp_db) + + # We want to make sure the operation completes. + create_op.result(240) # raises on failure / timeout. + + self.assertIsNone(temp_db.version_retention_period) + + ddl_statements = DDL_STATEMENTS + [ + "ALTER DATABASE {}" + " SET OPTIONS (version_retention_period = '{}')".format( + temp_db_id, retention_period + ) + ] + operation = temp_db.update_ddl(ddl_statements) + + # We want to make sure the operation completes. + operation.result(240) # raises on failure / timeout. + + temp_db.reload() + self.assertEqual(temp_db.version_retention_period, retention_period) + self.assertEqual(len(temp_db.ddl_statements), len(ddl_statements)) + def test_db_batch_insert_then_db_snapshot_read(self): retry = RetryInstanceState(_has_all_ddl) retry(self._db.reload)() @@ -486,6 +598,8 @@ class TestBackupAPI(unittest.TestCase, _TestData): @classmethod def setUpClass(cls): + from datetime import datetime + pool = BurstyPool(labels={"testcase": "database_api"}) ddl_statements = EMULATOR_DDL_STATEMENTS if USE_EMULATOR else DDL_STATEMENTS db1 = Config.INSTANCE.database( @@ -498,6 +612,7 @@ def setUpClass(cls): op2 = db2.create() op1.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS) # raises on failure / timeout. op2.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS) # raises on failure / timeout. + cls.database_version_time = datetime.utcnow().replace(tzinfo=UTC) current_config = Config.INSTANCE.configuration_name same_config_instance_id = "same-config" + unique_resource_id("-") @@ -573,7 +688,12 @@ def test_backup_workflow(self): expire_time = expire_time.replace(tzinfo=UTC) # Create backup. - backup = instance.backup(backup_id, database=self._db, expire_time=expire_time) + backup = instance.backup( + backup_id, + database=self._db, + expire_time=expire_time, + version_time=self.database_version_time, + ) operation = backup.create() self.to_delete.append(backup) @@ -588,6 +708,7 @@ def test_backup_workflow(self): self.assertEqual(self._db.name, backup._database) self.assertEqual(expire_time, backup.expire_time) self.assertIsNotNone(backup.create_time) + self.assertEqual(self.database_version_time, backup.version_time) self.assertIsNotNone(backup.size_bytes) self.assertIsNotNone(backup.state) @@ -602,12 +723,92 @@ def test_backup_workflow(self): database = instance.database(restored_id) self.to_drop.append(database) operation = database.restore(source=backup) - operation.result() + restored_db = operation.result() + self.assertEqual( + self.database_version_time, restored_db.restore_info.backup_info.create_time + ) + + metadata = operation.metadata + self.assertEqual(self.database_version_time, metadata.backup_info.create_time) database.drop() backup.delete() self.assertFalse(backup.exists()) + def test_backup_version_time_defaults_to_create_time(self): + from datetime import datetime + from datetime import timedelta + from pytz import UTC + + instance = Config.INSTANCE + backup_id = "backup_id" + unique_resource_id("_") + expire_time = datetime.utcnow() + timedelta(days=3) + expire_time = expire_time.replace(tzinfo=UTC) + + # Create backup. + backup = instance.backup(backup_id, database=self._db, expire_time=expire_time,) + operation = backup.create() + self.to_delete.append(backup) + + # Check metadata. + metadata = operation.metadata + self.assertEqual(backup.name, metadata.name) + self.assertEqual(self._db.name, metadata.database) + operation.result() + + # Check backup object. + backup.reload() + self.assertEqual(self._db.name, backup._database) + self.assertIsNotNone(backup.create_time) + self.assertEqual(backup.create_time, backup.version_time) + + backup.delete() + self.assertFalse(backup.exists()) + + def test_create_backup_invalid_version_time_past(self): + from datetime import datetime + from datetime import timedelta + from pytz import UTC + + backup_id = "backup_id" + unique_resource_id("_") + expire_time = datetime.utcnow() + timedelta(days=3) + expire_time = expire_time.replace(tzinfo=UTC) + version_time = datetime.utcnow() - timedelta(days=10) + version_time = version_time.replace(tzinfo=UTC) + + backup = Config.INSTANCE.backup( + backup_id, + database=self._db, + expire_time=expire_time, + version_time=version_time, + ) + + with self.assertRaises(exceptions.InvalidArgument): + op = backup.create() + op.result() + + def test_create_backup_invalid_version_time_future(self): + from datetime import datetime + from datetime import timedelta + from pytz import UTC + + backup_id = "backup_id" + unique_resource_id("_") + expire_time = datetime.utcnow() + timedelta(days=3) + expire_time = expire_time.replace(tzinfo=UTC) + version_time = datetime.utcnow() + timedelta(days=2) + version_time = version_time.replace(tzinfo=UTC) + + backup = Config.INSTANCE.backup( + backup_id, + database=self._db, + expire_time=expire_time, + version_time=version_time, + ) + + with self.assertRaises(exceptions.InvalidArgument): + op = backup.create() + op.result() + def test_restore_to_diff_instance(self): from datetime import datetime from datetime import timedelta @@ -706,7 +907,10 @@ def test_list_backups(self): expire_time_1 = expire_time_1.replace(tzinfo=UTC) backup1 = Config.INSTANCE.backup( - backup_id_1, database=self._dbs[0], expire_time=expire_time_1 + backup_id_1, + database=self._dbs[0], + expire_time=expire_time_1, + version_time=self.database_version_time, ) expire_time_2 = datetime.utcnow() + timedelta(days=1) @@ -746,6 +950,13 @@ def test_list_backups(self): for backup in instance.list_backups(filter_=filter_): self.assertEqual(backup.name, backup2.name) + # List backups filtered by version time. + filter_ = 'version_time > "{0}"'.format( + create_time_compare.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + ) + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup2.name) + # List backups filtered by expire time. filter_ = 'expire_time > "{0}"'.format( expire_time_1.strftime("%Y-%m-%dT%H:%M:%S.%fZ") diff --git a/tests/unit/test_backup.py b/tests/unit/test_backup.py index 748c460291..bf6ce68a84 100644 --- a/tests/unit/test_backup.py +++ b/tests/unit/test_backup.py @@ -266,6 +266,9 @@ def test_create_database_not_set(self): def test_create_success(self): from google.cloud.spanner_admin_database_v1 import Backup + from datetime import datetime + from datetime import timedelta + from pytz import UTC op_future = object() client = _Client() @@ -273,12 +276,22 @@ def test_create_success(self): api.create_backup.return_value = op_future instance = _Instance(self.INSTANCE_NAME, client=client) - timestamp = self._make_timestamp() + version_timestamp = datetime.utcnow() - timedelta(minutes=5) + version_timestamp = version_timestamp.replace(tzinfo=UTC) + expire_timestamp = self._make_timestamp() backup = self._make_one( - self.BACKUP_ID, instance, database=self.DATABASE_NAME, expire_time=timestamp + self.BACKUP_ID, + instance, + database=self.DATABASE_NAME, + expire_time=expire_timestamp, + version_time=version_timestamp, ) - backup_pb = Backup(database=self.DATABASE_NAME, expire_time=timestamp,) + backup_pb = Backup( + database=self.DATABASE_NAME, + expire_time=expire_timestamp, + version_time=version_timestamp, + ) future = backup.create() self.assertIs(future, op_future) @@ -437,6 +450,7 @@ def test_reload_success(self): name=self.BACKUP_NAME, database=self.DATABASE_NAME, expire_time=timestamp, + version_time=timestamp, create_time=timestamp, size_bytes=10, state=1, @@ -452,6 +466,7 @@ def test_reload_success(self): self.assertEqual(backup.database, self.DATABASE_NAME) self.assertEqual(backup.expire_time, timestamp) self.assertEqual(backup.create_time, timestamp) + self.assertEqual(backup.version_time, timestamp) self.assertEqual(backup.size_bytes, 10) self.assertEqual(backup.state, Backup.State.CREATING) self.assertEqual(backup.referencing_databases, []) diff --git a/tests/unit/test_database.py b/tests/unit/test_database.py index 175c269d50..a2a5b84b2f 100644 --- a/tests/unit/test_database.py +++ b/tests/unit/test_database.py @@ -249,6 +249,20 @@ def test_restore_info(self): ) self.assertEqual(database.restore_info, restore_info) + def test_version_retention_period(self): + instance = _Instance(self.INSTANCE_NAME) + pool = _Pool() + database = self._make_one(self.DATABASE_ID, instance, pool=pool) + version_retention_period = database._version_retention_period = "1d" + self.assertEqual(database.version_retention_period, version_retention_period) + + def test_earliest_version_time(self): + instance = _Instance(self.INSTANCE_NAME) + pool = _Pool() + database = self._make_one(self.DATABASE_ID, instance, pool=pool) + earliest_version_time = database._earliest_version_time = self._make_timestamp() + self.assertEqual(database.earliest_version_time, earliest_version_time) + def test_spanner_api_property_w_scopeless_creds(self): client = _Client() @@ -581,6 +595,8 @@ def test_reload_success(self): state=2, create_time=_datetime_to_pb_timestamp(timestamp), restore_info=restore_info, + version_retention_period="1d", + earliest_version_time=_datetime_to_pb_timestamp(timestamp), ) api.get_database.return_value = db_pb instance = _Instance(self.INSTANCE_NAME, client=client) @@ -591,6 +607,8 @@ def test_reload_success(self): self.assertEqual(database._state, Database.State.READY) self.assertEqual(database._create_time, timestamp) self.assertEqual(database._restore_info, restore_info) + self.assertEqual(database._version_retention_period, "1d") + self.assertEqual(database._earliest_version_time, timestamp) self.assertEqual(database._ddl_statements, tuple(DDL_STATEMENTS)) api.get_database_ddl.assert_called_once_with(