Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for 'Blob.custom_time' and lifecycle rules #199

Merged
merged 22 commits into from Aug 26, 2020
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
c03d6ae
feat(storage): add support of custom time metadata and timestamp
HemangChothani Jul 1, 2020
7347cf2
Merge branch 'master' of https://github.com/googleapis/python-storage…
HemangChothani Jul 1, 2020
ef32ca2
Merge branch 'master' into storage_issue_196
jkwlui Jul 13, 2020
0212ca4
feat(storage): change the return type of custom_time_before
HemangChothani Jul 17, 2020
a6c830c
feat(storage): add setter method
HemangChothani Jul 24, 2020
3dbda6a
Merge branch 'master' of https://github.com/googleapis/python-storage…
HemangChothani Jul 24, 2020
04e3a10
Merge branch 'master' into storage_issue_196
tseaver Jul 24, 2020
75e6067
feat(storage): add test for None value
HemangChothani Jul 27, 2020
7ee2796
Merge branch 'storage_issue_196' of https://github.com/q-logic/python…
HemangChothani Jul 27, 2020
e2793ce
Merge branch 'master' of https://github.com/googleapis/python-storage…
HemangChothani Jul 27, 2020
43f1f5a
Merge branch 'master' of https://github.com/googleapis/python-storage…
HemangChothani Aug 5, 2020
0bd9bf4
feat(storage): changes in unittest
HemangChothani Aug 5, 2020
fee993f
feat(storage): change custom_time type to date
HemangChothani Aug 5, 2020
76457b1
Merge branch 'master' of https://github.com/googleapis/python-storage…
HemangChothani Aug 13, 2020
ae71f29
feat: change custom_time to datetime
HemangChothani Aug 17, 2020
d99fcb9
Merge branch 'master' of https://github.com/googleapis/python-storage…
HemangChothani Aug 17, 2020
62781a3
Merge branch 'master' into storage_issue_196
frankyn Aug 24, 2020
0fcf160
Merge branch 'master' into storage_issue_196
frankyn Aug 24, 2020
39a12f0
feat: nit
HemangChothani Aug 25, 2020
285e15b
Merge branch 'storage_issue_196' of https://github.com/q-logic/python…
HemangChothani Aug 25, 2020
780a616
feat: resolve conflict and nit
HemangChothani Aug 26, 2020
e76b8bb
Merge branch 'master' into storage_issue_196
frankyn Aug 26, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
15 changes: 15 additions & 0 deletions google/cloud/storage/blob.py
Expand Up @@ -3003,6 +3003,21 @@ def updated(self):
if value is not None:
return _rfc3339_to_datetime(value)

@property
def custom_time(self):
"""Retrieve the custom time for the object.

See https://cloud.google.com/storage/docs/json_api/v1/objects

:rtype: :class:`datetime.datetime` or ``NoneType``
:returns: Datetime object parsed from RFC3339 valid timestamp, or
``None`` if the blob's resource has not been loaded from
the server (see :meth:`reload`).
"""
value = self._properties.get("customTime")
if value is not None:
return _rfc3339_to_datetime(value)
tseaver marked this conversation as resolved.
Show resolved Hide resolved


def _get_encryption_headers(key, source=False):
"""Builds customer encryption key headers
Expand Down
30 changes: 30 additions & 0 deletions google/cloud/storage/bucket.py
Expand Up @@ -170,6 +170,18 @@ class LifecycleRuleConditions(dict):
:param number_of_newer_versions: (Optional) Apply rule action to versioned
items having N newer versions.

:type days_since_custom_time: int
:param days_since_custom_time: (Optional) Apply rule action to items whose number of days
elapsed since the custom timestamp. This condition is relevant
only for versioned objects. The value of the field must be a non
negative integer. If it's zero, the object version will become
eligible for lifecycle action as soon as it becomes custom.

:type custom_time_before: :class:`datetime.datetime`
:param custom_time_before: (Optional) Datetime object parsed from RFC3339 valid timestamp, apply
rule action to items whose custom time is before this timestamp.
This condition is relevant only for versioned objects.

:raises ValueError: if no arguments are passed.
"""

Expand All @@ -180,6 +192,8 @@ def __init__(
is_live=None,
matches_storage_class=None,
number_of_newer_versions=None,
days_since_custom_time=None,
custom_time_before=None,
_factory=False,
):
conditions = {}
Expand All @@ -199,6 +213,12 @@ def __init__(
if number_of_newer_versions is not None:
conditions["numNewerVersions"] = number_of_newer_versions

if days_since_custom_time is not None:
conditions["daysSinceCustomTime"] = days_since_custom_time

if custom_time_before is not None:
conditions["customTimeBefore"] = _datetime_to_rfc3339(custom_time_before)

if not _factory and not conditions:
raise ValueError("Supply at least one condition")

Expand Down Expand Up @@ -245,6 +265,16 @@ def number_of_newer_versions(self):
"""Conditon's 'number_of_newer_versions' value."""
return self.get("numNewerVersions")

@property
def days_since_custom_time(self):
"""Conditon's 'days_since_custom_time' value."""
return self.get("daysSinceCustomTime")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Setter for this property?


@property
def custom_time_before(self):
"""Conditon's 'custom_time_before' value."""
return self.get("customTimeBefore")


class LifecycleRuleDelete(dict):
"""Map a lifecycle rule deleting matching items.
Expand Down
16 changes: 14 additions & 2 deletions tests/system/test_system.py
Expand Up @@ -191,22 +191,34 @@ def test_bucket_create_w_alt_storage_class(self):
self.assertEqual(created.storage_class, constants.ARCHIVE_STORAGE_CLASS)

def test_lifecycle_rules(self):
import datetime
from google.cloud.storage import constants

new_bucket_name = "w-lifcycle-rules" + unique_resource_id("-")
custom_time_before = datetime.datetime.now() + datetime.timedelta(days=10)
self.assertRaises(
exceptions.NotFound, Config.CLIENT.get_bucket, new_bucket_name
)
bucket = Config.CLIENT.bucket(new_bucket_name)
bucket.add_lifecycle_delete_rule(age=42)
bucket.add_lifecycle_delete_rule(
age=42,
number_of_newer_versions=3,
days_since_custom_time=2,
custom_time_before=custom_time_before,
)
bucket.add_lifecycle_set_storage_class_rule(
constants.COLDLINE_STORAGE_CLASS,
is_live=False,
matches_storage_class=[constants.NEARLINE_STORAGE_CLASS],
)

expected_rules = [
LifecycleRuleDelete(age=42),
LifecycleRuleDelete(
age=42,
number_of_newer_versions=3,
days_since_custom_time=2,
custom_time_before=custom_time_before,
),
LifecycleRuleSetStorageClass(
constants.COLDLINE_STORAGE_CLASS,
is_live=False,
Expand Down
19 changes: 19 additions & 0 deletions tests/unit/test_blob.py
Expand Up @@ -157,6 +157,7 @@ def _set_properties_helper(self, kms_key_name=None):
"crc32c": CRC32C,
"componentCount": COMPONENT_COUNT,
"etag": ETAG,
"customTime": NOW,
}

if kms_key_name is not None:
Expand Down Expand Up @@ -188,6 +189,7 @@ def _set_properties_helper(self, kms_key_name=None):
self.assertEqual(blob.crc32c, CRC32C)
self.assertEqual(blob.component_count, COMPONENT_COUNT)
self.assertEqual(blob.etag, ETAG)
self.assertEqual(blob.custom_time, now)

if kms_key_name is not None:
self.assertEqual(blob.kms_key_name, kms_key_name)
Expand Down Expand Up @@ -3790,6 +3792,23 @@ def test_updated_unset(self):
blob = self._make_one("blob-name", bucket=BUCKET)
self.assertIsNone(blob.updated)

def test_custom_time(self):
from google.cloud._helpers import _RFC3339_MICROS
from google.cloud._helpers import UTC

BLOB_NAME = "blob-name"
bucket = _Bucket()
TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC)
TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS)
properties = {"customTime": TIME_CREATED}
blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties)
self.assertEqual(blob.custom_time, TIMESTAMP)

def test_custom_time_unset(self):
BUCKET = object()
blob = self._make_one("blob-name", bucket=BUCKET)
self.assertIsNone(blob.custom_time)

def test_from_string_w_valid_uri(self):
from google.cloud.storage.blob import Blob

Expand Down
45 changes: 45 additions & 0 deletions tests/unit/test_bucket.py
Expand Up @@ -88,8 +88,47 @@ def test_ctor_w_number_of_newer_versions(self):
self.assertIsNone(conditions.matches_storage_class)
self.assertEqual(conditions.number_of_newer_versions, 3)

def test_ctor_w_days_since_custom_time(self):
conditions = self._make_one(
number_of_newer_versions=3, days_since_custom_time=2
)
expected = {"numNewerVersions": 3, "daysSinceCustomTime": 2}
self.assertEqual(dict(conditions), expected)
self.assertIsNone(conditions.age)
self.assertIsNone(conditions.created_before)
self.assertIsNone(conditions.is_live)
self.assertIsNone(conditions.matches_storage_class)
self.assertEqual(conditions.number_of_newer_versions, 3)
self.assertEqual(conditions.days_since_custom_time, 2)

def test_ctor_w_custom_time_before(self):
import datetime
from google.cloud._helpers import _datetime_to_rfc3339

custom_time_before = datetime.datetime.now() + datetime.timedelta(days=10)
conditions = self._make_one(
number_of_newer_versions=3, custom_time_before=custom_time_before
)
expected = {
"numNewerVersions": 3,
"customTimeBefore": _datetime_to_rfc3339(custom_time_before),
}

self.assertEqual(dict(conditions), expected)
self.assertIsNone(conditions.age)
self.assertIsNone(conditions.created_before)
self.assertIsNone(conditions.is_live)
self.assertIsNone(conditions.matches_storage_class)
self.assertEqual(conditions.number_of_newer_versions, 3)
self.assertEqual(
conditions.custom_time_before, _datetime_to_rfc3339(custom_time_before)
)

def test_from_api_repr(self):
import datetime
from google.cloud._helpers import _datetime_to_rfc3339

custom_time_before = datetime.datetime.now() + datetime.timedelta(days=10)

before = datetime.date(2018, 8, 1)
klass = self._get_target_class()
Expand All @@ -99,13 +138,19 @@ def test_from_api_repr(self):
"isLive": True,
"matchesStorageClass": ["COLDLINE"],
"numNewerVersions": 3,
"daysSinceCustomTime": 2,
"customTimeBefore": _datetime_to_rfc3339(custom_time_before),
}
conditions = klass.from_api_repr(resource)
self.assertEqual(conditions.age, 10)
self.assertEqual(conditions.created_before, before)
self.assertEqual(conditions.is_live, True)
self.assertEqual(conditions.matches_storage_class, ["COLDLINE"])
self.assertEqual(conditions.number_of_newer_versions, 3)
self.assertEqual(conditions.days_since_custom_time, 2)
self.assertEqual(
conditions.custom_time_before, _datetime_to_rfc3339(custom_time_before)
)


class Test_LifecycleRuleDelete(unittest.TestCase):
Expand Down