Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: add offset and includeTrailingPrefix options to list_blobs #125

Merged
merged 4 commits into from May 5, 2020
Merged

feat: add offset and includeTrailingPrefix options to list_blobs #125

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c9ca926
feat: add offset and includeTrailingPrefix options to list_blobs
andrewsg May 1, 2020
8697dc7
lint
andrewsg May 1, 2020
bbc32b7
fix comment typo in system tests
andrewsg May 4, 2020
e3d6cd9
Merge branch 'master' into new-list-flags
frankyn May 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
32 changes: 32 additions & 0 deletions google/cloud/storage/bucket.py
View file
Open in desktop
Expand Up @@ -894,6 +894,9 @@ def list_blobs(
page_token=None,
prefix=None,
delimiter=None,
start_offset=None,
end_offset=None,
include_trailing_delimiter=None,
versions=None,
projection="noAcl",
fields=None,
Expand Down Expand Up @@ -926,6 +929,26 @@ def list_blobs(
:param delimiter: (Optional) Delimiter, used with ``prefix`` to
emulate hierarchy.

:type start_offset: str
:param start_offset:
(Optional) Filter results to objects whose names are
lexicographically equal to or after ``startOffset``. If
``endOffset`` is also set, the objects listed will have names
between ``startOffset`` (inclusive) and ``endOffset`` (exclusive).

:type end_offset: str
:param end_offset:
(Optional) Filter results to objects whose names are
lexicographically before ``endOffset``. If ``startOffset`` is also
set, the objects listed will have names between ``startOffset``
(inclusive) and ``endOffset`` (exclusive).

:type include_trailing_delimiter: boolean
:param include_trailing_delimiter:
(Optional) If true, objects that end in exactly one instance of
``delimiter`` will have their metadata included in ``items`` in
addition to ``prefixes``.

:type versions: bool
:param versions: (Optional) Whether object versions should be returned
as separate blobs.
Expand Down Expand Up @@ -967,6 +990,15 @@ def list_blobs(
if delimiter is not None:
extra_params["delimiter"] = delimiter

if start_offset is not None:
extra_params["startOffset"] = start_offset

if end_offset is not None:
extra_params["endOffset"] = end_offset

if include_trailing_delimiter is not None:
extra_params["includeTrailingDelimiter"] = include_trailing_delimiter

if versions is not None:
extra_params["versions"] = versions

Expand Down
24 changes: 24 additions & 0 deletions google/cloud/storage/client.py
View file
Open in desktop
Expand Up @@ -540,6 +540,9 @@ def list_blobs(
page_token=None,
prefix=None,
delimiter=None,
start_offset=None,
end_offset=None,
include_trailing_delimiter=None,
versions=None,
projection="noAcl",
fields=None,
Expand Down Expand Up @@ -573,6 +576,24 @@ def list_blobs(
(Optional) Delimiter, used with ``prefix`` to
emulate hierarchy.

start_offset (str):
(Optional) Filter results to objects whose names are
lexicographically equal to or after ``startOffset``. If
``endOffset`` is also set, the objects listed will have names
between ``startOffset`` (inclusive) and ``endOffset``
(exclusive).

end_offset (str):
(Optional) Filter results to objects whose names are
lexicographically before ``endOffset``. If ``startOffset`` is
also set, the objects listed will have names between
``startOffset`` (inclusive) and ``endOffset`` (exclusive).

include_trailing_delimiter (boolean):
(Optional) If true, objects that end in exactly one instance of
``delimiter`` will have their metadata included in ``items`` in
addition to ``prefixes``.

versions (bool):
(Optional) Whether object versions should be returned
as separate blobs.
Expand Down Expand Up @@ -606,6 +627,9 @@ def list_blobs(
page_token=page_token,
prefix=prefix,
delimiter=delimiter,
start_offset=start_offset,
end_offset=end_offset,
include_trailing_delimiter=include_trailing_delimiter,
versions=versions,
projection=projection,
fields=fields,
Expand Down
48 changes: 46 additions & 2 deletions tests/system/test_system.py
View file
Open in desktop
Expand Up @@ -761,7 +761,7 @@ def test_fetch_object_and_check_content(self):

class TestStorageListFiles(TestStorageFiles):

FILENAMES = ("CloudLogo1", "CloudLogo2", "CloudLogo3")
FILENAMES = ("CloudLogo1", "CloudLogo2", "CloudLogo3", "CloudLogo4")

@classmethod
def setUpClass(cls):
Expand Down Expand Up @@ -825,11 +825,42 @@ def test_paginate_files(self):
last_blobs = list(page2)
self.assertEqual(len(last_blobs), truncation_size)

@RetryErrors(unittest.TestCase.failureException)
def test_paginate_files_with_offset(self):
truncation_size = 1
inclusive_start_offset = self.FILENAMES[1]
exclusive_end_offset = self.FILENAMES[-1]
desired_files = self.FILENAMES[1:-1]
count = len(desired_files) - truncation_size
iterator = self.bucket.list_blobs(
max_results=count,
start_offset=inclusive_start_offset,
end_offset=exclusive_end_offset,
)
page_iter = iterator.pages

page1 = six.next(page_iter)
blobs = list(page1)
self.assertEqual(len(blobs), count)
self.assertEqual(blobs[0].name, desired_files[0])
self.assertIsNotNone(iterator.next_page_token)
# Technically the iterator is exhausted.
self.assertEqual(iterator.num_results, iterator.max_results)
# But we modify the iterator to continue paging after
# articially stopping after ``count`` items.
andrewsg marked this conversation as resolved.
Show resolved Hide resolved
iterator.max_results = None

page2 = six.next(page_iter)
last_blobs = list(page2)
self.assertEqual(len(last_blobs), truncation_size)
self.assertEqual(last_blobs[-1].name, desired_files[-1])


class TestStoragePseudoHierarchy(TestStorageFiles):

FILENAMES = (
"file01.txt",
"parent/",
"parent/file11.txt",
"parent/child/file21.txt",
"parent/child/file22.txt",
Expand Down Expand Up @@ -877,7 +908,9 @@ def test_first_level(self):
iterator = self.bucket.list_blobs(delimiter="/", prefix="parent/")
page = six.next(iterator.pages)
blobs = list(page)
self.assertEqual([blob.name for blob in blobs], ["parent/file11.txt"])
self.assertEqual(
[blob.name for blob in blobs], ["parent/", "parent/file11.txt"]
)
self.assertIsNone(iterator.next_page_token)
self.assertEqual(iterator.prefixes, set(["parent/child/"]))

Expand Down Expand Up @@ -909,6 +942,17 @@ def test_third_level(self):
self.assertIsNone(iterator.next_page_token)
self.assertEqual(iterator.prefixes, set())

@RetryErrors(unittest.TestCase.failureException)
def test_include_trailing_delimiter(self):
iterator = self.bucket.list_blobs(
delimiter="/", include_trailing_delimiter=True
)
page = six.next(iterator.pages)
blobs = list(page)
self.assertEqual([blob.name for blob in blobs], ["file01.txt", "parent/"])
self.assertIsNone(iterator.next_page_token)
self.assertEqual(iterator.prefixes, set(["parent/"]))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how is this request different when include_trailinig_delimitier=False? I don't see the difference between the system test in this case.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The difference is that this is the only name that includes parent/ in self.assertEqual([blob.name for blob in blobs], ["file01.txt", "parent/"]).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll leave this for you to resolve if you're satisfied with that explanation



class TestStorageSignURLs(unittest.TestCase):
BLOB_CONTENT = b"This time for sure, Rocky!"
Expand Down
9 changes: 9 additions & 0 deletions tests/unit/test_bucket.py
View file
Open in desktop
Expand Up @@ -733,6 +733,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
PAGE_TOKEN = "ABCD"
PREFIX = "subfolder"
DELIMITER = "/"
START_OFFSET = "c"
END_OFFSET = "g"
INCLUDE_TRAILING_DELIMITER = True
VERSIONS = True
PROJECTION = "full"
FIELDS = "items/contentLanguage,nextPageToken"
Expand All @@ -741,6 +744,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
"pageToken": PAGE_TOKEN,
"prefix": PREFIX,
"delimiter": DELIMITER,
"startOffset": START_OFFSET,
"endOffset": END_OFFSET,
"includeTrailingDelimiter": INCLUDE_TRAILING_DELIMITER,
"versions": VERSIONS,
"projection": PROJECTION,
"fields": FIELDS,
Expand All @@ -754,6 +760,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
page_token=PAGE_TOKEN,
prefix=PREFIX,
delimiter=DELIMITER,
start_offset=START_OFFSET,
end_offset=END_OFFSET,
include_trailing_delimiter=INCLUDE_TRAILING_DELIMITER,
versions=VERSIONS,
projection=PROJECTION,
fields=FIELDS,
Expand Down
9 changes: 9 additions & 0 deletions tests/unit/test_client.py
View file
Open in desktop
Expand Up @@ -970,6 +970,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
PAGE_TOKEN = "ABCD"
PREFIX = "subfolder"
DELIMITER = "/"
START_OFFSET = "c"
END_OFFSET = "g"
INCLUDE_TRAILING_DELIMITER = True
VERSIONS = True
PROJECTION = "full"
FIELDS = "items/contentLanguage,nextPageToken"
Expand All @@ -978,6 +981,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
"pageToken": PAGE_TOKEN,
"prefix": PREFIX,
"delimiter": DELIMITER,
"startOffset": START_OFFSET,
"endOffset": END_OFFSET,
"includeTrailingDelimiter": INCLUDE_TRAILING_DELIMITER,
"versions": VERSIONS,
"projection": PROJECTION,
"fields": FIELDS,
Expand All @@ -1001,6 +1007,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
page_token=PAGE_TOKEN,
prefix=PREFIX,
delimiter=DELIMITER,
start_offset=START_OFFSET,
end_offset=END_OFFSET,
include_trailing_delimiter=INCLUDE_TRAILING_DELIMITER,
versions=VERSIONS,
projection=PROJECTION,
fields=FIELDS,
Expand Down