Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(storage): add projection parameter for BucketHandle.Objects() #3549

Merged
merged 11 commits into from Jan 22, 2021
6 changes: 5 additions & 1 deletion storage/bucket.go
Expand Up @@ -1204,7 +1204,11 @@ func (it *ObjectIterator) Next() (*ObjectAttrs, error) {
func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) {
req := it.bucket.c.raw.Objects.List(it.bucket.name)
setClientHeader(req.Header())
req.Projection("full")
projection := it.query.Projection
if projection == ProjectionDefault {
projection = ProjectionFull
}
req.Projection(projection.String())
req.Delimiter(it.query.Delimiter)
req.Prefix(it.query.Prefix)
req.StartOffset(it.query.StartOffset)
Expand Down
37 changes: 37 additions & 0 deletions storage/integration_test.go
Expand Up @@ -698,6 +698,7 @@ func TestIntegration_Objects(t *testing.T) {
testObjectsIterateSelectedAttrs(t, bkt, objects)
testObjectsIterateAllSelectedAttrs(t, bkt, objects)
testObjectIteratorWithOffset(t, bkt, objects)
testObjectsIterateWithProjection(t, bkt)
t.Run("testObjectsIterateSelectedAttrsDelimiter", func(t *testing.T) {
query := &Query{Prefix: "", Delimiter: "/"}
if err := query.SetAttrSelection([]string{"Name"}); err != nil {
Expand Down Expand Up @@ -1236,6 +1237,42 @@ func testObjectsIterateAllSelectedAttrs(t *testing.T, bkt *BucketHandle, objects
}
}

func testObjectsIterateWithProjection(t *testing.T, bkt *BucketHandle) {
projections := map[Projection]bool{
ProjectionDefault: true,
ProjectionFull: true,
ProjectionNoACL: false,
}

for projection, expectACL := range projections {
query := &Query{Projection: projection}
it := bkt.Objects(context.Background(), query)
attrs, err := it.Next()
if err == iterator.Done {
t.Fatalf("no objects")
}
if err != nil {
t.Fatalf("iterator.Next: %v", err)
}

if expectACL {
if attrs.Owner == "" {
t.Errorf("projection: %q, empty Owner", projection)
ava12 marked this conversation as resolved.
Show resolved Hide resolved
}
if len(attrs.ACL) == 0 {
t.Errorf("projection: %q, empty ACL", projection)
}
} else {
if attrs.Owner != "" {
t.Errorf("projection: %q, expected empty Owner, got %q", projection, attrs.Owner)
}
if len(attrs.ACL) != 0 {
t.Errorf("projection: %q, expected empty ACL, got %d entries", projection, len(attrs.ACL))
}
}
}
}

func TestIntegration_SignedURL(t *testing.T) {
if testing.Short() { // do not test during replay
t.Skip("Integration tests skipped in short mode")
Expand Down
30 changes: 30 additions & 0 deletions storage/storage.go
Expand Up @@ -1306,6 +1306,31 @@ func encodeUint32(u uint32) string {
return base64.StdEncoding.EncodeToString(b)
}

// Projection is enumerated type for Query.Projection.
type Projection int

const (
// ProjectionDefault returns all fields of objects.
ProjectionDefault Projection = iota

// ProjectionFull returns all fields of objects.
ProjectionFull

// ProjectionNoACL returns all fields of objects except for Owner and ACL.
ProjectionNoACL
)

func (p Projection) String() string {
switch p {
case ProjectionFull:
return "full"
case ProjectionNoACL:
return "noAcl"
default:
return ""
}
}

// Query represents a query to filter objects from a bucket.
type Query struct {
// Delimiter returns results in a directory-like fashion.
Expand Down Expand Up @@ -1341,6 +1366,11 @@ type Query struct {
// lexicographically before endOffset. If startOffset is also set, the objects
// listed will have names between startOffset (inclusive) and endOffset (exclusive).
EndOffset string

// Projection defines the set of properties to return. It will default to ProjectionFull,
// which returns all properties. Passing ProjectionNoACL will omit Owner and ACL,
// which may improve performance when listing many objects.
Projection Projection
}

// attrToFieldMap maps the field names of ObjectAttrs to the underlying field
Expand Down