Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(storage): revise Reader to send XML preconditions #4479

Merged
merged 12 commits into from Jul 23, 2021
30 changes: 30 additions & 0 deletions storage/integration_test.go
Expand Up @@ -744,6 +744,36 @@ func TestIntegration_ObjectsRangeReader(t *testing.T) {
}
}

func TestIntegration_ConditionalDownload(t *testing.T) {
ctx := context.Background()
client := testConfig(ctx, t)
defer client.Close()
h := testHelper{t}

o := client.Bucket(bucketName).Object("condread")
defer o.Delete(ctx)

wc := o.NewWriter(ctx)
wc.ContentType = "text/plain"
h.mustWrite(wc, []byte("foo"))

gen := wc.Attrs().Generation
metaGen := wc.Attrs().Metageneration

if _, err := o.Generation(gen + 1).NewReader(ctx); err == nil {
t.Fatalf("Unexpected successful download with nonexistent Generation")
cojenco marked this conversation as resolved.
Show resolved Hide resolved
}
if _, err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).NewReader(ctx); err == nil {
t.Fatalf("Unexpected successful download with failed preconditions IfMetaGenerationMatch")
}
if _, err := o.If(Conditions{GenerationMatch: gen + 1}).NewReader(ctx); err == nil {
t.Fatalf("Unexpected successful download with failed preconditions IfGenerationMatch")
}
if _, err := o.If(Conditions{GenerationMatch: gen}).NewReader(ctx); err != nil {
t.Fatalf("Download failed: %v", err)
}
}

func TestIntegration_Objects(t *testing.T) {
// TODO(jba): Use subtests (Go 1.7).
ctx := context.Background()
Expand Down
8 changes: 7 additions & 1 deletion storage/reader.go
Expand Up @@ -149,7 +149,13 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, offset+length-1))
}
// We wait to assign conditions here because the generation number can change in between reopen() runs.
req.URL.RawQuery = conditionsQuery(gen, o.conds)
cojenco marked this conversation as resolved.
Show resolved Hide resolved
if err := setConditionsHeaders(req.Header, o.conds); err != nil {
return nil, err
}
if gen >= 0 {
cojenco marked this conversation as resolved.
Show resolved Hide resolved
req.URL.RawQuery = fmt.Sprintf("generation=%d", gen)
}

var res *http.Response
err = runWithRetry(ctx, func() error {
res, err = o.c.hc.Do(req)
Expand Down
18 changes: 18 additions & 0 deletions storage/storage.go
Expand Up @@ -1634,6 +1634,24 @@ func conditionsQuery(gen int64, conds *Conditions) string {
return string(buf)
}

// setConditionsHeaders sets precondition request headers for downloads
// using the XML API. It assumes that the conditions have been validated.
cojenco marked this conversation as resolved.
Show resolved Hide resolved
func setConditionsHeaders(headers http.Header, conds *Conditions) error {
if conds == nil {
return nil
}
if conds.MetagenerationMatch != 0 {
headers.Set("x-goog-if-metageneration-match", fmt.Sprint(conds.MetagenerationMatch))
}
switch {
case conds.GenerationMatch != 0:
headers.Set("x-goog-if-generation-match", fmt.Sprint(conds.GenerationMatch))
case conds.DoesNotExist:
headers.Set("x-goog-if-generation-match", "0")
}
return nil
}

// composeSourceObj wraps a *raw.ComposeRequestSourceObjects, but adds the methods
// that modifyCall searches for by name.
type composeSourceObj struct {
Expand Down
81 changes: 53 additions & 28 deletions storage/storage_test.go
Expand Up @@ -644,34 +644,6 @@ func TestCondition(t *testing.T) {
},
"GET /buck/obj?generation=1234",
},
{
func() error {
_, err := obj.If(Conditions{GenerationMatch: 1234}).NewReader(ctx)
return err
},
"GET /buck/obj?ifGenerationMatch=1234",
},
{
func() error {
_, err := obj.If(Conditions{GenerationNotMatch: 1234}).NewReader(ctx)
return err
},
"GET /buck/obj?ifGenerationNotMatch=1234",
},
{
func() error {
_, err := obj.If(Conditions{MetagenerationMatch: 1234}).NewReader(ctx)
return err
},
"GET /buck/obj?ifMetagenerationMatch=1234",
},
{
func() error {
_, err := obj.If(Conditions{MetagenerationNotMatch: 1234}).NewReader(ctx)
return err
},
"GET /buck/obj?ifMetagenerationNotMatch=1234",
},
{
func() error {
_, err := obj.If(Conditions{MetagenerationNotMatch: 1234}).Attrs(ctx)
Expand Down Expand Up @@ -734,6 +706,59 @@ func TestCondition(t *testing.T) {
}
}

readerTests := []struct {
fn func() error
want string
}{
{
func() error {
_, err := obj.If(Conditions{GenerationMatch: 1234}).NewReader(ctx)
return err
},
"x-goog-if-generation-match: 1234, x-goog-if-metageneration-match: ",
},
{
func() error {
_, err := obj.If(Conditions{MetagenerationMatch: 5}).NewReader(ctx)
return err
},
"x-goog-if-generation-match: , x-goog-if-metageneration-match: 5",
},
{
func() error {
_, err := obj.If(Conditions{GenerationMatch: 1234, MetagenerationMatch: 5}).NewReader(ctx)
return err
},
"x-goog-if-generation-match: 1234, x-goog-if-metageneration-match: 5",
},
}

for i, tt := range readerTests {
if err := tt.fn(); err != nil && err != io.EOF {
t.Error(err)
continue
}

select {
case r := <-gotReq:
generationConds := r.Header.Get("x-goog-if-generation-match")
metagenerationConds := r.Header.Get("x-goog-if-metageneration-match")
got := fmt.Sprintf(
"x-goog-if-generation-match: %s, x-goog-if-metageneration-match: %s",
generationConds,
metagenerationConds,
)
if got != tt.want {
t.Errorf("%d. RequestHeaders = %q; want %q", i, got, tt.want)
}
case <-time.After(5 * time.Second):
t.Fatalf("%d. timeout", i)
}
if err != nil {
t.Fatal(err)
}
}

// Test an error, too:
err = obj.Generation(1234).NewWriter(ctx).Close()
if err == nil || !strings.Contains(err.Error(), "NewWriter: generation not supported") {
Expand Down