test(httpreplay): fix TestIntegration_RecordAndReplay (#3096)

A bucket this test relied upon no longer exists. Refactored test
to not assume objects exist somewhere else. The test now uploads
and cleans-up gcs objects.

Fixes: #3015

httpreplay/httpreplay_test.go

@@ -19,11 +19,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -33,6 +35,11 @@ import (
 	"google.golang.org/api/option"
 )
 
+const (
+	compressedFile   = "httpreplay_compressed.txt"
+	uncompressedFile = "httpreplay_uncompressed.txt"
+)
+
 func TestIntegration_RecordAndReplay(t *testing.T) {
 	httpreplay.DebugHeaders()
 	if testing.Short() {
@@ -45,6 +52,11 @@ func TestIntegration_RecordAndReplay(t *testing.T) {
 		t.Skip("Need project ID. See CONTRIBUTING.md for details.")
 	}
 	ctx := context.Background()
+	cleanup, err := setup(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
 
 	// Record.
 	initial := time.Now()
@@ -95,6 +107,49 @@ func TestIntegration_RecordAndReplay(t *testing.T) {
 	}
 }
 
+func setup(ctx context.Context) (cleanup func(), err error) {
+	ts := testutil.TokenSource(ctx, storage.ScopeFullControl)
+	client, err := storage.NewClient(ctx, option.WithTokenSource(ts))
+	if err != nil {
+		return nil, err
+	}
+	bucket := testutil.ProjID()
+
+	// upload compressed object
+	f1, err := os.Open(filepath.Join("internal", "testdata", "compressed.txt"))
+	if err != nil {
+		return nil, err
+	}
+	defer f1.Close()
+	w := client.Bucket(bucket).Object(compressedFile).NewWriter(ctx)
+	w.ContentEncoding = "gzip"
+	w.ContentType = "text/plain"
+	if _, err = io.Copy(w, f1); err != nil {
+		return nil, err
+	}
+	if err := w.Close(); err != nil {
+		return nil, err
+	}
+	// upload uncompressed object
+	f2, err := os.Open(filepath.Join("internal", "testdata", "uncompressed.txt"))
+	if err != nil {
+		return nil, err
+	}
+	defer f2.Close()
+	w = client.Bucket(testutil.ProjID()).Object(uncompressedFile).NewWriter(ctx)
+	if _, err = io.Copy(w, f2); err != nil {
+		return nil, err
+	}
+	if err := w.Close(); err != nil {
+		return nil, err
+	}
+	return func() {
+		client.Bucket(bucket).Object(compressedFile).Delete(ctx)
+		client.Bucket(bucket).Object(uncompressedFile).Delete(ctx)
+		client.Close()
+	}, nil
+}
+
 // TODO(jba): test errors
 
 func run(t *testing.T, hc *http.Client) (*storage.BucketAttrs, []byte) {
@@ -133,24 +188,16 @@ func run(t *testing.T, hc *http.Client) (*storage.BucketAttrs, []byte) {
 }
 
 func testReadCRC(t *testing.T, hc *http.Client, mode string) {
-	const (
-		// This is an uncompressed file.
-		// See https://cloud.google.com/storage/docs/public-datasets/landsat
-		uncompressedBucket = "gcp-public-data-landsat"
-		uncompressedObject = "LC08/PRE/044/034/LC80440342016259LGN00/LC80440342016259LGN00_MTL.txt"
-
-		gzippedBucket = "storage-library-test-bucket"
-		gzippedObject = "gzipped-text.txt"
-	)
 	ctx := context.Background()
 	client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
 	if err != nil {
 		t.Fatalf("%s: %v", mode, err)
 	}
 	defer client.Close()
 
-	uncompressedObj := client.Bucket(uncompressedBucket).Object(uncompressedObject)
-	gzippedObj := client.Bucket(gzippedBucket).Object(gzippedObject)
+	bucket := testutil.ProjID()
+	uncompressedObj := client.Bucket(bucket).Object(uncompressedFile)
+	gzippedObj := client.Bucket(bucket).Object(compressedFile)
 
 	for _, test := range []struct {
 		desc           string
@@ -167,23 +214,23 @@ func testReadCRC(t *testing.T, hc *http.Client, mode string) {
 			offset:         0,
 			length:         -1,
 			readCompressed: false,
-			wantLen:        7903,
+			wantLen:        179,
 		},
 		{
 			desc:           "uncompressed, entire file, don't decompress",
 			obj:            uncompressedObj,
 			offset:         0,
 			length:         -1,
 			readCompressed: true,
-			wantLen:        7903,
+			wantLen:        179,
 		},
 		{
 			desc:           "uncompressed, suffix",
 			obj:            uncompressedObj,
-			offset:         3,
+			offset:         9,
 			length:         -1,
 			readCompressed: false,
-			wantLen:        7900,
+			wantLen:        170,
 		},
 		{
 			desc:           "uncompressed, prefix",
@@ -204,7 +251,7 @@ func testReadCRC(t *testing.T, hc *http.Client, mode string) {
 			offset:         0,
 			length:         -1,
 			readCompressed: false,
-			wantLen:        11,
+			wantLen:        179,
 		},
 		{
 			// When we read a gzipped file uncompressed, it's like reading a regular file:
@@ -214,7 +261,7 @@ func testReadCRC(t *testing.T, hc *http.Client, mode string) {
 			offset:         0,
 			length:         -1,
 			readCompressed: true,
-			wantLen:        31,
+			wantLen:        128,
 		},
 		{
 			desc:           "compressed, partial, read compressed",

httpreplay/internal/testdata/uncompressed.txt

@@ -0,0 +1,15 @@
+This    is  a
+
+test    file    that
+
+includes    some
+
+uncompressed    text.
+
+It's    sister  compressed
+
+file    is  compressed.txt.
+
+This    is  used    to
+
+test    httpreplay.