test(bigtable): run against live endpoint for continuous tests (#4159)

* test(bigtable): fix #4157 by attempting cleanup of old resources
* test(bigtable): fix #4158 by using live bigtable for continuous IT

bigtable/export_test.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
@@ -103,7 +104,18 @@ type IntegrationEnv interface {
 
 // NewIntegrationEnv creates a new environment based on the command line args
 func NewIntegrationEnv() (IntegrationEnv, error) {
-	c := integrationConfig
+	c := &integrationConfig
+
+	// Check if config settings aren't set. If not, populate from env vars.
+	if c.Project == "" {
+		c.Project = os.Getenv("GCLOUD_TESTS_GOLANG_PROJECT_ID")
+	}
+	if c.Instance == "" {
+		c.Instance = os.Getenv("GCLOUD_TESTS_BIGTABLE_INSTANCE")
+	}
+	if c.Cluster == "" {
+		c.Cluster = os.Getenv("GCLOUD_TESTS_BIGTABLE_CLUSTER")
+	}
 
 	if legacyUseProd != "" {
 		fmt.Println("WARNING: using legacy commandline arg -use_prod, please switch to -it.*")
@@ -114,10 +126,15 @@ func NewIntegrationEnv() (IntegrationEnv, error) {
 		c.Table = parts[2]
 	}
 
+	if c.Instance != "" || c.Cluster != "" {
+		// If commandline args were specified for a live instance, set UseProd
+		c.UseProd = true
+	}
+
 	if integrationConfig.UseProd {
-		return NewProdEnv(c)
+		return NewProdEnv(*c)
 	}
-	return NewEmulatedEnv(c)
+	return NewEmulatedEnv(*c)
 }
 
 // EmulatedEnv encapsulates the state of an emulator
@@ -241,6 +258,9 @@ func NewProdEnv(config IntegrationTestConfig) (*ProdEnv, error) {
 	if config.Instance == "" {
 		return nil, errors.New("Instance not set")
 	}
+	if config.Cluster == "" {
+		return nil, errors.New("Cluster not set")
+	}
 	if config.Table == "" {
 		return nil, errors.New("Table not set")
 	}

bigtable/integration_test.go

@@ -20,12 +20,14 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log"
 	"math"
 	"math/rand"
 	"os"
 	"os/exec"
 	"reflect"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -47,8 +49,10 @@ import (
 )
 
 const (
-	directPathIPV6Prefix = "[2001:4860:8040"
-	directPathIPV4Prefix = "34.126"
+	directPathIPV6Prefix      = "[2001:4860:8040"
+	directPathIPV4Prefix      = "34.126"
+	timeUntilResourceCleanup  = time.Hour * 48 // two days
+	prefixOfInstanceResources = "bt-it-"
 )
 
 var (
@@ -80,14 +84,18 @@ var instanceToCreate string
 
 func init() {
 	if runCreateInstanceTests {
-		instanceToCreate = fmt.Sprintf("create-%d", time.Now().Unix())
+		instanceToCreate = fmt.Sprintf("bt-it-%d", time.Now().Unix())
 	}
 }
 
 func TestMain(m *testing.M) {
 	flag.Parse()
 
-	c := integrationConfig
+	env, err := NewIntegrationEnv()
+	if err != nil {
+		panic(fmt.Sprintf("there was an issue creating an integration env: %v", err))
+	}
+	c := env.Config()
 	if c.UseProd {
 		fmt.Printf(
 			"Note: when using prod, you must first create an instance:\n"+
@@ -96,7 +104,43 @@ func TestMain(m *testing.M) {
 			c.Cluster, "us-central1-b", "1",
 		)
 	}
-	os.Exit(m.Run())
+	exit := m.Run()
+	if err := cleanup(c); err != nil {
+		log.Printf("Post-test cleanup failed: %v", err)
+	}
+	os.Exit(exit)
+}
+
+func cleanup(c IntegrationTestConfig) error {
+	// Cleanup resources marked with bt-it- after a time delay
+	if !c.UseProd {
+		return nil
+	}
+	ctx := context.Background()
+	iac, err := NewInstanceAdminClient(ctx, c.Project)
+	if err != nil {
+		return err
+	}
+	instances, err := iac.Instances(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, info := range instances {
+		if strings.HasPrefix(info.Name, prefixOfInstanceResources) {
+			timestamp := info.Name[len(prefixOfInstanceResources):]
+			t, err := strconv.ParseInt(timestamp, 10, 64)
+			if err != nil {
+				return err
+			}
+			uT := time.Unix(t, 0)
+			if time.Now().After(uT.Add(timeUntilResourceCleanup)) {
+				iac.DeleteInstance(ctx, info.Name)
+			}
+		}
+	}
+
+	return nil
 }
 
 func TestIntegration_ConditionalMutations(t *testing.T) {
@@ -1315,7 +1359,7 @@ func TestIntegration_TableIam(t *testing.T) {
 	iamHandle := adminClient.TableIAM("mytable")
 	p, err := iamHandle.Policy(ctx)
 	if err != nil {
-		t.Errorf("Iam GetPolicy mytable: %v", err)
+		t.Fatalf("Iam GetPolicy mytable: %v", err)
 	}
 	if err = iamHandle.SetPolicy(ctx, p); err != nil {
 		t.Errorf("Iam SetPolicy mytable: %v", err)
@@ -2151,7 +2195,6 @@ func TestIntegration_InstanceAdminClient_AppProfile(t *testing.T) {
 	}
 
 	err = iAdminClient.DeleteAppProfile(ctx, adminClient.instance, "app_profile1")
-	err = iAdminClient.DeleteAppProfile(ctx, adminClient.instance, "default")
 
 	defer iAdminClient.Close()
 	profile := ProfileConf{

internal/kokoro/continuous.sh

@@ -37,6 +37,12 @@ export GCLOUD_TESTS_API_KEY=`cat $KOKORO_KEYSTORE_DIR/72523_go_gcloud_tests_api_
 export GCLOUD_TESTS_GOLANG_KEYRING=projects/dulcet-port-762/locations/us/keyRings/go-integration-test
 export GCLOUD_TESTS_GOLANG_PROFILER_ZONE="us-west1-b"
 
+# Bigtable integration tests expect an existing instance and cluster
+#  ❯ cbt createinstance gcloud-bt-it-tests-instance "Bigtable IT Instance" \
+#    gcloud-bt-it-tests-cluster us-west1-b 1 SSD
+export GCLOUD_TESTS_BIGTABLE_CLUSTER="gcloud-bt-it-tests-cluster"
+export GCLOUD_TESTS_BIGTABLE_INSTANCE="gcloud-bt-it-tests-instance"
+
 # Fail on any error
 set -eo pipefail