Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(bigtable): Backup Level IAM #3222

Merged
merged 17 commits into from Dec 29, 2020
Merged
7 changes: 7 additions & 0 deletions bigtable/admin.go
Expand Up @@ -603,6 +603,13 @@ func (ac *AdminClient) TableIAM(tableID string) *iam.Handle {
"projects/"+ac.project+"/instances/"+ac.instance+"/tables/"+tableID)
}

// BackupIAM creates an IAM client specific to a given Cluster and Backup.
AlisskaPie marked this conversation as resolved.
Show resolved Hide resolved
func (ac *AdminClient) BackupIAM(ctx context.Context, cluster, backup string) *iam.Handle {
prefix := ac.instancePrefix()
backupPath := prefix + "/clusters/" + cluster + "/backups/" + backup
AlisskaPie marked this conversation as resolved.
Show resolved Hide resolved
return iam.InternalNewHandleGRPCClient(ac.tClient, backupPath)
}

const instanceAdminAddr = "bigtableadmin.googleapis.com:443"
const mtlsInstanceAdminAddr = "bigtableadmin.mtls.googleapis.com:443"

Expand Down
196 changes: 189 additions & 7 deletions bigtable/integration_test.go
Expand Up @@ -29,6 +29,7 @@ import (
"testing"
"time"

"cloud.google.com/go/iam"
"cloud.google.com/go/internal"
"cloud.google.com/go/internal/testutil"
"cloud.google.com/go/internal/uid"
Expand All @@ -37,6 +38,10 @@ import (
gax "github.com/googleapis/gax-go/v2"
"google.golang.org/api/iterator"
btapb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
iampb "google.golang.org/genproto/googleapis/iam/v1"
"google.golang.org/genproto/googleapis/longrunning"
"google.golang.org/grpc"
"google.golang.org/protobuf/types/known/emptypb"
)

var (
Expand Down Expand Up @@ -1243,6 +1248,88 @@ func TestIntegration_TableIam(t *testing.T) {
}
}

func TestIntegration_BackupIAM(t *testing.T) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given this is an integration test file, it feels like we shouldn't be mocking out the interactions. It's probably enough to follow the pattern of TestIntegration_TableIam (which should be IAM, but that's another story 🙃).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1. @kolea2 do you know how other languages are handling this testing, and if we will face any quota-related issues?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's strange, but I got an error while comparing policies without mocking. I had an Etag mismatch, even in the read-modify-write cycle.
From documentation:

It is strongly suggested that systems make use of the etag in the read-modify-write cycle to perform policy updates in order to avoid race conditions: An etag is returned in the response to getIamPolicy, and systems are expected to put that etag in the request to setIamPolicy to ensure that their change will be applied to the same version of the policy.

image

Will wait for response!

P.S. Anyway, I can also rename TestIntegration_TableIam to TestIntegration_TableIAM here :)
It's related to this PR.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The Etag should only change when there is a modification. So, something else must have modified the value between the initial read and the subsequent attempted write. Mind uploading the updated test without the mock, even if it isn't passing yet?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, sure!
Uploaded.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tritone for Java we take the existing IAM policy and reset it. Probably could be improved: https://github.com/googleapis/java-bigtable/blob/master/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java#L194-L217. I haven't seen any quota related issues so far.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tbpg @tritone any additional thoughts here?

testEnv, err := NewIntegrationEnv()
if err != nil {
t.Fatalf("IntegrationEnv: %v", err)
}
defer testEnv.Close()

if !testEnv.Config().UseProd {
t.Skip("emulator doesn't support IAM Policy creation")
}
timeout := 5 * time.Minute
ctx, _ := context.WithTimeout(context.Background(), timeout)

adminClient, err := testEnv.NewAdminClient()
if err != nil {
t.Fatalf("NewAdminClient: %v", err)
}
defer adminClient.Close()

table := testEnv.Config().Table
cluster := testEnv.Config().Cluster

defer deleteTable(ctx, t, adminClient, table)
if err := adminClient.CreateTable(ctx, table); err != nil {
t.Fatalf("Creating table: %v", err)
}
// Create backup.
backup := "backup"
defer adminClient.DeleteBackup(ctx, cluster, backup)
if err = adminClient.CreateBackup(ctx, table, cluster, backup, time.Now().Add(8*time.Hour)); err != nil {
t.Fatalf("Creating backup: %v", err)
}
// Mock setup.
iamPolicy := &iam.Policy{
InternalProto: &iampb.Policy{
Version: 1,
Bindings: []*iampb.Binding{
{
Role: "roles/bigtable.viewer",
Members: []string{"serviceAccount:service_acc1@test.com", "user:user1@test.com"},
},
},
Etag: []byte("etag_v1"),
},
}
permissions := []string{"bigtable.backups.create", "bigtable.backups.list"}

adminClient.tClient = &adminClientMock{
Policy: iamPolicy.InternalProto,
Permissions: permissions,
}
iamHandle := adminClient.BackupIAM(ctx, cluster, backup)
// Get backup policy.
p, err := iamHandle.Policy(ctx)
if err != nil {
t.Errorf("iamHandle.Policy: %v", err)
}
if diff := testutil.Diff(p, iamPolicy); diff != "" {
t.Fatalf("iamHandle.Policy: got - want +\n%s", diff)
}
// Set backup policy.
p.Add("user:user2@test.com", "roles/bigtable.viewer")
if err = iamHandle.SetPolicy(ctx, p); err != nil {
t.Errorf("iamHandle.SetPolicy: %v", err)
}
newPolicy, err := iamHandle.Policy(ctx)
if err != nil {
t.Errorf("iamHandle.Policy: %v", err)
}
if diff := testutil.Diff(newPolicy, p); diff != "" {
t.Fatalf("iamHandle.Policy: got - want +\n%s", diff)
}
// Test backup permissions.
gotPermissions, err := iamHandle.TestPermissions(ctx, permissions)
if err != nil {
t.Errorf("iamHandle.TestPermissions: %v", err)
}
if !testutil.Equal(gotPermissions, permissions) {
t.Fatalf("TestIAMPermissions: got %#v\nwant %#v", gotPermissions, permissions)
}
}

func TestIntegration_AdminCreateInstance(t *testing.T) {
if instanceToCreate == "" {
t.Skip("instanceToCreate not set, skipping instance creation testing")
Expand Down Expand Up @@ -2045,9 +2132,10 @@ func TestIntegration_AdminBackup(t *testing.T) {
}

// Create backup
backupName := "mybackup"
defer adminClient.DeleteBackup(ctx, cluster, "mybackup")

if err = adminClient.CreateBackup(ctx, table, cluster, "mybackup", time.Now().Add(8*time.Hour)); err != nil {
if err = adminClient.CreateBackup(ctx, table, cluster, backupName, time.Now().Add(8*time.Hour)); err != nil {
t.Fatalf("Creating backup: %v", err)
}

Expand All @@ -2059,7 +2147,7 @@ func TestIntegration_AdminBackup(t *testing.T) {
if got, want := len(backups), 1; got != want {
t.Fatalf("Listing backup count: %d, want: %d", got, want)
}
if got, want := backups[0].Name, "mybackup"; got != want {
if got, want := backups[0].Name, backupName; got != want {
t.Fatalf("Backup name: %s, want: %s", got, want)
}
if got, want := backups[0].SourceTable, table; got != want {
Expand All @@ -2070,7 +2158,7 @@ func TestIntegration_AdminBackup(t *testing.T) {
}

// Get backup
backup, err := adminClient.BackupInfo(ctx, cluster, "mybackup")
backup, err := adminClient.BackupInfo(ctx, cluster, backupName)
if err != nil {
t.Fatalf("BackupInfo: %v", backup)
}
Expand All @@ -2080,13 +2168,13 @@ func TestIntegration_AdminBackup(t *testing.T) {

// Update backup
newExpireTime := time.Now().Add(10 * time.Hour)
err = adminClient.UpdateBackup(ctx, cluster, "mybackup", newExpireTime)
err = adminClient.UpdateBackup(ctx, cluster, backupName, newExpireTime)
if err != nil {
t.Fatalf("UpdateBackup failed: %v", err)
}

// Check that updated backup has the correct expire time
updatedBackup, err := adminClient.BackupInfo(ctx, cluster, "mybackup")
updatedBackup, err := adminClient.BackupInfo(ctx, cluster, backupName)
if err != nil {
t.Fatalf("BackupInfo: %v", err)
}
Expand All @@ -2099,15 +2187,15 @@ func TestIntegration_AdminBackup(t *testing.T) {
// Restore backup
restoredTable := table + "-restored"
defer deleteTable(ctx, t, adminClient, restoredTable)
if err = adminClient.RestoreTable(ctx, restoredTable, cluster, "mybackup"); err != nil {
if err = adminClient.RestoreTable(ctx, restoredTable, cluster, backupName); err != nil {
t.Fatalf("RestoreTable: %v", err)
}
if _, err := adminClient.TableInfo(ctx, restoredTable); err != nil {
t.Fatalf("Restored TableInfo: %v", err)
}

// Delete backup
if err = adminClient.DeleteBackup(ctx, cluster, "mybackup"); err != nil {
if err = adminClient.DeleteBackup(ctx, cluster, backupName); err != nil {
t.Fatalf("DeleteBackup: %v", err)
}
backups, err = list(cluster)
Expand Down Expand Up @@ -2212,3 +2300,97 @@ func deleteTable(ctx context.Context, t *testing.T, ac *AdminClient, name string
t.Logf("DeleteTable: %v", err)
}
}

// adminClientMock is used to test backup level IAM.
type adminClientMock struct {
Policy *iampb.Policy
Permissions []string
}

func (acm *adminClientMock) CreateTable(ctx context.Context, in *btapb.CreateTableRequest, opts ...grpc.CallOption) (*btapb.Table, error) {
return nil, nil
}

func (acm *adminClientMock) CreateTableFromSnapshot(ctx context.Context, in *btapb.CreateTableFromSnapshotRequest, opts ...grpc.CallOption) (*longrunning.Operation, error) {
return nil, nil
}

func (acm *adminClientMock) ListTables(ctx context.Context, in *btapb.ListTablesRequest, opts ...grpc.CallOption) (*btapb.ListTablesResponse, error) {
return nil, nil
}

func (acm *adminClientMock) GetTable(ctx context.Context, in *btapb.GetTableRequest, opts ...grpc.CallOption) (*btapb.Table, error) {
return nil, nil
}

func (acm *adminClientMock) DeleteTable(ctx context.Context, in *btapb.DeleteTableRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
return nil, nil
}

func (acm *adminClientMock) ModifyColumnFamilies(ctx context.Context, in *btapb.ModifyColumnFamiliesRequest, opts ...grpc.CallOption) (*btapb.Table, error) {
return nil, nil
}

func (acm *adminClientMock) DropRowRange(ctx context.Context, in *btapb.DropRowRangeRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
return nil, nil
}

func (acm *adminClientMock) GenerateConsistencyToken(ctx context.Context, in *btapb.GenerateConsistencyTokenRequest, opts ...grpc.CallOption) (*btapb.GenerateConsistencyTokenResponse, error) {
return nil, nil
}

func (acm *adminClientMock) CheckConsistency(ctx context.Context, in *btapb.CheckConsistencyRequest, opts ...grpc.CallOption) (*btapb.CheckConsistencyResponse, error) {
return nil, nil
}

func (acm *adminClientMock) SnapshotTable(ctx context.Context, in *btapb.SnapshotTableRequest, opts ...grpc.CallOption) (*longrunning.Operation, error) {
return nil, nil
}

func (acm *adminClientMock) GetSnapshot(ctx context.Context, in *btapb.GetSnapshotRequest, opts ...grpc.CallOption) (*btapb.Snapshot, error) {
return nil, nil
}

func (acm *adminClientMock) ListSnapshots(ctx context.Context, in *btapb.ListSnapshotsRequest, opts ...grpc.CallOption) (*btapb.ListSnapshotsResponse, error) {
return nil, nil
}

func (acm *adminClientMock) DeleteSnapshot(ctx context.Context, in *btapb.DeleteSnapshotRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
return nil, nil
}

func (acm *adminClientMock) CreateBackup(ctx context.Context, in *btapb.CreateBackupRequest, opts ...grpc.CallOption) (*longrunning.Operation, error) {
return nil, nil
}

func (acm *adminClientMock) GetBackup(ctx context.Context, in *btapb.GetBackupRequest, opts ...grpc.CallOption) (*btapb.Backup, error) {
return nil, nil
}

func (acm *adminClientMock) UpdateBackup(ctx context.Context, in *btapb.UpdateBackupRequest, opts ...grpc.CallOption) (*btapb.Backup, error) {
return nil, nil
}

func (acm *adminClientMock) DeleteBackup(ctx context.Context, in *btapb.DeleteBackupRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
return nil, nil
}

func (acm *adminClientMock) ListBackups(ctx context.Context, in *btapb.ListBackupsRequest, opts ...grpc.CallOption) (*btapb.ListBackupsResponse, error) {
return nil, nil
}

func (acm *adminClientMock) RestoreTable(ctx context.Context, in *btapb.RestoreTableRequest, opts ...grpc.CallOption) (*longrunning.Operation, error) {
return nil, nil
}

func (acm *adminClientMock) GetIamPolicy(ctx context.Context, in *iampb.GetIamPolicyRequest, opts ...grpc.CallOption) (*iampb.Policy, error) {
return acm.Policy, nil
}

func (acm *adminClientMock) SetIamPolicy(ctx context.Context, in *iampb.SetIamPolicyRequest, opts ...grpc.CallOption) (*iampb.Policy, error) {
return acm.Policy, nil
}

func (acm *adminClientMock) TestIamPermissions(ctx context.Context, in *iampb.TestIamPermissionsRequest, opts ...grpc.CallOption) (*iampb.TestIamPermissionsResponse, error) {
return &iampb.TestIamPermissionsResponse{Permissions: acm.Permissions}, nil
}