New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(bigtable): Backup Level IAM #3222
Changes from 12 commits
1819733
127937c
1b4b43b
bcf684c
ab7d4f2
da59b03
085c20c
df99660
28277fc
c4d1dbc
30edfa8
1297f50
b31e4dd
f68c9dd
b1a59aa
df139af
ab73206
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -29,6 +29,7 @@ import ( | |
"testing" | ||
"time" | ||
|
||
"cloud.google.com/go/iam" | ||
"cloud.google.com/go/internal" | ||
"cloud.google.com/go/internal/testutil" | ||
"cloud.google.com/go/internal/uid" | ||
|
@@ -37,6 +38,10 @@ import ( | |
gax "github.com/googleapis/gax-go/v2" | ||
"google.golang.org/api/iterator" | ||
btapb "google.golang.org/genproto/googleapis/bigtable/admin/v2" | ||
iampb "google.golang.org/genproto/googleapis/iam/v1" | ||
"google.golang.org/genproto/googleapis/longrunning" | ||
"google.golang.org/grpc" | ||
"google.golang.org/protobuf/types/known/emptypb" | ||
) | ||
|
||
var ( | ||
|
@@ -1243,6 +1248,88 @@ func TestIntegration_TableIam(t *testing.T) { | |
} | ||
} | ||
|
||
func TestIntegration_BackupIAM(t *testing.T) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Given this is an integration test file, it feels like we shouldn't be mocking out the interactions. It's probably enough to follow the pattern of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. +1. @kolea2 do you know how other languages are handling this testing, and if we will face any quota-related issues? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's strange, but I got an error while comparing policies without mocking. I had an
Will wait for response! P.S. Anyway, I can also rename There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The Etag should only change when there is a modification. So, something else must have modified the value between the initial read and the subsequent attempted write. Mind uploading the updated test without the mock, even if it isn't passing yet? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, sure! There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @tritone for Java we take the existing IAM policy and reset it. Probably could be improved: https://github.com/googleapis/java-bigtable/blob/master/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java#L194-L217. I haven't seen any quota related issues so far. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
testEnv, err := NewIntegrationEnv() | ||
if err != nil { | ||
t.Fatalf("IntegrationEnv: %v", err) | ||
} | ||
defer testEnv.Close() | ||
|
||
if !testEnv.Config().UseProd { | ||
t.Skip("emulator doesn't support IAM Policy creation") | ||
} | ||
timeout := 5 * time.Minute | ||
ctx, _ := context.WithTimeout(context.Background(), timeout) | ||
|
||
adminClient, err := testEnv.NewAdminClient() | ||
if err != nil { | ||
t.Fatalf("NewAdminClient: %v", err) | ||
} | ||
defer adminClient.Close() | ||
|
||
table := testEnv.Config().Table | ||
cluster := testEnv.Config().Cluster | ||
|
||
defer deleteTable(ctx, t, adminClient, table) | ||
if err := adminClient.CreateTable(ctx, table); err != nil { | ||
t.Fatalf("Creating table: %v", err) | ||
} | ||
// Create backup. | ||
backup := "backup" | ||
defer adminClient.DeleteBackup(ctx, cluster, backup) | ||
if err = adminClient.CreateBackup(ctx, table, cluster, backup, time.Now().Add(8*time.Hour)); err != nil { | ||
t.Fatalf("Creating backup: %v", err) | ||
} | ||
// Mock setup. | ||
iamPolicy := &iam.Policy{ | ||
InternalProto: &iampb.Policy{ | ||
Version: 1, | ||
Bindings: []*iampb.Binding{ | ||
{ | ||
Role: "roles/bigtable.viewer", | ||
Members: []string{"serviceAccount:service_acc1@test.com", "user:user1@test.com"}, | ||
}, | ||
}, | ||
Etag: []byte("etag_v1"), | ||
}, | ||
} | ||
permissions := []string{"bigtable.backups.create", "bigtable.backups.list"} | ||
|
||
adminClient.tClient = &adminClientMock{ | ||
Policy: iamPolicy.InternalProto, | ||
Permissions: permissions, | ||
} | ||
iamHandle := adminClient.BackupIAM(ctx, cluster, backup) | ||
// Get backup policy. | ||
p, err := iamHandle.Policy(ctx) | ||
if err != nil { | ||
t.Errorf("iamHandle.Policy: %v", err) | ||
} | ||
if diff := testutil.Diff(p, iamPolicy); diff != "" { | ||
t.Fatalf("iamHandle.Policy: got - want +\n%s", diff) | ||
} | ||
// Set backup policy. | ||
p.Add("user:user2@test.com", "roles/bigtable.viewer") | ||
if err = iamHandle.SetPolicy(ctx, p); err != nil { | ||
t.Errorf("iamHandle.SetPolicy: %v", err) | ||
} | ||
newPolicy, err := iamHandle.Policy(ctx) | ||
if err != nil { | ||
t.Errorf("iamHandle.Policy: %v", err) | ||
} | ||
if diff := testutil.Diff(newPolicy, p); diff != "" { | ||
t.Fatalf("iamHandle.Policy: got - want +\n%s", diff) | ||
} | ||
// Test backup permissions. | ||
gotPermissions, err := iamHandle.TestPermissions(ctx, permissions) | ||
if err != nil { | ||
t.Errorf("iamHandle.TestPermissions: %v", err) | ||
} | ||
if !testutil.Equal(gotPermissions, permissions) { | ||
t.Fatalf("TestIAMPermissions: got %#v\nwant %#v", gotPermissions, permissions) | ||
} | ||
} | ||
|
||
func TestIntegration_AdminCreateInstance(t *testing.T) { | ||
if instanceToCreate == "" { | ||
t.Skip("instanceToCreate not set, skipping instance creation testing") | ||
|
@@ -2045,9 +2132,10 @@ func TestIntegration_AdminBackup(t *testing.T) { | |
} | ||
|
||
// Create backup | ||
backupName := "mybackup" | ||
defer adminClient.DeleteBackup(ctx, cluster, "mybackup") | ||
|
||
if err = adminClient.CreateBackup(ctx, table, cluster, "mybackup", time.Now().Add(8*time.Hour)); err != nil { | ||
if err = adminClient.CreateBackup(ctx, table, cluster, backupName, time.Now().Add(8*time.Hour)); err != nil { | ||
t.Fatalf("Creating backup: %v", err) | ||
} | ||
|
||
|
@@ -2059,7 +2147,7 @@ func TestIntegration_AdminBackup(t *testing.T) { | |
if got, want := len(backups), 1; got != want { | ||
t.Fatalf("Listing backup count: %d, want: %d", got, want) | ||
} | ||
if got, want := backups[0].Name, "mybackup"; got != want { | ||
if got, want := backups[0].Name, backupName; got != want { | ||
t.Fatalf("Backup name: %s, want: %s", got, want) | ||
} | ||
if got, want := backups[0].SourceTable, table; got != want { | ||
|
@@ -2070,7 +2158,7 @@ func TestIntegration_AdminBackup(t *testing.T) { | |
} | ||
|
||
// Get backup | ||
backup, err := adminClient.BackupInfo(ctx, cluster, "mybackup") | ||
backup, err := adminClient.BackupInfo(ctx, cluster, backupName) | ||
if err != nil { | ||
t.Fatalf("BackupInfo: %v", backup) | ||
} | ||
|
@@ -2080,13 +2168,13 @@ func TestIntegration_AdminBackup(t *testing.T) { | |
|
||
// Update backup | ||
newExpireTime := time.Now().Add(10 * time.Hour) | ||
err = adminClient.UpdateBackup(ctx, cluster, "mybackup", newExpireTime) | ||
err = adminClient.UpdateBackup(ctx, cluster, backupName, newExpireTime) | ||
if err != nil { | ||
t.Fatalf("UpdateBackup failed: %v", err) | ||
} | ||
|
||
// Check that updated backup has the correct expire time | ||
updatedBackup, err := adminClient.BackupInfo(ctx, cluster, "mybackup") | ||
updatedBackup, err := adminClient.BackupInfo(ctx, cluster, backupName) | ||
if err != nil { | ||
t.Fatalf("BackupInfo: %v", err) | ||
} | ||
|
@@ -2099,15 +2187,15 @@ func TestIntegration_AdminBackup(t *testing.T) { | |
// Restore backup | ||
restoredTable := table + "-restored" | ||
defer deleteTable(ctx, t, adminClient, restoredTable) | ||
if err = adminClient.RestoreTable(ctx, restoredTable, cluster, "mybackup"); err != nil { | ||
if err = adminClient.RestoreTable(ctx, restoredTable, cluster, backupName); err != nil { | ||
t.Fatalf("RestoreTable: %v", err) | ||
} | ||
if _, err := adminClient.TableInfo(ctx, restoredTable); err != nil { | ||
t.Fatalf("Restored TableInfo: %v", err) | ||
} | ||
|
||
// Delete backup | ||
if err = adminClient.DeleteBackup(ctx, cluster, "mybackup"); err != nil { | ||
if err = adminClient.DeleteBackup(ctx, cluster, backupName); err != nil { | ||
t.Fatalf("DeleteBackup: %v", err) | ||
} | ||
backups, err = list(cluster) | ||
|
@@ -2212,3 +2300,97 @@ func deleteTable(ctx context.Context, t *testing.T, ac *AdminClient, name string | |
t.Logf("DeleteTable: %v", err) | ||
} | ||
} | ||
|
||
// adminClientMock is used to test backup level IAM. | ||
type adminClientMock struct { | ||
Policy *iampb.Policy | ||
Permissions []string | ||
} | ||
|
||
func (acm *adminClientMock) CreateTable(ctx context.Context, in *btapb.CreateTableRequest, opts ...grpc.CallOption) (*btapb.Table, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) CreateTableFromSnapshot(ctx context.Context, in *btapb.CreateTableFromSnapshotRequest, opts ...grpc.CallOption) (*longrunning.Operation, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) ListTables(ctx context.Context, in *btapb.ListTablesRequest, opts ...grpc.CallOption) (*btapb.ListTablesResponse, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) GetTable(ctx context.Context, in *btapb.GetTableRequest, opts ...grpc.CallOption) (*btapb.Table, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) DeleteTable(ctx context.Context, in *btapb.DeleteTableRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) ModifyColumnFamilies(ctx context.Context, in *btapb.ModifyColumnFamiliesRequest, opts ...grpc.CallOption) (*btapb.Table, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) DropRowRange(ctx context.Context, in *btapb.DropRowRangeRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) GenerateConsistencyToken(ctx context.Context, in *btapb.GenerateConsistencyTokenRequest, opts ...grpc.CallOption) (*btapb.GenerateConsistencyTokenResponse, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) CheckConsistency(ctx context.Context, in *btapb.CheckConsistencyRequest, opts ...grpc.CallOption) (*btapb.CheckConsistencyResponse, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) SnapshotTable(ctx context.Context, in *btapb.SnapshotTableRequest, opts ...grpc.CallOption) (*longrunning.Operation, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) GetSnapshot(ctx context.Context, in *btapb.GetSnapshotRequest, opts ...grpc.CallOption) (*btapb.Snapshot, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) ListSnapshots(ctx context.Context, in *btapb.ListSnapshotsRequest, opts ...grpc.CallOption) (*btapb.ListSnapshotsResponse, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) DeleteSnapshot(ctx context.Context, in *btapb.DeleteSnapshotRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) CreateBackup(ctx context.Context, in *btapb.CreateBackupRequest, opts ...grpc.CallOption) (*longrunning.Operation, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) GetBackup(ctx context.Context, in *btapb.GetBackupRequest, opts ...grpc.CallOption) (*btapb.Backup, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) UpdateBackup(ctx context.Context, in *btapb.UpdateBackupRequest, opts ...grpc.CallOption) (*btapb.Backup, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) DeleteBackup(ctx context.Context, in *btapb.DeleteBackupRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) ListBackups(ctx context.Context, in *btapb.ListBackupsRequest, opts ...grpc.CallOption) (*btapb.ListBackupsResponse, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) RestoreTable(ctx context.Context, in *btapb.RestoreTableRequest, opts ...grpc.CallOption) (*longrunning.Operation, error) { | ||
return nil, nil | ||
} | ||
|
||
func (acm *adminClientMock) GetIamPolicy(ctx context.Context, in *iampb.GetIamPolicyRequest, opts ...grpc.CallOption) (*iampb.Policy, error) { | ||
return acm.Policy, nil | ||
} | ||
|
||
func (acm *adminClientMock) SetIamPolicy(ctx context.Context, in *iampb.SetIamPolicyRequest, opts ...grpc.CallOption) (*iampb.Policy, error) { | ||
return acm.Policy, nil | ||
} | ||
|
||
func (acm *adminClientMock) TestIamPermissions(ctx context.Context, in *iampb.TestIamPermissionsRequest, opts ...grpc.CallOption) (*iampb.TestIamPermissionsResponse, error) { | ||
return &iampb.TestIamPermissionsResponse{Permissions: acm.Permissions}, nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
is this anywhere else in the code? If not, it seems useful as a common utility.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I've met this code in few places. In UpdateBackup, BackupInfo, RestoreTable, DeleteBackup methods.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
cool - I think it would be worth it to refactor this as a utility helper then in this PR
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Sorry for the delay here, I had an emergency.
Done, thanks for advice!
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No worries, lgtm thank you!