Skip to content

Commit

Permalink
Fix ManagedSeed bootstrapping for Kubernetes 1.24+ clusters (#7315)
Browse files Browse the repository at this point in the history 
* Bump Kubernetes version in `Shoot` example

* Use `TokenRequest` API instead of expecting static secret

* Make parameter names more concrete

Also, the option to not submit a namespace is not used anywhere, so it's dropped completely now.
  • Loading branch information
@rfranzke
rfranzke committed Jan 13, 2023
1 parent 6f824f6 commit b6d3850
Show file tree
Hide file tree
Showing 7 changed files with 110 additions and 131 deletions.
2 changes: 1 addition & 1 deletion example/provider-local/managedseeds/shoot-managedseed.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ spec:
maxSurge: 1
maxUnavailable: 0
kubernetes:
version: 1.24.0
version: 1.24.8
kubelet:
serializeImagePulls: false
registryPullQPS: 10
Expand Down
2 changes: 1 addition & 1 deletion pkg/admissioncontroller/webhook/admission/seedrestriction/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ func (h *Handler) admitClusterRoleBinding(ctx context.Context, seedName string,
return admission.Errored(http.StatusForbidden, fmt.Errorf("can only bindings referring to the bootstrapper role"))
}

managedSeedNamespace, managedSeedName := gardenletbootstraputil.MetadataFromClusterRoleBindingName(request.Name)
managedSeedNamespace, managedSeedName := gardenletbootstraputil.ManagedSeedInfoFromClusterRoleBindingName(request.Name)
return h.allowIfManagedSeedIsNotYetBootstrapped(ctx, seedName, managedSeedNamespace, managedSeedName)
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/admissioncontroller/webhook/auth/seed/authorizer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func (a *authorizer) authorizeClusterRoleBinding(log logr.Logger, seedName strin
if attrs.GetVerb() == "delete" &&
strings.HasPrefix(attrs.GetName(), gardenletbootstraputil.ClusterRoleBindingNamePrefix) {

managedSeedNamespace, managedSeedName := gardenletbootstraputil.MetadataFromClusterRoleBindingName(attrs.GetName())
managedSeedNamespace, managedSeedName := gardenletbootstraputil.ManagedSeedInfoFromClusterRoleBindingName(attrs.GetName())
if managedSeedNamespace == v1beta1constants.GardenNamespace && managedSeedName == seedName {
return auth.DecisionAllow, "", nil
}
Expand Down
8 changes: 4 additions & 4 deletions pkg/gardenlet/bootstrap/bootstrap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,21 +104,21 @@ func DeleteBootstrapAuth(ctx context.Context, reader client.Reader, writer clien
)

case strings.HasPrefix(csr.Spec.Username, serviceaccount.ServiceAccountUsernamePrefix):
namespace, name, err := serviceaccount.SplitUsername(csr.Spec.Username)
serviceAccountNamespace, serviceAccountName, err := serviceaccount.SplitUsername(csr.Spec.Username)
if err != nil {
return err
}

resourcesToDelete = append(resourcesToDelete,
&corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Name: serviceAccountName,
Namespace: serviceAccountNamespace,
},
},
&rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: gardenletbootstraputil.ClusterRoleBindingName(v1beta1constants.GardenNamespace, seedName),
Name: gardenletbootstraputil.ClusterRoleBindingName(serviceAccountNamespace, serviceAccountName),
},
},
)
Expand Down
73 changes: 36 additions & 37 deletions pkg/gardenlet/bootstrap/util/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,25 +26,27 @@ import (
"time"

"github.com/go-logr/logr"

v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
"github.com/gardener/gardener/pkg/client/kubernetes"
"github.com/gardener/gardener/pkg/controllerutils"
"github.com/gardener/gardener/pkg/gardenlet/apis/config"
"github.com/gardener/gardener/pkg/utils"
kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
"github.com/gardener/gardener/pkg/utils/kubernetes/bootstraptoken"

authenticationv1 "k8s.io/api/authentication/v1"
certificatesv1 "k8s.io/api/certificates/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
corev1clientset "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
bootstraptokenapi "k8s.io/cluster-bootstrap/token/api"
bootstraptokenutil "k8s.io/cluster-bootstrap/token/util"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"

v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
"github.com/gardener/gardener/pkg/client/kubernetes"
"github.com/gardener/gardener/pkg/controllerutils"
"github.com/gardener/gardener/pkg/gardenlet/apis/config"
"github.com/gardener/gardener/pkg/utils"
kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
"github.com/gardener/gardener/pkg/utils/kubernetes/bootstraptoken"
)

// GetSeedName returns the seed name from the SeedConfig or the default Seed name
Expand Down Expand Up @@ -255,31 +257,33 @@ func ComputeGardenletKubeconfigWithBootstrapToken(ctx context.Context, gardenCli
// ComputeGardenletKubeconfigWithServiceAccountToken creates a kubeconfig containing the token of a service account
// Creates the required service account in the Garden cluster and puts the associated token into a Kubeconfig
// tailored to the Gardenlet
func ComputeGardenletKubeconfigWithServiceAccountToken(ctx context.Context, gardenClient client.Client, gardenClientRestConfig *rest.Config, serviceAccountName, serviceAccountNamespace string) ([]byte, error) {
func ComputeGardenletKubeconfigWithServiceAccountToken(ctx context.Context, gardenClient client.Client, coreV1Client corev1clientset.CoreV1Interface, gardenClientRestConfig *rest.Config, serviceAccountName, serviceAccountNamespace string) ([]byte, error) {
// Create a temporary service account
sa := &corev1.ServiceAccount{
serviceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
Namespace: serviceAccountNamespace,
},
}
if _, err := controllerutils.CreateOrGetAndStrategicMergePatch(ctx, gardenClient, sa, func() error { return nil }); err != nil {
if _, err := controllerutils.CreateOrGetAndStrategicMergePatch(ctx, gardenClient, serviceAccount, func() error { return nil }); err != nil {
return nil, err
}

// Get the service account secret
if len(sa.Secrets) == 0 {
return nil, fmt.Errorf("service account token controller has not yet created a secret for the service account")
// Get a token for this service account
tokenRequest := &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
ExpirationSeconds: pointer.Int64(600),
},
}
saSecret := &corev1.Secret{}
if err := gardenClient.Get(ctx, kubernetesutils.Key(sa.Namespace, sa.Secrets[0].Name), saSecret); err != nil {
return nil, err
result, err := coreV1Client.ServiceAccounts(serviceAccount.Namespace).CreateToken(ctx, serviceAccount.Name, tokenRequest, metav1.CreateOptions{})
if err != nil {
return nil, fmt.Errorf("failed creating a token for ServiceAccount %q: %w", client.ObjectKeyFromObject(serviceAccount), err)
}

// Create a ClusterRoleBinding
clusterRoleBinding := &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: ClusterRoleBindingName(sa.Namespace, sa.Name),
Name: ClusterRoleBindingName(serviceAccount.Namespace, serviceAccount.Name),
},
}
if _, err := controllerutils.CreateOrGetAndStrategicMergePatch(ctx, gardenClient, clusterRoleBinding, func() error {
Expand All @@ -291,17 +295,17 @@ func ComputeGardenletKubeconfigWithServiceAccountToken(ctx context.Context, gard
clusterRoleBinding.Subjects = []rbacv1.Subject{
{
Kind: rbacv1.ServiceAccountKind,
Name: sa.Name,
Namespace: sa.Namespace,
Name: serviceAccount.Name,
Namespace: serviceAccount.Namespace,
},
}
return nil
}); err != nil {
return nil, err
return nil, fmt.Errorf("failed creating a ClusterRoleBinding for ServiceAccount %q: %w", client.ObjectKeyFromObject(serviceAccount), err)
}

// Get bootstrap kubeconfig from service account secret
return CreateGardenletKubeconfigWithToken(gardenClientRestConfig, string(saSecret.Data[corev1.ServiceAccountTokenKey]))
return CreateGardenletKubeconfigWithToken(gardenClientRestConfig, result.Status.Token)
}

// TokenID returns the token id based on the given metadata.
Expand All @@ -315,28 +319,25 @@ func TokenID(meta metav1.ObjectMeta) string {
}

// ClusterRoleBindingName concatenates the gardener seed bootstrapper group with the given name, separated by a colon.
func ClusterRoleBindingName(namespace, name string) string {
suffix := name
if namespace != "" {
suffix = namespace + clusterRoleBindingNameDelimiter + name
}
return ClusterRoleBindingNamePrefix + suffix
func ClusterRoleBindingName(managedSeedNamespace, serviceAccountName string) string {
return ClusterRoleBindingNamePrefix + managedSeedNamespace + clusterRoleBindingNameDelimiter + serviceAccountName
}

// MetadataFromClusterRoleBindingName returns the namespace and name for a given cluster role binding name.
func MetadataFromClusterRoleBindingName(clusterRoleBindingName string) (namespace, name string) {
// ManagedSeedInfoFromClusterRoleBindingName returns the namespace and name of the related ManagedSeed for a given
// cluster role binding name.
func ManagedSeedInfoFromClusterRoleBindingName(clusterRoleBindingName string) (managedSeedNamespace, managedSeedName string) {
var (
metadata = strings.TrimPrefix(clusterRoleBindingName, ClusterRoleBindingNamePrefix)
split = strings.Split(metadata, clusterRoleBindingNameDelimiter)
)

managedSeedName = split[0]
if len(split) > 1 {
namespace = split[0]
name = split[1]
return
managedSeedNamespace = split[0]
managedSeedName = split[1]
}

name = split[0]
managedSeedName = strings.TrimPrefix(managedSeedName, ServiceAccountNamePrefix)
return
}

Expand All @@ -346,8 +347,6 @@ func ServiceAccountName(name string) string {
}

const (
// KindSeed is a constant for the "seed" kind.
KindSeed = "seed"
// KindManagedSeed is a constant for the "managed seed" kind.
KindManagedSeed = "managed seed"
// ServiceAccountNamePrefix is the prefix used for service account names.
Expand Down
124 changes: 49 additions & 75 deletions pkg/gardenlet/bootstrap/util/util_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,24 @@ import (
"github.com/golang/mock/gomock"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
authenticationv1 "k8s.io/api/authentication/v1"
certificatesv1 "k8s.io/api/certificates/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
kubernetesscheme "k8s.io/client-go/kubernetes/scheme"
corev1fake "k8s.io/client-go/kubernetes/typed/core/v1/fake"
"k8s.io/client-go/rest"
"k8s.io/client-go/testing"
"k8s.io/client-go/util/keyutil"
bootstraptokenapi "k8s.io/cluster-bootstrap/token/api"
bootstraptokenutil "k8s.io/cluster-bootstrap/token/util"
"sigs.k8s.io/controller-runtime/pkg/client"
fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"

gardencore "github.com/gardener/gardener/pkg/apis/core"
"github.com/gardener/gardener/pkg/client/kubernetes"
Expand Down Expand Up @@ -404,78 +410,57 @@ var _ = Describe("Util", func() {
})

Describe("#ComputeGardenletKubeconfigWithServiceAccountToken", func() {
var (
restConfig = &rest.Config{
Host: "apiserver.dummy",
}
serviceAccountName = "gardenlet"
serviceAccountNamespace = "garden"
serviceAccountSecretName = "service-account-secret"
)

It("should fail because the service account token controller has not yet created a secret for the service account", func() {
c.EXPECT().Create(ctx, gomock.AssignableToTypeOf(&corev1.ServiceAccount{})).DoAndReturn(func(_ context.Context, s *corev1.ServiceAccount, _ ...client.CreateOption) error {
s.Name = serviceAccountName
s.Namespace = "garden"
s.Secrets = []corev1.ObjectReference{}
return nil
})

_, err := ComputeGardenletKubeconfigWithServiceAccountToken(ctx, c, restConfig, serviceAccountName, serviceAccountNamespace)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("service account token controller has not yet created a secret for the service account"))
})

It("should succeed", func() {
// create service account
c.EXPECT().Create(ctx, gomock.AssignableToTypeOf(&corev1.ServiceAccount{})).DoAndReturn(func(_ context.Context, s *corev1.ServiceAccount, _ ...client.CreateOption) error {
Expect(s.Name).To(Equal(serviceAccountName))
Expect(s.Namespace).To(Equal("garden"))
s.Secrets = []corev1.ObjectReference{
{
Name: serviceAccountSecretName,
var (
restConfig = &rest.Config{
Host: "apiserver.dummy",
}
serviceAccount = &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "gardenlet",
Namespace: "garden",
},
}
return nil
})
fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetesscheme.Scheme).Build()
coreV1Client = &corev1fake.FakeCoreV1{Fake: &testing.Fake{}}
)

// mock existing service account secret
c.EXPECT().Get(ctx, kubernetesutils.Key("garden", serviceAccountSecretName), gomock.AssignableToTypeOf(&corev1.Secret{})).DoAndReturn(func(_ context.Context, _ client.ObjectKey, s *corev1.Secret, _ ...client.GetOption) error {
s.Data = map[string][]byte{
"token": []byte("tokenizer"),
coreV1Client.AddReactor("create", "serviceaccounts", func(action testing.Action) (bool, runtime.Object, error) {
if action.GetSubresource() != "token" {
return false, nil, fmt.Errorf("subresource should be 'token'")
}
return nil
})

// create cluster role binding
clusterRoleBinding := &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("gardener.cloud:system:seed-bootstrapper:%s:%s", serviceAccountNamespace, serviceAccountName),
},
}
c.EXPECT().Create(ctx, gomock.AssignableToTypeOf(&rbacv1.ClusterRoleBinding{})).DoAndReturn(func(_ context.Context, s *rbacv1.ClusterRoleBinding, _ ...client.CreateOption) error {
expectedClusterRoleBinding := clusterRoleBinding
expectedClusterRoleBinding.RoleRef = rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "gardener.cloud:system:seed-bootstrapper",
}
expectedClusterRoleBinding.Subjects = []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: serviceAccountName,
Namespace: serviceAccountNamespace,
},
cAction, ok := action.(testing.CreateAction)
if !ok {
return false, nil, fmt.Errorf("could not convert action (type %T) to type testing.CreateAction", cAction)
}

Expect(s).To(Equal(expectedClusterRoleBinding))
return nil
return true, &authenticationv1.TokenRequest{
Status: authenticationv1.TokenRequestStatus{
Token: "some-token",
},
}, nil
})

kubeconfig, err := ComputeGardenletKubeconfigWithServiceAccountToken(ctx, c, restConfig, serviceAccountName, serviceAccountNamespace)
kubeconfig, err := ComputeGardenletKubeconfigWithServiceAccountToken(ctx, fakeClient, coreV1Client, restConfig, serviceAccount.Name, serviceAccount.Namespace)
Expect(err).ToNot(HaveOccurred())
Expect(kubeconfig).ToNot(BeNil())

Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(serviceAccount), &corev1.ServiceAccount{})).To(Succeed())

clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
Expect(fakeClient.Get(ctx, client.ObjectKey{Name: fmt.Sprintf("gardener.cloud:system:seed-bootstrapper:%s:%s", serviceAccount.Namespace, serviceAccount.Name)}, clusterRoleBinding)).To(Succeed())
Expect(clusterRoleBinding.RoleRef).To(Equal(rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "gardener.cloud:system:seed-bootstrapper",
}))
Expect(clusterRoleBinding.Subjects).To(ConsistOf(rbacv1.Subject{
Kind: "ServiceAccount",
Name: serviceAccount.Name,
Namespace: serviceAccount.Namespace,
}))

rest, err := kubernetes.RESTConfigFromKubeconfig(kubeconfig)
Expect(err).ToNot(HaveOccurred())
Expect(rest.Host).To(Equal(restConfig.Host))
Expand All @@ -501,32 +486,21 @@ var _ = Describe("Util", func() {
namespace = "bar"
name = "baz"

clusterRoleNameWithoutNamespace = "gardener.cloud:system:seed-bootstrapper:" + name
clusterRoleNameWithNamespace = "gardener.cloud:system:seed-bootstrapper:" + namespace + ":" + name
clusterRoleNameWithNamespace = "gardener.cloud:system:seed-bootstrapper:" + namespace + ":" + name

descriptionWithoutNamespace = fmt.Sprintf("A bootstrap token for the Gardenlet for %s %s.", kind, name)
descriptionWithNamespace = fmt.Sprintf("A bootstrap token for the Gardenlet for %s %s/%s.", kind, namespace, name)
)

Describe("#ClusterRoleBindingName", func() {
It("should return the correct name (w/o namespace)", func() {
Expect(ClusterRoleBindingName("", name)).To(Equal(fmt.Sprintf("gardener.cloud:system:seed-bootstrapper:%s", name)))
})

It("should return the correct name (w/ namespace)", func() {
It("should return the correct name", func() {
Expect(ClusterRoleBindingName(namespace, name)).To(Equal(fmt.Sprintf("gardener.cloud:system:seed-bootstrapper:%s:%s", namespace, name)))
})
})

Describe("#MetadataFromClusterRoleBindingName", func() {
It("should return the expected namespace/name from a cluster role binding name (w/o namespace)", func() {
resultNamespace, resultName := MetadataFromClusterRoleBindingName(clusterRoleNameWithoutNamespace)
Expect(resultNamespace).To(BeEmpty())
Expect(resultName).To(Equal(name))
})

It("should return the expected namespace/name from a cluster role binding name (w/ namespace)", func() {
resultNamespace, resultName := MetadataFromClusterRoleBindingName(clusterRoleNameWithNamespace)
Describe("#ManagedSeedInfoFromClusterRoleBindingName", func() {
It("should return the expected namespace/name from a cluster role binding name", func() {
resultNamespace, resultName := ManagedSeedInfoFromClusterRoleBindingName(clusterRoleNameWithNamespace)
Expect(resultNamespace).To(Equal(namespace))
Expect(resultName).To(Equal(name))
})
Expand Down

0 comments on commit b6d3850

Please sign in to comment.