Generate kubelet server cert with cluster CA

cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults.go

@@ -45,6 +45,11 @@ const (
 	// DefaultClusterName defines the default cluster name
 	DefaultClusterName = "kubernetes"
 
+	// DefaultKubeletServerCertFileName defines file name for kubelet server cert
+	DefaultKubeletServerCertFileName = "kubelet-server.crt"
+	// DefaultKubeletServerCertFileName defines file name for kubelet server key
+	DefaultKubeletServerKeyFileName = "kubelet-server.key"
+
 	// DefaultEtcdDataDir defines default location of etcd where static pods will save data to
 	DefaultEtcdDataDir = "/var/lib/etcd"
 	// DefaultProxyBindAddressv4 is the default bind address when the advertise address is v4

cmd/kubeadm/app/cmd/init.go

@@ -176,6 +176,8 @@ func NewCmdInit(out io.Writer, initOptions *initOptions) *cobra.Command {
 	initRunner.AppendPhase(phases.NewControlPlanePhase())
 	initRunner.AppendPhase(phases.NewEtcdPhase())
 	initRunner.AppendPhase(phases.NewWaitControlPlanePhase())
+	initRunner.AppendPhase(phases.NewKubeletServerCertAndKey())
+	initRunner.AppendPhase(phases.NewKubeletStartPhase())
 	initRunner.AppendPhase(phases.NewUploadConfigPhase())
 	initRunner.AppendPhase(phases.NewUploadCertsPhase())
 	initRunner.AppendPhase(phases.NewMarkControlPlanePhase())

cmd/kubeadm/app/cmd/join.go

@@ -202,6 +202,7 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command {
 	joinRunner.AppendPhase(phases.NewCheckEtcdPhase())
 	joinRunner.AppendPhase(phases.NewKubeletStartPhase())
 	joinRunner.AppendPhase(phases.NewControlPlaneJoinPhase())
+	joinRunner.AppendPhase(phases.NewKubeletServerCertAndKey())
 
 	// sets the data builder function, that will be used by the runner
 	// both when running the entire workflow or single phases

cmd/kubeadm/app/cmd/phases/init/bootstraptoken.go

@@ -90,6 +90,11 @@ func runBootstrapToken(c workflow.RunData) error {
 	if err := nodebootstraptokenphase.AllowBootstrapTokensToPostCSRs(client); err != nil {
 		return errors.Wrap(err, "error allowing bootstrap tokens to post CSRs")
 	}
+
+	if err := nodebootstraptokenphase.AllowBootstrapTokensApproveCertificates(client); err != nil {
+		return errors.Wrap(err, "error allowing bootstrap tokens to post CSRs")
+	}
+
 	// Create RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
 	if err := nodebootstraptokenphase.AutoApproveNodeBootstrapTokens(client); err != nil {
 		return errors.Wrap(err, "error auto-approving node bootstrap tokens")

cmd/kubeadm/app/cmd/phases/init/kubelet_server.go

@@ -0,0 +1,76 @@
+package phases
+
+import (
+	"fmt"
+	"github.com/pkg/errors"
+
+	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
+
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeletserver"
+	"k8s.io/kubernetes/pkg/util/normalizer"
+)
+
+var (
+	kubeletServerCertExample = normalizer.Examples(`
+		# Generates key and certificate signing request and approves it with kube-apiserver
+		kubeadm init phase kubelet-server
+		`)
+)
+
+// NewKubeletStartPhase creates a kubeadm workflow phase that start kubelet on a node.
+func NewKubeletServerCertAndKey() workflow.Phase {
+	return workflow.Phase{
+		Name:    "kubelet-server",
+		Short:   "Generates, approves and writes kubelet server cert and key to file",
+		Long:    "Generates, approves and writes kubelet server cert and key by submitting certificate signing request to kube-apiserver.",
+		Example: kubeletServerCertExample,
+		Run:     kubeletServerCerts,
+		InheritFlags: []string{
+			options.CfgPath,
+			options.NodeName,
+			options.KubeconfigDir,
+			options.KubeconfigPath,
+			options.DryRun,
+		},
+	}
+}
+
+// runKubeletStart executes kubelet start logic.
+func kubeletServerCerts(c workflow.RunData) error {
+	fmt.Println("[kubelet-server] Generate kubelet server certificates")
+	data, ok := c.(InitData)
+	if !ok {
+		return errors.New("kubelet-start phase invoked with an invalid data struct")
+	}
+
+	data.Cfg().ComponentConfigs.Kubelet.TLSCertFile = kubeadmapiv1beta1.DefaultKubeletServerCertFileName
+	data.Cfg().ComponentConfigs.Kubelet.TLSPrivateKeyFile = kubeadmapiv1beta1.DefaultKubeletServerKeyFileName
+
+	kubeClient, err := data.Client()
+
+	if err != nil {
+		return errors.Wrap(err, "error getting kubernetes client")
+	}
+
+	certDataPem, keyDataPem, err := kubeletserver.KubeletServerCert(kubeClient, data.Cfg().LocalAPIEndpoint, data.Cfg().NodeRegistration)
+
+	if err != nil {
+		return errors.Wrap(err, "init phase kubelet server cert")
+	}
+
+	err = kubeletserver.WriteKeyToFile(data.Cfg().CertificatesDir, data.Cfg().ComponentConfigs.Kubelet.TLSPrivateKeyFile, keyDataPem)
+
+	if err != nil {
+		return errors.Wrap(err, "init phase kubelet server cert")
+	}
+
+	err = kubeletserver.WriteCertToFile(data.Cfg().CertificatesDir, data.Cfg().ComponentConfigs.Kubelet.TLSCertFile, certDataPem)
+
+	if err != nil {
+		return errors.Wrap(err, "init phase kubelet server cert")
+	}
+
+	return nil
+}

cmd/kubeadm/app/cmd/phases/init/uploadconfig.go

@@ -120,6 +120,11 @@ func runUploadKubeletConfig(c workflow.RunData) error {
 		return err
 	}
 
+	// Erase these keys form component config, kubelets on workers must start without
+	// these keys because they will appear only after approving and downloading csr
+	cfg.ComponentConfigs.Kubelet.TLSCertFile = ""
+	cfg.ComponentConfigs.Kubelet.TLSPrivateKeyFile = ""
+
 	klog.V(1).Infoln("[upload-config] Uploading the kubelet component config to a ConfigMap")
 	if err = kubeletphase.CreateConfigMap(cfg.ClusterConfiguration.ComponentConfigs.Kubelet, cfg.KubernetesVersion, client); err != nil {
 		return errors.Wrap(err, "error creating kubelet configuration ConfigMap")

cmd/kubeadm/app/cmd/phases/join/kubelet_server.go

@@ -0,0 +1,99 @@
+package phases
+
+import (
+	"fmt"
+	"github.com/pkg/errors"
+
+	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeletserver"
+	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
+	"k8s.io/kubernetes/pkg/util/normalizer"
+)
+
+var (
+	kubeletServerCertExample = normalizer.Examples(`
+		# Generates key and certificate signing request and approves it with kube-apiserver
+		kubeadm init phase kubelet-server
+		`)
+)
+
+// NewKubeletStartPhase creates a kubeadm workflow phase that start kubelet on a node.
+func NewKubeletServerCertAndKey() workflow.Phase {
+	return workflow.Phase{
+		Name:    "kubelet-server",
+		Short:   "Generates, approves and writes kubelet server cert and key to file",
+		Long:    "Generates, approves and writes kubelet server cert and key by submitting certificate signing request to kube-apiserver.",
+		Example: kubeletServerCertExample,
+		Run:     kubeletServerCerts,
+		InheritFlags: []string{
+			options.CfgPath,
+			options.NodeName,
+			options.KubeconfigDir,
+			options.KubeconfigPath,
+			options.DryRun,
+		},
+	}
+}
+
+// kubeletServerCerts generates kubelet server certs, update kubelet addtional flags file
+// and restart kubelet with new params
+func kubeletServerCerts(c workflow.RunData) error {
+	fmt.Println("[kubelet-server] Generate kubelet server certificates")
+	data, ok := c.(JoinData)
+	if !ok {
+		return errors.New("kubelet-server phase invoked with an invalid data struct")
+	}
+
+	initCfg, err := data.InitCfg()
+
+	initCfg.ComponentConfigs.Kubelet.TLSCertFile = kubeadmapiv1beta1.DefaultKubeletServerCertFileName
+	initCfg.ComponentConfigs.Kubelet.TLSPrivateKeyFile = kubeadmapiv1beta1.DefaultKubeletServerKeyFileName
+
+	if !ok {
+		return errors.New("kubelet-server join phase get InitCfg")
+	}
+
+	kubeClient, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetBootstrapKubeletKubeConfigPath())
+	if err != nil {
+		return errors.Errorf("couldn't create client from kubeconfig file %q", kubeadmconstants.GetBootstrapKubeletKubeConfigPath())
+	}
+
+	certDataPem, keyDataPem, err := kubeletserver.KubeletServerCert(kubeClient, initCfg.LocalAPIEndpoint, data.Cfg().NodeRegistration)
+
+	if err != nil {
+		return errors.Wrap(err, "join phase kubelet server cert")
+	}
+
+	err = kubeletserver.WriteKeyToFile(initCfg.CertificatesDir, initCfg.ComponentConfigs.Kubelet.TLSPrivateKeyFile, keyDataPem)
+
+	if err != nil {
+		return errors.Wrap(err, "join phase kubelet server cert")
+	}
+
+	err = kubeletserver.WriteCertToFile(initCfg.CertificatesDir, initCfg.ComponentConfigs.Kubelet.TLSCertFile, certDataPem)
+
+	if err != nil {
+		return errors.Wrap(err, "join phase kubelet server cert")
+	}
+
+	// Configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet
+	// Try to stop the kubelet service so no race conditions occur when configuring it
+	fmt.Println("[kubelet-server] Stopping the kubelet")
+	kubeletphase.TryStopKubelet()
+
+	// New config will have kubelet server key and cert
+	registerTaintsUsingFlags := data.Cfg().ControlPlane == nil
+	if err := kubeletphase.WriteKubeletDynamicEnvFile(&initCfg.ClusterConfiguration, &initCfg.NodeRegistration, registerTaintsUsingFlags, kubeadmconstants.KubeletRunDirectory); err != nil {
+		return err
+	}
+
+	// Try to start the kubelet service in case it's inactive
+	fmt.Println("[kubelet-server] Starting the kubelet")
+	kubeletphase.TryStartKubelet()
+
+	return nil
+}

cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go

@@ -33,6 +33,10 @@ const (
 	// NodeKubeletBootstrap defines the name of the ClusterRoleBinding that lets kubelets post CSRs
 	NodeKubeletBootstrap = "kubeadm:kubelet-bootstrap"
 
+	// NodeKubeletBootstrap defines the name of the ClusterRoleBinding that lets kubelets approve CSRs
+	NodeApproveCertificate     = "kubeadm:node-certificate-approve"
+	CertificateRControllerRole = "system:controller:certificate-controller"
+
 	// CSRAutoApprovalClusterRoleName defines the name of the auto-bootstrapped ClusterRole for making the csrapprover controller auto-approve the CSR
 	// TODO: This value should be defined in an other, generic authz package instead of here
 	// Starting from v1.8, CSRAutoApprovalClusterRoleName is automatically created by the API server on startup
@@ -67,6 +71,29 @@ func AllowBootstrapTokensToPostCSRs(client clientset.Interface) error {
 	})
 }
 
+// AllowBootstrapTokensToPostCSRs creates RBAC rules in a way the makes Node Bootstrap Tokens able to post CSRs
+func AllowBootstrapTokensApproveCertificates(client clientset.Interface) error {
+	fmt.Println("[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to approve certificate signing requests")
+
+	return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: NodeApproveCertificate,
+		},
+		RoleRef: rbac.RoleRef{
+			APIGroup: rbac.GroupName,
+			Kind:     "ClusterRole",
+			Name:     CertificateRControllerRole,
+		},
+		Subjects: []rbac.Subject{
+			{
+				Kind: rbac.GroupKind,
+				Name: constants.NodeBootstrapTokenAuthGroup,
+			},
+		},
+	})
+}
+
+
 // AutoApproveNodeBootstrapTokens creates RBAC rules in a way that makes Node Bootstrap Tokens' CSR auto-approved by the csrapprover controller
 func AutoApproveNodeBootstrapTokens(client clientset.Interface) error {
 	fmt.Println("[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token")

cmd/kubeadm/app/phases/kubelet/flags.go

@@ -38,6 +38,9 @@ type kubeletFlagsOpts struct {
 	nodeRegOpts              *kubeadmapi.NodeRegistrationOptions
 	featureGates             map[string]bool
 	pauseImage               string
+	tlsPrivateKey            string
+	certDir                  string
+	tlsCert                  string
 	registerTaintsUsingFlags bool
 	execer                   utilsexec.Interface
 	pidOfFunc                func(string) ([]int, error)
@@ -60,6 +63,9 @@ func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.ClusterConfiguration, nodeReg *k
 		execer:                   utilsexec.New(),
 		pidOfFunc:                procfs.PidOf,
 		defaultHostname:          hostName,
+		tlsPrivateKey:            cfg.ComponentConfigs.Kubelet.TLSPrivateKeyFile,
+		tlsCert:                  cfg.ComponentConfigs.Kubelet.TLSCertFile,
+		certDir:                  cfg.CertificatesDir,
 	}
 	stringMap := buildKubeletArgMap(flagOpts)
 	argList := kubeadmutil.BuildArgumentListFromMap(stringMap, nodeReg.KubeletExtraArgs)
@@ -110,6 +116,10 @@ func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string {
 		kubeletFlags["hostname-override"] = opts.nodeRegOpts.Name
 	}
 
+	if opts.tlsPrivateKey != "" && opts.tlsCert != "" {
+		kubeletFlags["tls-private-key-file"] = filepath.Join(opts.certDir, opts.tlsPrivateKey)
+		kubeletFlags["tls-cert-file"] = filepath.Join(opts.certDir, opts.tlsCert)
+	}
 	// TODO: Conditionally set `--cgroup-driver` to either `systemd` or `cgroupfs` for CRI other than Docker
 
 	return kubeletFlags