enable kubelet server to dynamically load tls certificate files

pkg/features/kube_features.go

@@ -671,6 +671,14 @@ const (
 	// Allow almost all printable ASCII characters in environment variables
 	RelaxedEnvironmentVariableValidation featuregate.Feature = "RelaxedEnvironmentVariableValidation"
 
+	// owner: @zhangweikop
+	// beta: v1.31
+	//
+	// Enable kubelet tls server to update certificate if the specified certificate files are changed.
+	// This feature is useful when specifying tlsCertFile & tlsPrivateKeyFile in kubelet Configuration.
+	// No effect for other cases such as using serverTLSbootstap.
+	ReloadKubeletServerCertificateFile featuregate.Feature = "ReloadKubeletServerCertificateFile"
+
 	// owner: @mikedanese
 	// alpha: v1.7
 	// beta: v1.12
@@ -1159,6 +1167,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	RelaxedEnvironmentVariableValidation: {Default: false, PreRelease: featuregate.Alpha},
 
+	ReloadKubeletServerCertificateFile: {Default: true, PreRelease: featuregate.Beta},
+
 	RotateKubeletServerCertificate: {Default: true, PreRelease: featuregate.Beta},
 
 	RuntimeClassInImageCriAPI: {Default: false, PreRelease: featuregate.Alpha},

pkg/kubelet/certificate/kubelet.go

@@ -17,22 +17,26 @@ limitations under the License.
 package certificate
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"fmt"
 	"math"
 	"net"
 	"sort"
+	"sync/atomic"
 	"time"
 
 	certificates "k8s.io/api/certificates/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/util/certificate"
 	compbasemetrics "k8s.io/component-base/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/klog/v2"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	"k8s.io/kubernetes/pkg/kubelet/metrics"
 	netutils "k8s.io/utils/net"
@@ -234,3 +238,66 @@ func NewKubeletClientCertificateManager(
 
 	return m, nil
 }
+
+// NewKubeletServerCertificateDynamicFileManager creates a certificate manager based on reading and watching certificate and key files.
+// The returned struct implements certificate.Manager interface, enabling using it like other CertificateManager in this package.
+// But the struct doesn't communicate with API server to perform certificate request at all.
+func NewKubeletServerCertificateDynamicFileManager(certFile, keyFile string) (certificate.Manager, error) {
+	c, err := dynamiccertificates.NewDynamicServingContentFromFiles("kubelet-server-cert-files", certFile, keyFile)
+	if err != nil {
+		return nil, fmt.Errorf("no certificate available: %w", err)
+	}
+	m := &kubeletServerCertificateDynamicFileManager{
+		dynamicCertificateContent: c,
+		certFile:                  certFile,
+		keyFile:                   keyFile,
+	}
+	m.Enqueue()
+	c.AddListener(m)
+	return m, nil
+}
+
+// kubeletServerCertificateDynamicFileManager uses a dynamic CertKeyContentProvider based on cert and key files.
+type kubeletServerCertificateDynamicFileManager struct {
+	cancelFn                  context.CancelFunc
+	certFile                  string
+	keyFile                   string
+	dynamicCertificateContent *dynamiccertificates.DynamicCertKeyPairContent
+	currentTLSCertificate     atomic.Pointer[tls.Certificate]
+}
+
+// Enqueue implements the functions to be notified when the serving cert content changes.
+func (m *kubeletServerCertificateDynamicFileManager) Enqueue() {
+	certContent, keyContent := m.dynamicCertificateContent.CurrentCertKeyContent()
+	cert, err := tls.X509KeyPair(certContent, keyContent)
+	if err != nil {
+		klog.ErrorS(err, "invalid certificate and key pair from file", "certFile", m.certFile, "keyFile", m.keyFile)
+	} else {
+		m.currentTLSCertificate.Store(&cert)
+		klog.InfoS("loaded certificate and key pair in kubelet server certificate manager", "certFile", m.certFile, "keyFile", m.keyFile)
+	}
+}
+
+// Current returns the last valid certificate key pair loaded from files.
+func (m *kubeletServerCertificateDynamicFileManager) Current() *tls.Certificate {
+	return m.currentTLSCertificate.Load()
+}
+
+// Start starts watching the certificate and key files
+func (m *kubeletServerCertificateDynamicFileManager) Start() {
+	var ctx context.Context
+	ctx, m.cancelFn = context.WithCancel(context.Background())
+	go m.dynamicCertificateContent.Run(ctx, 1)
+}
+
+// Stop stops watching the certificate and key files
+func (m *kubeletServerCertificateDynamicFileManager) Stop() {
+	if m.cancelFn != nil {
+		m.cancelFn()
+	}
+}
+
+// ServerHealthy always return true since the file manager doesn't communicate with any server
+func (m *kubeletServerCertificateDynamicFileManager) ServerHealthy() bool {
+	return true
+}

pkg/kubelet/certificate/kubelet_test.go

@@ -17,11 +17,16 @@ limitations under the License.
 package certificate
 
 import (
+	"bytes"
 	"net"
+	"os"
+	"path/filepath"
 	"reflect"
 	"testing"
+	"time"
 
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/util/cert"
 	netutils "k8s.io/utils/net"
 )
 
@@ -100,3 +105,123 @@ func TestAddressesToHostnamesAndIPs(t *testing.T) {
 		})
 	}
 }
+
+func atomicWriteFile(name string, data []byte, perm os.FileMode) error {
+	tmp := name + ".tmp"
+	if err := os.WriteFile(tmp, data, perm); err != nil {
+		return err
+	}
+	return os.Rename(tmp, name)
+}
+
+func removeThenCreate(name string, data []byte, perm os.FileMode) error {
+	if err := os.Remove(name); err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+	}
+	return os.WriteFile(name, data, perm)
+}
+
+func createCertAndKeyFiles(certPath, keyPath string, useRename bool) error {
+	cert, key, err := cert.GenerateSelfSignedCertKey("k8s.io", nil, nil)
+	if err != nil {
+		return err
+	}
+
+	createFile := removeThenCreate
+	if useRename {
+		createFile = atomicWriteFile
+	}
+
+	if err := createFile(certPath, cert, os.FileMode(0644)); err != nil {
+		return err
+	}
+
+	if err := createFile(keyPath, key, os.FileMode(0600)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func TestKubeletServerCertificateFromFiles(t *testing.T) {
+	tests := []struct {
+		name           string
+		useAtomicWrite bool
+	}{
+		{
+			name:           "use atomic write",
+			useAtomicWrite: true,
+		},
+		{
+			name:           "remove and create",
+			useAtomicWrite: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			certDir := t.TempDir()
+			rotateCertErrs := make(chan error, 10)
+			defer func() {
+				if len(rotateCertErrs) != 0 {
+					t.Errorf("got errors when rotating certificate files in the test")
+				}
+				close(rotateCertErrs)
+			}()
+
+			certPath := filepath.Join(certDir, "kubelet.cert")
+			keyPath := filepath.Join(certDir, "kubelet.key")
+
+			err := createCertAndKeyFiles(certPath, keyPath, tt.useAtomicWrite)
+			if err != nil {
+				t.Fatalf("Unable to setup cert files: %v", err)
+			}
+
+			// simulate certificate files update in the background
+			go func() {
+				time.Sleep(1 * time.Second)
+				if err := createCertAndKeyFiles(certPath, keyPath, tt.useAtomicWrite); err != nil {
+					rotateCertErrs <- err
+				}
+			}()
+
+			m, err := NewKubeletServerCertificateDynamicFileManager(certPath, keyPath)
+			if err != nil {
+				t.Fatalf("Unable to create certificte provider: %v", err)
+			}
+
+			m.Start()
+			defer m.Stop()
+
+			c := m.Current()
+			if c == nil {
+				t.Fatal("failed to provide valid certificate")
+			}
+			time.Sleep(100 * time.Millisecond)
+			c2 := m.Current()
+			if c2 == nil {
+				t.Fatal("failed to provide valid certificate")
+			}
+			if c2 != c {
+				t.Errorf("expected the same loaded certificate object when there is no cert file change, got different")
+			}
+
+			time.Sleep(2 * time.Second)
+			c3 := m.Current()
+			if c3 == nil {
+				t.Errorf("failed to provide valid certificate after file update")
+			} else if bytes.Equal(c.Certificate[0], c3.Certificate[0]) {
+				t.Errorf("failed to provide the updated certificate")
+			}
+
+			if err = os.Remove(certPath); err != nil {
+				t.Errorf("could not delete file in order to perform test")
+			}
+
+			if m.Current() == nil {
+				t.Errorf("expected the manager still provide cached content when certificate file was not available")
+			}
+		})
+	}
+}

pkg/kubelet/kubelet.go

@@ -774,17 +774,28 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	}
 	klet.imageManager = imageManager
 
-	if kubeCfg.ServerTLSBootstrap && kubeDeps.TLSOptions != nil && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
-		klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, klet.getLastObservedNodeAddresses, certDirectory)
-		if err != nil {
-			return nil, fmt.Errorf("failed to initialize certificate manager: %v", err)
+	if kubeDeps.TLSOptions != nil {
+		if kubeCfg.ServerTLSBootstrap && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
+			klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, klet.getLastObservedNodeAddresses, certDirectory)
+			if err != nil {
+				return nil, fmt.Errorf("failed to initialize certificate manager: %w", err)
+			}
+
+		} else if kubeDeps.TLSOptions.CertFile != "" && kubeDeps.TLSOptions.KeyFile != "" && utilfeature.DefaultFeatureGate.Enabled(features.ReloadKubeletServerCertificateFile) {
+			klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateDynamicFileManager(kubeDeps.TLSOptions.CertFile, kubeDeps.TLSOptions.KeyFile)
+			if err != nil {
+				return nil, fmt.Errorf("failed to initialize file based certificate manager: %w", err)
+			}
 		}
-		kubeDeps.TLSOptions.Config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
-			cert := klet.serverCertificateManager.Current()
-			if cert == nil {
-				return nil, fmt.Errorf("no serving certificate available for the kubelet")
+
+		if klet.serverCertificateManager != nil {
+			kubeDeps.TLSOptions.Config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+				cert := klet.serverCertificateManager.Current()
+				if cert == nil {
+					return nil, fmt.Errorf("no serving certificate available for the kubelet")
+				}
+				return cert, nil
 			}
-			return cert, nil
 		}
 	}