Skip to content

Commit

Permalink
Deprecate the podEvictionTimeout field in favor of newly introduced…
Browse files Browse the repository at this point in the history 
… kube-apiserver fields

The kube-controller-manager flag `--pod-eviction-timeout` is deprecated in favor of the kube-apiserver flags `--default-not-ready-toleration-seconds` and `--default-unreachable-toleration-seconds`. The `--pod-eviction-timeout` flag does not have effect when the taint besed eviction is enabled. The taint  based eviction is beta (enabled by default) since Kubernetes 1.13 and GA since Kubernetes 1.18. For more details, see kubernetes/kubernetes#74651.

This commit allows configuring the kube-apiserver flags `--default-not-ready-toleration-seconds` and `--default-unreachable-toleration-seconds`. The `podEvictionTimeout` field is deprecated in favor of the newly introduced fields. gardener-apiserver no longer defaults the `podEvictionTimeout` field. gardener-apiserver also returns a warning when the `podEvictionTimeout` field is set.
  • Loading branch information
@ialidzhikov
ialidzhikov committed Jan 13, 2023
1 parent cf7175d commit 8c9bebf
Show file tree
Hide file tree
Showing 28 changed files with 1,966 additions and 1,517 deletions.
36 changes: 36 additions & 0 deletions docs/api-reference/core.md
View file
Expand Up @@ -4450,6 +4450,36 @@ KubeAPIServerLogging
<p>Logging contains configuration for the log level and HTTP access logs.</p>
</td>
</tr>
<tr>
<td>
<code>defaultNotReadyTolerationSeconds</code></br>
<em>
int64
</em>
</td>
<td>
<em>(Optional)</em>
<p>DefaultNotReadyTolerationSeconds indicates the tolerationSeconds of the toleration for notReady:NoExecute
that is added by default to every pod that does not already have such a toleration (flag <code>--default-not-ready-toleration-seconds</code>).
The field has effect only when the <code>DefaultTolerationSeconds</code> admission plugin is enabled.
Defaults to 300.</p>
</td>
</tr>
<tr>
<td>
<code>defaultUnreachableTolerationSeconds</code></br>
<em>
int64
</em>
</td>
<td>
<em>(Optional)</em>
<p>DefaultUnreachableTolerationSeconds indicates the tolerationSeconds of the toleration for unreachable:NoExecute
that is added by default to every pod that does not already have such a toleration (flag <code>--default-unreachable-toleration-seconds</code>).
The field has effect only when the <code>DefaultTolerationSeconds</code> admission plugin is enabled.
Defaults to 300.</p>
</td>
</tr>
</tbody>
</table>
<h3 id="core.gardener.cloud/v1beta1.KubeAPIServerLogging">KubeAPIServerLogging
Expand Down Expand Up @@ -4611,6 +4641,12 @@ Kubernetes meta/v1.Duration
<td>
<em>(Optional)</em>
<p>PodEvictionTimeout defines the grace period for deleting pods on failed nodes. Defaults to 2m.</p>
<p>Deprecated: The corresponding kube-controller-manager flag <code>--pod-eviction-timeout</code> is deprecated
in favor of the kube-apiserver flags <code>--default-not-ready-toleration-seconds</code> and <code>--default-unreachable-toleration-seconds</code>.
The <code>--pod-eviction-timeout</code> flag does not have effect when the taint besed eviction is enabled. The taint
based eviction is beta (enabled by default) since Kubernetes 1.13 and GA since Kubernetes 1.18. Hence,
instead of setting this field, set the <code>spec.kubernetes.kubeAPIServer.defaultNotReadyTolerationSeconds</code> and
<code>spec.kubernetes.kubeAPIServer.defaultUnreachableTolerationSeconds</code>.</p>
</td>
</tr>
<tr>
Expand Down
4 changes: 3 additions & 1 deletion example/90-shoot.yaml
View file
Expand Up @@ -197,7 +197,9 @@ spec:
# maxTokenExpiration: 45d
# logging:
# verbosity: 2
# httpAccessLogs: 3
# httpAccessLogs: 3
# defaultNotReadyTolerationSeconds: 300
# defaultUnreachableTolerationSeconds: 300
# kubeControllerManager:
# nodeCIDRMaskSize: 24
# podEvictionTimeout: 2m0s
Expand Down
4 changes: 4 additions & 0 deletions pkg/api/core/shoot/warnings.go
View file
Expand Up @@ -52,6 +52,10 @@ func GetWarnings(_ context.Context, shoot, oldShoot *core.Shoot, credentialsRota
}
}

if kubeControllerManager := shoot.Spec.Kubernetes.KubeControllerManager; kubeControllerManager != nil && kubeControllerManager.PodEvictionTimeout != nil {
warnings = append(warnings, "you are setting the spec.kubernetes.kubeControllerManager.podEvictionTimeout field. The field does not have effect since Kubernetes 1.13. Instead, use the spec.kubernetes.kubeAPIServer.(defaultNotReadyTolerationSeconds/defaultUnreachableTolerationSeconds) fields.")
}

return warnings
}

Expand Down
6 changes: 6 additions & 0 deletions pkg/api/core/shoot/warnings_test.go
View file
Expand Up @@ -318,5 +318,11 @@ var _ = Describe("Warnings", func() {
})
})

It("should return a warning when podEvictionTimeout is set", func() {
shoot.Spec.Kubernetes.KubeControllerManager = &core.KubeControllerManagerConfig{
PodEvictionTimeout: &metav1.Duration{Duration: 2 * time.Minute},
}
Expect(GetWarnings(ctx, shoot, nil, credentialsRotationInterval)).To(ContainElement(Equal("you are setting the spec.kubernetes.kubeControllerManager.podEvictionTimeout field. The field does not have effect since Kubernetes 1.13. Instead, use the spec.kubernetes.kubeAPIServer.(defaultNotReadyTolerationSeconds/defaultUnreachableTolerationSeconds) fields.")))
})
})
})
15 changes: 15 additions & 0 deletions pkg/apis/core/types_shoot.go
View file
Expand Up @@ -569,6 +569,14 @@ type KubeAPIServerConfig struct {
EventTTL *metav1.Duration
// Logging contains configuration settings for the log verbosity and access logging
Logging *KubeAPIServerLogging
// DefaultNotReadyTolerationSeconds indicates the tolerationSeconds of the toleration for notReady:NoExecute
// that is added by default to every pod that does not already have such a toleration (flag `--default-not-ready-toleration-seconds`).
// The field has effect only when the `DefaultTolerationSeconds` admission plugin is enabled.
DefaultNotReadyTolerationSeconds *int64
// DefaultUnreachableTolerationSeconds indicates the tolerationSeconds of the toleration for unreachable:NoExecute
// that is added by default to every pod that does not already have such a toleration (flag `--default-unreachable-toleration-seconds`).
// The field has effect only when the `DefaultTolerationSeconds` admission plugin is enabled.
DefaultUnreachableTolerationSeconds *int64
}

// KubeAPIServerLogging contains configuration for the logs level and http access logs
Expand Down Expand Up @@ -701,6 +709,13 @@ type KubeControllerManagerConfig struct {
// NodeCIDRMaskSize defines the mask size for node cidr in cluster (default is 24). This field is immutable.
NodeCIDRMaskSize *int32
// PodEvictionTimeout defines the grace period for deleting pods on failed nodes.
//
// Deprecated: The corresponding kube-controller-manager flag `--pod-eviction-timeout` is deprecated
// in favor of the kube-apiserver flags `--default-not-ready-toleration-seconds` and `--default-unreachable-toleration-seconds`.
// The `--pod-eviction-timeout` flag does not have effect when the taint besed eviction is enabled. The taint
// based eviction is beta (enabled by default) since Kubernetes 1.13 and GA since Kubernetes 1.18. Hence,
// instead of setting this field, set the `spec.kubernetes.kubeAPIServer.defaultNotReadyTolerationSeconds` and
// `spec.kubernetes.kubeAPIServer.defaultUnreachableTolerationSeconds`.
PodEvictionTimeout *metav1.Duration
// NodeMonitorGracePeriod defines the grace period before an unresponsive node is marked unhealthy.
NodeMonitorGracePeriod *metav1.Duration
Expand Down
30 changes: 15 additions & 15 deletions pkg/apis/core/v1alpha1/defaults.go
View file
Expand Up @@ -175,19 +175,31 @@ func SetDefaults_Shoot(obj *Shoot) {
if obj.Spec.Kubernetes.KubeAPIServer.Requests.MaxMutatingInflight == nil {
obj.Spec.Kubernetes.KubeAPIServer.Requests.MaxMutatingInflight = pointer.Int32(200)
}
if obj.Spec.Kubernetes.KubeAPIServer.EnableAnonymousAuthentication == nil {
obj.Spec.Kubernetes.KubeAPIServer.EnableAnonymousAuthentication = pointer.Bool(false)
}
if obj.Spec.Kubernetes.KubeAPIServer.EventTTL == nil {
obj.Spec.Kubernetes.KubeAPIServer.EventTTL = &metav1.Duration{Duration: time.Hour}
}
if obj.Spec.Kubernetes.KubeAPIServer.Logging == nil {
obj.Spec.Kubernetes.KubeAPIServer.Logging = &KubeAPIServerLogging{}
}
if obj.Spec.Kubernetes.KubeAPIServer.Logging.Verbosity == nil {
obj.Spec.Kubernetes.KubeAPIServer.Logging.Verbosity = pointer.Int32(2)
}
if obj.Spec.Kubernetes.KubeAPIServer.DefaultNotReadyTolerationSeconds == nil {
obj.Spec.Kubernetes.KubeAPIServer.DefaultNotReadyTolerationSeconds = pointer.Int64(300)
}
if obj.Spec.Kubernetes.KubeAPIServer.DefaultUnreachableTolerationSeconds == nil {
obj.Spec.Kubernetes.KubeAPIServer.DefaultUnreachableTolerationSeconds = pointer.Int64(300)
}

if obj.Spec.Kubernetes.KubeControllerManager == nil {
obj.Spec.Kubernetes.KubeControllerManager = &KubeControllerManagerConfig{}
}
if obj.Spec.Kubernetes.KubeControllerManager.NodeCIDRMaskSize == nil {
obj.Spec.Kubernetes.KubeControllerManager.NodeCIDRMaskSize = calculateDefaultNodeCIDRMaskSize(obj.Spec.Kubernetes.Kubelet, obj.Spec.Provider.Workers)
}
if obj.Spec.Kubernetes.KubeControllerManager.PodEvictionTimeout == nil {
obj.Spec.Kubernetes.KubeControllerManager.PodEvictionTimeout = &metav1.Duration{Duration: 2 * time.Minute}
}
if obj.Spec.Kubernetes.KubeControllerManager.NodeMonitorGracePeriod == nil {
obj.Spec.Kubernetes.KubeControllerManager.NodeMonitorGracePeriod = &metav1.Duration{Duration: 2 * time.Minute}
}
Expand Down Expand Up @@ -284,18 +296,6 @@ func SetDefaults_Shoot(obj *Shoot) {
obj.Spec.Maintenance = &Maintenance{}
}

if obj.Spec.Kubernetes.KubeAPIServer.EnableAnonymousAuthentication == nil {
obj.Spec.Kubernetes.KubeAPIServer.EnableAnonymousAuthentication = pointer.Bool(false)
}

if obj.Spec.Kubernetes.KubeAPIServer.Logging == nil {
obj.Spec.Kubernetes.KubeAPIServer.Logging = &KubeAPIServerLogging{}
}

if obj.Spec.Kubernetes.KubeAPIServer.Logging.Verbosity == nil {
obj.Spec.Kubernetes.KubeAPIServer.Logging.Verbosity = pointer.Int32(2)
}

for i, worker := range obj.Spec.Provider.Workers {
kubernetesVersion := obj.Spec.Kubernetes.Version
if worker.Kubernetes != nil && worker.Kubernetes.Version != nil {
Expand Down
67 changes: 38 additions & 29 deletions pkg/apis/core/v1alpha1/defaults_test.go
View file
Expand Up @@ -487,23 +487,6 @@ var _ = Describe("Defaults", func() {
Expect(obj.Spec.Kubernetes.Kubelet.SerializeImagePulls).To(PointTo(BeFalse()))
})

It("should not default the kube-controller-manager's pod eviction timeout field", func() {
podEvictionTimeout := &metav1.Duration{Duration: time.Minute}
obj.Spec.Kubernetes.KubeControllerManager = &KubeControllerManagerConfig{PodEvictionTimeout: podEvictionTimeout}

SetDefaults_Shoot(obj)

Expect(obj.Spec.Kubernetes.KubeControllerManager.PodEvictionTimeout).To(Equal(podEvictionTimeout))
})

It("should default the kube-controller-manager's pod eviction timeout field", func() {
obj.Spec.Kubernetes.KubeControllerManager = &KubeControllerManagerConfig{}

SetDefaults_Shoot(obj)

Expect(obj.Spec.Kubernetes.KubeControllerManager.PodEvictionTimeout).To(Equal(&metav1.Duration{Duration: 2 * time.Minute}))
})

It("should not default the kube-controller-manager's node monitor grace period", func() {
nodeMonitorGracePeriod := &metav1.Duration{Duration: time.Minute}
obj.Spec.Kubernetes.KubeControllerManager = &KubeControllerManagerConfig{NodeMonitorGracePeriod: nodeMonitorGracePeriod}
Expand Down Expand Up @@ -613,6 +596,18 @@ var _ = Describe("Defaults", func() {
Expect(obj.Spec.Kubernetes.KubeAPIServer.Requests.MaxMutatingInflight).To(Equal(&maxMutatingRequestsInflight))
})

It("should disable anonymous authentication by default", func() {
SetDefaults_Shoot(obj)
Expect(obj.Spec.Kubernetes.KubeAPIServer.EnableAnonymousAuthentication).To(PointTo(BeFalse()))
})

It("should not default the anonymous authentication field if it is explicitly set", func() {
trueVar := true
obj.Spec.Kubernetes.KubeAPIServer = &KubeAPIServerConfig{EnableAnonymousAuthentication: &trueVar}
SetDefaults_Shoot(obj)
Expect(obj.Spec.Kubernetes.KubeAPIServer.EnableAnonymousAuthentication).To(PointTo(BeTrue()))
})

It("should default the event ttl field", func() {
SetDefaults_Shoot(obj)
Expect(obj.Spec.Kubernetes.KubeAPIServer.EventTTL).To(Equal(&metav1.Duration{Duration: time.Hour}))
Expand All @@ -626,18 +621,6 @@ var _ = Describe("Defaults", func() {
Expect(obj.Spec.Kubernetes.KubeAPIServer.EventTTL).To(Equal(eventTTL))
})

It("should disable anonymous authentication by default", func() {
SetDefaults_Shoot(obj)
Expect(obj.Spec.Kubernetes.KubeAPIServer.EnableAnonymousAuthentication).To(PointTo(BeFalse()))
})

It("should not default the anonymous authentication field if it is explicitly set", func() {
trueVar := true
obj.Spec.Kubernetes.KubeAPIServer = &KubeAPIServerConfig{EnableAnonymousAuthentication: &trueVar}
SetDefaults_Shoot(obj)
Expect(obj.Spec.Kubernetes.KubeAPIServer.EnableAnonymousAuthentication).To(PointTo(BeTrue()))
})

It("should default the log verbosity level", func() {
SetDefaults_Shoot(obj)
Expect(obj.Spec.Kubernetes.KubeAPIServer.Logging.Verbosity).To(PointTo(Equal(int32(2))))
Expand All @@ -654,6 +637,32 @@ var _ = Describe("Defaults", func() {
Expect(obj.Spec.Kubernetes.KubeAPIServer.Logging.HTTPAccessVerbosity).To(BeNil())
})

It("should default the defaultNotReadyTolerationSeconds field", func() {
SetDefaults_Shoot(obj)
Expect(obj.Spec.Kubernetes.KubeAPIServer.DefaultNotReadyTolerationSeconds).To(PointTo(Equal(int64(300))))
})

It("should not default the defaultNotReadyTolerationSeconds field", func() {
var tolerationSeconds int64 = 120
obj.Spec.Kubernetes.KubeAPIServer = &KubeAPIServerConfig{DefaultNotReadyTolerationSeconds: pointer.Int64(tolerationSeconds)}

SetDefaults_Shoot(obj)
Expect(obj.Spec.Kubernetes.KubeAPIServer.DefaultNotReadyTolerationSeconds).To(PointTo(Equal(tolerationSeconds)))
})

It("should default the defaultUnreachableTolerationSeconds field", func() {
SetDefaults_Shoot(obj)
Expect(obj.Spec.Kubernetes.KubeAPIServer.DefaultUnreachableTolerationSeconds).To(PointTo(Equal(int64(300))))
})

It("should not default the defaultUnreachableTolerationSeconds field", func() {
var tolerationSeconds int64 = 120
obj.Spec.Kubernetes.KubeAPIServer = &KubeAPIServerConfig{DefaultUnreachableTolerationSeconds: pointer.Int64(tolerationSeconds)}

SetDefaults_Shoot(obj)
Expect(obj.Spec.Kubernetes.KubeAPIServer.DefaultUnreachableTolerationSeconds).To(PointTo(Equal(tolerationSeconds)))
})

It("should default architecture of worker's machine to amd64", func() {
obj.Spec.Provider.Workers = []Worker{
{Name: "Default Worker"},
Expand Down

0 comments on commit 8c9bebf

Please sign in to comment.