Skip to content

Commit

Permalink
Support configurable tags and enableMultipleStandardLoadBalancers
Browse files Browse the repository at this point in the history
  • Loading branch information
@nilo19
nilo19 committed Nov 18, 2020
1 parent 71de031 commit 1c3a098
Show file tree
Hide file tree
Showing 7 changed files with 257 additions and 222 deletions.
4 changes: 3 additions & 1 deletion parts/k8s/cloud-init/artifacts/cse_config.sh
View file
Expand Up @@ -208,7 +208,9 @@ configureK8s() {
"providerVaultName": "${KMS_PROVIDER_VAULT_NAME}",
"maximumLoadBalancerRuleCount": ${MAXIMUM_LOADBALANCER_RULE_COUNT},
"providerKeyName": "k8s",
"providerKeyVersion": ""
"providerKeyVersion": "",
"enableMultipleStandardLoadBalancers": ${ENABLE_MULTIPLE_STANDARD_LOAD_BALANCERS},
"tags": "${TAGS}"
}
EOF
set -x
Expand Down
2 changes: 2 additions & 0 deletions pkg/api/convertertoapi.go
View file
Expand Up @@ -313,6 +313,8 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
api.CloudProviderDisableOutboundSNAT = vlabs.CloudProviderDisableOutboundSNAT
api.KubeReservedCgroup = vlabs.KubeReservedCgroup
api.MicrosoftAptRepositoryURL = vlabs.MicrosoftAptRepositoryURL
api.EnableMultipleStandardLoadBalancers = vlabs.EnableMultipleStandardLoadBalancers
api.Tags = vlabs.Tags
convertComponentsToAPI(vlabs, api)
convertAddonsToAPI(vlabs, api)
convertKubeletConfigToAPI(vlabs, api)
Expand Down
286 changes: 145 additions & 141 deletions pkg/api/types.go
View file

Large diffs are not rendered by default.

160 changes: 81 additions & 79 deletions pkg/api/vlabs/types.go
View file
Expand Up @@ -323,85 +323,87 @@ const (
// KubernetesConfig contains the Kubernetes config structure, containing
// Kubernetes specific configuration
type KubernetesConfig struct {
KubernetesImageBase string `json:"kubernetesImageBase,omitempty"`
KubernetesImageBaseType string `json:"kubernetesImageBaseType,omitempty"`
MCRKubernetesImageBase string `json:"mcrKubernetesImageBase,omitempty"`
ClusterSubnet string `json:"clusterSubnet,omitempty"`
DNSServiceIP string `json:"dnsServiceIP,omitempty"`
ServiceCidr string `json:"serviceCidr,omitempty"`
NetworkPolicy string `json:"networkPolicy,omitempty"`
NetworkPlugin string `json:"networkPlugin,omitempty"`
NetworkMode string `json:"networkMode,omitempty"`
ContainerRuntime string `json:"containerRuntime,omitempty"`
MaxPods int `json:"maxPods,omitempty"`
DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"`
UseManagedIdentity *bool `json:"useManagedIdentity,omitempty"`
UserAssignedID string `json:"userAssignedID,omitempty"`
UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json.
CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"`
CustomKubeAPIServerImage string `json:"customKubeAPIServerImage,omitempty"`
CustomKubeControllerManagerImage string `json:"customKubeControllerManagerImage,omitempty"`
CustomKubeProxyImage string `json:"customKubeProxyImage,omitempty"`
CustomKubeSchedulerImage string `json:"customKubeSchedulerImage,omitempty"`
CustomKubeBinaryURL string `json:"customKubeBinaryURL,omitempty"`
DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated
MobyVersion string `json:"mobyVersion,omitempty"`
ContainerdVersion string `json:"containerdVersion,omitempty"`
LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"`
CustomCcmImage string `json:"customCcmImage,omitempty"`
UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"`
CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"`
WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"`
WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"`
WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"`
UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"`
EnableRbac *bool `json:"enableRbac,omitempty"`
EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"`
EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"`
PrivateCluster *PrivateCluster `json:"privateCluster,omitempty"`
GCHighThreshold int `json:"gchighthreshold,omitempty"`
GCLowThreshold int `json:"gclowthreshold,omitempty"`
EtcdVersion string `json:"etcdVersion,omitempty"`
EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"`
EtcdStorageLimitGB int `json:"etcdStorageLimitGB,omitempty"`
EtcdEncryptionKey string `json:"etcdEncryptionKey,omitempty"`
EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"`
EnableEncryptionWithExternalKms *bool `json:"enableEncryptionWithExternalKms,omitempty"`
EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"`
Addons []KubernetesAddon `json:"addons,omitempty"`
Components []KubernetesComponent `json:"components,omitempty"`
ContainerRuntimeConfig map[string]string `json:"containerRuntimeConfig,omitempty"`
KubeletConfig map[string]string `json:"kubeletConfig,omitempty"`
ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"`
CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"`
APIServerConfig map[string]string `json:"apiServerConfig,omitempty"`
SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"`
PodSecurityPolicyConfig map[string]string `json:"podSecurityPolicyConfig,omitempty"` // Deprecated
KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"`
CloudProviderBackoffMode string `json:"cloudProviderBackoffMode"`
CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"`
CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"`
CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"`
CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"`
CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"`
CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"`
CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"`
CloudProviderRateLimitQPSWrite float64 `json:"cloudProviderRateLimitQPSWrite,omitempty"`
CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"`
CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty"`
CloudProviderDisableOutboundSNAT *bool `json:"cloudProviderDisableOutboundSNAT,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"`
LoadBalancerOutboundIPs *int `json:"loadBalancerOutboundIPs,omitempty"`
AzureCNIVersion string `json:"azureCNIVersion,omitempty"`
AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"`
AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"`
KeyVaultSku string `json:"keyVaultSku,omitempty"`
MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty"`
ProxyMode KubeProxyMode `json:"kubeProxyMode,omitempty"`
PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"`
OutboundRuleIdleTimeoutInMinutes int32 `json:"outboundRuleIdleTimeoutInMinutes,omitempty"`
MicrosoftAptRepositoryURL string `json:"microsoftAptRepositoryURL,omitempty"`
KubernetesImageBase string `json:"kubernetesImageBase,omitempty"`
KubernetesImageBaseType string `json:"kubernetesImageBaseType,omitempty"`
MCRKubernetesImageBase string `json:"mcrKubernetesImageBase,omitempty"`
ClusterSubnet string `json:"clusterSubnet,omitempty"`
DNSServiceIP string `json:"dnsServiceIP,omitempty"`
ServiceCidr string `json:"serviceCidr,omitempty"`
NetworkPolicy string `json:"networkPolicy,omitempty"`
NetworkPlugin string `json:"networkPlugin,omitempty"`
NetworkMode string `json:"networkMode,omitempty"`
ContainerRuntime string `json:"containerRuntime,omitempty"`
MaxPods int `json:"maxPods,omitempty"`
DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"`
UseManagedIdentity *bool `json:"useManagedIdentity,omitempty"`
UserAssignedID string `json:"userAssignedID,omitempty"`
UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json.
CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"`
CustomKubeAPIServerImage string `json:"customKubeAPIServerImage,omitempty"`
CustomKubeControllerManagerImage string `json:"customKubeControllerManagerImage,omitempty"`
CustomKubeProxyImage string `json:"customKubeProxyImage,omitempty"`
CustomKubeSchedulerImage string `json:"customKubeSchedulerImage,omitempty"`
CustomKubeBinaryURL string `json:"customKubeBinaryURL,omitempty"`
DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated
MobyVersion string `json:"mobyVersion,omitempty"`
ContainerdVersion string `json:"containerdVersion,omitempty"`
LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"`
CustomCcmImage string `json:"customCcmImage,omitempty"`
UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"`
CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"`
WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"`
WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"`
WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"`
UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"`
EnableRbac *bool `json:"enableRbac,omitempty"`
EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"`
EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"`
PrivateCluster *PrivateCluster `json:"privateCluster,omitempty"`
GCHighThreshold int `json:"gchighthreshold,omitempty"`
GCLowThreshold int `json:"gclowthreshold,omitempty"`
EtcdVersion string `json:"etcdVersion,omitempty"`
EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"`
EtcdStorageLimitGB int `json:"etcdStorageLimitGB,omitempty"`
EtcdEncryptionKey string `json:"etcdEncryptionKey,omitempty"`
EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"`
EnableEncryptionWithExternalKms *bool `json:"enableEncryptionWithExternalKms,omitempty"`
EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"`
Addons []KubernetesAddon `json:"addons,omitempty"`
Components []KubernetesComponent `json:"components,omitempty"`
ContainerRuntimeConfig map[string]string `json:"containerRuntimeConfig,omitempty"`
KubeletConfig map[string]string `json:"kubeletConfig,omitempty"`
ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"`
CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"`
APIServerConfig map[string]string `json:"apiServerConfig,omitempty"`
SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"`
PodSecurityPolicyConfig map[string]string `json:"podSecurityPolicyConfig,omitempty"` // Deprecated
KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"`
CloudProviderBackoffMode string `json:"cloudProviderBackoffMode"`
CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"`
CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"`
CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"`
CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"`
CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"`
CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"`
CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"`
CloudProviderRateLimitQPSWrite float64 `json:"cloudProviderRateLimitQPSWrite,omitempty"`
CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"`
CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty"`
CloudProviderDisableOutboundSNAT *bool `json:"cloudProviderDisableOutboundSNAT,omitempty"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty"`
ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"`
LoadBalancerOutboundIPs *int `json:"loadBalancerOutboundIPs,omitempty"`
AzureCNIVersion string `json:"azureCNIVersion,omitempty"`
AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"`
AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"`
KeyVaultSku string `json:"keyVaultSku,omitempty"`
MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty"`
ProxyMode KubeProxyMode `json:"kubeProxyMode,omitempty"`
PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"`
OutboundRuleIdleTimeoutInMinutes int32 `json:"outboundRuleIdleTimeoutInMinutes,omitempty"`
MicrosoftAptRepositoryURL string `json:"microsoftAptRepositoryURL,omitempty"`
EnableMultipleStandardLoadBalancers *bool `json:"enableMultipleStandardLoadBalancers,omitempty"`
Tags string `json:"tags,omitempty"`
}

// CustomFile has source as the full absolute source path to a file and dest
Expand Down
7 changes: 7 additions & 0 deletions pkg/api/vlabs/validate.go
View file
Expand Up @@ -1507,6 +1507,13 @@ func (k *KubernetesConfig) Validate(k8sVersion string, hasWindows, ipv6DualStack
if e := k.validateKubernetesImageBaseType(); e != nil {
return e
}

if to.Bool(k.EnableMultipleStandardLoadBalancers) && !common.IsKubernetesVersionGe(k8sVersion, "1.20.0-beta.1") {
return errors.Errorf("OrchestratorProfile.KubernetesConfig.EnableMultipleStandardLoadBalancers is available since kubernetes version v1.20.0-beta.1, current version is %s", k8sVersion)
}
if k.Tags != "" && !common.IsKubernetesVersionGe(k8sVersion, "1.20.0-beta.1") {
return errors.Errorf("OrchestratorProfile.KubernetesConfig.Tags is available since kubernetes version v1.20.0-beta.1, current version is %s", k8sVersion)
}
return k.validateContainerRuntimeConfig()
}

Expand Down
16 changes: 16 additions & 0 deletions pkg/api/vlabs/validate_test.go
View file
Expand Up @@ -706,6 +706,22 @@ func Test_KubernetesConfig_Validate(t *testing.T) {
t.Error("should error when dual stack and single stack IPv6 enabled simultaneously")
}
}

// Tests that apply to 1.20 and later releases
for _, k8sVersion := range common.GetVersionsLt(common.GetAllSupportedKubernetesVersions(false, false, false), "1.20.0", false, false) {
c := KubernetesConfig{
EnableMultipleStandardLoadBalancers: to.BoolPtr(true),
}
if err := c.Validate(k8sVersion, false, false, false); err == nil {
t.Errorf("should error when enable multiple standard load balancer before v1.20.0")
}
c = KubernetesConfig{
Tags: "a=b",
}
if err := c.Validate(k8sVersion, false, false, false); err == nil {
t.Errorf("should error when setting tags before v1.20.0")
}
}
}

func Test_Properties_ValidateCustomKubeComponent(t *testing.T) {
Expand Down
4 changes: 3 additions & 1 deletion pkg/engine/templates_generated.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 1c3a098

Please sign in to comment.