From aba42bdd7fda5115926e67482e191622159eabad Mon Sep 17 00:00:00 2001 From: Angus Lees Date: Tue, 30 Mar 2021 18:31:44 +1100 Subject: [PATCH] feat(eks): Pass bootstrap.sh args to avoid DescribeCluster call Pass sufficient information to bootstrap.sh in ASG user-data, to prevent it from making a DescribeCluster AWS API call. This per-node API call can become an issue in large clusters, and we already have all the information to avoid it. --- packages/@aws-cdk/aws-eks/README.md | 4 +- packages/@aws-cdk/aws-eks/lib/cluster.ts | 2 +- packages/@aws-cdk/aws-eks/lib/user-data.ts | 20 ++++- .../test/integ.eks-cluster.expected.json | 82 ++++++++++++++++--- .../@aws-cdk/aws-eks/test/test.cluster.ts | 6 +- .../@aws-cdk/aws-eks/test/test.user-data.ts | 70 +++++++++------- 6 files changed, 132 insertions(+), 52 deletions(-) diff --git a/packages/@aws-cdk/aws-eks/README.md b/packages/@aws-cdk/aws-eks/README.md index 59909c9ea9a0b..a8b6fd1ccc5b3 100644 --- a/packages/@aws-cdk/aws-eks/README.md +++ b/packages/@aws-cdk/aws-eks/README.md @@ -202,7 +202,7 @@ cluster.addNodegroupCapacity('custom-node-group', { #### Spot Instances Support Use `capacityType` to create managed node groups comprised of spot instances. To maximize the availability of your applications while using -Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property. +Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property. > For more details visit [Managed node group capacity types](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types). @@ -395,7 +395,7 @@ const asg = new ec2.AutoScalingGroup(...) cluster.connectAutoScalingGroupCapacity(asg); ``` -This will add the necessary user-data and configure all connections, roles, and tags needed for the instances in the auto-scaling group to properly join the cluster. +This will add the necessary user-data to access the apiserver and configure all connections, roles, and tags needed for the instances in the auto-scaling group to properly join the cluster. #### Spot Instances diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index c578b90fa388b..8451d9e2efba3 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -1264,7 +1264,7 @@ export class Cluster extends ClusterBase { if (bootstrapEnabled) { const userData = options.machineImageType === MachineImageType.BOTTLEROCKET ? renderBottlerocketUserData(this) : - renderAmazonLinuxUserData(this.clusterName, autoScalingGroup, options.bootstrapOptions); + renderAmazonLinuxUserData(this, autoScalingGroup, options.bootstrapOptions); autoScalingGroup.addUserData(...userData); } diff --git a/packages/@aws-cdk/aws-eks/lib/user-data.ts b/packages/@aws-cdk/aws-eks/lib/user-data.ts index 3b8d997535771..a3614659bc0df 100644 --- a/packages/@aws-cdk/aws-eks/lib/user-data.ts +++ b/packages/@aws-cdk/aws-eks/lib/user-data.ts @@ -3,7 +3,7 @@ import { Stack } from '@aws-cdk/core'; import { BootstrapOptions, ICluster } from './cluster'; // eslint-disable-next-line max-len -export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup: autoscaling.AutoScalingGroup, options: BootstrapOptions = {}): string[] { +export function renderAmazonLinuxUserData(cluster: ICluster, autoScalingGroup: autoscaling.AutoScalingGroup, options: BootstrapOptions = {}): string[] { const stack = Stack.of(autoScalingGroup); @@ -13,6 +13,22 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup: const extraArgs = new Array(); + try { + extraArgs.push(`--apiserver-endpoint '${cluster.clusterEndpoint}'`); + } catch (e) { + if (!e.message.includes('is not defined for this imported cluster')) { + throw e; + } + } + + try { + extraArgs.push(`--b64-cluster-ca '${cluster.clusterCertificateAuthorityData}'`); + } catch (e) { + if (!e.message.includes('is not defined for this imported cluster')) { + throw e; + } + } + extraArgs.push(`--use-max-pods ${options.useMaxPods ?? true}`); if (options.awsApiRetryAttempts) { @@ -41,7 +57,7 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup: return [ 'set -o xtrace', - `/etc/eks/bootstrap.sh ${clusterName} --kubelet-extra-args "${kubeletExtraArgs}" ${commandLineSuffix}`.trim(), + `/etc/eks/bootstrap.sh ${cluster.clusterName} --kubelet-extra-args "${kubeletExtraArgs}" ${commandLineSuffix}`.trim(), `/opt/aws/bin/cfn-signal --exit-code $? --stack ${stack.stackName} --resource ${asgLogicalId} --region ${stack.region}`, ]; } diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json index 1ea62568197d4..de8e6fe0bf8ae 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json @@ -1682,7 +1682,21 @@ { "Ref": "Cluster9EE0221C" }, - " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesASGF172BD19 --region test-region" + " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "Endpoint" + ] + }, + "' --b64-cluster-ca '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "CertificateAuthorityData" + ] + }, + "' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesASGF172BD19 --region test-region" ] ] } @@ -1993,7 +2007,21 @@ { "Ref": "Cluster9EE0221C" }, - " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesArmASG40A593D0 --region test-region" + " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "Endpoint" + ] + }, + "' --b64-cluster-ca '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "CertificateAuthorityData" + ] + }, + "' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesArmASG40A593D0 --region test-region" ] ] } @@ -2630,7 +2658,21 @@ { "Ref": "Cluster9EE0221C" }, - " --kubelet-extra-args \"--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels foo=bar,goo=far\" --use-max-pods true --aws-api-retry-attempts 5\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterspotASG857494B6 --region test-region" + " --kubelet-extra-args \"--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels foo=bar,goo=far\" --apiserver-endpoint '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "Endpoint" + ] + }, + "' --b64-cluster-ca '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "CertificateAuthorityData" + ] + }, + "' --use-max-pods true --aws-api-retry-attempts 5\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterspotASG857494B6 --region test-region" ] ] } @@ -2973,7 +3015,21 @@ { "Ref": "Cluster9EE0221C" }, - " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterInferenceInstancesASGE90717C7 --region test-region" + " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "Endpoint" + ] + }, + "' --b64-cluster-ca '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "CertificateAuthorityData" + ] + }, + "' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterInferenceInstancesASGE90717C7 --region test-region" ] ] } @@ -4146,7 +4202,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3Bucket055DC235" + "Ref": "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3BucketB7E1A9C0" }, "S3Key": { "Fn::Join": [ @@ -4159,7 +4215,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3VersionKey2FFFA299" + "Ref": "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3VersionKey542FDEBD" } ] } @@ -4172,7 +4228,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3VersionKey2FFFA299" + "Ref": "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3VersionKey542FDEBD" } ] } @@ -4674,17 +4730,17 @@ "Type": "String", "Description": "Artifact hash for asset \"b7d38dc0eeb2c5d024919020e09d2590b68559eab4a5264c3b1aa7a429d1edd4\"" }, - "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3Bucket055DC235": { + "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3BucketB7E1A9C0": { "Type": "String", - "Description": "S3 bucket for asset \"952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344\"" + "Description": "S3 bucket for asset \"6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3\"" }, - "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3VersionKey2FFFA299": { + "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3VersionKey542FDEBD": { "Type": "String", - "Description": "S3 key for asset version \"952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344\"" + "Description": "S3 key for asset version \"6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3\"" }, - "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344ArtifactHash1AB042BC": { + "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3ArtifactHash5E61FCA5": { "Type": "String", - "Description": "Artifact hash for asset \"952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344\"" + "Description": "Artifact hash for asset \"6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3\"" }, "AssetParameters5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636dS3BucketA6642550": { "Type": "String", diff --git a/packages/@aws-cdk/aws-eks/test/test.cluster.ts b/packages/@aws-cdk/aws-eks/test/test.cluster.ts index 46554d34dceb0..3c18396ff9bb2 100644 --- a/packages/@aws-cdk/aws-eks/test/test.cluster.ts +++ b/packages/@aws-cdk/aws-eks/test/test.cluster.ts @@ -1294,7 +1294,7 @@ export = { // THEN const template = app.synth().getStackByName(stack.stackName).template; const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData; - test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); + test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); test.done(); }, @@ -1333,7 +1333,7 @@ export = { // THEN const template = app.synth().getStackByName(stack.stackName).template; const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData; - test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand --node-labels FOO=42" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); + test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand --node-labels FOO=42" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); test.done(); }, @@ -1353,7 +1353,7 @@ export = { // THEN const template = app.synth().getStackByName(stack.stackName).template; const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData; - test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); + test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); test.done(); }, diff --git a/packages/@aws-cdk/aws-eks/test/test.user-data.ts b/packages/@aws-cdk/aws-eks/test/test.user-data.ts index 044d1ff55f7ca..5accc1f839d69 100644 --- a/packages/@aws-cdk/aws-eks/test/test.user-data.ts +++ b/packages/@aws-cdk/aws-eks/test/test.user-data.ts @@ -2,6 +2,7 @@ import * as autoscaling from '@aws-cdk/aws-autoscaling'; import * as ec2 from '@aws-cdk/aws-ec2'; import { App, Stack } from '@aws-cdk/core'; import { Test } from 'nodeunit'; +import { Cluster } from '../lib/cluster'; import { renderAmazonLinuxUserData } from '../lib/user-data'; /* eslint-disable max-len */ @@ -9,15 +10,15 @@ import { renderAmazonLinuxUserData } from '../lib/user-data'; export = { 'default user data'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg)); + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg)); // THEN test.deepEqual(userData, [ 'set -o xtrace', - '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true', + '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true', '/opt/aws/bin/cfn-signal --exit-code $? --stack my-stack --resource ASG46ED3070 --region us-west-33', ]); @@ -26,127 +27,128 @@ export = { '--use-max-pods=true'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { useMaxPods: true, })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true'); test.done(); }, '--use-max-pods=false'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { useMaxPods: false, })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods false'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods false'); test.done(); }, '--aws-api-retry-attempts'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { awsApiRetryAttempts: 123, })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true --aws-api-retry-attempts 123'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true --aws-api-retry-attempts 123'); test.done(); }, '--docker-config-json'(test: Test) { // GIVEN - const { asg } = newFixtures(); + const { asg, cluster } = newFixtures(); // WHEN - const userData = renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = renderAmazonLinuxUserData(cluster, asg, { dockerConfigJson: '{"docker":123}', }); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true --docker-config-json \'{"docker":123}\''); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true --docker-config-json \'{"docker":123}\''); test.done(); }, '--enable-docker-bridge=true'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { enableDockerBridge: true, })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true --enable-docker-bridge true'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true --enable-docker-bridge true'); test.done(); }, '--enable-docker-bridge=false'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { enableDockerBridge: false, })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true'); test.done(); }, '--kubelet-extra-args'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { kubeletExtraArgs: '--extra-args-for --kubelet', })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand --extra-args-for --kubelet" --use-max-pods true'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand --extra-args-for --kubelet" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true'); test.done(); }, 'arbitrary additional bootstrap arguments can be passed through "additionalArgs"'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { additionalArgs: '--apiserver-endpoint 1111 --foo-bar', })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true --apiserver-endpoint 1111 --foo-bar'); + // NB: duplicated --apiserver-endpoint is fine. Last wins. + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true --apiserver-endpoint 1111 --foo-bar'); test.done(); }, 'if asg has spot instances, the correct label and taint is used'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(true); + const { asg, stack, cluster } = newFixtures(true); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { kubeletExtraArgs: '--node-labels X=y', })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels X=y" --use-max-pods true'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels X=y" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true'); test.done(); }, }; @@ -155,6 +157,12 @@ function newFixtures(spot = false) { const app = new App(); const stack = new Stack(app, 'my-stack', { env: { region: 'us-west-33' } }); const vpc = new ec2.Vpc(stack, 'vpc'); + const cluster = Cluster.fromClusterAttributes(stack, 'cluster', { + clusterName: 'my-cluster-name', + clusterCertificateAuthorityData: 'CADATA', + clusterEndpoint: 'https://mycluster:443', + vpc, + }); const asg = new autoscaling.AutoScalingGroup(stack, 'ASG', { instanceType: new ec2.InstanceType('m4.xlarge'), machineImage: new ec2.AmazonLinuxImage(), @@ -162,5 +170,5 @@ function newFixtures(spot = false) { vpc, }); - return { stack, vpc, asg }; + return { stack, vpc, cluster, asg }; }