Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(eks): Pass bootstrap.sh args to avoid DescribeCluster call and make nodes join the cluster faster #12659

Merged
merged 3 commits into from Apr 15, 2021
Merged

feat(eks): Pass bootstrap.sh args to avoid DescribeCluster call and make nodes join the cluster faster #12659

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c95bc09
feat(eks): Pass bootstrap.sh args to avoid DescribeCluster call
anguslees Mar 30, 2021
e11179c
Merge branch 'master' into eks-bootstrap-args
iliapolo Apr 15, 2021
d9ed213
Merge branch 'master' into eks-bootstrap-args
mergify[bot] Apr 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/@aws-cdk/aws-eks/README.md
View file
Open in desktop
Expand Up @@ -202,7 +202,7 @@ cluster.addNodegroupCapacity('custom-node-group', {
#### Spot Instances Support

Use `capacityType` to create managed node groups comprised of spot instances. To maximize the availability of your applications while using
Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property.
Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property.

> For more details visit [Managed node group capacity types](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types).

Expand Down Expand Up @@ -395,7 +395,7 @@ const asg = new ec2.AutoScalingGroup(...)
cluster.connectAutoScalingGroupCapacity(asg);
```

This will add the necessary user-data and configure all connections, roles, and tags needed for the instances in the auto-scaling group to properly join the cluster.
This will add the necessary user-data to access the apiserver and configure all connections, roles, and tags needed for the instances in the auto-scaling group to properly join the cluster.

#### Spot Instances

Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-eks/lib/cluster.ts
View file
Open in desktop
Expand Up @@ -1264,7 +1264,7 @@ export class Cluster extends ClusterBase {
if (bootstrapEnabled) {
const userData = options.machineImageType === MachineImageType.BOTTLEROCKET ?
renderBottlerocketUserData(this) :
renderAmazonLinuxUserData(this.clusterName, autoScalingGroup, options.bootstrapOptions);
renderAmazonLinuxUserData(this, autoScalingGroup, options.bootstrapOptions);
autoScalingGroup.addUserData(...userData);
}

Expand Down
9 changes: 6 additions & 3 deletions packages/@aws-cdk/aws-eks/lib/user-data.ts
View file
Open in desktop
@@ -1,9 +1,9 @@
import * as autoscaling from '@aws-cdk/aws-autoscaling';
import { Stack } from '@aws-cdk/core';
import { BootstrapOptions, ICluster } from './cluster';
import { BootstrapOptions, ICluster, Cluster } from './cluster';

// eslint-disable-next-line max-len
export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup: autoscaling.AutoScalingGroup, options: BootstrapOptions = {}): string[] {
export function renderAmazonLinuxUserData(cluster: Cluster, autoScalingGroup: autoscaling.AutoScalingGroup, options: BootstrapOptions = {}): string[] {

const stack = Stack.of(autoScalingGroup);

Expand All @@ -13,6 +13,9 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup:

const extraArgs = new Array<string>();

extraArgs.push(`--apiserver-endpoint '${cluster.clusterEndpoint}'`);
extraArgs.push(`--b64-cluster-ca '${cluster.clusterCertificateAuthorityData}'`);
iliapolo marked this conversation as resolved.
Show resolved Hide resolved

extraArgs.push(`--use-max-pods ${options.useMaxPods ?? true}`);

if (options.awsApiRetryAttempts) {
Expand Down Expand Up @@ -45,7 +48,7 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup:

return [
'set -o xtrace',
`/etc/eks/bootstrap.sh ${clusterName} --kubelet-extra-args "${kubeletExtraArgs}" ${commandLineSuffix}`.trim(),
`/etc/eks/bootstrap.sh ${cluster.clusterName} --kubelet-extra-args "${kubeletExtraArgs}" ${commandLineSuffix}`.trim(),
`/opt/aws/bin/cfn-signal --exit-code $? --stack ${stack.stackName} --resource ${asgLogicalId} --region ${stack.region}`,
];
}
Expand Down
82 changes: 69 additions & 13 deletions packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json
View file
Open in desktop
Expand Up @@ -1682,7 +1682,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesASGF172BD19 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesASGF172BD19 --region test-region"
]
]
}
Expand Down Expand Up @@ -1993,7 +2007,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesArmASG40A593D0 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesArmASG40A593D0 --region test-region"
]
]
}
Expand Down Expand Up @@ -2630,7 +2658,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels foo=bar,goo=far\" --use-max-pods true --aws-api-retry-attempts 5\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterspotASG857494B6 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels foo=bar,goo=far\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true --aws-api-retry-attempts 5\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterspotASG857494B6 --region test-region"
]
]
}
Expand Down Expand Up @@ -2973,7 +3015,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterInferenceInstancesASGE90717C7 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterInferenceInstancesASGE90717C7 --region test-region"
]
]
}
Expand Down Expand Up @@ -4146,7 +4202,7 @@
"Properties": {
"Code": {
"S3Bucket": {
"Ref": "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3Bucket055DC235"
"Ref": "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3BucketB7E1A9C0"
},
"S3Key": {
"Fn::Join": [
Expand All @@ -4159,7 +4215,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3VersionKey2FFFA299"
"Ref": "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3VersionKey542FDEBD"
}
]
}
Expand All @@ -4172,7 +4228,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3VersionKey2FFFA299"
"Ref": "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3VersionKey542FDEBD"
}
]
}
Expand Down Expand Up @@ -4674,17 +4730,17 @@
"Type": "String",
"Description": "Artifact hash for asset \"b7d38dc0eeb2c5d024919020e09d2590b68559eab4a5264c3b1aa7a429d1edd4\""
},
"AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3Bucket055DC235": {
"AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3BucketB7E1A9C0": {
"Type": "String",
"Description": "S3 bucket for asset \"952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344\""
"Description": "S3 bucket for asset \"6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3\""
},
"AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3VersionKey2FFFA299": {
"AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3VersionKey542FDEBD": {
"Type": "String",
"Description": "S3 key for asset version \"952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344\""
"Description": "S3 key for asset version \"6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3\""
},
"AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344ArtifactHash1AB042BC": {
"AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3ArtifactHash5E61FCA5": {
"Type": "String",
"Description": "Artifact hash for asset \"952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344\""
"Description": "Artifact hash for asset \"6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3\""
},
"AssetParameters5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636dS3BucketA6642550": {
"Type": "String",
Expand Down
6 changes: 3 additions & 3 deletions packages/@aws-cdk/aws-eks/test/test.cluster.ts
View file
Open in desktop
Expand Up @@ -1294,7 +1294,7 @@ export = {
// THEN
const template = app.synth().getStackByName(stack.stackName).template;
const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData;
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.done();
},

Expand Down Expand Up @@ -1333,7 +1333,7 @@ export = {
// THEN
const template = app.synth().getStackByName(stack.stackName).template;
const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData;
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand --node-labels FOO=42" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand --node-labels FOO=42" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.done();
},

Expand All @@ -1353,7 +1353,7 @@ export = {
// THEN
const template = app.synth().getStackByName(stack.stackName).template;
const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData;
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.done();
},

Expand Down