Ability to use existing route tables for controller and workers

multi-node/aws/pkg/config/config.go

@@ -129,6 +129,7 @@ type Cluster struct {
 	VPCCIDR                  string            `yaml:"vpcCIDR,omitempty"`
 	InstanceCIDR             string            `yaml:"instanceCIDR,omitempty"`
 	ControllerIP             string            `yaml:"controllerIP,omitempty"`
+	ControllerSubnets        []Subnet          `yaml:"controllerSubnets,omitempty"`
 	PodCIDR                  string            `yaml:"podCIDR,omitempty"`
 	ServiceCIDR              string            `yaml:"serviceCIDR,omitempty"`
 	DNSServiceIP             string            `yaml:"dnsServiceIP,omitempty"`
@@ -150,6 +151,7 @@ type Cluster struct {
 type Subnet struct {
 	AvailabilityZone string `yaml:"availabilityZone,omitempty"`
 	InstanceCIDR     string `yaml:"instanceCIDR,omitempty"`
+	RouteTableID     string `yaml:"routeTableId,omitempty"`
 }
 
 const (
@@ -293,8 +295,15 @@ func (c Cluster) stackConfig(opts StackTemplateOptions, compressUserData bool) (
 	if controllerIPAddr == nil {
 		return nil, fmt.Errorf("invalid controllerIP: %s", stackConfig.ControllerIP)
 	}
+
+	// if no controller subnet specified, the controller IP should match one of the worker subnet CIDRs
+	controllerSubnets := stackConfig.Subnets
+	if &stackConfig.ControllerSubnets != nil && len(stackConfig.ControllerSubnets) > 0 {
+		controllerSubnets = stackConfig.ControllerSubnets
+	}
+
 	controllerSubnetFound := false
-	for i, subnet := range stackConfig.Subnets {
+	for i, subnet := range controllerSubnets {
 		_, instanceCIDR, err := net.ParseCIDR(subnet.InstanceCIDR)
 		if err != nil {
 			return nil, fmt.Errorf("invalid instanceCIDR: %v", err)
@@ -305,7 +314,7 @@ func (c Cluster) stackConfig(opts StackTemplateOptions, compressUserData bool) (
 		}
 	}
 	if !controllerSubnetFound {
-		return nil, fmt.Errorf("Fail-fast occurred possibly because of a bug: ControllerSubnetIndex couldn't be determined for subnets (%v) and controllerIP (%v)", stackConfig.Subnets, stackConfig.ControllerIP)
+		return nil, fmt.Errorf("Fail-fast occurred possibly because of a bug: ControllerSubnetIndex couldn't be determined for subnets (%v) and controllerIP (%v)", controllerSubnets, stackConfig.ControllerIP)
 	}
 
 	if stackConfig.UserDataWorker, err = execute(opts.WorkerTmplFile, stackConfig.Config, compressUserData); err != nil {
@@ -526,7 +535,13 @@ func (c Cluster) valid() error {
 		}
 
 		var instanceCIDRs = make([]*net.IPNet, 0)
-		for i, subnet := range c.Subnets {
+		var subnets = c.Subnets
+		if &c.ControllerSubnets != nil {
+			// controller subnets are defined, add to the list
+			subnets = append(subnets, c.ControllerSubnets...)
+		}
+
+		for i, subnet := range subnets {
 			if subnet.AvailabilityZone == "" {
 				return fmt.Errorf("availabilityZone must be set for subnet #%d", i)
 			}

multi-node/aws/pkg/config/templates/cluster.yaml

@@ -73,27 +73,38 @@ kmsKeyArn: "{{.KMSKeyARN}}"
 # workerSpotPrice: "0.05"
 
 # ID of existing VPC to create subnet in. Leave blank to create a new VPC
-# vpcId:
+# vpcId: vpc-UNIQUEID
 
 # ID of existing route table in existing VPC to attach subnet to. Leave blank to use the VPC's main route table.
-# routeTableId:
+# routeTableId: rtb-UNIQUEID
 
 # CIDR for Kubernetes VPC. If vpcId is specified, must match the CIDR of existing vpc.
 # vpcCIDR: "10.0.0.0/16"
 
 # CIDR for Kubernetes subnet when placing nodes in a single availability zone (not highly-available) Leave commented out for multi availability zone setting and use the below `subnets` section instead.
 # instanceCIDR: "10.0.0.0/24"
 
-# Kubernetes subnets with their CIDRs and availability zones. Differentiating availability zone for 2 or more subnets result in high-availability (failures of a single availability zone won't result in immediate downtimes)
+# Kubernetes subnets with their CIDRs and availability zones. Differentiating availability zone for 2 or more subnets result in high-availability (failures of a single availability zone won't result in immediate downtimes). You may also optionally specify existing route tables within the VPC to use, if not specified drop down to the `routeTableId` above then the VPC's main route table.
 # subnets:
 #   - availabilityZone: us-west-1a
 #     instanceCIDR: "10.0.0.0/24"
+#     routeTableId: rtb-UNIQUEID
 #   - availabilityZone: us-west-1b
 #     instanceCIDR: "10.0.1.0/24"
+#     routeTableId: rtb-UNIQUEID
 
-# IP Address for the controller in Kubernetes subnet. When we have 2 or more subnets, the controller is placed in the first subnet and controllerIP must be included in the instanceCIDR of the first subnet. This convention will change once we have H/A controllers
+# IP Address for the controller in Kubernetes subnet. The controllerIP must be included in the instanceCIDRs of the `controllerSubnets` if specified or if not specifed then the instanceCIDRs of `subnets` if specified. This convention will change once we have H/A controllers
 # controllerIP: 10.0.0.50
 
+# Optionally override the details used for any controller subnets. You may also optionally specify existing route tables within the VPC to use, if not specified drop down to the `routeTableId` above then the VPC's main route table.
+# controllerSubnets:
+#   - availabilityZone: us-west-1a
+#     instanceCIDR: "10.0.2.0/24"
+#     routeTableId: rtb-UNIQUEID
+#   - availabilityZone: us-west-1b
+#     instanceCIDR: "10.0.3.0/24"
+#     routeTableId: rtb-UNIQUEID
+
 # CIDR for all service IP addresses
 # serviceCIDR: "10.3.0.0/24"
 

multi-node/aws/pkg/config/templates/stack-template.json

@@ -1,6 +1,6 @@
 {
   "AWSTemplateFormatVersion": "2010-09-09",
-  "Description": "kube-aws Kubernetes cluster {{.ClusterName}}",
+  "Description": "kube-aws Kubernetes cluster {{$.ClusterName}}",
   "Resources": {
     "AlarmControllerRecover": {
       "Properties": {
@@ -57,12 +57,12 @@
           {
             "Key": "KubernetesCluster",
             "PropagateAtLaunch": "true",
-            "Value": "{{.ClusterName}}"
+            "Value": "{{$.ClusterName}}"
           },
           {
             "Key": "Name",
             "PropagateAtLaunch": "true",
-            "Value": "{{.ClusterName}}-kube-aws-worker"
+            "Value": "{{$.ClusterName}}-kube-aws-worker"
           }
         ],
         "VPCZoneIdentifier": [
@@ -251,7 +251,12 @@
     },
     "InstanceController": {
       "Properties": {
-        "AvailabilityZone": "{{(index .Subnets .ControllerSubnetIndex).AvailabilityZone}}",
+        "AvailabilityZone":
+        {{if .ControllerSubnets}}
+        "{{(index .ControllerSubnets .ControllerSubnetIndex).AvailabilityZone}}"
+        {{else}}
+        "{{(index .Subnets .ControllerSubnetIndex).AvailabilityZone}}"
+        {{end}},
         "BlockDeviceMappings": [
           {
             "DeviceName": "/dev/xvda",
@@ -282,18 +287,23 @@
             ],
             "PrivateIpAddress": "{{.ControllerIP}}",
             "SubnetId": {
-              "Ref": "Subnet{{.ControllerSubnetIndex}}"
+              "Ref": 
+              {{if .ControllerSubnets}}
+              "ControllerSubnet{{.ControllerSubnetIndex}}"
+              {{else}}
+              "Subnet{{.ControllerSubnetIndex}}"
+              {{end}}
             }
           }
         ],
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "{{.ClusterName}}"
+            "Value": "{{$.ClusterName}}"
           },
           {
             "Key": "Name",
-            "Value": "{{.ClusterName}}-kube-aws-controller"
+            "Value": "{{$.ClusterName}}-kube-aws-controller"
           }
         ],
         "UserData": "{{ .UserDataController }}"
@@ -380,7 +390,7 @@
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "{{.ClusterName}}"
+            "Value": "{{$.ClusterName}}"
           }
         ],
         "VpcId": {{.VPCRef}}
@@ -443,7 +453,7 @@
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "{{.ClusterName}}"
+            "Value": "{{$.ClusterName}}"
           }
         ],
         "VpcId": {{.VPCRef}}
@@ -568,6 +578,46 @@
     }
     {{end}}
     {{end}}
+
+    {{if .ControllerSubnets}}
+      {{range $index, $subnet := .ControllerSubnets}}
+      {{with $subnetLogicalName := printf "ControllerSubnet%d" $index}}
+      ,
+      "{{$subnetLogicalName}}": {
+        "Properties": {
+          "AvailabilityZone": "{{$subnet.AvailabilityZone}}",
+          "CidrBlock": "{{$subnet.InstanceCIDR}}",
+          "MapPublicIpOnLaunch": true,
+          "Tags": [
+            {
+              "Key": "Name",
+              "Value": "{{$.ClusterName}}-controller-subnet"
+            },
+            {
+              "Key": "KubernetesCluster",
+              "Value": "{{$.ClusterName}}"
+            }
+          ],
+          "VpcId": {{$.VPCRef}}
+        },
+        "Type": "AWS::EC2::Subnet"
+      }
+      {{if $subnet.RouteTableID}}
+        ,
+        "{{$subnetLogicalName}}RouteTableAssociation": {
+          "Properties": {
+            "RouteTableId": "{{$subnet.RouteTableID}}",
+            "SubnetId": {
+              "Ref": "{{$subnetLogicalName}}"
+            }
+          },
+          "Type": "AWS::EC2::SubnetRouteTableAssociation"
+        }
+      {{end}}
+      {{end}}
+      {{end}}
+    {{end}}
+
     {{if not .VPCID}}
     ,
     "{{.VPCLogicalName}}": {
@@ -579,11 +629,11 @@
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "{{.ClusterName}}"
+            "Value": "{{$.ClusterName}}"
           },
           {
             "Key": "Name",
-            "Value": "kubernetes-{{.ClusterName}}-vpc"
+            "Value": "kubernetes-{{$.ClusterName}}-vpc"
           }
         ]
       },
@@ -594,7 +644,7 @@
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "{{.ClusterName}}"
+            "Value": "{{$.ClusterName}}"
           }
         ],
         "VpcId": {{.VPCRef}}
@@ -616,7 +666,7 @@
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "{{.ClusterName}}"
+            "Value": "{{$.ClusterName}}"
           }
         ]
       },
@@ -645,10 +695,22 @@
     }
     {{end}}
     {{end}}
-    {{else}}
-    {{if .RouteTableID}}
+
+    {{else}} {{/* else VPC ID set */}}
     {{range $index, $subnet := .Subnets}}
     {{with $subnetLogicalName := printf "Subnet%d" $index}}
+    {{if $subnet.RouteTableID}} {{/* subnet specific route table set */}}
+    ,
+    "{{$subnetLogicalName}}RouteTableAssociation": {
+      "Properties": {
+        "RouteTableId": "{{$subnet.RouteTableID}}",
+        "SubnetId": {
+          "Ref": "{{$subnetLogicalName}}"
+        }
+      },
+      "Type": "AWS::EC2::SubnetRouteTableAssociation"
+    }
+    {{else if $.RouteTableID}} {{/* default route table set */}}
     ,
     "{{$subnetLogicalName}}RouteTableAssociation": {
       "Properties": {