NVMe support for ec2 provider

Fixes: andsens#452

Adds support for building on EC2 hosts that have NVMe EBS devices.

Introduces the DescribeInstances permission requirement for the calling
role.

Changes the device naming logic on the host during build time.
- The new target volume will be mounted with the highest available
  DeviceName in the BlockDeviceMapping object, taking care to avoid
  assignments that also are allocated at launch to ephemeral devices
  (C5d, I3, F1, and M5d currently).
- The system device name will be identifed by the difference between the
  existing block devices prior to and after the AttachVolume call has
  finished. This race condition already existed with the previous logic.

bootstrapvz/providers/ec2/ebsvolume.py

@@ -33,19 +33,49 @@ def attach(self, instance_id):
         self.fsm.attach(instance_id=instance_id)
 
     def _before_attach(self, e):
-        import os.path
         import string
+        import subprocess
+        import urllib2
+
+        def name_mapped(path):
+            return path.split('/')[-1].replace('xvd', 'sd')[:3]
 
         self.instance_id = e.instance_id
-        for letter in string.ascii_lowercase[5:]:
-            dev_path = os.path.join('/dev', 'xvd' + letter)
-            if not os.path.exists(dev_path):
-                self.device_path = dev_path
-                self.ec2_device_path = os.path.join('/dev', 'sd' + letter)
+
+        dev_map_names = set()
+        launch_map_url = 'http://169.254.169.254/latest/meta-data/block-device-mapping/'
+        launch_map_response = urllib2.urlopen(url=launch_map_url, timeout=5)
+        for map_name in [d.strip() for d in launch_map_response.readlines()]:
+            dev_url = launch_map_url + map_name
+            dev_response = urllib2.urlopen(url=dev_url, timeout=5)
+            dev_map_names.add(name_mapped(dev_response.read().strip()))
+
+        try:
+            instance = self.conn.describe_instances(
+                Filters=[
+                    {'Name': 'instance-id', 'Values': [self.instance_id]}
+                ]
+            )['Reservations'][0]['Instances'][0]
+        except (IndexError, KeyError):
+            raise VolumeError('Unable to fetch EC2 instance volume data')
+
+        for mapped_dev in instance.get('BlockDeviceMappings', list()):
+            dev_map_names.add(name_mapped(mapped_dev['DeviceName']))
+
+        for letter in reversed(string.ascii_lowercase[1:]):
+            if 'sd' + letter not in dev_map_names:
+                self.ec2_device_path = '/dev/sd' + letter
                 break
 
-        if self.device_path is None:
-            raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume')
+        if self.ec2_device_path is None:
+            raise VolumeError('Unable to find a free block device mapping for bootstrap volume')
+
+        self.device_path = None
+
+        lsblk_command = ['lsblk', '--noheadings', '--list', '--nodeps', '--output', 'NAME']
+
+        lsblk_start = subprocess.check_output(lsblk_command)
+        start_dev_names = set(lsblk_start.split())
 
         self.conn.attach_volume(VolumeId=self.vol_id,
                                 InstanceId=self.instance_id,
@@ -54,6 +84,19 @@ def _before_attach(self, e):
         waiter.wait(VolumeIds=[self.vol_id],
                     Filters=[{'Name': 'attachment.status', 'Values': ['attached']}])
 
+        subprocess.check_call(['udevadm', 'settle'])
+
+        lsblk_end = subprocess.check_output(lsblk_command)
+        end_dev_names = set(lsblk_end.split())
+
+        if len(start_dev_names ^ end_dev_names) != 1:
+            raise VolumeError('Could not determine the device name for bootstrap volume')
+
+        udev_name = (start_dev_names ^ end_dev_names).pop()
+        self.device_path = subprocess.check_output(['udevadm', 'info',
+                                                    '--root', '--query=name',
+                                                    '--name', udev_name]).strip()
+
     def _before_detach(self, e):
         self.conn.detach_volume(VolumeId=self.vol_id,
                                 InstanceId=self.instance_id,