Merge pull request #5443 from vz10/cf_deploy_parameters_file

Add support for cf and codepipeline parameters file to cf deploy

.changes/next-release/feature-cloudformation-43713.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "``cloudformation``",
+  "description": "CloudFormation ``deploy`` command now supports various JSON file formats as an input for ``--parameter-overrides`` option `#2828 <https://github.com/aws/aws-cli/issues/2828>`__"
+}

awscli/customizations/cloudformation/deploy.py

@@ -11,13 +11,16 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 
+import functools
+import json
 import os
 import sys
 import logging
 
 from botocore.client import Config
 
 from awscli.compat import compat_open
+from awscli.customizations.exceptions import ParamValidationError
 from awscli.customizations.cloudformation import exceptions
 from awscli.customizations.cloudformation.deployer import Deployer
 from awscli.customizations.s3uploader import S3Uploader
@@ -30,6 +33,70 @@
 LOG = logging.getLogger(__name__)
 
 
+class BaseParameterOverrideParser:
+    def can_parse(self, data):
+        # Returns true/false if it can parse
+        raise NotImplementedError('can_parse')
+
+    def parse(self, data):
+        # Return the properly formatted parameter dictionary
+        raise NotImplementedError('parse')
+
+
+class CodePipelineLikeParameterOverrideParser(BaseParameterOverrideParser):
+    def can_parse(self, data):
+        return isinstance(data, dict) and 'Parameters' in data
+
+    def parse(self, data):
+        # Parse parameter_overrides if they were given in
+        # CodePipeline params file format
+        # {
+        #     "Parameters": {
+        #         "ParameterKey": "ParameterValue"
+        #     }
+        # }
+        return data['Parameters']
+
+
+class CloudFormationLikeParameterOverrideParser(BaseParameterOverrideParser):
+    def can_parse(self, data):
+        for param_pair in data:
+            if ('ParameterKey' not in param_pair or
+                    'ParameterValue' not in param_pair):
+                return False
+            if len(param_pair.keys()) > 2:
+                return False
+        return True
+
+    def parse(self, data):
+        # Parse parameter_overrides if they were given in
+        # CloudFormation params file format
+        # [{
+        #   "ParameterKey": "string",
+        #   "ParameterValue": "string",
+        # }]
+        return {
+            param['ParameterKey']: param['ParameterValue']
+            for param in data
+        }
+
+
+class StringEqualsParameterOverrideParser(BaseParameterOverrideParser):
+    def can_parse(self, data):
+        return all(
+            isinstance(param, str) and len(param.split("=", 1)) == 2
+            for param in data
+        )
+
+    def parse(self, data):
+        result = {}
+        for param in data:
+            # Split at first '=' from left
+            key_value_pair = param.split("=", 1)
+            result[key_value_pair[0]] = key_value_pair[1]
+        return result
+
+
 class DeployCommand(BasicCommand):
 
     MSG_NO_EXECUTE_CHANGESET = \
@@ -107,12 +174,7 @@ class DeployCommand(BasicCommand):
             'name': PARAMETER_OVERRIDE_CMD,
             'action': 'store',
             'required': False,
-            'schema': {
-                'type': 'array',
-                'items': {
-                    'type': 'string'
-                }
-            },
+            'nargs': '+',
             'default': [],
             'help_text': (
                 'A list of parameter structures that specify input parameters'
@@ -121,7 +183,7 @@ class DeployCommand(BasicCommand):
                 ' existing value. For new stacks, you must specify'
                 ' parameters that don\'t have a default value.'
                 ' Syntax: ParameterKey1=ParameterValue1'
-                ' ParameterKey2=ParameterValue2 ...'
+                ' ParameterKey2=ParameterValue2 ... or JSON file (see Examples)'
             )
         },
         {
@@ -255,10 +317,9 @@ def _run_main(self, parsed_args, parsed_globals):
             template_str = handle.read()
 
         stack_name = parsed_args.stack_name
-        parameter_overrides = self.parse_key_value_arg(
-                parsed_args.parameter_overrides,
-                self.PARAMETER_OVERRIDE_CMD)
-
+        parameter_overrides = self.parse_parameter_overrides(
+            parsed_args.parameter_overrides
+        )
         tags_dict = self.parse_key_value_arg(parsed_args.tags, self.TAGS_CMD)
         tags = [{"Key": key, "Value": value}
                 for key, value in tags_dict.items()]
@@ -359,6 +420,42 @@ def merge_parameters(self, template_dict, parameter_overrides):
 
         return parameter_values
 
+    def _parse_input_as_json(self, arg_value):
+        # In case of reading from file it'll be string and in case
+        # of inline json input it'll be list where json string
+        # will be the first element
+        if arg_value:
+            if isinstance(arg_value, str):
+                return json.loads(arg_value)
+            try:
+                return json.loads(arg_value[0])
+            except json.JSONDecodeError:
+                return None
+
+    def parse_parameter_overrides(self, arg_value):
+        data = self._parse_input_as_json(arg_value)
+        if data is not None:
+            parsers = [
+                CloudFormationLikeParameterOverrideParser(),
+                CodePipelineLikeParameterOverrideParser(),
+                StringEqualsParameterOverrideParser()
+            ]
+            for parser in parsers:
+                if parser.can_parse(data):
+                    return parser.parse(data)
+            raise ParamValidationError(
+                'JSON passed to --parameter-overrides must be one of '
+                'the formats: ["Key1=Value1","Key2=Value2", ...] , '
+                '[{"ParameterKey": "Key1", "ParameterValue": "Value1"}, ...] , '
+                '["Parameters": {"Key1": "Value1", "Key2": "Value2", ...}]')
+        else:
+            # In case it was in deploy command format
+            # and was input via command line
+            return self.parse_key_value_arg(
+                arg_value,
+                self.PARAMETER_OVERRIDE_CMD
+            )
+
     def parse_key_value_arg(self, arg_value, argname):
         """
         Converts arguments that are passed as list of "Key=Value" strings

awscli/examples/cloudformation/deploy.rst

@@ -4,3 +4,42 @@ Following command deploys template named ``template.json`` to a stack named
 
     aws cloudformation deploy --template-file /path_to_template/template.json --stack-name my-new-stack --parameter-overrides Key1=Value1 Key2=Value2 --tags Key1=Value1 Key2=Value2
 
+or the same command using parameters from JSON file ``parameters.json``::
+
+    aws cloudformation deploy --template-file /path_to_template/template.json --stack-name my-new-stack --parameter-overrides file://path_to_parameters/parameters.json --tags Key1=Value1 Key2=Value2
+
+Supported JSON syntax
+~~~~~~~~~~~~~~~~~~~~~
+
+Original format::
+
+    [
+        "Key1=Value1",
+        "Key2=Value2"
+    ]
+
+CloudFormation like format::
+
+    [
+       {
+            "ParameterKey": "Key1",
+            "ParameterValue": "Value1"
+        },
+        {
+            "ParameterKey": "Key2",
+            "ParameterValue": "Value2"
+        }
+    ]
+
+.. note::
+
+ Only ParameterKey and ParameterValue are expected keys, command will throw an exception if receives unexpected keys (e.g. UsePreviousValue or ResolvedValue).
+
+CodePipeline like format::
+
+    [
+        "Parameters": {
+            "Key1": "Value1",
+            "Key2": "Value2"
+        }
+    ]

tests/functional/cloudformation/test_deploy.py

@@ -10,6 +10,8 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+import json
+
 from awscli.testutils import BaseAWSCommandParamsTest
 from awscli.testutils import FileCreator
 
@@ -60,3 +62,119 @@ def test_does_return_zero_exit_code_on_empty_changeset(self):
     def test_does_return_non_zero_exit_code_on_empty_changeset(self):
         self.command += ' --fail-on-empty-changeset'
         self.run_cmd(self.command, expected_rc=255)
+
+
+class TestDeployCommandParameterOverrides(TestDeployCommand):
+    def setUp(self):
+        super(TestDeployCommandParameterOverrides, self).setUp()
+        template = '''{
+          "AWSTemplateFormatVersion": "2010-09-09",
+          "Parameters": {
+            "Key1": {
+              "Type": "String"
+            },
+            "Key2": {
+              "Type": "String"
+            }
+          }
+        }'''
+        path = self.files.create_file('template.json', template)
+        self.command = (
+            'cloudformation deploy --template-file %s '
+            '--stack-name Stack '
+        ) % path
+
+    def _assert_parameters_parsed(self):
+        self.assertEqual(
+            self.operations_called[1][1]['Parameters'],
+            [
+                {'ParameterKey': 'Key1', 'ParameterValue': 'Value1'},
+                {'ParameterKey': 'Key2', 'ParameterValue': 'Value2'}
+            ]
+        )
+
+    def create_json_file(self, filename, data):
+        return self.files.create_file(filename, json.dumps(data))
+
+    def test_parameter_overrides_shorthand(self):
+        self.command += ' --parameter-overrides Key1=Value1 Key2=Value2'
+        self.run_cmd(self.command)
+        self._assert_parameters_parsed()
+
+    def test_parameter_overrides_from_inline_original_json(self):
+        original_like_json = ['Key1=Value1', 'Key2=Value2']
+        path = self.create_json_file('param.json', original_like_json)
+        self.command += ' --parameter-overrides file://%s' % path
+        self.run_cmd(self.command)
+        self._assert_parameters_parsed()
+
+    def test_parameter_overrides_from_inline_cf_like_json(self):
+        cf_like_json = ('[{"ParameterKey":"Key1",'
+                        '"ParameterValue":"Value1"},'
+                        '{"ParameterKey":"Key2",'
+                        '"ParameterValue":"Value2"}]')
+        self.command += ' --parameter-overrides %s' % cf_like_json
+        self.run_cmd(self.command)
+        self._assert_parameters_parsed()
+
+    def test_parameter_overrides_from_cf_like_json_file(self):
+        cf_like_json = [
+            {'ParameterKey': 'Key1', 'ParameterValue': 'Value1'},
+            {'ParameterKey': 'Key2', 'ParameterValue': 'Value2'}
+        ]
+        path = self.create_json_file('param.json', cf_like_json)
+        self.command += ' --parameter-overrides file://%s' % path
+        self.run_cmd(self.command)
+        self._assert_parameters_parsed()
+
+    def test_parameter_overrides_from_inline_codepipeline_like_json(self):
+        codepipeline_like_json = ('{"Parameters":{"Key1":"Value1",'
+                                  '"Key2":"Value2"}}')
+        self.command += ' --parameter-overrides %s' % codepipeline_like_json
+        self.run_cmd(self.command)
+        self._assert_parameters_parsed()
+
+    def test_parameter_overrides_from_codepipeline_like_json_file(self):
+        codepipeline_like_json = {
+            'Parameters': {
+                'Key1': 'Value1',
+                'Key2': 'Value2'
+            }
+        }
+        path = self.create_json_file('param.json', codepipeline_like_json)
+        self.command += ' --parameter-overrides file://%s' % path
+        self.run_cmd(self.command)
+        self._assert_parameters_parsed()
+
+    def test_parameter_overrides_from_original_json_file(self):
+        original_like_json = ['Key1=Value1', 'Key2=Value2']
+        path = self.create_json_file('param.json', original_like_json)
+        self.command += ' --parameter-overrides file://%s' % path
+        self.run_cmd(self.command, expected_rc=0)
+        self._assert_parameters_parsed()
+
+    def test_parameter_overrides_from_invalid_cf_like_json_file(self):
+        invalid_cf_like_json = [
+            {
+                'ParameterKey': 'Key1',
+                'ParameterValue': 'Value1',
+                'RedundantKey': 'RedundantValue'
+            },
+            {
+                'ParameterKey': 'Key2',
+                'ParameterValue': 'Value2'
+            }
+        ]
+        path = self.create_json_file('param.json', invalid_cf_like_json)
+        self.command += ' --parameter-overrides file://%s' % path
+        _, err, _ = self.run_cmd(self.command, expected_rc=252)
+        self.assertTrue('JSON passed to --parameter-overrides must be'
+                        in err)
+
+    def test_parameter_overrides_from_invalid_json(self):
+        cf_like_json = {'SomeKey': [{'RedundantKey': 'RedundantValue'}]}
+        path = self.create_json_file('param.json', cf_like_json)
+        self.command += ' --parameter-overrides file://%s' % path
+        _, err, _ = self.run_cmd(self.command, expected_rc=252)
+        self.assertTrue('JSON passed to --parameter-overrides must be'
+                        in err)