New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add support for cf and codepipeline parameters file to cf deploy #5443
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
{ | ||
"type": "enhancement", | ||
"category": "``cloudformation``", | ||
"description": "CloudFormation ``deploy`` command now supports various JSON file formats as an input for ``--parameter-overrides`` option `#2828 <https://github.com/aws/aws-cli/issues/2828>`__" | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -11,13 +11,16 @@ | |
# ANY KIND, either express or implied. See the License for the specific | ||
# language governing permissions and limitations under the License. | ||
|
||
import functools | ||
import json | ||
import os | ||
import sys | ||
import logging | ||
|
||
from botocore.client import Config | ||
|
||
from awscli.compat import compat_open | ||
from awscli.customizations.exceptions import ParamValidationError | ||
from awscli.customizations.cloudformation import exceptions | ||
from awscli.customizations.cloudformation.deployer import Deployer | ||
from awscli.customizations.s3uploader import S3Uploader | ||
|
@@ -30,6 +33,70 @@ | |
LOG = logging.getLogger(__name__) | ||
|
||
|
||
class BaseParameterOverrideParser: | ||
def can_parse(self, data): | ||
# Returns true/false if it can parse | ||
raise NotImplementedError('can_parse') | ||
|
||
def parse(self, data): | ||
# Return the properly formatted parameter dictionary | ||
raise NotImplementedError('parse') | ||
|
||
|
||
class CodePipelineLikeParameterOverrideParser(BaseParameterOverrideParser): | ||
def can_parse(self, data): | ||
return isinstance(data, dict) and 'Parameters' in data | ||
|
||
def parse(self, data): | ||
# Parse parameter_overrides if they were given in | ||
# CodePipeline params file format | ||
# { | ||
# "Parameters": { | ||
# "ParameterKey": "ParameterValue" | ||
# } | ||
# } | ||
return data['Parameters'] | ||
|
||
|
||
class CloudFormationLikeParameterOverrideParser(BaseParameterOverrideParser): | ||
def can_parse(self, data): | ||
for param_pair in data: | ||
if ('ParameterKey' not in param_pair or | ||
'ParameterValue' not in param_pair): | ||
return False | ||
if len(param_pair.keys()) > 2: | ||
return False | ||
return True | ||
|
||
def parse(self, data): | ||
# Parse parameter_overrides if they were given in | ||
# CloudFormation params file format | ||
# [{ | ||
# "ParameterKey": "string", | ||
# "ParameterValue": "string", | ||
# }] | ||
return { | ||
param['ParameterKey']: param['ParameterValue'] | ||
for param in data | ||
} | ||
|
||
|
||
class StringEqualsParameterOverrideParser(BaseParameterOverrideParser): | ||
def can_parse(self, data): | ||
return all( | ||
isinstance(param, str) and len(param.split("=", 1)) == 2 | ||
for param in data | ||
) | ||
|
||
def parse(self, data): | ||
result = {} | ||
for param in data: | ||
# Split at first '=' from left | ||
key_value_pair = param.split("=", 1) | ||
result[key_value_pair[0]] = key_value_pair[1] | ||
return result | ||
|
||
|
||
class DeployCommand(BasicCommand): | ||
|
||
MSG_NO_EXECUTE_CHANGESET = \ | ||
|
@@ -107,12 +174,7 @@ class DeployCommand(BasicCommand): | |
'name': PARAMETER_OVERRIDE_CMD, | ||
'action': 'store', | ||
'required': False, | ||
'schema': { | ||
'type': 'array', | ||
'items': { | ||
'type': 'string' | ||
} | ||
}, | ||
'nargs': '+', | ||
'default': [], | ||
'help_text': ( | ||
'A list of parameter structures that specify input parameters' | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We also should add a changelog entry for this. |
||
|
@@ -121,7 +183,7 @@ class DeployCommand(BasicCommand): | |
' existing value. For new stacks, you must specify' | ||
' parameters that don\'t have a default value.' | ||
' Syntax: ParameterKey1=ParameterValue1' | ||
' ParameterKey2=ParameterValue2 ...' | ||
' ParameterKey2=ParameterValue2 ... or JSON file (see Examples)' | ||
) | ||
}, | ||
{ | ||
|
@@ -255,10 +317,9 @@ def _run_main(self, parsed_args, parsed_globals): | |
template_str = handle.read() | ||
|
||
stack_name = parsed_args.stack_name | ||
parameter_overrides = self.parse_key_value_arg( | ||
parsed_args.parameter_overrides, | ||
self.PARAMETER_OVERRIDE_CMD) | ||
|
||
parameter_overrides = self.parse_parameter_overrides( | ||
parsed_args.parameter_overrides | ||
) | ||
tags_dict = self.parse_key_value_arg(parsed_args.tags, self.TAGS_CMD) | ||
tags = [{"Key": key, "Value": value} | ||
for key, value in tags_dict.items()] | ||
|
@@ -359,6 +420,42 @@ def merge_parameters(self, template_dict, parameter_overrides): | |
|
||
return parameter_values | ||
|
||
def _parse_input_as_json(self, arg_value): | ||
# In case of reading from file it'll be string and in case | ||
# of inline json input it'll be list where json string | ||
# will be the first element | ||
if arg_value: | ||
if isinstance(arg_value, str): | ||
return json.loads(arg_value) | ||
Comment on lines
+428
to
+429
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Then is this the case where it got loaded from a file? If so, it might be good to include that in the comment. |
||
try: | ||
return json.loads(arg_value[0]) | ||
except json.JSONDecodeError: | ||
return None | ||
|
||
def parse_parameter_overrides(self, arg_value): | ||
data = self._parse_input_as_json(arg_value) | ||
if data is not None: | ||
parsers = [ | ||
CloudFormationLikeParameterOverrideParser(), | ||
CodePipelineLikeParameterOverrideParser(), | ||
StringEqualsParameterOverrideParser() | ||
] | ||
for parser in parsers: | ||
if parser.can_parse(data): | ||
return parser.parse(data) | ||
raise ParamValidationError( | ||
'JSON passed to --parameter-overrides must be one of ' | ||
'the formats: ["Key1=Value1","Key2=Value2", ...] , ' | ||
'[{"ParameterKey": "Key1", "ParameterValue": "Value1"}, ...] , ' | ||
'["Parameters": {"Key1": "Value1", "Key2": "Value2", ...}]') | ||
else: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm wondering if we should allow users to be able to send these new formats through the command line as JSON? For example, this should probably work:
Generally, the expectation for CLI usage is that if it can be specified as a file, it can also be specified directly to the command line. While I expect most people to be using the file, I do not think we should be limiting the ability to pass the new formats in directly to the command line as users may be confused by this inconsistency and they may not necessarily want to write out a file in order to run a command. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm not sure anyone will do it but to be consistent we need to allow it, I think |
||
# In case it was in deploy command format | ||
# and was input via command line | ||
return self.parse_key_value_arg( | ||
arg_value, | ||
self.PARAMETER_OVERRIDE_CMD | ||
) | ||
|
||
def parse_key_value_arg(self, arg_value, argname): | ||
""" | ||
Converts arguments that are passed as list of "Key=Value" strings | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -10,6 +10,8 @@ | |
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF | ||
# ANY KIND, either express or implied. See the License for the specific | ||
# language governing permissions and limitations under the License. | ||
import json | ||
|
||
from awscli.testutils import BaseAWSCommandParamsTest | ||
from awscli.testutils import FileCreator | ||
|
||
|
@@ -60,3 +62,119 @@ def test_does_return_zero_exit_code_on_empty_changeset(self): | |
def test_does_return_non_zero_exit_code_on_empty_changeset(self): | ||
self.command += ' --fail-on-empty-changeset' | ||
self.run_cmd(self.command, expected_rc=255) | ||
|
||
|
||
class TestDeployCommandParameterOverrides(TestDeployCommand): | ||
def setUp(self): | ||
super(TestDeployCommandParameterOverrides, self).setUp() | ||
template = '''{ | ||
"AWSTemplateFormatVersion": "2010-09-09", | ||
"Parameters": { | ||
"Key1": { | ||
"Type": "String" | ||
}, | ||
"Key2": { | ||
"Type": "String" | ||
} | ||
} | ||
}''' | ||
Comment on lines
+70
to
+80
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For any time we are writing a JSON document out to a file, it may make sense to expose a template = {
"AWSTemplateFormatVersion": "2010-09-09",
"Parameters": {
"Key1": {
"Type": "String"
},
"Key2": {
"Type": "String"
}
}
}
path = self.create_json_file('template.json', template)
... Specifically when the template or parameters is in the form of python dictionary it is a bit easier to work with (e.g. you don't have to worry about getting JSON syntax correctly) and expand on it (e.g you want to add or remove keys from some default JSON document). |
||
path = self.files.create_file('template.json', template) | ||
self.command = ( | ||
'cloudformation deploy --template-file %s ' | ||
'--stack-name Stack ' | ||
) % path | ||
|
||
def _assert_parameters_parsed(self): | ||
self.assertEqual( | ||
self.operations_called[1][1]['Parameters'], | ||
[ | ||
{'ParameterKey': 'Key1', 'ParameterValue': 'Value1'}, | ||
{'ParameterKey': 'Key2', 'ParameterValue': 'Value2'} | ||
] | ||
) | ||
|
||
def create_json_file(self, filename, data): | ||
return self.files.create_file(filename, json.dumps(data)) | ||
|
||
def test_parameter_overrides_shorthand(self): | ||
self.command += ' --parameter-overrides Key1=Value1 Key2=Value2' | ||
self.run_cmd(self.command) | ||
self._assert_parameters_parsed() | ||
|
||
def test_parameter_overrides_from_inline_original_json(self): | ||
original_like_json = ['Key1=Value1', 'Key2=Value2'] | ||
path = self.create_json_file('param.json', original_like_json) | ||
self.command += ' --parameter-overrides file://%s' % path | ||
self.run_cmd(self.command) | ||
self._assert_parameters_parsed() | ||
|
||
def test_parameter_overrides_from_inline_cf_like_json(self): | ||
cf_like_json = ('[{"ParameterKey":"Key1",' | ||
'"ParameterValue":"Value1"},' | ||
'{"ParameterKey":"Key2",' | ||
'"ParameterValue":"Value2"}]') | ||
self.command += ' --parameter-overrides %s' % cf_like_json | ||
self.run_cmd(self.command) | ||
self._assert_parameters_parsed() | ||
|
||
def test_parameter_overrides_from_cf_like_json_file(self): | ||
cf_like_json = [ | ||
{'ParameterKey': 'Key1', 'ParameterValue': 'Value1'}, | ||
{'ParameterKey': 'Key2', 'ParameterValue': 'Value2'} | ||
] | ||
path = self.create_json_file('param.json', cf_like_json) | ||
self.command += ' --parameter-overrides file://%s' % path | ||
self.run_cmd(self.command) | ||
self._assert_parameters_parsed() | ||
|
||
def test_parameter_overrides_from_inline_codepipeline_like_json(self): | ||
codepipeline_like_json = ('{"Parameters":{"Key1":"Value1",' | ||
'"Key2":"Value2"}}') | ||
self.command += ' --parameter-overrides %s' % codepipeline_like_json | ||
self.run_cmd(self.command) | ||
self._assert_parameters_parsed() | ||
|
||
def test_parameter_overrides_from_codepipeline_like_json_file(self): | ||
codepipeline_like_json = { | ||
'Parameters': { | ||
'Key1': 'Value1', | ||
'Key2': 'Value2' | ||
} | ||
} | ||
path = self.create_json_file('param.json', codepipeline_like_json) | ||
self.command += ' --parameter-overrides file://%s' % path | ||
self.run_cmd(self.command) | ||
self._assert_parameters_parsed() | ||
|
||
def test_parameter_overrides_from_original_json_file(self): | ||
original_like_json = ['Key1=Value1', 'Key2=Value2'] | ||
path = self.create_json_file('param.json', original_like_json) | ||
self.command += ' --parameter-overrides file://%s' % path | ||
self.run_cmd(self.command, expected_rc=0) | ||
self._assert_parameters_parsed() | ||
|
||
def test_parameter_overrides_from_invalid_cf_like_json_file(self): | ||
invalid_cf_like_json = [ | ||
{ | ||
'ParameterKey': 'Key1', | ||
'ParameterValue': 'Value1', | ||
'RedundantKey': 'RedundantValue' | ||
}, | ||
{ | ||
'ParameterKey': 'Key2', | ||
'ParameterValue': 'Value2' | ||
} | ||
] | ||
path = self.create_json_file('param.json', invalid_cf_like_json) | ||
self.command += ' --parameter-overrides file://%s' % path | ||
_, err, _ = self.run_cmd(self.command, expected_rc=252) | ||
self.assertTrue('JSON passed to --parameter-overrides must be' | ||
in err) | ||
|
||
def test_parameter_overrides_from_invalid_json(self): | ||
cf_like_json = {'SomeKey': [{'RedundantKey': 'RedundantValue'}]} | ||
path = self.create_json_file('param.json', cf_like_json) | ||
self.command += ' --parameter-overrides file://%s' % path | ||
_, err, _ = self.run_cmd(self.command, expected_rc=252) | ||
self.assertTrue('JSON passed to --parameter-overrides must be' | ||
in err) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Just a reminder we need doc updates for these new formats.