Merge pull request #40 from fabfuel/release/1.4.0

Release 1.4.0

.travis.yml

@@ -2,7 +2,8 @@ language: python
 python:
   - "2.7"
   - "3.5"
-install: pip install tox-travis
+  - "3.6"
+install: pip install tox-travis boto3
 script: tox
 after_script:
     - pip install scrutinizer-ocular

README.rst

@@ -253,6 +253,26 @@ Or implicitly via environment variables ``NEW_RELIC_API_KEY`` and ``NEW_RELIC_AP
 Optionally you can provide an additional comment to the deployment via ``--comment "New feature X"`` and the name of the user who deployed with ``--user john.doe``
 
 
+Troubleshooting
+---------------
+If the service configuration in ECS is not optimally set, you might be seeing
+timeout or other errors during the deployment.
+
+Timeout
+=======
+The timeout error means, that AWS ECS takes longer for the full deployment cycle then ecs-deploy is told to wait. The deployment itself still might finish successfully, if there are no other problems with the deployed containers.
+
+You can increase the time (in seconds) to wait for finishing the deployment via the ``--timeout`` parameter. This time includes the full cycle of stopping all old containers and (re)starting all new containers. Different stacks require different timeout values, the default is 300 seconds.
+
+The overall deployment time depends on different things:
+
+- the type of the application. For example node.js containers tend to take a long time to get stopped. But nginx containers tend to stop immediately, etc.
+- are old and new containers able to run in parallel (e.g. using dynamic ports)?
+- the deployment options and strategy (Maximum percent > 100)?
+- the desired count of running tasks, compared to
+- the number of ECS instances in the cluster
+
+
 Alternative Implementation
 --------------------------
 There are some other libraries/tools available on GitHub, which also handle the deployment of containers in AWS ECS. If you prefer another language over Python, have a look at these projects:

ecs_deploy/ecs.py

@@ -5,6 +5,9 @@
 from dateutil.tz.tz import tzlocal
 
 
+VERSION = '1.4.0'
+
+
 class EcsClient(object):
     def __init__(self, access_key_id=None, secret_access_key=None,
                  region=None, profile=None):
@@ -375,9 +378,11 @@ def update_task_definition(self, task_definition):
             additional_properties=task_definition.additional_properties
         )
         new_task_definition = EcsTaskDefinition(**response[u'taskDefinition'])
-        self._client.deregister_task_definition(task_definition.arn)
         return new_task_definition
 
+    def deregister_task_definition(self, task_definition):
+        self._client.deregister_task_definition(task_definition.arn)
+
     def update_service(self, service):
         response = self._client.update_service(
             cluster=service.cluster,
@@ -434,14 +439,20 @@ def service_name(self):
 
 class DeployAction(EcsAction):
     def deploy(self, task_definition):
-        self._service.set_task_definition(task_definition)
-        return self.update_service(self._service)
+        try:
+            self._service.set_task_definition(task_definition)
+            return self.update_service(self._service)
+        except ClientError as e:
+            raise EcsError(str(e))
 
 
 class ScaleAction(EcsAction):
     def scale(self, desired_count):
-        self._service.set_desired_count(desired_count)
-        return self.update_service(self._service)
+        try:
+            self._service.set_desired_count(desired_count)
+            return self.update_service(self._service)
+        except ClientError as e:
+            raise EcsError(str(e))
 
 
 class RunAction(EcsAction):
@@ -452,15 +463,18 @@ def __init__(self, client, cluster_name):
         self.started_tasks = []
 
     def run(self, task_definition, count, started_by):
-        result = self._client.run_task(
-            cluster=self._cluster_name,
-            task_definition=task_definition.family_revision,
-            count=count,
-            started_by=started_by,
-            overrides=dict(containerOverrides=task_definition.get_overrides())
-        )
-        self.started_tasks = result['tasks']
-        return True
+        try:
+            result = self._client.run_task(
+                cluster=self._cluster_name,
+                task_definition=task_definition.family_revision,
+                count=count,
+                started_by=started_by,
+                overrides=dict(containerOverrides=task_definition.get_overrides())
+            )
+            self.started_tasks = result['tasks']
+            return True
+        except ClientError as e:
+            raise EcsError(str(e))
 
 
 class EcsError(Exception):

setup.cfg

@@ -1,7 +1,7 @@
 [wheel]
 universal = 1
 
-[pytest]
+[tool:pytest]
 testpaths = tests
 flake8-max-line-length = 120
 

setup.py

@@ -3,13 +3,15 @@
 """
 from setuptools import find_packages, setup
 
+from ecs_deploy.ecs import VERSION
+
 dependencies = ['click', 'botocore', 'boto3>=1.4.7', 'future', 'requests']
 
 setup(
     name='ecs-deploy',
-    version='1.3.1',
+    version=VERSION,
     url='https://github.com/fabfuel/ecs-deploy',
-    download_url='https://github.com/fabfuel/ecs-deploy/archive/1.3.1.tar.gz',
+    download_url='https://github.com/fabfuel/ecs-deploy/archive/%s.tar.gz' % VERSION,
     license='BSD',
     author='Fabian Fuelling',
     author_email='pypi@fabfuel.de',
@@ -33,7 +35,6 @@
         'coverage'
     ],
     classifiers=[
-        'Development Status :: 3 - Alpha',
         'Environment :: Console',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: BSD License',

tests/test_cli.py

@@ -63,19 +63,53 @@ def test_deploy(get_client, runner):
     result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME))
     assert result.exit_code == 0
     assert not result.exception
+    assert u"Deploying based on task definition: test-task:1" in result.output
     assert u'Successfully created revision: 2' in result.output
     assert u'Successfully deregistered revision: 1' in result.output
     assert u'Successfully changed task definition to: test-task:2' in result.output
     assert u'Deployment successful' in result.output
     assert u"Updating task definition" not in result.output
 
 
+@patch('ecs_deploy.cli.get_client')
+def test_deploy_with_rollback(get_client, runner):
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key', wait=2)
+    result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '--timeout=1', '--rollback'))
+
+    assert result.exit_code == 1
+    assert result.exception
+    assert u"Deploying based on task definition: test-task:1" in result.output
+
+    assert u"Deployment failed" in result.output
+    assert u"Rolling back to task definition: test-task:1" in result.output
+    assert u'Successfully changed task definition to: test-task:1' in result.output
+
+    assert u"Rollback successful" in result.output
+    assert u'Deployment failed, but service has been rolled back to ' \
+           u'previous task definition: test-task:1' in result.output
+
+
+@patch('ecs_deploy.cli.get_client')
+def test_deploy_without_deregister(get_client, runner):
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key')
+    result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '--no-deregister'))
+    assert result.exit_code == 0
+    assert not result.exception
+    assert u"Deploying based on task definition: test-task:1" in result.output
+    assert u'Successfully created revision: 2' in result.output
+    assert u'Successfully deregistered revision: 1' not in result.output
+    assert u'Successfully changed task definition to: test-task:2' in result.output
+    assert u'Deployment successful' in result.output
+    assert u"Updating task definition" not in result.output
+
+
 @patch('ecs_deploy.cli.get_client')
 def test_deploy_with_role_arn(get_client, runner):
     get_client.return_value = EcsTestClient('acces_key', 'secret_key')
     result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '-r', 'arn:new:role'))
     assert result.exit_code == 0
     assert not result.exception
+    assert u"Deploying based on task definition: test-task:1" in result.output
     assert u'Successfully created revision: 2' in result.output
     assert u'Successfully deregistered revision: 1' in result.output
     assert u'Successfully changed task definition to: test-task:2' in result.output
@@ -90,6 +124,7 @@ def test_deploy_new_tag(get_client, runner):
     result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '-t', 'latest'))
     assert result.exit_code == 0
     assert not result.exception
+    assert u"Deploying based on task definition: test-task:1" in result.output
     assert u"Updating task definition" in result.output
     assert u'Changed image of container "webserver" to: "webserver:latest" (was: "webserver:123")' in result.output
     assert u'Changed image of container "application" to: "application:latest" (was: "application:123")' in result.output
@@ -105,6 +140,7 @@ def test_deploy_one_new_image(get_client, runner):
     result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '-i', 'application', 'application:latest'))
     assert result.exit_code == 0
     assert not result.exception
+    assert u"Deploying based on task definition: test-task:1" in result.output
     assert u"Updating task definition" in result.output
     assert u'Changed image of container "application" to: "application:latest" (was: "application:123")' in result.output
     assert u'Successfully created revision: 2' in result.output
@@ -120,6 +156,7 @@ def test_deploy_two_new_images(get_client, runner):
                                         '-i', 'webserver', 'webserver:latest'))
     assert result.exit_code == 0
     assert not result.exception
+    assert u"Deploying based on task definition: test-task:1" in result.output
     assert u"Updating task definition" in result.output
     assert u'Changed image of container "webserver" to: "webserver:latest" (was: "webserver:123")' in result.output
     assert u'Changed image of container "application" to: "application:latest" (was: "application:123")' in result.output
@@ -135,6 +172,7 @@ def test_deploy_one_new_command(get_client, runner):
     result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '-c', 'application', 'foobar'))
     assert result.exit_code == 0
     assert not result.exception
+    assert u"Deploying based on task definition: test-task:1" in result.output
     assert u"Updating task definition" in result.output
     assert u'Changed command of container "application" to: "foobar" (was: "run")' in result.output
     assert u'Successfully created revision: 2' in result.output
@@ -153,6 +191,7 @@ def test_deploy_one_new_environment_variable(get_client, runner):
     assert result.exit_code == 0
     assert not result.exception
 
+    assert u"Deploying based on task definition: test-task:1" in result.output
     assert u"Updating task definition" in result.output
     assert u'Changed environment "foo" of container "application" to: "bar"' in result.output
     assert u'Changed environment "foo" of container "webserver" to: "baz"' in result.output
@@ -171,6 +210,7 @@ def test_deploy_without_changing_environment_value(get_client, runner):
     assert result.exit_code == 0
     assert not result.exception
 
+    assert u"Deploying based on task definition: test-task:1" in result.output
     assert u"Updating task definition" not in result.output
     assert u'Changed environment' not in result.output
     assert u'Successfully created revision: 2' in result.output
@@ -187,6 +227,7 @@ def test_deploy_without_diff(get_client, runner):
     assert result.exit_code == 0
     assert not result.exception
 
+    assert u"Deploying based on task definition: test-task:1" in result.output
     assert u"Updating task definition" not in result.output
     assert u'Changed environment' not in result.output
     assert u'Successfully created revision: 2' in result.output
@@ -197,20 +238,29 @@ def test_deploy_without_diff(get_client, runner):
 
 @patch('ecs_deploy.cli.get_client')
 def test_deploy_with_errors(get_client, runner):
-    get_client.return_value = EcsTestClient('acces_key', 'secret_key', errors=True)
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key', deployment_errors=True)
     result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME))
     assert result.exit_code == 1
     assert u"Deployment failed" in result.output
     assert u"ERROR: Service was unable to Lorem Ipsum" in result.output
 
 
+@patch('ecs_deploy.cli.get_client')
+def test_deploy_with_client_errors(get_client, runner):
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key', client_errors=True)
+    result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME))
+    assert result.exit_code == 1
+    assert u"Something went wrong" in result.output
+
+
 @patch('ecs_deploy.cli.get_client')
 def test_deploy_ignore_warnings(get_client, runner):
-    get_client.return_value = EcsTestClient('acces_key', 'secret_key', errors=True)
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key', deployment_errors=True)
     result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '--ignore-warnings'))
 
     assert result.exit_code == 0
     assert not result.exception
+    assert u"Deploying based on task definition: test-task:1" in result.output
     assert u'Successfully created revision: 2' in result.output
     assert u'Successfully deregistered revision: 1' in result.output
     assert u'Successfully changed task definition to: test-task:2' in result.output
@@ -219,6 +269,31 @@ def test_deploy_ignore_warnings(get_client, runner):
     assert u'Deployment successful' in result.output
 
 
+@patch('ecs_deploy.newrelic.Deployment.deploy')
+@patch('ecs_deploy.cli.get_client')
+def test_deploy_with_newrelic(get_client, newrelic, runner):
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key')
+    result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME,
+                                        '-t', 'my-tag',
+                                        '--newrelic-apikey', 'test',
+                                        '--newrelic-appid', 'test',
+                                        '--comment', 'Lorem Ipsum'))
+    assert result.exit_code == 0
+    assert not result.exception
+    assert u"Deploying based on task definition: test-task:1" in result.output
+    assert u'Successfully created revision: 2' in result.output
+    assert u'Successfully deregistered revision: 1' in result.output
+    assert u'Successfully changed task definition to: test-task:2' in result.output
+    assert u'Deployment successful' in result.output
+    assert u"Recording deployment in New Relic" in result.output
+
+    newrelic.assert_called_once_with(
+        'my-tag',
+        '',
+        'Lorem Ipsum'
+    )
+
+
 @patch('ecs_deploy.newrelic.Deployment.deploy')
 @patch('ecs_deploy.cli.get_client')
 def test_deploy_with_newrelic_errors(get_client, deploy, runner):
@@ -248,6 +323,15 @@ def test_deploy_task_definition_arn(get_client, runner):
     assert u'Deployment successful' in result.output
 
 
+@patch('ecs_deploy.cli.get_client')
+def test_deploy_with_timeout(get_client, runner):
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key', wait=2)
+    result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '--timeout', '1'))
+    assert result.exit_code == 1
+    assert u"Deployment failed due to timeout. Please see: " \
+           u"https://github.com/fabfuel/ecs-deploy#timeout" in result.output
+
+
 @patch('ecs_deploy.cli.get_client')
 def test_deploy_unknown_task_definition_arn(get_client, runner):
     get_client.return_value = EcsTestClient('acces_key', 'secret_key')
@@ -293,16 +377,24 @@ def test_scale(get_client, runner):
 
 @patch('ecs_deploy.cli.get_client')
 def test_scale_with_errors(get_client, runner):
-    get_client.return_value = EcsTestClient('acces_key', 'secret_key', errors=True)
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key', deployment_errors=True)
     result = runner.invoke(cli.scale, (CLUSTER_NAME, SERVICE_NAME, '2'))
     assert result.exit_code == 1
     assert u"Scaling failed" in result.output
     assert u"ERROR: Service was unable to Lorem Ipsum" in result.output
 
 
+@patch('ecs_deploy.cli.get_client')
+def test_scale_with_client_errors(get_client, runner):
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key', client_errors=True)
+    result = runner.invoke(cli.scale, (CLUSTER_NAME, SERVICE_NAME, '2'))
+    assert result.exit_code == 1
+    assert u"Something went wrong" in result.output
+
+
 @patch('ecs_deploy.cli.get_client')
 def test_scale_ignore_warnings(get_client, runner):
-    get_client.return_value = EcsTestClient('acces_key', 'secret_key', errors=True)
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key', deployment_errors=True)
     result = runner.invoke(cli.scale, (CLUSTER_NAME, SERVICE_NAME, '2', '--ignore-warnings'))
 
     assert not result.exception
@@ -318,7 +410,8 @@ def test_scale_with_timeout(get_client, runner):
     get_client.return_value = EcsTestClient('acces_key', 'secret_key', wait=2)
     result = runner.invoke(cli.scale, (CLUSTER_NAME, SERVICE_NAME, '2', '--timeout', '1'))
     assert result.exit_code == 1
-    assert u"Scaling failed (timeout)" in result.output
+    assert u"Scaling failed due to timeout. Please see: " \
+           u"https://github.com/fabfuel/ecs-deploy#timeout" in result.output
 
 
 @patch('ecs_deploy.cli.get_client')
@@ -389,8 +482,9 @@ def test_run_task_without_diff(get_client, runner):
 
 @patch('ecs_deploy.cli.get_client')
 def test_run_task_with_errors(get_client, runner):
-    get_client.return_value = EcsTestClient('acces_key', 'secret_key', errors=True)
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key', deployment_errors=True)
     result = runner.invoke(cli.run, (CLUSTER_NAME, 'test-task'))
+    assert result.exception
     assert result.exit_code == 1
     assert u"An error occurred (123) when calling the fake_error operation: Something went wrong" in result.output