ci: increase default process timeout for tests and update image valid…

…ation after build (aws#3374)

* increase default process timeout for tests and update image validation after build

* add images information to assertion message

* fix: update tag checking and replace "verify_pulling_only_latest_tag" with "verify_pulled_image"

tests/integration/buildcmd/build_integ_base.py

@@ -13,7 +13,7 @@
 import jmespath
 from pathlib import Path
 
-from samcli.lib.utils.architecture import X86_64
+from samcli.lib.utils.architecture import X86_64, has_runtime_multi_arch_image
 from samcli.local.docker.lambda_build_container import LambdaBuildContainer
 from samcli.yamlhelper import yaml_parse
 from tests.testing_utils import IS_WINDOWS, run_command, SKIP_DOCKER_TESTS, SKIP_DOCKER_MESSAGE, SKIP_DOCKER_BUILD
@@ -135,23 +135,24 @@ def verify_docker_container_cleanedup(self, runtime):
         )
         self.assertFalse(bool(samcli_containers), "Build containers have not been removed")
 
-    def verify_pulled_image(self, runtime, architecture):
+    def verify_pulled_image(self, runtime, architecture=X86_64):
         docker_client = docker.from_env()
         image_name = f"{LambdaBuildContainer._IMAGE_URI_PREFIX}-{runtime}"
         images = docker_client.images.list(name=image_name)
-        arch = architecture if architecture else X86_64
+        architecture = architecture if architecture and "provided" not in runtime else X86_64
+        tag_name = LambdaBuildContainer.get_image_tag(architecture)
         self.assertGreater(
             len(images),
             0,
             f"Image {image_name} was not pulled",
         )
-        self.assertEqual(
+        self.assertIn(
             len(images),
-            1,
-            f"Other version of the build image {image_name} was pulled",
+            [1, 2],
+            f"Other version of the build image {image_name} was pulled. Currently pulled images: {images}",
         )
-        image_tag = f"{image_name}:latest-{arch}"
-        for t in images[0].tags:
+        image_tag = f"{image_name}:{tag_name}"
+        for t in [tag for image in images for tag in image.tags]:
             if t == image_tag:
                 # Found, pass
                 return

tests/integration/buildcmd/test_build_cmd.py

@@ -37,8 +37,6 @@
 
 LOG = logging.getLogger(__name__)
 
-TIMEOUT = 420  # 7 mins
-
 # SAR tests require credentials. This is to skip running the test where credentials are not available.
 SKIP_SAR_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY
 
@@ -760,7 +758,7 @@ def test_build_single_function(self, runtime, use_container, function_identifier
 
         if use_container:
             self.verify_docker_container_cleanedup(runtime)
-            self.verify_pulling_only_latest_tag(runtime)
+            self.verify_pulled_image(runtime)
 
     def _verify_built_artifact(self, build_dir, function_logical_id, expected_files):
         self.assertTrue(build_dir.exists(), "Build directory should be created")
@@ -891,7 +889,7 @@ def test_build_function_and_layer(self, runtime, use_container):
             )
         if use_container:
             self.verify_docker_container_cleanedup(runtime)
-            self.verify_pulling_only_latest_tag(runtime)
+            self.verify_pulled_image(runtime)
 
     @parameterized.expand([("python3.7", False), ("python3.7", "use_container")])
     def test_build_function_with_dependent_layer(self, runtime, use_container):
@@ -928,7 +926,7 @@ def test_build_function_with_dependent_layer(self, runtime, use_container):
             )
         if use_container:
             self.verify_docker_container_cleanedup(runtime)
-            self.verify_pulling_only_latest_tag(runtime)
+            self.verify_pulled_image(runtime)
 
     def _verify_built_artifact(
         self, build_dir, resource_logical_id, expected_files, code_property_name, artifact_subfolder=""
@@ -1050,7 +1048,7 @@ def test_with_makefile_builder_specified_python_runtime(self, use_container, man
 
         if use_container:
             self.verify_docker_container_cleanedup(runtime)
-            self.verify_pulling_only_latest_tag(runtime)
+            self.verify_pulled_image(runtime)
 
     @parameterized.expand([(False,), ("use_container")])
     @pytest.mark.flaky(reruns=3)
@@ -1087,7 +1085,7 @@ def test_with_native_builder_specified_python_runtime(self, use_container):
 
         if use_container:
             self.verify_docker_container_cleanedup(runtime)
-            self.verify_pulling_only_latest_tag(runtime)
+            self.verify_pulled_image(runtime)
 
     @parameterized.expand([(False,), ("use_container")])
     @pytest.mark.flaky(reruns=3)
@@ -1440,7 +1438,7 @@ def test_inline_not_built(self, use_container):
 
         if use_container:
             self.verify_docker_container_cleanedup("python3.7")
-            self.verify_pulling_only_latest_tag("python3.7")
+            self.verify_pulled_image("python3.7")
 
     def _verify_built_artifact(self, build_dir):
         self.assertTrue(build_dir.exists(), "Build directory should be created")
@@ -1490,7 +1488,7 @@ def test_json_env_vars_passed(self, use_container, env_vars_file):
 
         if use_container:
             self.verify_docker_container_cleanedup("python3.7")
-            self.verify_pulling_only_latest_tag("python3.7")
+            self.verify_pulled_image("python3.7")
 
     @staticmethod
     def get_env_file(filename):
@@ -1539,7 +1537,7 @@ def test_inline_env_vars_passed(self, use_container, inline_env_var):
 
         if use_container:
             self.verify_docker_container_cleanedup("python3.7")
-            self.verify_pulling_only_latest_tag("python3.7")
+            self.verify_pulled_image("python3.7")
 
     def _verify_built_env_var(self, build_dir):
         self.assertTrue(build_dir.exists(), "Build directory should be created")

tests/integration/delete/test_delete_command.py

@@ -24,7 +24,6 @@
 # This is to restrict package tests to run outside of CI/CD, when the branch is not master or tests are not run by Canary
 SKIP_DELETE_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY
 CFN_SLEEP = 3
-TIMEOUT = 300
 CFN_PYTHON_VERSION_SUFFIX = os.environ.get("PYTHON_VERSION", "0.0.0").replace(".", "-")
 
 

tests/integration/deploy/test_deploy_command.py

@@ -24,7 +24,6 @@
 # This is to restrict package tests to run outside of CI/CD, when the branch is not master or tests are not run by Canary
 SKIP_DEPLOY_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY
 CFN_SLEEP = 3
-TIMEOUT = 300
 CFN_PYTHON_VERSION_SUFFIX = os.environ.get("PYTHON_VERSION", "0.0.0").replace(".", "-")
 
 

tests/integration/deploy/test_managed_stack_deploy.py

@@ -25,7 +25,6 @@
 
 CFN_PYTHON_VERSION_SUFFIX = PYTHON_VERSION.replace(".", "-")
 CFN_SLEEP = 3
-TIMEOUT = 300
 # Set region for managed stacks to be in a different region than the ones in deploy
 DEFAULT_REGION = "us-west-2"
 

tests/testing_utils.py

@@ -26,7 +26,7 @@
 LOG = logging.getLogger(__name__)
 
 CommandResult = namedtuple("CommandResult", "process stdout stderr")
-TIMEOUT = 300
+TIMEOUT = 600
 
 
 def run_command(command_list, cwd=None, env=None, timeout=TIMEOUT) -> CommandResult: