Merge branch 'release-v1.50.0' into main

aws · Aug 17, 2022 · c8d94aa · c8d94aa
2 parents 8505451 + 7e20cce
commit c8d94aa
Show file tree

Hide file tree

Showing 37 changed files with 806 additions and 35 deletions.
diff --git a/docs/cloudformation_compatibility.rst b/docs/cloudformation_compatibility.rst
@@ -120,13 +120,28 @@ SourceAccessConfigurations      All
 
 MSK
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-======================== ================================== ========================
-     Property Name        Intrinsic(s) Supported            Reasons
-======================== ================================== ========================
-Stream                   All
-Topics                   All
-StartingPosition         All
-======================== ================================== ========================
+================================ ================================== ========================
+     Property Name                Intrinsic(s) Supported            Reasons
+================================ ================================== ========================
+MaximumBatchingWindowInSeconds   All
+Stream                           All
+Topics                           All
+StartingPosition                 All
+ConsumerGroupId                  All
+================================ ================================== ========================
+
+SelfManagedKafka
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+================================ ================================== ========================
+     Property Name                Intrinsic(s) Supported            Reasons
+================================ ================================== ========================
+BatchSize                        All
+Topics                           All
+KafkaBootstrapServers            All
+SourceAccessConfigurations       All
+Enabled                          All
+ConsumerGroupId                  All
+================================ ================================= ========================
 
 DynamoDB
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

diff --git a/integration/combination/test_api_with_authorizer_apikey.py b/integration/combination/test_api_with_authorizer_apikey.py
@@ -1,8 +1,15 @@
+from unittest.case import skipIf
+
 from integration.helpers.base_test import BaseTest
 from integration.helpers.deployer.utils.retry import retry
 from integration.helpers.exception import StatusCodeError
 
 
+from integration.helpers.resource import current_region_does_not_support
+from integration.config.service_names import REST_API
+
+
+@skipIf(current_region_does_not_support([REST_API]), "RestApi is not supported in this testing region")
 class TestApiWithAuthorizerApiKey(BaseTest):
     def test_authorizer_apikey(self):
         self.create_and_verify_stack("combination/api_with_authorizer_apikey")
@@ -12,12 +19,9 @@ def test_authorizer_apikey(self):
         apigw_client = self.client_provider.api_client
 
         authorizers = apigw_client.get_authorizers(restApiId=rest_api_id)["items"]
-        lambda_authorizer_uri = (
-            "arn:aws:apigateway:"
-            + self.my_region
-            + ":lambda:path/2015-03-31/functions/"
-            + stack_outputs["AuthorizerFunctionArn"]
-            + "/invocations"
+
+        lambda_authorizer_uri = "arn:{}:apigateway:{}:lambda:path/2015-03-31/functions/{}/invocations".format(
+            self.partition, self.my_region, stack_outputs["AuthorizerFunctionArn"]
         )
 
         lambda_token_authorizer = get_authorizer_by_name(authorizers, "MyLambdaTokenAuth")

diff --git a/integration/combination/test_api_with_disable_execute_api_endpoint.py b/integration/combination/test_api_with_disable_execute_api_endpoint.py
@@ -1,8 +1,14 @@
+from unittest.case import skipIf
 from parameterized import parameterized
 
 from integration.helpers.base_test import BaseTest
 
 
+from integration.helpers.resource import current_region_does_not_support
+from integration.config.service_names import REST_API
+
+
+@skipIf(current_region_does_not_support([REST_API]), "RestApi is not supported in this testing region")
 class TestApiWithDisableExecuteApiEndpoint(BaseTest):
     @parameterized.expand(
         [

diff --git a/integration/combination/test_api_with_fail_on_warnings.py b/integration/combination/test_api_with_fail_on_warnings.py
@@ -1,8 +1,12 @@
+from unittest.case import skipIf
 from parameterized import parameterized
 
 from integration.helpers.base_test import BaseTest
+from integration.helpers.resource import current_region_does_not_support
+from integration.config.service_names import REST_API
 
 
+@skipIf(current_region_does_not_support([REST_API]), "RestApi is not supported in this testing region")
 class TestApiWithFailOnWarnings(BaseTest):
     @parameterized.expand(
         [

diff --git a/integration/combination/test_function_with_kinesis.py b/integration/combination/test_function_with_kinesis.py
@@ -29,6 +29,7 @@ def test_function_with_kinesis_trigger(self):
         self.assertEqual(event_source_mapping_kinesis_stream_arn, kinesis_stream["StreamARN"])
 
 
+@skipIf(current_region_does_not_support([KINESIS]), "Kinesis is not supported in this testing region")
 class TestFunctionWithKinesisIntrinsics(BaseTest):
     def test_function_with_kinesis_trigger(self):
         self.create_and_verify_stack("combination/function_with_kinesis_intrinsics")

diff --git a/integration/combination/test_function_with_self_managed_kafka.py b/integration/combination/test_function_with_self_managed_kafka.py
@@ -1,8 +1,19 @@
+from unittest.case import skipIf
+import pytest
+
 from integration.helpers.base_test import BaseTest
+from integration.helpers.resource import current_region_not_included
+
+from integration.config.service_names import SELF_MANAGED_KAFKA
 from parameterized import parameterized
 
 
+@skipIf(
+    current_region_not_included([SELF_MANAGED_KAFKA]),
+    "SelfManagedKafka testing is not performed in this testing region",
+)
 class TestFunctionWithSelfManagedKafka(BaseTest):
+    @pytest.mark.flaky(reruns=5)
     @parameterized.expand(
         [
             "combination/function_with_self_managed_kafka",

diff --git a/integration/combination/test_state_machine_with_api.py b/integration/combination/test_state_machine_with_api.py
@@ -1,7 +1,16 @@
+from unittest.case import skipIf
 from integration.helpers.base_test import BaseTest
 from integration.helpers.common_api import get_policy_statements
 
 
+from integration.helpers.resource import current_region_does_not_support
+from integration.config.service_names import STATE_MACHINE_WITH_APIS
+
+
+@skipIf(
+    current_region_does_not_support([STATE_MACHINE_WITH_APIS]),
+    "StateMachine with APIs is not supported in this testing region",
+)
 class TestStateMachineWithApi(BaseTest):
     def test_state_machine_with_api(self):
         self.create_and_verify_stack("combination/state_machine_with_api")

diff --git a/integration/combination/test_state_machine_with_cwe.py b/integration/combination/test_state_machine_with_cwe.py
@@ -1,7 +1,15 @@
+from unittest.case import skipIf
 from integration.helpers.base_test import BaseTest
 from integration.helpers.common_api import get_policy_statements
 
+from integration.helpers.resource import current_region_does_not_support
+from integration.config.service_names import STATE_MACHINE_CWE_CWS
 
+
+@skipIf(
+    current_region_does_not_support([STATE_MACHINE_CWE_CWS]),
+    "StateMachineCWECWS is not supported in this testing region",
+)
 class TestStateMachineWithCwe(BaseTest):
     def test_state_machine_with_cwe(self):
         self.create_and_verify_stack("combination/state_machine_with_cwe")

diff --git a/integration/combination/test_state_machine_with_cwe_dlq_and_retry_policy.py b/integration/combination/test_state_machine_with_cwe_dlq_and_retry_policy.py
@@ -2,10 +2,13 @@
 
 from integration.helpers.base_test import BaseTest
 from integration.helpers.resource import current_region_does_not_support
-from integration.config.service_names import CWE_CWS_DLQ
+from integration.config.service_names import CWE_CWS_DLQ, STATE_MACHINE_CWE_CWS
 
 
-@skipIf(current_region_does_not_support([CWE_CWS_DLQ]), "CweCwsDlq is not supported in this testing region")
+@skipIf(
+    current_region_does_not_support([CWE_CWS_DLQ, STATE_MACHINE_CWE_CWS]),
+    "CweCwsDlq is not supported in this testing region",
+)
 class TestStateMachineWithCweDlqAndRetryPolicy(BaseTest):
     def test_state_machine_with_api(self):
         self.create_and_verify_stack("combination/state_machine_with_cwe_with_dlq_and_retry_policy")

diff --git a/integration/combination/test_state_machine_with_cwe_dlq_generated.py b/integration/combination/test_state_machine_with_cwe_dlq_generated.py
@@ -3,10 +3,13 @@
 from integration.helpers.base_test import BaseTest
 from integration.helpers.common_api import get_policy_statements, get_queue_policy
 from integration.helpers.resource import current_region_does_not_support
-from integration.config.service_names import CWE_CWS_DLQ
+from integration.config.service_names import CWE_CWS_DLQ, STATE_MACHINE_CWE_CWS
 
 
-@skipIf(current_region_does_not_support([CWE_CWS_DLQ]), "CweCwsDlq is not supported in this testing region")
+@skipIf(
+    current_region_does_not_support([CWE_CWS_DLQ, STATE_MACHINE_CWE_CWS]),
+    "CweCwsDlq is not supported in this testing region",
+)
 class TestStateMachineWithCweDlqGenerated(BaseTest):
     def test_state_machine_with_cwe(self):
         self.create_and_verify_stack("combination/state_machine_with_cwe_dlq_generated")

diff --git a/integration/combination/test_state_machine_with_schedule.py b/integration/combination/test_state_machine_with_schedule.py
@@ -1,7 +1,16 @@
+from unittest.case import skipIf
 from integration.helpers.base_test import BaseTest
 from integration.helpers.common_api import get_policy_statements
 
 
+from integration.helpers.resource import current_region_does_not_support
+from integration.config.service_names import STATE_MACHINE_CWE_CWS
+
+
+@skipIf(
+    current_region_does_not_support([STATE_MACHINE_CWE_CWS]),
+    "StateMachine CweCws is not supported in this testing region",
+)
 class TestStateMachineWithSchedule(BaseTest):
     def test_state_machine_with_schedule(self):
         self.create_and_verify_stack("combination/state_machine_with_schedule")

diff --git a/integration/config/service_names.py b/integration/config/service_names.py
@@ -24,3 +24,12 @@
 ARM = "ARM"
 EFS = "EFS"
 S3_EVENTS = "S3Events"
+SELF_MANAGED_KAFKA = "SelfManagedKafka"
+SECRET_MANAGER = "SecretsManager"
+STATE_MACHINE_INLINE_DEFINITION = "StateMachineInlineDefinition"
+STATE_MACHINE_CWE_CWS = "StateMachineCweCws"
+STATE_MACHINE_WITH_APIS = "StateMachineWithApis"
+LAMBDA_URL = "LambdaUrl"
+LAMBDA_ENV_VARS = "LambdaEnvVars"
+EVENT_INVOKE_CONFIG = "EventInvokeConfig"
+EPHEMERAL_STORAGE = "EphemeralStorage"
diff --git a/integration/helpers/base_test.py b/integration/helpers/base_test.py
@@ -72,6 +72,7 @@ def setUpClass(cls):
         cls.code_dir = Path(cls.resources_dir, "code")
         cls.session = boto3.session.Session()
         cls.my_region = cls.session.region_name
+        cls.partition = cls.session.get_partition_for_region(cls.my_region)
         cls.client_provider = ClientProvider()
         cls.file_to_s3_uri_map = read_test_config_file("file_to_s3_map_modified.json")
         cls.code_key_to_file = read_test_config_file("code_key_to_file_map.json")

diff --git a/integration/single/test_basic_function.py b/integration/single/test_basic_function.py
@@ -3,7 +3,16 @@
 
 import pytest
 
-from integration.config.service_names import KMS, XRAY, ARM, CODE_DEPLOY, HTTP_API
+from integration.config.service_names import (
+    EVENT_INVOKE_CONFIG,
+    KMS,
+    LAMBDA_URL,
+    XRAY,
+    ARM,
+    CODE_DEPLOY,
+    HTTP_API,
+    EPHEMERAL_STORAGE,
+)
 from integration.helpers.resource import current_region_does_not_support
 from parameterized import parameterized
 from integration.helpers.base_test import BaseTest
@@ -77,7 +86,7 @@ def test_basic_function_with_architecture(self, file_name, architecture):
             ("single/basic_function_with_function_url_with_autopuplishalias", "live"),
         ]
     )
-    @skipIf(current_region_does_not_support(["Url"]), "Url is not supported in this testing region")
+    @skipIf(current_region_does_not_support([LAMBDA_URL]), "Lambda Url is not supported in this testing region")
     def test_basic_function_with_url_config(self, file_name, qualifier):
         """
         Creates a basic lambda function with Function Url enabled
@@ -172,6 +181,10 @@ def test_basic_function_with_tags(self):
         self.assertTrue("TagKey2" in tags)
         self.assertEqual(tags["TagKey2"], "")
 
+    @skipIf(
+        current_region_does_not_support([EVENT_INVOKE_CONFIG]),
+        "EventInvokeConfig is not supported in this testing region",
+    )
     def test_basic_function_event_destinations(self):
         """
         Creates a basic lambda function with event destinations
@@ -250,6 +263,10 @@ def test_basic_function_with_tracing(self):
             "single/function_with_ephemeral_storage",
         ]
     )
+    @skipIf(
+        current_region_does_not_support([EPHEMERAL_STORAGE]),
+        "Lambda ephemeral storage is not supported in this testing region",
+    )
     def test_function_with_ephemeral_storage(self, file_name):
         """
         Creates a basic function with ephemeral storage

diff --git a/integration/single/test_basic_state_machine.py b/integration/single/test_basic_state_machine.py
@@ -2,20 +2,25 @@
 
 from integration.helpers.base_test import BaseTest
 from integration.helpers.resource import current_region_does_not_support
+from integration.config.service_names import XRAY, STATE_MACHINE_INLINE_DEFINITION
 
 
 class TestBasicLayerVersion(BaseTest):
     """
     Basic AWS::Serverless::StateMachine tests
     """
 
+    @skipIf(
+        current_region_does_not_support([STATE_MACHINE_INLINE_DEFINITION]),
+        "StateMachine with inline definition is not supported in this testing region",
+    )
     def test_basic_state_machine_inline_definition(self):
         """
         Creates a State Machine from inline definition
         """
         self.create_and_verify_stack("single/basic_state_machine_inline_definition")
 
-    @skipIf(current_region_does_not_support(["XRay"]), "XRay is not supported in this testing region")
+    @skipIf(current_region_does_not_support([XRAY]), "XRay is not supported in this testing region")
     def test_basic_state_machine_with_tags(self):
         """
         Creates a State Machine with tags

diff --git a/integration/single/test_function_with_http_api_and_auth.py b/integration/single/test_function_with_http_api_and_auth.py
@@ -1,27 +1,19 @@
-import logging
 from unittest.case import skipIf
 
-from tenacity import stop_after_attempt, retry_if_exception_type, after_log, wait_exponential, retry, wait_random
+import pytest
 
 from integration.helpers.base_test import BaseTest
 from integration.helpers.resource import current_region_does_not_support
+from integration.config.service_names import HTTP_API
 
-LOG = logging.getLogger(__name__)
 
-
-@skipIf(current_region_does_not_support(["HttpApi"]), "HttpApi is not supported in this testing region")
+@skipIf(current_region_does_not_support([HTTP_API]), "HttpApi is not supported in this testing region")
 class TestFunctionWithHttpApiAndAuth(BaseTest):
     """
     AWS::Lambda::Function tests with http api events and auth
     """
 
-    @retry(
-        stop=stop_after_attempt(5),
-        wait=wait_exponential(multiplier=1, min=4, max=10) + wait_random(0, 1),
-        retry=retry_if_exception_type(AssertionError),
-        after=after_log(LOG, logging.WARNING),
-        reraise=True,
-    )
+    @pytest.mark.flaky(reruns=5)
     def test_function_with_http_api_and_auth(self):
         # If the request is not signed, which none of the below are, IAM will respond with a "Forbidden" message.
         # We are not testing that IAM auth works here, we are simply testing if it was applied.

diff --git a/samtranslator/__init__.py b/samtranslator/__init__.py
@@ -1 +1 @@
-__version__ = "1.49.0"
+__version__ = "1.50.0"
diff --git a/samtranslator/model/eventsources/pull.py b/samtranslator/model/eventsources/pull.py
@@ -46,6 +46,7 @@ class PullEventSource(ResourceMacro):
         "FunctionResponseTypes": PropertyType(False, is_type(list)),
         "KafkaBootstrapServers": PropertyType(False, is_type(list)),
         "FilterCriteria": PropertyType(False, is_type(dict)),
+        "ConsumerGroupId": PropertyType(False, is_str()),
     }
 
     def get_policy_arn(self):
@@ -112,6 +113,17 @@ def to_cloudformation(self, **kwargs):
             lambda_eventsourcemapping.SelfManagedEventSource = {
                 "Endpoints": {"KafkaBootstrapServers": self.KafkaBootstrapServers}
             }
+        if self.ConsumerGroupId:
+            consumer_group_id_structure = {"ConsumerGroupId": self.ConsumerGroupId}
+            if self.resource_type == "MSK":
+                lambda_eventsourcemapping.AmazonManagedKafkaConfig = consumer_group_id_structure
+            elif self.resource_type == "SelfManagedKafka":
+                lambda_eventsourcemapping.SelfManagedKafkaConfig = consumer_group_id_structure
+            else:
+                raise InvalidEventException(
+                    self.logical_id,
+                    "Property ConsumerGroupId not defined for resource of type {}.".format(self.resource_type),
+                )
 
         destination_config_policy = None
         if self.DestinationConfig:

diff --git a/samtranslator/model/lambda_.py b/samtranslator/model/lambda_.py
@@ -81,6 +81,8 @@ class LambdaEventSourceMapping(Resource):
         "FunctionResponseTypes": PropertyType(False, is_type(list)),
         "SelfManagedEventSource": PropertyType(False, is_type(dict)),
         "FilterCriteria": PropertyType(False, is_type(dict)),
+        "AmazonManagedKafkaConfig": PropertyType(False, is_type(dict)),
+        "SelfManagedKafkaConfig": PropertyType(False, is_type(dict)),
     }
 
     runtime_attrs = {"name": lambda self: ref(self.logical_id)}