Add support for no_proxy environment variable for .NET Framework targ…

…et. (#3200)

The .NET Core targets rely on the .NET runtime to process the no_proxy environment variable.

generator/.DevConfigs/C8926FB7-FC77-4478-ACC9-249E75851953.json

@@ -0,0 +1,9 @@
+{
+    "core": {
+      "changeLogMessages": [
+        "Add support for no_proxy environment variable for .NET Framework targets."
+      ],
+      "type": "patch",
+      "updateMinimum": true
+    }
+  }

sdk/src/Core/Amazon.Runtime/Pipeline/HttpHandler/_bcl/HttpWebRequestFactory.cs

@@ -16,6 +16,7 @@
 using Amazon.Runtime.Internal.Transform;
 using Amazon.Runtime.Internal.Util;
 using Amazon.Util;
+using Amazon.Util.Internal;
 using System;
 using System.Collections.Generic;
 using System.Globalization;
@@ -474,13 +475,16 @@ public virtual void ConfigureRequest(IRequestContext requestContext)
                 requestContext.Metrics.AddProperty(Metric.ProxyPort, requestContext.ClientConfig.ProxyPort);
                 _request.Proxy = proxy;
             }
-            else if (_request.RequestUri.Scheme == Uri.UriSchemeHttp)
+            else if(!NoProxyFilter.Instance.Match(_request.RequestUri))
             {
-                _request.Proxy = requestContext.ClientConfig.GetHttpProxy();
-            }
-            else if (_request.RequestUri.Scheme == Uri.UriSchemeHttps)
-            {
-                _request.Proxy = requestContext.ClientConfig.GetHttpsProxy();
+                if (_request.RequestUri.Scheme == Uri.UriSchemeHttp)
+                {
+                    _request.Proxy = requestContext.ClientConfig.GetHttpProxy();
+                }
+                else if (_request.RequestUri.Scheme == Uri.UriSchemeHttps)
+                {
+                    _request.Proxy = requestContext.ClientConfig.GetHttpsProxy();
+                }
             }
 
             // Set service point properties.

sdk/src/Core/Amazon.Util/Internal/_bcl/NoProxyFilter.cs

@@ -0,0 +1,87 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ * 
+ *  http://aws.amazon.com/apache2.0
+ * 
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+using Amazon.Runtime.Internal.Util;
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+
+namespace Amazon.Util.Internal
+{
+    /// <summary>
+    /// This type is used to check if an out going request should not use the https_proxy or http_proxy environment variable if the
+    /// host matches a pattern in the no_proxy environment variable.
+    /// 
+    /// The SDK does not do a full implementation of the no_proxy evaluation. It handles host name match with wildcard support. A full
+    /// implementation would also support DNS IP resolution and CIDR pattern matching.
+    /// </summary>
+    public class NoProxyFilter
+    {
+        internal const string NO_PROXY_ENV_NAME = "no_proxy";
+
+        public static readonly NoProxyFilter Instance = new NoProxyFilter(new EnvironmentVariableRetriever());
+
+        private IList<Regex> _proxyFilterRegex = new List<Regex>();
+
+        public NoProxyFilter(IEnvironmentVariableRetriever environmentVariableRetriever) 
+        {
+            var filters = environmentVariableRetriever.GetEnvironmentVariable(NO_PROXY_ENV_NAME)?.Split(',');
+            if (filters == null)
+            {
+                return;
+            }
+
+            foreach (var filter in filters)
+            {
+                if (string.IsNullOrEmpty(filter))
+                {
+                    continue;
+                }
+
+                try
+                {
+                    var regExPattern = string.Concat("^", Regex.Escape(filter.Trim()).Replace("\\*", ".*"), "$");
+                    var regEx = new Regex(regExPattern, RegexOptions.Compiled);
+                    _proxyFilterRegex.Add(regEx);
+                }
+                catch(ArgumentException e)
+                {
+                    // We don't have any control if users have put invalid information in the no_proxy and the SDK's implementation is not a full
+                    // implementation of no_proxy, we will skip any segements in the no_proxy env var that we fail to parse. Since no_proxy has existed
+                    // before the SDK added this support we could introduce a breaking change if we start throwing fatal exceptions at this point.
+                    Logger.GetLogger(typeof(NoProxyFilter)).Error(e, "Failed parse segment \"{0}\" in no_proxy environment variable.", filter);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Evaluates if the host in the Uri should be exluded from proxy if it matches a pattern in the no_proxy environment variable.
+        /// </summary>
+        /// <param name="uri"></param>
+        /// <returns></returns>
+        public bool Match(Uri uri)
+        {
+            var host = uri.Host;
+            foreach(var filter in _proxyFilterRegex)
+            {
+                if(filter.IsMatch(host))
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+    }
+}

sdk/src/Core/Amazon.Util/Internal/_netstandard/NoProxyFilter.cs

@@ -0,0 +1,41 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ * 
+ *  http://aws.amazon.com/apache2.0
+ * 
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+using System;
+
+
+namespace Amazon.Util.Internal
+{
+    /// <summary>
+    /// This is a stub implementation because in .NET Core we rely on the .NET runtime to evaluate the no_proxy environment variable.
+    /// </summary>
+    public class NoProxyFilter
+    {
+        public static readonly NoProxyFilter Instance = new NoProxyFilter();
+
+        public NoProxyFilter() 
+        {
+        }
+
+        /// <summary>
+        /// This is a stub method because in .NET Core we rely on the .NET runtime to evaluate the no_proxy environment variable.
+        /// </summary>
+        /// <param name="uri"></param>
+        /// <returns></returns>
+        public bool Match(Uri uri)
+        {
+            return false;
+        }
+    }
+}

sdk/src/Services/EC2/Custom/Util/_async/ImageUtilities.async.cs

@@ -30,6 +30,7 @@
 using ThirdParty.Json.LitJson;
 using Amazon.Runtime;
 using System.Threading.Tasks;
+using Amazon.Util.Internal;
 
 #pragma warning disable 1591
 
@@ -103,7 +104,7 @@ private static async Task LoadDefinitionsFromWebAsync(AmazonEC2Config ec2Config)
                     foreach (var location in DownloadLocations)
                     {
                         var useProxy = webProxy;
-                        if (useProxy == null)
+                        if (useProxy == null && !NoProxyFilter.Instance.Match(new Uri(location)))
                         {
                             if (location.StartsWith(httpPrefix))
                             {

sdk/src/Services/EC2/Custom/Util/_bcl/ImageUtilities.bcl.cs

@@ -29,6 +29,7 @@
 
 using ThirdParty.Json.LitJson;
 using Amazon.Runtime;
+using Amazon.Util.Internal;
 
 #pragma warning disable 1591
 
@@ -103,7 +104,7 @@ private static void LoadDefinitionsFromWeb(AmazonEC2Config ec2Config)
                     foreach (var location in DownloadLocations)
                     {
                         var useProxy = webProxy;
-                        if (useProxy == null)
+                        if (useProxy == null && !NoProxyFilter.Instance.Match(new Uri(location)))
                         {
                             if (location.StartsWith(httpPrefix))
                             {

sdk/src/Services/S3/Custom/AmazonS3Client.Extensions.cs

@@ -482,7 +482,7 @@ internal void ConfigureProxy(HttpWebRequest httpRequest)
             {
                 httpRequest.Proxy.Credentials = Config.ProxyCredentials;
             }
-            if (httpRequest.Proxy == null)
+            if (httpRequest.Proxy == null && !NoProxyFilter.Instance.Match(httpRequest.RequestUri))
             {
                 if (httpRequest.RequestUri.Scheme == Uri.UriSchemeHttp)
                 {

sdk/src/Services/S3/Custom/Util/AmazonS3HttpUtil.cs

@@ -15,8 +15,7 @@
 using System;
 using System.Net;
 using Amazon.Runtime;
-using Amazon.Util;
-using Amazon.Runtime.Internal.Util;
+using Amazon.Util.Internal;
 
 #if AWS_ASYNC_API
 using System.Threading.Tasks;
@@ -117,13 +116,16 @@ private static void SetProxyIfAvailableAndConfigured(IClientConfig config, HttpW
             {
                 httpWebRequest.Proxy = proxy;
             }
-            else if (httpWebRequest.RequestUri.Scheme == Uri.UriSchemeHttp)
+            else if (!NoProxyFilter.Instance.Match(httpWebRequest.RequestUri))
             {
-                httpWebRequest.Proxy = config.GetHttpProxy();
-            }
-            else if (httpWebRequest.RequestUri.Scheme == Uri.UriSchemeHttps)
-            {
-                httpWebRequest.Proxy = config.GetHttpsProxy();
+                if (httpWebRequest.RequestUri.Scheme == Uri.UriSchemeHttp)
+                {
+                    httpWebRequest.Proxy = config.GetHttpProxy();
+                }
+                else if (httpWebRequest.RequestUri.Scheme == Uri.UriSchemeHttps)
+                {
+                    httpWebRequest.Proxy = config.GetHttpsProxy();
+                }
             }
         }
     }

sdk/src/Services/SecurityToken/Custom/AmazonSecurityTokenServiceClient.Extension.cs

@@ -21,6 +21,8 @@
 using Amazon.Runtime.SharedInterfaces;
 using Amazon.SecurityToken.Model;
 using Amazon.SecurityToken.SAML;
+using Amazon.Util.Internal;
+
 #if AWS_ASYNC_API
 using System.Threading.Tasks;
 #endif
@@ -47,7 +49,7 @@ public partial class AmazonSecurityTokenServiceClient : AmazonServiceClient, IAm
             try
             {
                 var proxy = Config.GetWebProxy();
-                if (proxy == null)
+                if (proxy == null && !NoProxyFilter.Instance.Match(new Uri(endpoint)))
                 {
                     if (endpoint.StartsWith(httpPrefix))
                     {

sdk/test/UnitTests/Custom/NoProxyFilterTest.cs

@@ -0,0 +1,64 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+using Amazon.Runtime.Internal;
+using Amazon.Util.Internal;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+
+namespace AWSSDK.UnitTests
+{
+    [TestClass]
+    public class NoProxyFilterTest
+    {
+        [TestMethod]
+        [DataRow("dynamodb.*.amazonaws.com", "dynamodb.us-west-2.amazonaws.com", true)]
+        [DataRow("dynamodb.*.amazonaws.com", "ec2.us-west-2.amazonaws.com", false)]
+        [DataRow("169.254.169.254", "169.254.169.254", true)]
+        [DataRow("169.254.169.*", "169.254.169.254", true)]
+        [DataRow("www.foo.com", "www.foo.com", true)]
+        [DataRow(" www.foo.com ", "www.foo.com", true)]
+        [DataRow("*.foo.com", "www.foo.com", true)]
+        [DataRow("*foo.com", "foo.com", true)]
+        [DataRow("www.foo.com", "www.bar.com", false)]
+        [DataRow("*foo.com", "www.bar.com", false)]
+        [DataRow("*foo.com", "bar.com", false)]
+        [DataRow("", "bar.com", false)]
+        [DataRow(null, "bar.com", false)]
+        [DataRow("127.0.0.1,localhost,ssm.us-west-2.amazonaws.com", "ssm.us-west-2.amazonaws.com", true)]
+        [DataRow("127.0.0.1,localhost,ssm.us-west-2.amazonaws.com", "autoscaling.us-west-2.amazonaws.com", false)]
+        public void Match(string noProxyValue, string testUri, bool skip)
+        {
+            var filter = new NoProxyFilter(new EnvironmentVariableStub(noProxyValue));
+            Assert.AreEqual(skip, filter.Match(new Uri("https://" + testUri + "/")));
+        }
+
+        class EnvironmentVariableStub : IEnvironmentVariableRetriever
+        {
+            private readonly string _noProxyValue;
+            public EnvironmentVariableStub(string noProxyValue)
+            {
+                _noProxyValue = noProxyValue;
+            }
+
+            public string GetEnvironmentVariable(string key)
+            {
+                if (!string.Equals(key, NoProxyFilter.NO_PROXY_ENV_NAME, StringComparison.OrdinalIgnoreCase))
+                    return null;
+
+                return _noProxyValue;
+            }
+        }
+    }
+}