SE-1752 Added retry upon token refresh/get failure

sdk/Lusid.Sdk.Tests/TokenProviderTests.cs

@@ -101,5 +101,33 @@ public void CanUseTokenProviderConfiguration()
             var __ = config.AccessToken;
             mockTokenProvider.Verify(x => x.GetAuthenticationTokenAsync(), Times.Exactly(2));
         }
+        [Test]
+        public async Task CanGetNewTokenWhenRefreshTokenExpired()
+        {
+            var provider = new ClientCredentialsFlowTokenProvider(GetConfig());
+            var _ = await provider.GetAuthenticationTokenAsync();
+            var firstTokenDetails = provider.GetLastToken();
+
+            Assert.That(firstTokenDetails.RefreshToken, Is.Not.Null.And.Not.Empty, "refresh_token not returned so unable to verify refresh behaviour.");
+
+            Console.WriteLine($"Token expiring at {firstTokenDetails.ExpiresOn:o}");
+
+            // WHEN we pretend to delay until both...
+            // (1) the original token has expired (for expediency update the expires_on on the token)
+            provider.ExpireToken();
+            provider.ExpireRefreshToken();
+            // (2) the refresh token has expired (for expediency update the refresh_token to an invalid value that will not be found)
+            var oldToken = provider.GetLastToken().Token;
+            provider.GetLastToken().RefreshToken = "InvalidRefreshToken";
+            provider.GetLastToken().Token = "invalidToken";
+
+            Assert.That(DateTimeOffset.UtcNow, Is.GreaterThan(firstTokenDetails.ExpiresOn));
+            var refreshedToken = await provider.GetAuthenticationTokenAsync();
+
+            // THEN it should be populated, and the ExpiresOn should be in the future
+            Assert.That(refreshedToken, Is.Not.Empty);
+            Assert.That(provider.GetLastToken().ExpiresOn, Is.GreaterThan(DateTimeOffset.UtcNow));
+            Assert.That(oldToken != provider.GetLastToken().Token);
+        }
     }
 }

sdk/Lusid.Sdk/Utilities/ClientCredentialsFlowTokenProvider.cs

@@ -3,8 +3,11 @@
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Runtime.CompilerServices;
+using System.Threading;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
+using Polly;
+using Polly.Retry;
 
 [assembly: InternalsVisibleTo("Lusid.Sdk.Tests")]
 
@@ -32,18 +35,22 @@ public interface ITokenProvider
     public class ClientCredentialsFlowTokenProvider : ITokenProvider
     {
         private readonly ApiConfiguration _apiConfig;
+        private const int RefreshExpires = 5400; // Refresh token expires in 90 minutes - Speak to Xan if you think this has changed
+        private const string ExpireMessage = "refresh token is invalid or expired";
 
         internal class AuthenticationToken
         {
-            public AuthenticationToken(string token, DateTimeOffset expiresOn, string refreshToken)
+            public AuthenticationToken(string token, DateTimeOffset expiresOn, string refreshToken, DateTimeOffset refreshExpiresOn)
             {
                 Token = token;
                 ExpiresOn = expiresOn;
                 RefreshToken = refreshToken;
+                RefreshExpiresOn = refreshExpiresOn;
             }
-            public string Token { get; }
+            public string Token { get; internal set; }
             public DateTimeOffset ExpiresOn { get; internal set; }
-            public string RefreshToken { get; }
+            public string RefreshToken { get; internal set; }
+            public DateTimeOffset RefreshExpiresOn { get; internal set; }
         }
 
 
@@ -60,19 +67,37 @@ public ClientCredentialsFlowTokenProvider(ApiConfiguration configuration)
         /// <inheritdoc />
         public async Task<string> GetAuthenticationTokenAsync()
         {
-            if (_lastIssuedToken == null || _lastIssuedToken.ExpiresOn < DateTimeOffset.UtcNow)
+             var policy = 
+                Policy
+                    .Handle<HttpRequestException>()
+                    .WaitAndRetryAsync(5, retryAttempt =>
+                        TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), OnRetry);
+
+            return await policy.ExecuteAsync(context => GetAuthenticationTokenAsyncInternal(), CancellationToken.None);
+
+            async Task<string> GetAuthenticationTokenAsyncInternal()
             {
-                if (_lastIssuedToken?.RefreshToken != null)
+                if (_lastIssuedToken == null || _lastIssuedToken.ExpiresOn < DateTimeOffset.UtcNow)
                 {
-                    _lastIssuedToken = await RefreshToken(_apiConfig, _lastIssuedToken.RefreshToken);
-                }
-                else
-                {
-                    _lastIssuedToken = await GetNewToken(_apiConfig);
+                    if (_lastIssuedToken?.RefreshToken != null && _lastIssuedToken?.RefreshExpiresOn > DateTimeOffset.UtcNow)
+                    {
+                        _lastIssuedToken = await RefreshToken(_apiConfig, _lastIssuedToken.RefreshToken);
+                    }
+                    else
+                    {
+                        _lastIssuedToken = await GetNewToken(_apiConfig);
+                    }
                 }
+                return _lastIssuedToken.Token;
+            }
+        }
+
+        private void OnRetry(Exception arg1, TimeSpan arg2)
+        {
+            if (arg1.Message.ToLower().Contains(ExpireMessage))
+            {
+                ExpireRefreshToken();
             }
-
-            return _lastIssuedToken.Token;
         }
 
         /// <inheritdoc />
@@ -139,8 +164,12 @@ private static async Task<AuthenticationToken> GetNewToken(ApiConfiguration apiC
                 {
                     throw new InvalidOperationException("Failed to parse expires_in: " + expires);
                 }
+
+                // expiration is shorten to overcome a race condition where the token is still valid when retrieved from cache but expired when used
+                DateTimeOffset refreshExpiresAt;
+                refreshExpiresAt = DateTimeOffset.UtcNow.AddSeconds(RefreshExpires - 30);
 
-                return new AuthenticationToken(apiToken, expiresAt, refresh_token);
+                return new AuthenticationToken(apiToken, expiresAt, refresh_token, refreshExpiresAt);
             }
         }
 
@@ -199,8 +228,12 @@ private static async Task<AuthenticationToken> RefreshToken(ApiConfiguration api
                 {
                     throw new InvalidOperationException("Failed to parse expires_in: " + expires);
                 }
-
-                return new AuthenticationToken(apiToken, expiresAt, refresh_token);
+
+                // expiration is shorten to overcome a race condition where the token is still valid when retrieved from cache but expired when used
+                DateTimeOffset refreshExpiresAt;
+                refreshExpiresAt = DateTimeOffset.UtcNow.AddSeconds(RefreshExpires - 30);
+
+                return new AuthenticationToken(apiToken, expiresAt, refresh_token, refreshExpiresAt);
             }
         }
 
@@ -222,5 +255,11 @@ internal void ExpireToken()
         {
             _lastIssuedToken.ExpiresOn = DateTimeOffset.UtcNow.AddSeconds(-1);
         }
+
+        // todo For test purposes only, to be removed once upgrade to IHttpClientFactory
+        internal void ExpireRefreshToken()
+        {
+            _lastIssuedToken.RefreshExpiresOn = DateTimeOffset.UtcNow.AddSeconds(-1);
+        }
     }
 }