From ec68c1145a89753e7d42458bbea86737cad6090f Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 11 Aug 2021 21:38:08 +0530 Subject: [PATCH] feat: Updated `BigQueryRetryAlgorithm` so that it can retry on RateLimit Errors using RegEx (#1499) * Modified RATE_LIMIT_ERROR_MSG and added testRateLimitRegEx for testing regex matching * Added JOB_RATE_LIMIT_EXCEEDED_MSG and RATE_LIMIT_EXCEEDED_REGEX pattern * Implemented retryOnRegEx to include retry on RegEx patterns * Updated DEFAULT_RETRY_CONFIG to include retry on JOB_RATE_LIMIT_EXCEEDED_MSG and RATE_LIMIT_EXCEEDED_REGEX * Modified shouldRetryBasedOnBigQueryRetryConfig method to try RegEx matching if all the contains checks fail --- .../cloud/bigquery/BigQueryErrorMessages.java | 5 +++ .../google/cloud/bigquery/BigQueryImpl.java | 4 ++- .../bigquery/BigQueryRetryAlgorithm.java | 21 ++++++++++++- .../cloud/bigquery/BigQueryRetryConfig.java | 14 +++++++++ .../cloud/bigquery/BigQueryImplTest.java | 31 ++++++++++++++++--- 5 files changed, 69 insertions(+), 6 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryErrorMessages.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryErrorMessages.java index 04cabfc67..5c86e0806 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryErrorMessages.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryErrorMessages.java @@ -19,4 +19,9 @@ public class BigQueryErrorMessages { public static final String RATE_LIMIT_EXCEEDED_MSG = "Exceeded rate limits:"; // Error Message for RateLimitExceeded Error + public static final String JOB_RATE_LIMIT_EXCEEDED_MSG = "Job exceeded rate limits:"; + + public class RetryRegExPatterns { + public static final String RATE_LIMIT_EXCEEDED_REGEX = ".*exceed.*rate.*limit.*"; + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index c871eb551..03635a89e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -240,7 +240,9 @@ public Page getNextPage() { private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG = BigQueryRetryConfig.newBuilder() .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG) - .build(); // retry config with Error Message for RateLimitExceeded Error + .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG) + .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX) + .build(); // retry config with Error Messages and RegEx for RateLimitExceeded Error BigQueryImpl(BigQueryOptions options) { super(options); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java index 1a75a9c08..c694e2c06 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java @@ -27,6 +27,7 @@ import com.google.api.gax.retrying.TimedRetryAlgorithmWithContext; import java.util.Iterator; import java.util.concurrent.CancellationException; +import java.util.regex.Pattern; public class BigQueryRetryAlgorithm extends RetryAlgorithm { private final BigQueryRetryConfig bigQueryRetryConfig; @@ -69,10 +70,23 @@ private boolean shouldRetryBasedOnBigQueryRetryConfig( */ String errorDesc; if (previousThrowable != null && (errorDesc = previousThrowable.getMessage()) != null) { + errorDesc = errorDesc.toLowerCase(); // for case insensitive comparison for (Iterator retriableMessages = bigQueryRetryConfig.getRetriableErrorMessages().iterator(); retriableMessages.hasNext(); ) { - if (errorDesc.contains(retriableMessages.next())) { // Error message should be retried + if (errorDesc.contains( + retriableMessages + .next() + .toLowerCase())) { // Error message should be retried, implementing cases + // insensitive match + return true; + } + } + // Check if there's a regex which matches the error message. This avoids too many regex + // matches which is expensive + for (Iterator retriableRegExes = bigQueryRetryConfig.getRetriableRegExes().iterator(); + retriableRegExes.hasNext(); ) { + if (matchRegEx(retriableRegExes.next(), errorDesc)) { return true; } } @@ -80,6 +94,11 @@ private boolean shouldRetryBasedOnBigQueryRetryConfig( return false; } + public static boolean matchRegEx( + String retriableRegEx, String errorDesc) { // cases insensitive match regex matching + return Pattern.matches(retriableRegEx.toLowerCase(), errorDesc.toLowerCase()); + } + /*Duplicating this method as it can not be inherited from the RetryAlgorithm due to the default access modifier*/ boolean shouldRetryBasedOnResult( RetryingContext context, Throwable previousThrowable, ResponseT previousResponse) { diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryConfig.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryConfig.java index 7e28e5707..2e1f7c0bd 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryConfig.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryConfig.java @@ -21,18 +21,25 @@ public class BigQueryRetryConfig { private final ImmutableSet retriableErrorMessages; + private final ImmutableSet retriableRegExes; private BigQueryRetryConfig(Builder builder) { retriableErrorMessages = builder.retriableErrorMessages.build(); + retriableRegExes = builder.retriableRegExes.build(); } public ImmutableSet getRetriableErrorMessages() { return retriableErrorMessages; } + public ImmutableSet getRetriableRegExes() { + return retriableRegExes; + } + // BigQueryRetryConfig builder public static class Builder { private final ImmutableSet.Builder retriableErrorMessages = ImmutableSet.builder(); + private final ImmutableSet.Builder retriableRegExes = ImmutableSet.builder(); private Builder() {} @@ -43,6 +50,13 @@ public final Builder retryOnMessage(String... errorMessages) { return this; } + public final Builder retryOnRegEx(String... regExPatterns) { + for (String regExPattern : regExPatterns) { + retriableRegExes.add(checkNotNull(regExPattern)); + } + return this; + } + public BigQueryRetryConfig build() { return new BigQueryRetryConfig(this); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java index 654cc0266..a12800c87 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java @@ -508,11 +508,12 @@ public class BigQueryImplTest { .setEtag(ETAG) .setVersion(1) .build(); - private BigQueryOptions options; private BigQueryRpcFactory rpcFactoryMock; private BigQueryRpc bigqueryRpcMock; private BigQuery bigquery; + private static final String RATE_LIMIT_ERROR_MSG = + "Job exceeded rate limits: Your table exceeded quota for table update operations. For more information, see https://cloud.google.com/bigquery/docs/troubleshoot-quotas"; @Captor private ArgumentCaptor> capturedOptions; @Captor private ArgumentCaptor jobCapture; @@ -2439,9 +2440,7 @@ public void testFastQueryRateLimitIdempotency() throws Exception { .thenThrow(new BigQueryException(504, "Gateway Timeout")) .thenThrow( new BigQueryException( - 400, - BigQueryErrorMessages - .RATE_LIMIT_EXCEEDED_MSG)) // retrial on based on RATE_LIMIT_EXCEEDED_MSG + 400, RATE_LIMIT_ERROR_MSG)) // retrial on based on RATE_LIMIT_EXCEEDED_MSG .thenReturn(responsePb); bigquery = @@ -2470,6 +2469,30 @@ public void testFastQueryRateLimitIdempotency() throws Exception { verify(bigqueryRpcMock, times(6)).queryRpc(eq(PROJECT), requestPbCapture.capture()); } + @Test + public void testRateLimitRegEx() throws Exception { + String msg2 = + "Job eceeded rate limits: Your table exceeded quota for table update operations. For more information, see https://cloud.google.com/bigquery/docs/troubleshoot-quotas"; + String msg3 = "exceeded rate exceeded quota for table update"; + String msg4 = "exceeded rate limits"; + assertTrue( + BigQueryRetryAlgorithm.matchRegEx( + BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX, + RATE_LIMIT_ERROR_MSG)); + assertFalse( + BigQueryRetryAlgorithm.matchRegEx( + BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX, + msg2.toLowerCase())); + assertFalse( + BigQueryRetryAlgorithm.matchRegEx( + BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX, + msg3.toLowerCase())); + assertTrue( + BigQueryRetryAlgorithm.matchRegEx( + BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX, + msg4.toLowerCase())); + } + @Test public void testFastQueryDDLShouldRetry() throws Exception { com.google.api.services.bigquery.model.QueryResponse responsePb =