Merge pull request #411 from blackducksoftware/dev/idetect-4068-retry…

…-after Enhance retry mechanism for scan starts when BlackDuck specifies retry-after header
blackducksoftware · Oct 6, 2023 · e3f16f1 · e3f16f1
2 parents da222d6 + ccd5dc0
commit e3f16f1
Show file tree

Hide file tree

Showing 11 changed files with 109 additions and 48 deletions.
diff --git a/src/main/java/com/synopsys/integration/blackduck/bdio2/Bdio2FileUploadService.java b/src/main/java/com/synopsys/integration/blackduck/bdio2/Bdio2FileUploadService.java
@@ -48,18 +48,18 @@ public Bdio2FileUploadService(
         this.bdio2RetryAwareStreamUploader = bdio2RetryAwareStreamUploader;
     }
 
-    public Bdio2UploadResult uploadFile(UploadTarget uploadTarget, long timeout) throws IntegrationException, InterruptedException {
-        return uploadFile(uploadTarget, timeout, true, true);
+    public Bdio2UploadResult uploadFile(UploadTarget uploadTarget, long timeout, long clientStartTime) throws IntegrationException, InterruptedException {
+        return uploadFile(uploadTarget, timeout, true, true, clientStartTime);
     }
 
-    public Bdio2UploadResult uploadFile(UploadTarget uploadTarget, long timeout, boolean shouldUploadEntries, boolean shouldFinishUpload)
+    public Bdio2UploadResult uploadFile(UploadTarget uploadTarget, long timeout, boolean shouldUploadEntries, boolean shouldFinishUpload, long clientStartTime)
         throws IntegrationException, InterruptedException {
         logger.debug(String.format("Uploading BDIO file %s", uploadTarget.getUploadFile()));
         List<BdioFileContent> bdioFileContentList = bdio2Extractor.extractContent(uploadTarget.getUploadFile());
-        return uploadFiles(bdioFileContentList, uploadTarget.getProjectAndVersion().orElse(null), timeout, shouldUploadEntries, shouldFinishUpload);
+        return uploadFiles(bdioFileContentList, uploadTarget.getProjectAndVersion().orElse(null), timeout, shouldUploadEntries, shouldFinishUpload, clientStartTime);
     }
 
-    private Bdio2UploadResult uploadFiles(List<BdioFileContent> bdioFiles, @Nullable NameVersion nameVersion, long timeout, boolean shouldUploadEntries, boolean shouldFinishUpload)
+    private Bdio2UploadResult uploadFiles(List<BdioFileContent> bdioFiles, @Nullable NameVersion nameVersion, long timeout, boolean shouldUploadEntries, boolean shouldFinishUpload, long clientStartTime)
         throws IntegrationException, InterruptedException {
         if (bdioFiles.isEmpty()) {
             throw new IllegalArgumentException("BDIO files cannot be empty.");
@@ -85,7 +85,7 @@ private Bdio2UploadResult uploadFiles(List<BdioFileContent> bdioFiles, @Nullable
 
         WaitIntervalTracker waitIntervalTracker = WaitIntervalTrackerFactory.createConstant(timeout, BD_WAIT_AND_RETRY_INTERVAL);
         ResilientJobConfig jobConfig = new ResilientJobConfig(logger, System.currentTimeMillis(), waitIntervalTracker);
-        Bdio2UploadJob bdio2UploadJob = new Bdio2UploadJob(bdio2RetryAwareStreamUploader, header, remainingFiles, editor, count, shouldUploadEntries, shouldFinishUpload);
+        Bdio2UploadJob bdio2UploadJob = new Bdio2UploadJob(bdio2RetryAwareStreamUploader, header, remainingFiles, editor, count, shouldUploadEntries, shouldFinishUpload, clientStartTime, timeout);
         ResilientJobExecutor jobExecutor = new ResilientJobExecutor(jobConfig);
 
         return jobExecutor.executeJob(bdio2UploadJob);

diff --git a/src/main/java/com/synopsys/integration/blackduck/bdio2/Bdio2RetryAwareStreamUploader.java b/src/main/java/com/synopsys/integration/blackduck/bdio2/Bdio2RetryAwareStreamUploader.java
@@ -14,26 +14,45 @@
 import org.slf4j.LoggerFactory;
 
 import com.synopsys.integration.blackduck.bdio2.model.BdioFileContent;
+import com.synopsys.integration.blackduck.exception.BlackDuckIntegrationException;
 import com.synopsys.integration.blackduck.service.request.BlackDuckRequestBuilderEditor;
 import com.synopsys.integration.exception.IntegrationException;
 import com.synopsys.integration.rest.HttpUrl;
 import com.synopsys.integration.rest.exception.IntegrationRestException;
 import com.synopsys.integration.rest.response.Response;
 
 public class Bdio2RetryAwareStreamUploader {
-    private static final List<Integer> NON_RETRYABLE_EXIT_CODES = Arrays.asList(401, 402, 403, 404, 500);
+    private static final List<Integer> NON_RETRYABLE_EXIT_CODES = Arrays.asList(401, 402, 403, 404);
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
     private final Bdio2StreamUploader bdio2StreamUploader;
 
     public Bdio2RetryAwareStreamUploader(Bdio2StreamUploader bdio2StreamUploader) {
         this.bdio2StreamUploader = bdio2StreamUploader;
     }
 
-    public Response start(BdioFileContent header, BlackDuckRequestBuilderEditor editor)
-        throws RetriableBdioUploadException, IntegrationException {
+    public Response start(BdioFileContent header, BlackDuckRequestBuilderEditor editor, long clientStartTime, long clientTimeout)
+        throws RetriableBdioUploadException, IntegrationException, InterruptedException {
         logger.trace("Executing BDIO upload start operation; non-retryable status codes: {}", NON_RETRYABLE_EXIT_CODES);
         try {
-            return bdio2StreamUploader.start(header, editor);
+            Response response = bdio2StreamUploader.start(header, editor);
+
+            // Retry if BlackDuck specifically told us how long to wait and we don't exceed the client timeout.
+            if (!response.isStatusCodeSuccess() && isRetryableExitCode(response.getStatusCode())) {
+                String retryAfterInSeconds = response.getHeaderValue("retry-after");
+
+                if (null != retryAfterInSeconds && !retryAfterInSeconds.equals("0")) {
+                    long retryAfterInMillis = Integer.parseInt(retryAfterInSeconds) * 1000;
+
+                    if (isClientTimeoutExceededBy(clientStartTime, retryAfterInMillis, clientTimeout)) {
+                        throw new BlackDuckIntegrationException("Client timeout exceeded or will be exceeded due to server being busy.");
+                    }
+                    logger.trace("Waiting " + retryAfterInMillis + " milliseconds to retry BDIO upload start operation.");
+                    Thread.sleep(retryAfterInMillis);
+                    return start(header, editor, clientStartTime, clientTimeout);
+                }
+            }
+
+            return response;
         } catch (IntegrationRestException e) {
             return translateRetryableExceptions(e);
         }
@@ -74,6 +93,11 @@ public void onErrorThrowRetryableOrFailure(Response response) throws Integration
         }
         logger.trace("Response status code {} treated as success", response.getStatusCode());
     }
+
+    private boolean isClientTimeoutExceededBy(long startTime, long waitInMillis, long clientTimeout) {
+        long currentTime = System.currentTimeMillis();
+        return (currentTime - startTime + waitInMillis) > (clientTimeout * 1000);
+    }
 
     private Response translateRetryableExceptions(final IntegrationRestException e) throws RetriableBdioUploadException, IntegrationRestException {
         if (isRetryableExitCode(e.getHttpStatusCode())) {

diff --git a/src/main/java/com/synopsys/integration/blackduck/bdio2/Bdio2StreamUploader.java b/src/main/java/com/synopsys/integration/blackduck/bdio2/Bdio2StreamUploader.java
@@ -58,7 +58,7 @@ public Response start(BdioFileContent header, BlackDuckRequestBuilderEditor edit
             .addHeader(HEADER_CONTENT_TYPE, contentType)
             .apply(editor)
             .buildBlackDuckResponseRequest(url);
-        return blackDuckApiClient.execute(request);
+        return blackDuckApiClient.executeAndRetrieveResponse(request);
     }
 
     public Response append(HttpUrl url, int count, BdioFileContent bdioFileContent, BlackDuckRequestBuilderEditor editor) throws IntegrationException {

diff --git a/src/main/java/com/synopsys/integration/blackduck/bdio2/Bdio2UploadJob.java b/src/main/java/com/synopsys/integration/blackduck/bdio2/Bdio2UploadJob.java
@@ -35,6 +35,8 @@ public class Bdio2UploadJob implements ResilientJob<Bdio2UploadResult> {
     private HttpUrl uploadUrl;
     private String scanId;
     private boolean complete;
+    private long startTime;
+    private long timeout;
 
     public Bdio2UploadJob(
         Bdio2RetryAwareStreamUploader bdio2RetryAwareStreamUploader,
@@ -43,7 +45,9 @@ public Bdio2UploadJob(
         BlackDuckRequestBuilderEditor editor,
         int count,
         boolean onlyUploadHeader,
-        boolean shouldFinishUpload
+        boolean shouldFinishUpload,
+        long startTime,
+        long timeout
     ) {
         this.bdio2RetryAwareStreamUploader = bdio2RetryAwareStreamUploader;
         this.header = header;
@@ -52,12 +56,14 @@ public Bdio2UploadJob(
         this.count = count;
         this.shouldUploadEntries = onlyUploadHeader;
         this.shouldFinishUpload = shouldFinishUpload;
+        this.startTime = startTime;
+        this.timeout = timeout;
     }
 
     @Override
     public void attemptJob() throws IntegrationException {
         try {
-            Response headerResponse = bdio2RetryAwareStreamUploader.start(header, editor);
+            Response headerResponse = bdio2RetryAwareStreamUploader.start(header, editor, startTime, timeout);
             bdio2RetryAwareStreamUploader.onErrorThrowRetryableOrFailure(headerResponse);
             complete = true;
             uploadUrl = new HttpUrl(headerResponse.getHeaderValue("location"));
@@ -73,7 +79,7 @@ public void attemptJob() throws IntegrationException {
                 Response finishResponse = bdio2RetryAwareStreamUploader.finish(uploadUrl, count, editor);
                 bdio2RetryAwareStreamUploader.onErrorThrowRetryableOrFailure(finishResponse);
             }
-        } catch (RetriableBdioUploadException e) {
+        } catch (RetriableBdioUploadException | InterruptedException e) {
             complete = false;
         }
     }

diff --git a/...tion/blackduck/codelocation/intelligentpersistence/IntelligentPersistenceBatchRunner.java b/...tion/blackduck/codelocation/intelligentpersistence/IntelligentPersistenceBatchRunner.java
@@ -31,19 +31,19 @@ public IntelligentPersistenceBatchRunner(final IntLogger logger, final ExecutorS
         this.bdio2FileUploadService = bdio2FileUploadService;
     }
 
-    public UploadBatchOutput executeUploads(UploadBatch uploadBatch, long timeout) throws BlackDuckIntegrationException {
+    public UploadBatchOutput executeUploads(UploadBatch uploadBatch, long timeout, long clientStartTime) throws BlackDuckIntegrationException {
         logger.info("Starting the codelocation file uploads.");
-        UploadBatchOutput uploadBatchOutput = uploadTargets(uploadBatch, timeout);
+        UploadBatchOutput uploadBatchOutput = uploadTargets(uploadBatch, timeout, clientStartTime);
         logger.info("Completed the codelocation file uploads.");
 
         return uploadBatchOutput;
     }
 
-    private UploadBatchOutput uploadTargets(UploadBatch uploadBatch, long timeout) throws BlackDuckIntegrationException {
+    private UploadBatchOutput uploadTargets(UploadBatch uploadBatch, long timeout, long clientStartTime) throws BlackDuckIntegrationException {
         List<UploadOutput> uploadOutputs = new ArrayList<>();
 
         try {
-            List<IntelligentPersistenceCallable> callables = createCallables(uploadBatch, timeout);
+            List<IntelligentPersistenceCallable> callables = createCallables(uploadBatch, timeout, clientStartTime);
             List<Future<UploadOutput>> submitted = new ArrayList<>();
             for (IntelligentPersistenceCallable callable : callables) {
                 submitted.add(executorService.submit(callable));
@@ -59,10 +59,10 @@ private UploadBatchOutput uploadTargets(UploadBatch uploadBatch, long timeout) t
         return new UploadBatchOutput(uploadOutputs);
     }
 
-    private List<IntelligentPersistenceCallable> createCallables(UploadBatch uploadBatch, long timeout) {
+    private List<IntelligentPersistenceCallable> createCallables(UploadBatch uploadBatch, long timeout, long clientStartTime) {
         List<IntelligentPersistenceCallable> callables = uploadBatch.getUploadTargets()
                                                              .stream()
-                                                             .map(uploadTarget -> new IntelligentPersistenceCallable(bdio2FileUploadService, uploadTarget, timeout))
+                                                             .map(uploadTarget -> new IntelligentPersistenceCallable(bdio2FileUploadService, uploadTarget, timeout, clientStartTime))
                                                              .collect(Collectors.toList());
 
         return callables;

diff --git a/...gration/blackduck/codelocation/intelligentpersistence/IntelligentPersistenceCallable.java b/...gration/blackduck/codelocation/intelligentpersistence/IntelligentPersistenceCallable.java
@@ -21,11 +21,13 @@ public class IntelligentPersistenceCallable implements Callable<UploadOutput> {
     private final Bdio2FileUploadService bdio2FileUploadService;
     private final UploadTarget uploadTarget;
     private final long timeout;
+    private final long clientStartTime;
 
-    public IntelligentPersistenceCallable(Bdio2FileUploadService bdio2FileUploadService, UploadTarget uploadTarget, long timeout) {
+    public IntelligentPersistenceCallable(Bdio2FileUploadService bdio2FileUploadService, UploadTarget uploadTarget, long timeout, long clientStartTime) {
         this.bdio2FileUploadService = bdio2FileUploadService;
         this.uploadTarget = uploadTarget;
         this.timeout = timeout;
+        this.clientStartTime = clientStartTime;
     }
 
     @Override
@@ -34,7 +36,7 @@ public UploadOutput call() {
         NameVersion projectAndVersion = uploadTarget.getProjectAndVersion().orElse(null);
         String codeLocationName = uploadTarget.getCodeLocationName();
         try {
-            Bdio2UploadResult result = bdio2FileUploadService.uploadFile(uploadTarget, timeout);
+            Bdio2UploadResult result = bdio2FileUploadService.uploadFile(uploadTarget, timeout, clientStartTime);
             return UploadOutput.SUCCESS(projectAndVersion, codeLocationName, null, result.getScanId());
         } catch (Exception ex) {
             String errorMessage = String.format("Failed to upload file: %s because %s", uploadTarget.getUploadFile().getAbsolutePath(), ex.getMessage());

diff --git a/...odelocation/intelligentpersistence/IntelligentPersistenceCodeLocationCreationRequest.java b/...odelocation/intelligentpersistence/IntelligentPersistenceCodeLocationCreationRequest.java
@@ -16,15 +16,17 @@ public class IntelligentPersistenceCodeLocationCreationRequest extends CodeLocat
     private final IntelligentPersistenceBatchRunner uploadBatchRunner;
     private final UploadBatch uploadBatch;
     private final long timeout;
+    private final long clientStartTime;
 
-    public IntelligentPersistenceCodeLocationCreationRequest(final IntelligentPersistenceBatchRunner uploadBatchRunner, final UploadBatch uploadBatch, final long timeout) {
+    public IntelligentPersistenceCodeLocationCreationRequest(final IntelligentPersistenceBatchRunner uploadBatchRunner, final UploadBatch uploadBatch, final long timeout, final long clientStartTime) {
         this.uploadBatchRunner = uploadBatchRunner;
         this.uploadBatch = uploadBatch;
         this.timeout = timeout;
+        this.clientStartTime = clientStartTime;
     }
 
     @Override
     public UploadBatchOutput executeRequest() throws BlackDuckIntegrationException {
-        return uploadBatchRunner.executeUploads(uploadBatch, timeout);
+        return uploadBatchRunner.executeUploads(uploadBatch, timeout, clientStartTime);
     }
 }
diff --git a/...egration/blackduck/codelocation/intelligentpersistence/IntelligentPersistenceService.java b/...egration/blackduck/codelocation/intelligentpersistence/IntelligentPersistenceService.java
@@ -33,25 +33,25 @@ public IntelligentPersistenceService(BlackDuckApiClient blackDuckApiClient, ApiD
         this.codeLocationCreationService = codeLocationCreationService;
     }
 
-    public IntelligentPersistenceCodeLocationCreationRequest createUploadRequest(UploadBatch uploadBatch, long timeoutInSeconds) {
-        return new IntelligentPersistenceCodeLocationCreationRequest(uploadBatchRunner, uploadBatch, timeoutInSeconds);
+    public IntelligentPersistenceCodeLocationCreationRequest createUploadRequest(UploadBatch uploadBatch, long timeoutInSeconds, long clientStartTime) {
+        return new IntelligentPersistenceCodeLocationCreationRequest(uploadBatchRunner, uploadBatch, timeoutInSeconds, clientStartTime);
     }
 
     public CodeLocationCreationData<UploadBatchOutput> uploadBdio(CodeLocationCreationRequest<UploadBatchOutput> uploadRequest) throws IntegrationException {
         return codeLocationCreationService.createCodeLocations(uploadRequest);
     }
 
-    public CodeLocationCreationData<UploadBatchOutput> uploadBdio(UploadBatch uploadBatch, long timeoutInSeconds) throws IntegrationException {
-        IntelligentPersistenceCodeLocationCreationRequest uploadRequest = createUploadRequest(uploadBatch, timeoutInSeconds);
+    public CodeLocationCreationData<UploadBatchOutput> uploadBdio(UploadBatch uploadBatch, long timeoutInSeconds, long clientStartTime) throws IntegrationException {
+        IntelligentPersistenceCodeLocationCreationRequest uploadRequest = createUploadRequest(uploadBatch, timeoutInSeconds, clientStartTime);
         return uploadBdio(uploadRequest);
     }
 
     public UploadBatchOutput uploadBdioAndWait(CodeLocationCreationRequest<UploadBatchOutput> uploadRequest, long timeoutInSeconds) throws IntegrationException, InterruptedException {
         return codeLocationCreationService.createCodeLocationsAndWait(uploadRequest, timeoutInSeconds);
     }
 
-    public UploadBatchOutput uploadBdioAndWait(UploadBatch uploadBatch, long timeoutInSeconds) throws IntegrationException, InterruptedException {
-        IntelligentPersistenceCodeLocationCreationRequest uploadRequest = createUploadRequest(uploadBatch, (int) timeoutInSeconds);
+    public UploadBatchOutput uploadBdioAndWait(UploadBatch uploadBatch, long timeoutInSeconds, long clientStartTime) throws IntegrationException, InterruptedException {
+        IntelligentPersistenceCodeLocationCreationRequest uploadRequest = createUploadRequest(uploadBatch, (int) timeoutInSeconds, clientStartTime);
         return uploadBdioAndWait(uploadRequest, timeoutInSeconds);
     }