From a55d7ce64fff434151c1c3af0796d290e9db7470 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= <koloite@gmail.com>
Date: Fri, 19 Feb 2021 09:44:04 +0100
Subject: [PATCH] fix: wrong use of getRetryDelayInMillis() / 1000 in
 documentation and retry loops (#885)

Both the documentation for `TransactionManager` as well as some internal retry loops wrongly used `SpannerException#getRetryDelayInMillis() / 1000` as input for `Thread.sleep(long)`. The retry delay is already in milliseconds and should not be modified.

Fixes #874
---
 .../google/cloud/spanner/DatabaseClient.java  | 42 +++++++++---------
 .../com/google/cloud/spanner/SessionPool.java | 14 ++++--
 .../spanner/SpannerExceptionFactory.java      | 24 ++++++++++
 .../cloud/spanner/TransactionRunnerImpl.java  | 44 ++++++++++++-------
 .../connection/ReadWriteTransaction.java      |  5 ++-
 .../spanner/AsyncTransactionManagerTest.java  |  2 +-
 .../cloud/spanner/DatabaseClientImplTest.java |  4 +-
 .../cloud/spanner/MockSpannerServiceImpl.java |  2 +-
 .../RetryOnInvalidatedSessionTest.java        | 30 ++++++-------
 .../TransactionManagerAbortedTest.java        | 22 +++++-----
 .../ConnectionAsyncApiAbortedTest.java        | 31 ++++++++-----
 .../connection/ITAbstractSpannerTest.java     | 19 +++++++-
 .../connection/ReadWriteTransactionTest.java  | 22 +++++++++-
 .../cloud/spanner/it/ITClosedSessionTest.java |  2 +-
 .../it/ITTransactionManagerAsyncTest.java     |  8 ++--
 .../spanner/it/ITTransactionManagerTest.java  |  8 ++--
 16 files changed, 186 insertions(+), 93 deletions(-)

diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseClient.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseClient.java
index 8d0e0ea0e3..bd42aa307f 100644
--- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseClient.java
+++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseClient.java
@@ -321,19 +321,19 @@ CommitResponse writeAtLeastOnceWithOptions(
    * <pre>{@code
    * long singerId = my_singer_id;
    * try (TransactionManager manager = dbClient.transactionManager()) {
-   *   TransactionContext txn = manager.begin();
+   *   TransactionContext transaction = manager.begin();
    *   while (true) {
    *     String column = "FirstName";
-   *     Struct row = txn.readRow("Singers", Key.of(singerId), Collections.singleton(column));
+   *     Struct row = transaction.readRow("Singers", Key.of(singerId), Collections.singleton(column));
    *     String name = row.getString(column);
-   *     txn.buffer(
+   *     transaction.buffer(
    *         Mutation.newUpdateBuilder("Singers").set(column).to(name.toUpperCase()).build());
    *     try {
    *       manager.commit();
    *       break;
    *     } catch (AbortedException e) {
-   *       Thread.sleep(e.getRetryDelayInMillis() / 1000);
-   *       txn = manager.resetForRetry();
+   *       Thread.sleep(e.getRetryDelayInMillis());
+   *       transaction = manager.resetForRetry();
    *     }
    *   }
    * }
@@ -385,19 +385,19 @@ CommitResponse writeAtLeastOnceWithOptions(
    * <pre>{@code
    * long singerId = 1L;
    * try (AsyncTransactionManager manager = client.transactionManagerAsync()) {
-   *   TransactionContextFuture txnFut = manager.beginAsync();
+   *   TransactionContextFuture transactionFuture = manager.beginAsync();
    *   while (true) {
    *     String column = "FirstName";
    *     CommitTimestampFuture commitTimestamp =
-   *         txnFut
+   *         transactionFuture
    *             .then(
-   *                 (txn, __) ->
-   *                     txn.readRowAsync(
+   *                 (transaction, __) ->
+   *                     transaction.readRowAsync(
    *                         "Singers", Key.of(singerId), Collections.singleton(column)))
    *             .then(
-   *                 (txn, row) -> {
+   *                 (transaction, row) -> {
    *                   String name = row.getString(column);
-   *                   txn.buffer(
+   *                   transaction.buffer(
    *                       Mutation.newUpdateBuilder("Singers")
    *                           .set(column)
    *                           .to(name.toUpperCase())
@@ -409,8 +409,8 @@ CommitResponse writeAtLeastOnceWithOptions(
    *       commitTimestamp.get();
    *       break;
    *     } catch (AbortedException e) {
-   *       Thread.sleep(e.getRetryDelayInMillis() / 1000);
-   *       txnFut = manager.resetForRetryAsync();
+   *       Thread.sleep(e.getRetryDelayInMillis());
+   *       transactionFuture = manager.resetForRetryAsync();
    *     }
    *   }
    * }
@@ -421,26 +421,26 @@ CommitResponse writeAtLeastOnceWithOptions(
    * <pre>{@code
    * final long singerId = 1L;
    * try (AsyncTransactionManager manager = client().transactionManagerAsync()) {
-   *   TransactionContextFuture txn = manager.beginAsync();
+   *   TransactionContextFuture transactionFuture = manager.beginAsync();
    *   while (true) {
    *     final String column = "FirstName";
    *     CommitTimestampFuture commitTimestamp =
-   *         txn.then(
+   *         transactionFuture.then(
    *                 new AsyncTransactionFunction<Void, Struct>() {
    *                   @Override
-   *                   public ApiFuture<Struct> apply(TransactionContext txn, Void input)
+   *                   public ApiFuture<Struct> apply(TransactionContext transaction, Void input)
    *                       throws Exception {
-   *                     return txn.readRowAsync(
+   *                     return transaction.readRowAsync(
    *                         "Singers", Key.of(singerId), Collections.singleton(column));
    *                   }
    *                 })
    *             .then(
    *                 new AsyncTransactionFunction<Struct, Void>() {
    *                   @Override
-   *                   public ApiFuture<Void> apply(TransactionContext txn, Struct input)
+   *                   public ApiFuture<Void> apply(TransactionContext transaction, Struct input)
    *                       throws Exception {
    *                     String name = input.getString(column);
-   *                     txn.buffer(
+   *                     transaction.buffer(
    *                         Mutation.newUpdateBuilder("Singers")
    *                             .set(column)
    *                             .to(name.toUpperCase())
@@ -453,8 +453,8 @@ CommitResponse writeAtLeastOnceWithOptions(
    *       commitTimestamp.get();
    *       break;
    *     } catch (AbortedException e) {
-   *       Thread.sleep(e.getRetryDelayInMillis() / 1000);
-   *       txn = manager.resetForRetryAsync();
+   *       Thread.sleep(e.getRetryDelayInMillis());
+   *       transactionFuture = manager.resetForRetryAsync();
    *     }
    *   }
    * }
diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPool.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPool.java
index 51a42525bf..0d5f36b5c1 100644
--- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPool.java
+++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPool.java
@@ -816,13 +816,21 @@ private TransactionContext internalBegin() {
     }
 
     @Override
-    public SpannerException handleSessionNotFound(SessionNotFoundException notFound) {
-      session = sessionPool.replaceSession(notFound, session);
+    public SpannerException handleSessionNotFound(SessionNotFoundException notFoundException) {
+      session = sessionPool.replaceSession(notFoundException, session);
       PooledSession pooledSession = session.get();
       delegate = pooledSession.delegate.transactionManager(options);
       restartedAfterSessionNotFound = true;
+      return createAbortedExceptionWithMinimalRetryDelay(notFoundException);
+    }
+
+    private static SpannerException createAbortedExceptionWithMinimalRetryDelay(
+        SessionNotFoundException notFoundException) {
       return SpannerExceptionFactory.newSpannerException(
-          ErrorCode.ABORTED, notFound.getMessage(), notFound);
+          ErrorCode.ABORTED,
+          notFoundException.getMessage(),
+          SpannerExceptionFactory.createAbortedExceptionWithRetryDelay(
+              notFoundException.getMessage(), notFoundException, 0, 1));
     }
 
     @Override
diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerExceptionFactory.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerExceptionFactory.java
index 706ee87fd7..696ffc9a21 100644
--- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerExceptionFactory.java
+++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerExceptionFactory.java
@@ -24,9 +24,11 @@
 import com.google.common.base.Predicate;
 import com.google.rpc.ErrorInfo;
 import com.google.rpc.ResourceInfo;
+import com.google.rpc.RetryInfo;
 import io.grpc.Context;
 import io.grpc.Metadata;
 import io.grpc.Status;
+import io.grpc.StatusRuntimeException;
 import io.grpc.protobuf.ProtoUtils;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.TimeoutException;
@@ -226,6 +228,28 @@ private static ErrorInfo extractErrorInfo(Throwable cause) {
     return null;
   }
 
+  /**
+   * Creates a {@link StatusRuntimeException} that contains a {@link RetryInfo} with the specified
+   * retry delay.
+   */
+  static StatusRuntimeException createAbortedExceptionWithRetryDelay(
+      String message, Throwable cause, long retryDelaySeconds, int retryDelayNanos) {
+    Metadata.Key<RetryInfo> key = ProtoUtils.keyForProto(RetryInfo.getDefaultInstance());
+    Metadata trailers = new Metadata();
+    RetryInfo retryInfo =
+        RetryInfo.newBuilder()
+            .setRetryDelay(
+                com.google.protobuf.Duration.newBuilder()
+                    .setNanos(retryDelayNanos)
+                    .setSeconds(retryDelaySeconds))
+            .build();
+    trailers.put(key, retryInfo);
+    return io.grpc.Status.ABORTED
+        .withDescription(message)
+        .withCause(cause)
+        .asRuntimeException(trailers);
+  }
+
   static SpannerException newSpannerExceptionPreformatted(
       ErrorCode code, @Nullable String message, @Nullable Throwable cause) {
     // This is the one place in the codebase that is allowed to call constructors directly.
diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionRunnerImpl.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionRunnerImpl.java
index dbbbca068a..fef873112d 100644
--- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionRunnerImpl.java
+++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionRunnerImpl.java
@@ -531,7 +531,10 @@ public void onError(SpannerException e, boolean withBeginTransaction) {
         // Simulate an aborted transaction to force a retry with a new transaction.
         this.transactionIdFuture.setException(
             SpannerExceptionFactory.newSpannerException(
-                ErrorCode.ABORTED, "Aborted due to failed initial statement", e));
+                ErrorCode.ABORTED,
+                "Aborted due to failed initial statement",
+                SpannerExceptionFactory.createAbortedExceptionWithRetryDelay(
+                    "Aborted due to failed initial statement", e, 0, 1)));
       }
 
       if (e.getErrorCode() == ErrorCode.ABORTED) {
@@ -684,6 +687,19 @@ public void run() {
       return updateCount;
     }
 
+    private SpannerException createAbortedExceptionForBatchDml(ExecuteBatchDmlResponse response) {
+      // Manually construct an AbortedException with a 10ms retry delay for BatchDML responses that
+      // return an Aborted status (and not an AbortedException).
+      return newSpannerException(
+          ErrorCode.fromRpcStatus(response.getStatus()),
+          response.getStatus().getMessage(),
+          SpannerExceptionFactory.createAbortedExceptionWithRetryDelay(
+              response.getStatus().getMessage(),
+              /* cause = */ null,
+              /* retryDelaySeconds = */ 0,
+              /* retryDelayNanos = */ (int) TimeUnit.MILLISECONDS.toNanos(10L)));
+    }
+
     @Override
     public long[] batchUpdate(Iterable<Statement> statements, UpdateOption... options) {
       beforeReadOrQuery();
@@ -705,8 +721,7 @@ public long[] batchUpdate(Iterable<Statement> statements, UpdateOption... option
         // If one of the DML statements was aborted, we should throw an aborted exception.
         // In all other cases, we should throw a BatchUpdateException.
         if (response.getStatus().getCode() == Code.ABORTED_VALUE) {
-          throw newSpannerException(
-              ErrorCode.fromRpcStatus(response.getStatus()), response.getStatus().getMessage());
+          throw createAbortedExceptionForBatchDml(response);
         } else if (response.getStatus().getCode() != 0) {
           throw newSpannerBatchUpdateException(
               ErrorCode.fromRpcStatus(response.getStatus()),
@@ -741,25 +756,24 @@ public ApiFuture<long[]> batchUpdateAsync(
               response,
               new ApiFunction<ExecuteBatchDmlResponse, long[]>() {
                 @Override
-                public long[] apply(ExecuteBatchDmlResponse input) {
-                  long[] results = new long[input.getResultSetsCount()];
-                  for (int i = 0; i < input.getResultSetsCount(); ++i) {
-                    results[i] = input.getResultSets(i).getStats().getRowCountExact();
-                    if (input.getResultSets(i).getMetadata().hasTransaction()) {
+                public long[] apply(ExecuteBatchDmlResponse batchDmlResponse) {
+                  long[] results = new long[batchDmlResponse.getResultSetsCount()];
+                  for (int i = 0; i < batchDmlResponse.getResultSetsCount(); ++i) {
+                    results[i] = batchDmlResponse.getResultSets(i).getStats().getRowCountExact();
+                    if (batchDmlResponse.getResultSets(i).getMetadata().hasTransaction()) {
                       onTransactionMetadata(
-                          input.getResultSets(i).getMetadata().getTransaction(),
+                          batchDmlResponse.getResultSets(i).getMetadata().getTransaction(),
                           builder.getTransaction().hasBegin());
                     }
                   }
                   // If one of the DML statements was aborted, we should throw an aborted exception.
                   // In all other cases, we should throw a BatchUpdateException.
-                  if (input.getStatus().getCode() == Code.ABORTED_VALUE) {
-                    throw newSpannerException(
-                        ErrorCode.fromRpcStatus(input.getStatus()), input.getStatus().getMessage());
-                  } else if (input.getStatus().getCode() != 0) {
+                  if (batchDmlResponse.getStatus().getCode() == Code.ABORTED_VALUE) {
+                    throw createAbortedExceptionForBatchDml(batchDmlResponse);
+                  } else if (batchDmlResponse.getStatus().getCode() != 0) {
                     throw newSpannerBatchUpdateException(
-                        ErrorCode.fromRpcStatus(input.getStatus()),
-                        input.getStatus().getMessage(),
+                        ErrorCode.fromRpcStatus(batchDmlResponse.getStatus()),
+                        batchDmlResponse.getStatus().getMessage(),
                         results);
                   }
                   return results;
diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java
index 0a8e322e79..aa3f8e1ef2 100644
--- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java
+++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ReadWriteTransaction.java
@@ -725,8 +725,11 @@ private void handleAborted(AbortedException aborted) {
       logger.fine(toString() + ": Starting internal transaction retry");
       while (true) {
         // First back off and then restart the transaction.
+        long delay = aborted.getRetryDelayInMillis();
         try {
-          Thread.sleep(aborted.getRetryDelayInMillis() / 1000);
+          if (delay > 0L) {
+            Thread.sleep(delay);
+          }
         } catch (InterruptedException ie) {
           Thread.currentThread().interrupt();
           throw SpannerExceptionFactory.newSpannerException(
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/AsyncTransactionManagerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/AsyncTransactionManagerTest.java
index 748d6f7640..b5fa3ee581 100644
--- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/AsyncTransactionManagerTest.java
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/AsyncTransactionManagerTest.java
@@ -1137,7 +1137,7 @@ public ApiFuture<Void> apply(TransactionContext txn, Struct input)
           commitTimestamp.get();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetryAsync();
         }
       }
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseClientImplTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseClientImplTest.java
index 42a158d755..5cfdb5ea62 100644
--- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseClientImplTest.java
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseClientImplTest.java
@@ -662,7 +662,7 @@ public void transactionManagerIsNonBlocking() throws Exception {
           txManager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           tx = txManager.resetForRetry();
         }
       }
@@ -705,7 +705,7 @@ public CallbackResponse cursorReady(AsyncResultSet resultSet) {
           txManager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           tx = txManager.resetForRetry();
         }
       }
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MockSpannerServiceImpl.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MockSpannerServiceImpl.java
index 424e979692..0b68829604 100644
--- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MockSpannerServiceImpl.java
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MockSpannerServiceImpl.java
@@ -1732,7 +1732,7 @@ private void simulateAbort(Session session, ByteString transactionId) {
 
   public StatusRuntimeException createAbortedException(ByteString transactionId) {
     RetryInfo retryInfo =
-        RetryInfo.newBuilder().setRetryDelay(Duration.newBuilder().setNanos(100).build()).build();
+        RetryInfo.newBuilder().setRetryDelay(Duration.newBuilder().setNanos(1).build()).build();
     Metadata.Key<RetryInfo> key =
         Metadata.Key.of(
             retryInfo.getDescriptorForType().getFullName() + Metadata.BINARY_HEADER_SUFFIX,
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/RetryOnInvalidatedSessionTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/RetryOnInvalidatedSessionTest.java
index 157d71bf36..9ea77366b1 100644
--- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/RetryOnInvalidatedSessionTest.java
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/RetryOnInvalidatedSessionTest.java
@@ -1007,7 +1007,7 @@ public void transactionManagerReadOnlySessionInPool() throws InterruptedExceptio
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1035,7 +1035,7 @@ public void transactionManagerSelect() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1063,7 +1063,7 @@ public void transactionManagerRead() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1092,7 +1092,7 @@ public void transactionManagerReadUsingIndex() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1116,7 +1116,7 @@ public void transactionManagerReadRow() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1140,7 +1140,7 @@ public void transactionManagerReadRowUsingIndex() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1164,7 +1164,7 @@ public void transactionManagerUpdate() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1197,7 +1197,7 @@ public void transactionManagerAborted_thenSessionNotFoundOnBeginTransaction()
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1224,7 +1224,7 @@ public void transactionManagerBatchUpdate() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1248,7 +1248,7 @@ public void transactionManagerBuffer() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1286,7 +1286,7 @@ public void transactionManagerSelectInvalidatedDuringTransaction() throws Interr
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1324,7 +1324,7 @@ public void transactionManagerReadInvalidatedDuringTransaction() throws Interrup
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1365,7 +1365,7 @@ public void transactionManagerReadUsingIndexInvalidatedDuringTransaction()
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1394,7 +1394,7 @@ public void transactionManagerReadRowInvalidatedDuringTransaction() throws Inter
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
@@ -1424,7 +1424,7 @@ public void transactionManagerReadRowUsingIndexInvalidatedDuringTransaction()
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           transaction = manager.resetForRetry();
         }
       }
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerAbortedTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerAbortedTest.java
index 0291e67868..8917d2d2e5 100644
--- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerAbortedTest.java
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerAbortedTest.java
@@ -27,13 +27,13 @@
 import com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult;
 import com.google.cloud.spanner.v1.SpannerClient;
 import com.google.cloud.spanner.v1.SpannerSettings;
+import com.google.protobuf.ByteString;
 import com.google.protobuf.ListValue;
 import com.google.spanner.v1.ResultSetMetadata;
 import com.google.spanner.v1.StructType;
 import com.google.spanner.v1.StructType.Field;
 import com.google.spanner.v1.TypeCode;
 import io.grpc.Server;
-import io.grpc.Status;
 import io.grpc.inprocess.InProcessServerBuilder;
 import java.io.IOException;
 import java.util.Arrays;
@@ -139,7 +139,7 @@ public static void startStaticServer() throws IOException {
     mockSpanner.putStatementResult(
         StatementResult.exception(
             UPDATE_ABORTED_STATEMENT,
-            Status.ABORTED.withDescription("Transaction was aborted").asRuntimeException()));
+            mockSpanner.createAbortedException(ByteString.copyFromUtf8("test"))));
 
     String uniqueName = InProcessServerBuilder.generateName();
     server =
@@ -199,7 +199,7 @@ public void testTransactionManagerAbortOnCommit() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           manager.resetForRetry();
         }
       }
@@ -226,7 +226,7 @@ public void testTransactionManagerAbortOnUpdate() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         }
       }
@@ -253,7 +253,7 @@ public void testTransactionManagerAbortOnBatchUpdate() throws InterruptedExcepti
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         }
       }
@@ -281,7 +281,7 @@ public void testTransactionManagerAbortOnBatchUpdateHalfway() throws Interrupted
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         }
       }
@@ -313,7 +313,7 @@ public void testTransactionManagerAbortOnSelect() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         }
       }
@@ -345,7 +345,7 @@ public void testTransactionManagerAbortOnRead() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         }
       }
@@ -378,7 +378,7 @@ public void testTransactionManagerAbortOnReadUsingIndex() throws InterruptedExce
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         }
       }
@@ -405,7 +405,7 @@ public void testTransactionManagerAbortOnReadRow() throws InterruptedException {
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         }
       }
@@ -432,7 +432,7 @@ public void testTransactionManagerAbortOnReadRowUsingIndex() throws InterruptedE
           manager.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         }
       }
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionAsyncApiAbortedTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionAsyncApiAbortedTest.java
index a209bfa312..dcf5ac2f35 100644
--- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionAsyncApiAbortedTest.java
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionAsyncApiAbortedTest.java
@@ -39,8 +39,8 @@
 import com.google.common.collect.Lists;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.protobuf.AbstractMessage;
+import com.google.protobuf.ByteString;
 import com.google.spanner.v1.ExecuteSqlRequest;
-import io.grpc.Status;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
@@ -127,7 +127,8 @@ public void testSingleQueryAborted() {
     try (Connection connection = createConnection(counter)) {
       assertThat(counter.retryCount).isEqualTo(0);
       mockSpanner.setExecuteStreamingSqlExecutionTime(
-          SimulatedExecutionTime.ofException(Status.ABORTED.asRuntimeException()));
+          SimulatedExecutionTime.ofException(
+              mockSpanner.createAbortedException(ByteString.copyFromUtf8("test"))));
       QueryResult res = executeQueryAsync(connection, SELECT_RANDOM_STATEMENT);
 
       assertThat(get(res.finished)).isNull();
@@ -143,7 +144,8 @@ public void testTwoQueriesSecondAborted() {
       assertThat(counter.retryCount).isEqualTo(0);
       QueryResult res1 = executeQueryAsync(connection, SELECT_RANDOM_STATEMENT);
       mockSpanner.setExecuteStreamingSqlExecutionTime(
-          SimulatedExecutionTime.ofException(Status.ABORTED.asRuntimeException()));
+          SimulatedExecutionTime.ofException(
+              mockSpanner.createAbortedException(ByteString.copyFromUtf8("test"))));
       QueryResult res2 = executeQueryAsync(connection, SELECT_RANDOM_STATEMENT_2);
 
       assertThat(get(res1.finished)).isNull();
@@ -160,12 +162,14 @@ public void testTwoQueriesBothAborted() throws InterruptedException {
     try (Connection connection = createConnection(counter)) {
       assertThat(counter.retryCount).isEqualTo(0);
       mockSpanner.setExecuteStreamingSqlExecutionTime(
-          SimulatedExecutionTime.ofException(Status.ABORTED.asRuntimeException()));
+          SimulatedExecutionTime.ofException(
+              mockSpanner.createAbortedException(ByteString.copyFromUtf8("test"))));
       QueryResult res1 = executeQueryAsync(connection, SELECT_RANDOM_STATEMENT);
       // Wait until the first query aborted.
       assertThat(counter.latch.await(10L, TimeUnit.SECONDS)).isTrue();
       mockSpanner.setExecuteStreamingSqlExecutionTime(
-          SimulatedExecutionTime.ofException(Status.ABORTED.asRuntimeException()));
+          SimulatedExecutionTime.ofException(
+              mockSpanner.createAbortedException(ByteString.copyFromUtf8("test"))));
       QueryResult res2 = executeQueryAsync(connection, SELECT_RANDOM_STATEMENT_2);
 
       assertThat(get(res1.finished)).isNull();
@@ -180,7 +184,8 @@ public void testTwoQueriesBothAborted() throws InterruptedException {
   public void testSingleQueryAbortedMidway() {
     mockSpanner.setExecuteStreamingSqlExecutionTime(
         SimulatedExecutionTime.ofStreamException(
-            Status.ABORTED.asRuntimeException(), RANDOM_RESULT_SET_ROW_COUNT / 2));
+            mockSpanner.createAbortedException(ByteString.copyFromUtf8("test")),
+            RANDOM_RESULT_SET_ROW_COUNT / 2));
     RetryCounter counter = new RetryCounter();
     try (Connection connection = createConnection(counter)) {
       assertThat(counter.retryCount).isEqualTo(0);
@@ -200,7 +205,8 @@ public void testTwoQueriesSecondAbortedMidway() {
       QueryResult res1 = executeQueryAsync(connection, SELECT_RANDOM_STATEMENT);
       mockSpanner.setExecuteStreamingSqlExecutionTime(
           SimulatedExecutionTime.ofStreamException(
-              Status.ABORTED.asRuntimeException(), RANDOM_RESULT_SET_ROW_COUNT_2 / 2));
+              mockSpanner.createAbortedException(ByteString.copyFromUtf8("test")),
+              RANDOM_RESULT_SET_ROW_COUNT_2 / 2));
       QueryResult res2 = executeQueryAsync(connection, SELECT_RANDOM_STATEMENT_2);
 
       assertThat(get(res1.finished)).isNull();
@@ -215,7 +221,7 @@ public void testTwoQueriesSecondAbortedMidway() {
   public void testTwoQueriesOneAbortedMidway() {
     mockSpanner.setExecuteStreamingSqlExecutionTime(
         SimulatedExecutionTime.ofStreamException(
-            Status.ABORTED.asRuntimeException(),
+            mockSpanner.createAbortedException(ByteString.copyFromUtf8("test")),
             Math.min(RANDOM_RESULT_SET_ROW_COUNT / 2, RANDOM_RESULT_SET_ROW_COUNT_2 / 2)));
     RetryCounter counter = new RetryCounter();
     try (Connection connection = createConnection(counter)) {
@@ -239,7 +245,8 @@ public void testTwoQueriesOneAbortedMidway() {
   public void testUpdateAndQueryAbortedMidway() throws InterruptedException {
     mockSpanner.setExecuteStreamingSqlExecutionTime(
         SimulatedExecutionTime.ofStreamException(
-            Status.ABORTED.asRuntimeException(), RANDOM_RESULT_SET_ROW_COUNT / 2));
+            mockSpanner.createAbortedException(ByteString.copyFromUtf8("test")),
+            RANDOM_RESULT_SET_ROW_COUNT / 2));
     final RetryCounter counter = new RetryCounter();
     try (Connection connection = createConnection(counter)) {
       assertThat(counter.retryCount).isEqualTo(0);
@@ -334,7 +341,8 @@ public boolean apply(AbstractMessage input) {
   public void testUpdateAndQueryAbortedMidway_UpdateCountChanged() throws InterruptedException {
     mockSpanner.setExecuteStreamingSqlExecutionTime(
         SimulatedExecutionTime.ofStreamException(
-            Status.ABORTED.asRuntimeException(), RANDOM_RESULT_SET_ROW_COUNT / 2));
+            mockSpanner.createAbortedException(ByteString.copyFromUtf8("test")),
+            RANDOM_RESULT_SET_ROW_COUNT / 2));
     final RetryCounter counter = new RetryCounter();
     try (Connection connection = createConnection(counter)) {
       assertThat(counter.retryCount).isEqualTo(0);
@@ -423,7 +431,8 @@ public boolean apply(AbstractMessage input) {
   public void testQueriesAbortedMidway_ResultsChanged() throws InterruptedException {
     mockSpanner.setExecuteStreamingSqlExecutionTime(
         SimulatedExecutionTime.ofStreamException(
-            Status.ABORTED.asRuntimeException(), RANDOM_RESULT_SET_ROW_COUNT - 1));
+            mockSpanner.createAbortedException(ByteString.copyFromUtf8("test")),
+            RANDOM_RESULT_SET_ROW_COUNT - 1));
     final Statement statement = Statement.of("SELECT * FROM TEST_TABLE");
     final RandomResultSetGenerator generator =
         new RandomResultSetGenerator(RANDOM_RESULT_SET_ROW_COUNT - 10);
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ITAbstractSpannerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ITAbstractSpannerTest.java
index 88cbcc108b..aa633552a8 100644
--- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ITAbstractSpannerTest.java
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ITAbstractSpannerTest.java
@@ -34,6 +34,10 @@
 import com.google.common.base.Preconditions;
 import com.google.common.base.Stopwatch;
 import com.google.common.base.Strings;
+import com.google.rpc.RetryInfo;
+import io.grpc.Metadata;
+import io.grpc.StatusRuntimeException;
+import io.grpc.protobuf.ProtoUtils;
 import java.lang.reflect.Field;
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -158,10 +162,23 @@ public void intercept(
             probability = 0;
           }
           throw SpannerExceptionFactory.newSpannerException(
-              ErrorCode.ABORTED, "Transaction was aborted by interceptor");
+              ErrorCode.ABORTED,
+              "Transaction was aborted by interceptor",
+              createAbortedExceptionWithMinimalRetry());
         }
       }
     }
+
+    private static StatusRuntimeException createAbortedExceptionWithMinimalRetry() {
+      Metadata.Key<RetryInfo> key = ProtoUtils.keyForProto(RetryInfo.getDefaultInstance());
+      Metadata trailers = new Metadata();
+      RetryInfo retryInfo =
+          RetryInfo.newBuilder()
+              .setRetryDelay(com.google.protobuf.Duration.newBuilder().setNanos(1).setSeconds(0L))
+              .build();
+      trailers.put(key, retryInfo);
+      return io.grpc.Status.ABORTED.asRuntimeException(trailers);
+    }
   }
 
   @ClassRule public static IntegrationTestEnv env = new IntegrationTestEnv();
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ReadWriteTransactionTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ReadWriteTransactionTest.java
index de5c9fbdeb..ff44aa6538 100644
--- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ReadWriteTransactionTest.java
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ReadWriteTransactionTest.java
@@ -49,7 +49,11 @@
 import com.google.cloud.spanner.Type.StructField;
 import com.google.cloud.spanner.connection.StatementParser.ParsedStatement;
 import com.google.cloud.spanner.connection.StatementParser.StatementType;
+import com.google.rpc.RetryInfo;
 import com.google.spanner.v1.ResultSetStats;
+import io.grpc.Metadata;
+import io.grpc.StatusRuntimeException;
+import io.grpc.protobuf.ProtoUtils;
 import java.math.BigDecimal;
 import java.util.Arrays;
 import java.util.Collections;
@@ -98,7 +102,8 @@ public void commit() {
         case ABORT:
           state = TransactionState.COMMIT_FAILED;
           commitBehavior = CommitBehavior.SUCCEED;
-          throw SpannerExceptionFactory.newSpannerException(ErrorCode.ABORTED, "commit aborted");
+          throw SpannerExceptionFactory.newSpannerException(
+              ErrorCode.ABORTED, "commit aborted", createAbortedExceptionWithMinimalRetry());
         default:
           throw new IllegalStateException();
       }
@@ -448,7 +453,9 @@ public void testRetry() {
       }
 
       // first abort, then do nothing
-      doThrow(SpannerExceptionFactory.newSpannerException(ErrorCode.ABORTED, "commit aborted"))
+      doThrow(
+              SpannerExceptionFactory.newSpannerException(
+                  ErrorCode.ABORTED, "commit aborted", createAbortedExceptionWithMinimalRetry()))
           .doNothing()
           .when(txManager)
           .commit();
@@ -677,4 +684,15 @@ public void testChecksumResultSetWithArray() {
     rs2.next();
     assertThat(rs1.getChecksum(), is(not(equalTo(rs2.getChecksum()))));
   }
+
+  private static StatusRuntimeException createAbortedExceptionWithMinimalRetry() {
+    Metadata.Key<RetryInfo> key = ProtoUtils.keyForProto(RetryInfo.getDefaultInstance());
+    Metadata trailers = new Metadata();
+    RetryInfo retryInfo =
+        RetryInfo.newBuilder()
+            .setRetryDelay(com.google.protobuf.Duration.newBuilder().setNanos(1).setSeconds(0L))
+            .build();
+    trailers.put(key, retryInfo);
+    return io.grpc.Status.ABORTED.asRuntimeException(trailers);
+  }
 }
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITClosedSessionTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITClosedSessionTest.java
index 22dc4c5c45..f846dd3deb 100644
--- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITClosedSessionTest.java
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITClosedSessionTest.java
@@ -258,7 +258,7 @@ public void testTransactionManager() throws InterruptedException {
             break;
           }
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         }
       }
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITTransactionManagerAsyncTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITTransactionManagerAsyncTest.java
index f487fe4b0b..2ce0a5bc3f 100644
--- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITTransactionManagerAsyncTest.java
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITTransactionManagerAsyncTest.java
@@ -133,7 +133,7 @@ public ApiFuture<Void> apply(TransactionContext txn, Void input)
           assertThat(row.getBoolean(1)).isTrue();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetryAsync();
         }
       }
@@ -166,7 +166,7 @@ public ApiFuture<Void> apply(TransactionContext txn, Void input)
               .get();
           fail("Expected exception");
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetryAsync();
         } catch (ExecutionException e) {
           assertThat(e.getCause()).isInstanceOf(SpannerException.class);
@@ -211,7 +211,7 @@ public ApiFuture<Void> apply(TransactionContext txn, Void input) throws Exceptio
           manager.rollbackAsync();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetryAsync();
         }
       }
@@ -286,7 +286,7 @@ public ApiFuture<Void> apply(TransactionContext txn, Struct input)
           txn1Step2.commitAsync().get();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           // It is possible that it was txn2 that aborted.
           // In that case we should just retry without resetting anything.
           if (manager1.getState() == TransactionState.ABORTED) {
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITTransactionManagerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITTransactionManagerTest.java
index 3ea4a06771..870acda32d 100644
--- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITTransactionManagerTest.java
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITTransactionManagerTest.java
@@ -91,7 +91,7 @@ public void simpleInsert() throws InterruptedException {
           assertThat(row.getBoolean(1)).isTrue();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         }
       }
@@ -115,7 +115,7 @@ public void invalidInsert() throws InterruptedException {
           manager.commit();
           fail("Expected exception");
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         } catch (SpannerException e) {
           // expected
@@ -145,7 +145,7 @@ public void rollback() throws InterruptedException {
           manager.rollback();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           txn = manager.resetForRetry();
         }
       }
@@ -188,7 +188,7 @@ public void abortAndRetry() throws InterruptedException {
           manager1.commit();
           break;
         } catch (AbortedException e) {
-          Thread.sleep(e.getRetryDelayInMillis() / 1000);
+          Thread.sleep(e.getRetryDelayInMillis());
           // It is possible that it was txn2 that aborted.
           // In that case we should just retry without resetting anything.
           if (manager1.getState() == TransactionState.ABORTED) {