Reading from a non-existent database causes memory usage to grow

[sebright]

It seems like requests that fail with a NOT_FOUND error can cause the Spanner client to consume memory. I suspect that it is caused by the invalidatedDbClients field in SpannerImpl:

java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java

Line 96 in 71d7fd1

private final List<DatabaseClientImpl> invalidatedDbClients = new ArrayList<>();

I ran this test program to reproduce the issue with Java 11 and google-cloud-spanner 6.3.2:

import com.google.cloud.spanner.DatabaseId;
import com.google.cloud.spanner.InstanceConfigId;
import com.google.cloud.spanner.InstanceId;
import com.google.cloud.spanner.InstanceInfo;
import com.google.cloud.spanner.Key;
import com.google.cloud.spanner.Spanner;
import com.google.cloud.spanner.SpannerOptions;
import java.util.Arrays;
import org.junit.Test;

public class FailedRequestMemoryUsageTest {

  @Test
  public void testFailedRequestMemoryUsage() throws Exception {
    Spanner spanner = SpannerOptions.getDefaultInstance().getService();
    String projectId = spanner.getOptions().getProjectId();
    InstanceId instanceId = InstanceId.of(projectId, "my-instance");
    InstanceConfigId instanceConfigId = InstanceConfigId.of(projectId, "regional-us-central1");
    spanner
        .getInstanceAdminClient()
        .createInstance(
            InstanceInfo.newBuilder(instanceId)
                .setInstanceConfigId(instanceConfigId)
                .setDisplayName("my-instance")
                .setNodeCount(1)
                .build())
        .get();
    DatabaseId databaseId = DatabaseId.of(instanceId, "my-database");

    for (int i = 0; i <= 10000; i++) {
      if (i % 500 == 0) {
        Runtime.getRuntime().gc();
        System.out.println(
            i + "\t" + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()));
      }
      try {
        spanner
            .getDatabaseClient(databaseId)
            .singleUse()
            .readRow("MyTable", Key.of(0), Arrays.asList("MyColumn"));
      } catch (Exception e) {
        // Ignore NOT_FOUND error
      }
    }
  }
}

0	19573984
500	31193384
1000	35718312
1500	40337512
2000	44895408
2500	49444864
3000	54109824
3500	58690984
4000	63295664
4500	68078928
5000	72855264
5500	77463136
6000	82355544
6500	86576208
7000	91327472
7500	95821992
8000	100438752
8500	105055256
9000	109650504
9500	114148912
10000	118900040