getHeaders() {
- return ImmutableMap.of("User-Agent", fmtUserAgent("non-test"));
- }
- })
+ FixedHeaderProvider.create(ImmutableMap.of("User-Agent", fmtUserAgent("non-test"))))
.setRetrySettings(retrySettingsBuilder.setMaxAttempts(1).build());
}
return builder.build().getService();
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/TestBench.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/TestBench.java
index af79ffe74..4dd14f40f 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/TestBench.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/TestBench.java
@@ -30,6 +30,8 @@
import com.google.api.gax.retrying.BasicResultRetryAlgorithm;
import com.google.api.gax.retrying.RetrySettings;
import com.google.cloud.RetryHelper.RetryHelperException;
+import com.google.cloud.conformance.storage.v1.InstructionList;
+import com.google.cloud.conformance.storage.v1.Method;
import com.google.common.collect.ImmutableList;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
@@ -60,7 +62,7 @@
*
* This rule expects to be bound as an {@link org.junit.ClassRule @ClassRule} field.
*/
-final class TestBench implements TestRule {
+public final class TestBench implements TestRule {
private static final Logger LOGGER = Logger.getLogger(TestBench.class.getName());
@@ -69,6 +71,7 @@ final class TestBench implements TestRule {
private final String dockerImageName;
private final String dockerImageTag;
private final CleanupStrategy cleanupStrategy;
+ private final String containerName;
private final Gson gson;
private final HttpRequestFactory requestFactory;
@@ -78,12 +81,14 @@ private TestBench(
String baseUri,
String dockerImageName,
String dockerImageTag,
- CleanupStrategy cleanupStrategy) {
+ CleanupStrategy cleanupStrategy,
+ String containerName) {
this.ignorePullError = ignorePullError;
this.baseUri = baseUri;
this.dockerImageName = dockerImageName;
this.dockerImageTag = dockerImageTag;
this.cleanupStrategy = cleanupStrategy;
+ this.containerName = containerName;
this.gson = new Gson();
this.requestFactory =
new NetHttpTransport.Builder()
@@ -92,15 +97,17 @@ private TestBench(
request -> {
request.setCurlLoggingEnabled(false);
request.getHeaders().setAccept("application/json");
- request.getHeaders().setUserAgent("test-bench/ java-conformance-tests/");
+ request
+ .getHeaders()
+ .setUserAgent(String.format("%s/ test-bench/", this.containerName));
});
}
- String getBaseUri() {
+ public String getBaseUri() {
return baseUri;
}
- RetryTestResource createRetryTest(RetryTestResource retryTestResource) throws IOException {
+ public RetryTestResource createRetryTest(RetryTestResource retryTestResource) throws IOException {
GenericUrl url = new GenericUrl(baseUri + "/retry_test");
String jsonString = gson.toJson(retryTestResource);
HttpContent content =
@@ -112,14 +119,14 @@ RetryTestResource createRetryTest(RetryTestResource retryTestResource) throws IO
return result;
}
- void deleteRetryTest(RetryTestResource retryTestResource) throws IOException {
+ public void deleteRetryTest(RetryTestResource retryTestResource) throws IOException {
GenericUrl url = new GenericUrl(baseUri + "/retry_test/" + retryTestResource.id);
HttpRequest req = requestFactory.buildDeleteRequest(url);
HttpResponse resp = req.execute();
resp.disconnect();
}
- RetryTestResource getRetryTest(RetryTestResource retryTestResource) throws IOException {
+ public RetryTestResource getRetryTest(RetryTestResource retryTestResource) throws IOException {
GenericUrl url = new GenericUrl(baseUri + "/retry_test/" + retryTestResource.id);
HttpRequest req = requestFactory.buildGetRequest(url);
HttpResponse resp = req.execute();
@@ -128,7 +135,7 @@ RetryTestResource getRetryTest(RetryTestResource retryTestResource) throws IOExc
return result;
}
- List listRetryTests() throws IOException {
+ public List listRetryTests() throws IOException {
GenericUrl url = new GenericUrl(baseUri + "/retry_tests");
HttpRequest req = requestFactory.buildGetRequest(url);
HttpResponse resp = req.execute();
@@ -147,7 +154,8 @@ public Statement apply(final Statement base, Description description) {
return new Statement() {
@Override
public void evaluate() throws Throwable {
- Path tempDirectory = Files.createTempDirectory("retry-conformance-server");
+ String fullContainerName = String.format("storage-testbench_%s", containerName);
+ Path tempDirectory = Files.createTempDirectory(fullContainerName);
Path outPath = tempDirectory.resolve("stdout");
Path errPath = tempDirectory.resolve("stderr");
@@ -190,7 +198,7 @@ public void evaluate() throws Throwable {
"--rm",
"--publish",
port + ":9000",
- "--name=retry-conformance-server",
+ String.format("--name=%s", fullContainerName),
dockerImage)
.redirectOutput(outFile)
.redirectError(errFile)
@@ -261,15 +269,32 @@ private void dumpServerLog(String prefix, File out) throws IOException {
}
}
- static Builder newBuilder() {
+ public static Builder newBuilder() {
return new Builder();
}
- static final class RetryTestResource {
+ public static final class RetryTestResource {
public String id;
public Boolean completed;
public JsonObject instructions;
+ public RetryTestResource() {}
+
+ public RetryTestResource(JsonObject instructions) {
+ this.instructions = instructions;
+ }
+
+ public static RetryTestResource newRetryTestResource(Method m, InstructionList l) {
+ RetryTestResource resource = new RetryTestResource();
+ resource.instructions = new JsonObject();
+ JsonArray instructions = new JsonArray();
+ for (String s : l.getInstructionsList()) {
+ instructions.add(s);
+ }
+ resource.instructions.add(m.getName(), instructions);
+ return resource;
+ }
+
@Override
public String toString() {
return "RetryTestResource{"
@@ -284,36 +309,46 @@ public String toString() {
}
}
- static final class Builder {
+ public static final class Builder {
private static final String DEFAULT_BASE_URI = "http://localhost:9000";
private static final String DEFAULT_IMAGE_NAME =
"gcr.io/cloud-devrel-public-resources/storage-testbench";
- private static final String DEFAULT_IMAGE_TAG = "v0.8.0";
+ private static final String DEFAULT_IMAGE_TAG = "v0.10.0";
+ private static final String DEFAULT_CONTAINER_NAME = "default";
private boolean ignorePullError;
private String baseUri;
private String dockerImageName;
private String dockerImageTag;
private CleanupStrategy cleanupStrategy;
+ private String containerName;
- public Builder() {
- this(false, DEFAULT_BASE_URI, DEFAULT_IMAGE_NAME, DEFAULT_IMAGE_TAG, CleanupStrategy.ALWAYS);
+ private Builder() {
+ this(
+ false,
+ DEFAULT_BASE_URI,
+ DEFAULT_IMAGE_NAME,
+ DEFAULT_IMAGE_TAG,
+ CleanupStrategy.ALWAYS,
+ DEFAULT_CONTAINER_NAME);
}
- public Builder(
+ private Builder(
boolean ignorePullError,
String baseUri,
String dockerImageName,
String dockerImageTag,
- CleanupStrategy cleanupStrategy) {
+ CleanupStrategy cleanupStrategy,
+ String containerName) {
this.ignorePullError = ignorePullError;
this.baseUri = baseUri;
this.dockerImageName = dockerImageName;
this.dockerImageTag = dockerImageTag;
this.cleanupStrategy = cleanupStrategy;
+ this.containerName = containerName;
}
- public Builder setCleanupStraegy(CleanupStrategy cleanupStrategy) {
+ public Builder setCleanupStrategy(CleanupStrategy cleanupStrategy) {
this.cleanupStrategy = requireNonNull(cleanupStrategy, "cleanupStrategy must be non null");
return this;
}
@@ -338,9 +373,19 @@ public Builder setDockerImageTag(String dockerImageTag) {
return this;
}
+ public Builder setContainerName(String containerName) {
+ this.containerName = requireNonNull(containerName, "containerName must be non null");
+ return this;
+ }
+
public TestBench build() {
return new TestBench(
- ignorePullError, baseUri, dockerImageName, dockerImageTag, cleanupStrategy);
+ ignorePullError,
+ baseUri,
+ dockerImageName,
+ dockerImageTag,
+ cleanupStrategy,
+ containerName);
}
}
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobWriteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobWriteChannelTest.java
new file mode 100644
index 000000000..38f2e61e3
--- /dev/null
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobWriteChannelTest.java
@@ -0,0 +1,202 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage.it;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.google.api.client.json.JsonParser;
+import com.google.api.gax.rpc.FixedHeaderProvider;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.cloud.NoCredentials;
+import com.google.cloud.WriteChannel;
+import com.google.cloud.conformance.storage.v1.InstructionList;
+import com.google.cloud.conformance.storage.v1.Method;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.BucketInfo;
+import com.google.cloud.storage.DataGeneration;
+import com.google.cloud.storage.PackagePrivateMethodWorkarounds;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.Storage.BlobWriteOption;
+import com.google.cloud.storage.StorageOptions;
+import com.google.cloud.storage.conformance.retry.TestBench;
+import com.google.cloud.storage.conformance.retry.TestBench.RetryTestResource;
+import com.google.cloud.storage.spi.v1.StorageRpc;
+import com.google.cloud.storage.testing.RemoteStorageHelper;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.reflect.Reflection;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Optional;
+import java.util.Random;
+import java.util.logging.Logger;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.threeten.bp.Clock;
+import org.threeten.bp.Instant;
+import org.threeten.bp.ZoneId;
+import org.threeten.bp.ZoneOffset;
+import org.threeten.bp.format.DateTimeFormatter;
+
+public final class ITBlobWriteChannelTest {
+ private static final Logger LOGGER = Logger.getLogger(ITBlobWriteChannelTest.class.getName());
+ private static final String NOW_STRING;
+
+ static {
+ Instant now = Clock.systemUTC().instant();
+ DateTimeFormatter formatter =
+ DateTimeFormatter.ISO_LOCAL_DATE_TIME.withZone(ZoneId.from(ZoneOffset.UTC));
+ NOW_STRING = formatter.format(now);
+ }
+
+ private static final String BUCKET = RemoteStorageHelper.generateBucketName();
+
+ @ClassRule
+ public static final TestBench testBench =
+ TestBench.newBuilder()
+ .setContainerName("blob-write-channel-test")
+ // set a different base uri to prevent collision with conformance tests.
+ // It's possible for ITRetryConformanceTest to finish running and start shutting down
+ // when this test starts so the liveness check passes against the shutting down server.
+ .setBaseUri("http://localhost:9100")
+ .build();
+
+ @Rule public final TestName testName = new TestName();
+
+ @Rule public final DataGeneration dataGeneration = new DataGeneration(new Random(1234567890));
+
+ /**
+ * Test for unexpected EOF at the beginning of trying to read the json response.
+ *
+ * The error of this case shows up as an IllegalArgumentException rather than a json parsing
+ * error which comes from {@link JsonParser}{@code #startParsing()} which fails to find a node to
+ * start parsing.
+ */
+ @Test
+ public void testJsonEOF_0B() throws IOException {
+ int contentSize = 512 * 1024;
+ int cappedByteCount = 0;
+
+ doJsonUnexpectedEOFTest(contentSize, cappedByteCount);
+ }
+
+ /** Test for unexpected EOF 10 bytes into the json response */
+ @Test
+ public void testJsonEOF_10B() throws IOException {
+ int contentSize = 512 * 1024;
+ int cappedByteCount = 10;
+
+ doJsonUnexpectedEOFTest(contentSize, cappedByteCount);
+ }
+
+ private void doJsonUnexpectedEOFTest(int contentSize, int cappedByteCount) throws IOException {
+ String blobPath = String.format("%s/%s/blob", testName.getMethodName(), NOW_STRING);
+
+ BucketInfo bucketInfo = BucketInfo.of(BUCKET);
+ BlobInfo blobInfo = BlobInfo.newBuilder(bucketInfo, blobPath, 0L).build();
+
+ RetryTestResource retryTestResource =
+ RetryTestResource.newRetryTestResource(
+ Method.newBuilder().setName("storage.objects.insert").build(),
+ InstructionList.newBuilder()
+ .addInstructions(
+ String.format("return-broken-stream-final-chunk-after-%dB", cappedByteCount))
+ .build());
+ RetryTestResource retryTest = testBench.createRetryTest(retryTestResource);
+
+ StorageOptions baseOptions =
+ StorageOptions.newBuilder()
+ .setCredentials(NoCredentials.getInstance())
+ .setHost(testBench.getBaseUri())
+ .setProjectId("project-id")
+ .build();
+ StorageRpc noHeader = (StorageRpc) baseOptions.getRpc();
+ StorageRpc yesHeader =
+ (StorageRpc)
+ baseOptions
+ .toBuilder()
+ .setHeaderProvider(
+ FixedHeaderProvider.create(ImmutableMap.of("x-retry-test-id", retryTest.id)))
+ .build()
+ .getRpc();
+ //noinspection UnstableApiUsage
+ StorageOptions storageOptions =
+ baseOptions
+ .toBuilder()
+ .setServiceRpcFactory(
+ options ->
+ Reflection.newProxy(
+ StorageRpc.class,
+ (proxy, method, args) -> {
+ try {
+ if ("writeWithResponse".equals(method.getName())) {
+ boolean lastChunk = (boolean) args[5];
+ LOGGER.info(
+ String.format(
+ "writeWithResponse called. (lastChunk = %b)", lastChunk));
+ if (lastChunk) {
+ return method.invoke(yesHeader, args);
+ }
+ }
+ return method.invoke(noHeader, args);
+ } catch (Exception e) {
+ if (e.getCause() != null) {
+ throw e.getCause();
+ } else {
+ throw e;
+ }
+ }
+ }))
+ .build();
+
+ Storage testStorage = storageOptions.getService();
+
+ testStorage.create(bucketInfo);
+
+ ByteBuffer content = dataGeneration.randByteBuffer(contentSize);
+ // create a duplicate to preserve the initial offset and limit for assertion later
+ ByteBuffer expected = content.duplicate();
+
+ WriteChannel w = testStorage.writer(blobInfo, BlobWriteOption.generationMatch());
+ w.write(content);
+ w.close();
+
+ RetryTestResource postRunState = testBench.getRetryTest(retryTest);
+ assertTrue(postRunState.completed);
+
+ Optional optionalStorageObject =
+ PackagePrivateMethodWorkarounds.maybeGetStorageObjectFunction().apply(w);
+
+ assertTrue(optionalStorageObject.isPresent());
+ StorageObject storageObject = optionalStorageObject.get();
+ assertThat(storageObject.getName()).isEqualTo(blobInfo.getName());
+
+ Blob blobGen2 = testStorage.get(blobInfo.getBlobId());
+ assertEquals(contentSize, (long) blobGen2.getSize());
+ assertNotEquals(blobInfo.getGeneration(), blobGen2.getGeneration());
+ ByteArrayOutputStream actualData = new ByteArrayOutputStream();
+ blobGen2.downloadTo(actualData);
+ ByteBuffer actual = ByteBuffer.wrap(actualData.toByteArray());
+ assertEquals(expected, actual);
+ }
+}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java
index 73ef7e9b8..2307f0374 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java
@@ -67,6 +67,7 @@
import com.google.cloud.storage.BucketInfo.LifecycleRule.LifecycleCondition;
import com.google.cloud.storage.CopyWriter;
import com.google.cloud.storage.Cors;
+import com.google.cloud.storage.DataGeneration;
import com.google.cloud.storage.HmacKey;
import com.google.cloud.storage.HttpMethod;
import com.google.cloud.storage.PostPolicyV4;
@@ -116,7 +117,6 @@
import java.net.URL;
import java.net.URLConnection;
import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.Key;
@@ -216,6 +216,7 @@ public class ITStorageTest {
ImmutableList.of(LIFECYCLE_RULE_1, LIFECYCLE_RULE_2);
@Rule public final TestName testName = new TestName();
+ @Rule public final DataGeneration dataGeneration = new DataGeneration(new Random(1234567890));
@BeforeClass
public static void beforeClass() throws IOException {
@@ -3860,13 +3861,13 @@ private void blobWriteChannel_handlesRecoveryOnLastChunkWhenGenerationIsPresent(
BlobId blobId = BlobId.of(BUCKET, blobPath);
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
- Random rand = new Random(1234567890);
- String randString = randString(rand, contentSize);
- final byte[] randStringBytes = randString.getBytes(StandardCharsets.UTF_8);
+ ByteBuffer contentGen1 = dataGeneration.randByteBuffer(contentSize);
+ ByteBuffer contentGen2 = dataGeneration.randByteBuffer(contentSize);
+ ByteBuffer contentGen2Expected = contentGen2.duplicate();
Storage storage = StorageOptions.getDefaultInstance().getService();
WriteChannel ww = storage.writer(blobInfo);
ww.setChunkSize(chunkSize);
- ww.write(ByteBuffer.wrap(randStringBytes));
+ ww.write(contentGen1);
ww.close();
Blob blobGen1 = storage.get(blobId);
@@ -3926,8 +3927,7 @@ protected Object handleInvocation(
try (WriteChannel w = testStorage.writer(blobGen1, BlobWriteOption.generationMatch())) {
w.setChunkSize(chunkSize);
- ByteBuffer buffer = ByteBuffer.wrap(randStringBytes);
- w.write(buffer);
+ w.write(contentGen2);
}
assertTrue("Expected an exception to be thrown for the last chunk", exceptionThrown.get());
@@ -3937,18 +3937,6 @@ protected Object handleInvocation(
assertNotEquals(blobInfo.getGeneration(), blobGen2.getGeneration());
ByteArrayOutputStream actualData = new ByteArrayOutputStream();
blobGen2.downloadTo(actualData);
- assertArrayEquals(randStringBytes, actualData.toByteArray());
- }
-
- private static String randString(Random rand, int length) {
- final StringBuilder sb = new StringBuilder();
- while (sb.length() < length) {
- int i = rand.nextInt('z');
- char c = (char) i;
- if (Character.isLetter(c) || Character.isDigit(c)) {
- sb.append(c);
- }
- }
- return sb.toString();
+ assertEquals(contentGen2Expected, ByteBuffer.wrap(actualData.toByteArray()));
}
}