From 66d1eb793383b9e83992824b392cedd28d54609f Mon Sep 17 00:00:00 2001 From: Dmitry <58846611+dmitry-fa@users.noreply.github.com> Date: Thu, 16 Jul 2020 01:54:32 +0300 Subject: [PATCH] feat: auto content-type on blob creation (#338) * feat: auto content-type on blob creation * feat: auto content-type on blob creation * feat: auto content-type on blob creation * feat: auto content-type on blob creation * feat: auto content-type on blob creation * feat: auto content-type on blob creation * detection content type is made optional --- .../com/google/cloud/storage/Storage.java | 32 +++++++++-- .../cloud/storage/spi/v1/HttpStorageRpc.java | 23 ++++++-- .../cloud/storage/spi/v1/StorageRpc.java | 3 +- .../cloud/storage/it/ITStorageTest.java | 57 +++++++++++++++++++ 4 files changed, 104 insertions(+), 11 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 70863b17f..202615cdd 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -521,6 +521,15 @@ public static BlobTargetOption disableGzipContent() { return new BlobTargetOption(StorageRpc.Option.IF_DISABLE_GZIP_CONTENT, true); } + /** + * Returns an option for detecting content type. If this option is used, the content type is + * detected from the blob name if not explicitly set. This option is on the client side only, it + * does not appear in a RPC call. + */ + public static BlobTargetOption detectContentType() { + return new BlobTargetOption(StorageRpc.Option.DETECT_CONTENT_TYPE, true); + } + /** * Returns an option to set a customer-supplied AES256 key for server-side encryption of the * blob. @@ -593,6 +602,7 @@ enum Option { CUSTOMER_SUPPLIED_KEY, KMS_KEY_NAME, USER_PROJECT, + DETECT_CONTENT_TYPE, IF_DISABLE_GZIP_CONTENT; StorageRpc.Option toRpcOption() { @@ -733,6 +743,15 @@ public static BlobWriteOption userProject(String userProject) { public static BlobWriteOption disableGzipContent() { return new BlobWriteOption(Option.IF_DISABLE_GZIP_CONTENT, true); } + + /** + * Returns an option for detecting content type. If this option is used, the content type is + * detected from the blob name if not explicitly set. This option is on the client side only, it + * does not appear in a RPC call. + */ + public static BlobWriteOption detectContentType() { + return new BlobWriteOption(Option.DETECT_CONTENT_TYPE, true); + } } /** Class for specifying blob source options. */ @@ -1832,9 +1851,10 @@ public static Builder newBuilder() { * Creates a new blob. Direct upload is used to upload {@code content}. For large content, {@link * #writer} is recommended as it uses resumable upload. MD5 and CRC32C hashes of {@code content} * are computed and used for validating transferred data. Accepts an optional userProject {@link - * BlobGetOption} option which defines the project id to assign operational costs. + * BlobGetOption} option which defines the project id to assign operational costs. The content + * type is detected from the blob name if not explicitly set. * - *

Example of creating a blob from a byte array. + *

Example of creating a blob from a byte array: * *

{@code
    * String bucketName = "my-unique-bucket";
@@ -1857,7 +1877,7 @@ public static Builder newBuilder() {
    * Accepts a userProject {@link BlobGetOption} option, which defines the project id to assign
    * operational costs.
    *
-   * 

Example of creating a blob from a byte array. + *

Example of creating a blob from a byte array: * *

{@code
    * String bucketName = "my-unique-bucket";
@@ -1876,7 +1896,7 @@ Blob create(
 
   /**
    * Creates a new blob. Direct upload is used to upload {@code content}. For large content, {@link
-   * #writer} is recommended as it uses resumable upload. By default any md5 and crc32c values in
+   * #writer} is recommended as it uses resumable upload. By default any MD5 and CRC32C values in
    * the given {@code blobInfo} are ignored unless requested via the {@code
    * BlobWriteOption.md5Match} and {@code BlobWriteOption.crc32cMatch} options. The given input
    * stream is closed upon success.
@@ -2603,11 +2623,11 @@ Blob createFrom(
   ReadChannel reader(BlobId blob, BlobSourceOption... options);
 
   /**
-   * Creates a blob and return a channel for writing its content. By default any md5 and crc32c
+   * Creates a blob and returns a channel for writing its content. By default any MD5 and CRC32C
    * values in the given {@code blobInfo} are ignored unless requested via the {@code
    * BlobWriteOption.md5Match} and {@code BlobWriteOption.crc32cMatch} options.
    *
-   * 

Example of writing a blob's content through a writer. + *

Example of writing a blob's content through a writer: * *

{@code
    * String bucketName = "my-unique-bucket";
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java
index 519a9e9fa..ec849281e 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java
@@ -77,9 +77,12 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.math.BigInteger;
+import java.net.FileNameMap;
+import java.net.URLConnection;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 public class HttpStorageRpc implements StorageRpc {
@@ -98,6 +101,7 @@ public class HttpStorageRpc implements StorageRpc {
   private final HttpRequestInitializer batchRequestInitializer;
 
   private static final long MEGABYTE = 1024L * 1024L;
+  private static final FileNameMap FILE_NAME_MAP = URLConnection.getFileNameMap();
 
   public HttpStorageRpc(StorageOptions options) {
     HttpTransportOptions transportOptions = (HttpTransportOptions) options.getTransportOptions();
@@ -286,7 +290,7 @@ public StorageObject create(
               .insert(
                   storageObject.getBucket(),
                   storageObject,
-                  new InputStreamContent(storageObject.getContentType(), content));
+                  new InputStreamContent(detectContentType(storageObject, options), content));
       insert.getMediaHttpUploader().setDirectUploadEnabled(true);
       Boolean disableGzipContent = Option.IF_DISABLE_GZIP_CONTENT.getBoolean(options);
       if (disableGzipContent != null) {
@@ -372,6 +376,19 @@ public Tuple> list(final String bucket, Map options) {
+    String contentType = object.getContentType();
+    if (contentType != null) {
+      return contentType;
+    }
+
+    if (Boolean.TRUE == Option.DETECT_CONTENT_TYPE.get(options)) {
+      contentType = FILE_NAME_MAP.getContentTypeFor(object.getName().toLowerCase(Locale.ENGLISH));
+    }
+
+    return firstNonNull(contentType, "application/octet-stream");
+  }
+
   private static Function objectFromPrefix(final String bucket) {
     return new Function() {
       @Override
@@ -834,9 +851,7 @@ public String open(StorageObject object, Map options) {
       HttpRequest httpRequest =
           requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object));
       HttpHeaders requestHeaders = httpRequest.getHeaders();
-      requestHeaders.set(
-          "X-Upload-Content-Type",
-          firstNonNull(object.getContentType(), "application/octet-stream"));
+      requestHeaders.set("X-Upload-Content-Type", detectContentType(object, options));
       String key = Option.CUSTOMER_SUPPLIED_KEY.getString(options);
       if (key != null) {
         BaseEncoding base64 = BaseEncoding.base64();
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java
index 7ae9c8ec1..d1249f6df 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java
@@ -65,7 +65,8 @@ enum Option {
     KMS_KEY_NAME("kmsKeyName"),
     SERVICE_ACCOUNT_EMAIL("serviceAccount"),
     SHOW_DELETED_KEYS("showDeletedKeys"),
-    REQUESTED_POLICY_VERSION("optionsRequestedPolicyVersion");
+    REQUESTED_POLICY_VERSION("optionsRequestedPolicyVersion"),
+    DETECT_CONTENT_TYPE("detectContentType");
 
     private final String value;
 
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java
index ba4b3623f..fbce81db7 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java
@@ -3363,4 +3363,61 @@ public void testUploadWithEncryption() throws Exception {
     byte[] readBytes = blob.getContent(Blob.BlobSourceOption.decryptionKey(KEY));
     assertArrayEquals(BLOB_BYTE_CONTENT, readBytes);
   }
+
+  private Blob createBlob(String method, BlobInfo blobInfo, boolean detectType) throws IOException {
+    switch (method) {
+      case "create":
+        return detectType
+            ? storage.create(blobInfo, Storage.BlobTargetOption.detectContentType())
+            : storage.create(blobInfo);
+      case "createFrom":
+        InputStream inputStream = new ByteArrayInputStream(BLOB_BYTE_CONTENT);
+        return detectType
+            ? storage.createFrom(blobInfo, inputStream, Storage.BlobWriteOption.detectContentType())
+            : storage.createFrom(blobInfo, inputStream);
+      case "writer":
+        if (detectType) {
+          storage.writer(blobInfo, Storage.BlobWriteOption.detectContentType()).close();
+        } else {
+          storage.writer(blobInfo).close();
+        }
+        return storage.get(BlobId.of(blobInfo.getBucket(), blobInfo.getName()));
+      default:
+        throw new IllegalArgumentException("Unknown method " + method);
+    }
+  }
+
+  private void testAutoContentType(String method) throws IOException {
+    String[] names = {"file1.txt", "dir with spaces/Pic.Jpg", "no_extension"};
+    String[] types = {"text/plain", "image/jpeg", "application/octet-stream"};
+    for (int i = 0; i < names.length; i++) {
+      BlobId blobId = BlobId.of(BUCKET, names[i]);
+      BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
+      Blob blob_true = createBlob(method, blobInfo, true);
+      assertEquals(types[i], blob_true.getContentType());
+
+      Blob blob_false = createBlob(method, blobInfo, false);
+      assertEquals("application/octet-stream", blob_false.getContentType());
+    }
+    String customType = "custom/type";
+    BlobId blobId = BlobId.of(BUCKET, names[0]);
+    BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType(customType).build();
+    Blob blob = createBlob(method, blobInfo, true);
+    assertEquals(customType, blob.getContentType());
+  }
+
+  @Test
+  public void testAutoContentTypeCreate() throws IOException {
+    testAutoContentType("create");
+  }
+
+  @Test
+  public void testAutoContentTypeCreateFrom() throws IOException {
+    testAutoContentType("createFrom");
+  }
+
+  @Test
+  public void testAutoContentTypeWriter() throws IOException {
+    testAutoContentType("writer");
+  }
 }