Skip to content

Commit

Permalink
feat: auto content-type on blob creation (#338)
Browse files Browse the repository at this point in the history 
* feat: auto content-type on blob creation

* feat: auto content-type on blob creation

* feat: auto content-type on blob creation

* feat: auto content-type on blob creation

* feat: auto content-type on blob creation

* feat: auto content-type on blob creation

* detection content type is made optional
  • Loading branch information
@dmitry-fa
dmitry-fa committed Jul 15, 2020
1 parent 43202f6 commit 66d1eb7
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 11 deletions.
32 changes: 26 additions & 6 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java
View file
Expand Up @@ -521,6 +521,15 @@ public static BlobTargetOption disableGzipContent() {
return new BlobTargetOption(StorageRpc.Option.IF_DISABLE_GZIP_CONTENT, true);
}

/**
* Returns an option for detecting content type. If this option is used, the content type is
* detected from the blob name if not explicitly set. This option is on the client side only, it
* does not appear in a RPC call.
*/
public static BlobTargetOption detectContentType() {
return new BlobTargetOption(StorageRpc.Option.DETECT_CONTENT_TYPE, true);
}

/**
* Returns an option to set a customer-supplied AES256 key for server-side encryption of the
* blob.
Expand Down Expand Up @@ -593,6 +602,7 @@ enum Option {
CUSTOMER_SUPPLIED_KEY,
KMS_KEY_NAME,
USER_PROJECT,
DETECT_CONTENT_TYPE,
IF_DISABLE_GZIP_CONTENT;

StorageRpc.Option toRpcOption() {
Expand Down Expand Up @@ -733,6 +743,15 @@ public static BlobWriteOption userProject(String userProject) {
public static BlobWriteOption disableGzipContent() {
return new BlobWriteOption(Option.IF_DISABLE_GZIP_CONTENT, true);
}

/**
* Returns an option for detecting content type. If this option is used, the content type is
* detected from the blob name if not explicitly set. This option is on the client side only, it
* does not appear in a RPC call.
*/
public static BlobWriteOption detectContentType() {
return new BlobWriteOption(Option.DETECT_CONTENT_TYPE, true);
}
}

/** Class for specifying blob source options. */
Expand Down Expand Up @@ -1832,9 +1851,10 @@ public static Builder newBuilder() {
* Creates a new blob. Direct upload is used to upload {@code content}. For large content, {@link
* #writer} is recommended as it uses resumable upload. MD5 and CRC32C hashes of {@code content}
* are computed and used for validating transferred data. Accepts an optional userProject {@link
* BlobGetOption} option which defines the project id to assign operational costs.
* BlobGetOption} option which defines the project id to assign operational costs. The content
* type is detected from the blob name if not explicitly set.
*
* <p>Example of creating a blob from a byte array.
* <p>Example of creating a blob from a byte array:
*
* <pre>{@code
* String bucketName = "my-unique-bucket";
Expand All @@ -1857,7 +1877,7 @@ public static Builder newBuilder() {
* Accepts a userProject {@link BlobGetOption} option, which defines the project id to assign
* operational costs.
*
* <p>Example of creating a blob from a byte array.
* <p>Example of creating a blob from a byte array:
*
* <pre>{@code
* String bucketName = "my-unique-bucket";
Expand All @@ -1876,7 +1896,7 @@ Blob create(

/**
* Creates a new blob. Direct upload is used to upload {@code content}. For large content, {@link
* #writer} is recommended as it uses resumable upload. By default any md5 and crc32c values in
* #writer} is recommended as it uses resumable upload. By default any MD5 and CRC32C values in
* the given {@code blobInfo} are ignored unless requested via the {@code
* BlobWriteOption.md5Match} and {@code BlobWriteOption.crc32cMatch} options. The given input
* stream is closed upon success.
Expand Down Expand Up @@ -2603,11 +2623,11 @@ Blob createFrom(
ReadChannel reader(BlobId blob, BlobSourceOption... options);

/**
* Creates a blob and return a channel for writing its content. By default any md5 and crc32c
* Creates a blob and returns a channel for writing its content. By default any MD5 and CRC32C
* values in the given {@code blobInfo} are ignored unless requested via the {@code
* BlobWriteOption.md5Match} and {@code BlobWriteOption.crc32cMatch} options.
*
* <p>Example of writing a blob's content through a writer.
* <p>Example of writing a blob's content through a writer:
*
* <pre>{@code
* String bucketName = "my-unique-bucket";
Expand Down
23 changes: 19 additions & 4 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java
View file
Expand Up @@ -77,9 +77,12 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.math.BigInteger;
import java.net.FileNameMap;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;

public class HttpStorageRpc implements StorageRpc {
Expand All @@ -98,6 +101,7 @@ public class HttpStorageRpc implements StorageRpc {
private final HttpRequestInitializer batchRequestInitializer;

private static final long MEGABYTE = 1024L * 1024L;
private static final FileNameMap FILE_NAME_MAP = URLConnection.getFileNameMap();

public HttpStorageRpc(StorageOptions options) {
HttpTransportOptions transportOptions = (HttpTransportOptions) options.getTransportOptions();
Expand Down Expand Up @@ -286,7 +290,7 @@ public StorageObject create(
.insert(
storageObject.getBucket(),
storageObject,
new InputStreamContent(storageObject.getContentType(), content));
new InputStreamContent(detectContentType(storageObject, options), content));
insert.getMediaHttpUploader().setDirectUploadEnabled(true);
Boolean disableGzipContent = Option.IF_DISABLE_GZIP_CONTENT.getBoolean(options);
if (disableGzipContent != null) {
Expand Down Expand Up @@ -372,6 +376,19 @@ public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Opti
}
}

private static String detectContentType(StorageObject object, Map<Option, ?> options) {
String contentType = object.getContentType();
if (contentType != null) {
return contentType;
}

if (Boolean.TRUE == Option.DETECT_CONTENT_TYPE.get(options)) {
contentType = FILE_NAME_MAP.getContentTypeFor(object.getName().toLowerCase(Locale.ENGLISH));
}

return firstNonNull(contentType, "application/octet-stream");
}

private static Function<String, StorageObject> objectFromPrefix(final String bucket) {
return new Function<String, StorageObject>() {
@Override
Expand Down Expand Up @@ -834,9 +851,7 @@ public String open(StorageObject object, Map<Option, ?> options) {
HttpRequest httpRequest =
requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object));
HttpHeaders requestHeaders = httpRequest.getHeaders();
requestHeaders.set(
"X-Upload-Content-Type",
firstNonNull(object.getContentType(), "application/octet-stream"));
requestHeaders.set("X-Upload-Content-Type", detectContentType(object, options));
String key = Option.CUSTOMER_SUPPLIED_KEY.getString(options);
if (key != null) {
BaseEncoding base64 = BaseEncoding.base64();
Expand Down
3 changes: 2 additions & 1 deletion google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java
View file
Expand Up @@ -65,7 +65,8 @@ enum Option {
KMS_KEY_NAME("kmsKeyName"),
SERVICE_ACCOUNT_EMAIL("serviceAccount"),
SHOW_DELETED_KEYS("showDeletedKeys"),
REQUESTED_POLICY_VERSION("optionsRequestedPolicyVersion");
REQUESTED_POLICY_VERSION("optionsRequestedPolicyVersion"),
DETECT_CONTENT_TYPE("detectContentType");

private final String value;

Expand Down
57 changes: 57 additions & 0 deletions google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java
View file
Expand Up @@ -3363,4 +3363,61 @@ public void testUploadWithEncryption() throws Exception {
byte[] readBytes = blob.getContent(Blob.BlobSourceOption.decryptionKey(KEY));
assertArrayEquals(BLOB_BYTE_CONTENT, readBytes);
}

private Blob createBlob(String method, BlobInfo blobInfo, boolean detectType) throws IOException {
switch (method) {
case "create":
return detectType
? storage.create(blobInfo, Storage.BlobTargetOption.detectContentType())
: storage.create(blobInfo);
case "createFrom":
InputStream inputStream = new ByteArrayInputStream(BLOB_BYTE_CONTENT);
return detectType
? storage.createFrom(blobInfo, inputStream, Storage.BlobWriteOption.detectContentType())
: storage.createFrom(blobInfo, inputStream);
case "writer":
if (detectType) {
storage.writer(blobInfo, Storage.BlobWriteOption.detectContentType()).close();
} else {
storage.writer(blobInfo).close();
}
return storage.get(BlobId.of(blobInfo.getBucket(), blobInfo.getName()));
default:
throw new IllegalArgumentException("Unknown method " + method);
}
}

private void testAutoContentType(String method) throws IOException {
String[] names = {"file1.txt", "dir with spaces/Pic.Jpg", "no_extension"};
String[] types = {"text/plain", "image/jpeg", "application/octet-stream"};
for (int i = 0; i < names.length; i++) {
BlobId blobId = BlobId.of(BUCKET, names[i]);
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
Blob blob_true = createBlob(method, blobInfo, true);
assertEquals(types[i], blob_true.getContentType());

Blob blob_false = createBlob(method, blobInfo, false);
assertEquals("application/octet-stream", blob_false.getContentType());
}
String customType = "custom/type";
BlobId blobId = BlobId.of(BUCKET, names[0]);
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType(customType).build();
Blob blob = createBlob(method, blobInfo, true);
assertEquals(customType, blob.getContentType());
}

@Test
public void testAutoContentTypeCreate() throws IOException {
testAutoContentType("create");
}

@Test
public void testAutoContentTypeCreateFrom() throws IOException {
testAutoContentType("createFrom");
}

@Test
public void testAutoContentTypeWriter() throws IOException {
testAutoContentType("writer");
}
}

0 comments on commit 66d1eb7

Please sign in to comment.