feat: add storage.upload(path)

google-cloud-storage/clirr-ignored-differences.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- see http://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
+<differences>
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/storage/Storage</className>
+    <method>void upload(*.BlobInfo, java.nio.file.Path, *.Storage$BlobWriteOption[])</method>
+  </difference>
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/storage/Storage</className>
+    <method>void upload(*.BlobInfo, java.io.InputStream, *.Storage$BlobWriteOption[])</method>
+  </difference>
+</differences>

google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java

@@ -37,9 +37,11 @@
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.io.BaseEncoding;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.Serializable;
 import java.net.URL;
+import java.nio.file.Path;
 import java.security.Key;
 import java.util.Arrays;
 import java.util.Collections;
@@ -1811,6 +1813,113 @@ Blob create(
   @Deprecated
   Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options);
 
+  /**
+   * Uploads {@code path} to the blob using {@link #writer}. By default any MD5 and CRC32C values in
+   * the given {@code blobInfo} are ignored unless requested via the {@link
+   * BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options. Folder upload is
+   * not supported.
+   *
+   * <p>Example of uploading a file:
+   *
+   * <pre>{@code
+   * String bucketName = "my-unique-bucket";
+   * String fileName = "readme.txt";
+   * BlobId blobId = BlobId.of(bucketName, fileName);
+   * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
+   * storage.upload(blobInfo, Paths.get(fileName));
+   * }</pre>
+   *
+   * @param blobInfo blob to create
+   * @param path file to upload
+   * @param options blob write options
+   * @throws IOException on I/O error
+   * @throws StorageException on failure
+   * @see #upload(BlobInfo, Path, int, BlobWriteOption...)
+   */
+  void upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException;
+
+  /**
+   * Uploads {@code path} to the blob using {@link #writer} and {@code bufferSize}. By default any
+   * MD5 and CRC32C values in the given {@code blobInfo} are ignored unless requested via the {@link
+   * BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options. Folder upload is
+   * not supported.
+   *
+   * <p>{@link #upload(BlobInfo, Path, BlobWriteOption...)} invokes this method with a buffer size
+   * of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the upload
+   * performance but require more memory. This can cause an OutOfMemoryError or add significant
+   * garbage collection overhead. Smaller buffer sizes reduce memory consumption, that is noticeable
+   * when uploading many objects in parallel. Buffer sizes less than 256 KiB are treated as 256 KiB.
+   *
+   * <p>Example of uploading a humongous file:
+   *
+   * <pre>{@code
+   * BlobId blobId = BlobId.of(bucketName, blobName);
+   * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("video/webm").build();
+   *
+   * int largeBufferSize = 150 * 1024 * 1024;
+   * Path file = Paths.get("humongous.file");
+   * storage.upload(blobInfo, file, largeBufferSize);
+   * }</pre>
+   *
+   * @param blobInfo blob to create
+   * @param path file to upload
+   * @param bufferSize size of the buffer I/O operations
+   * @param options blob write options
+   * @throws IOException on I/O error
+   * @throws StorageException on failure
+   */
+  void upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options)
+      throws IOException;
+
+  /**
+   * Reads bytes from an input stream and uploads those bytes to the blob using {@link #writer}. By
+   * default any MD5 and CRC32C values in the given {@code blobInfo} are ignored unless requested
+   * via the {@link BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options.
+   *
+   * <p>Example of uploading data with CRC32C checksum:
+   *
+   * <pre>{@code
+   * BlobId blobId = BlobId.of(bucketName, blobName);
+   * byte[] content = "Hello, world".getBytes(StandardCharsets.UTF_8);
+   * Hasher hasher = Hashing.crc32c().newHasher().putBytes(content);
+   * String crc32c = BaseEncoding.base64().encode(Ints.toByteArray(hasher.hash().asInt()));
+   * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(crc32c).build();
+   * storage.upload(blobInfo, new ByteArrayInputStream(content), Storage.BlobWriteOption.crc32cMatch());
+   * }</pre>
+   *
+   * @param blobInfo blob to create
+   * @param content input stream to read from
+   * @param options blob write options
+   * @throws IOException on I/O error
+   * @throws StorageException on failure
+   * @see #upload(BlobInfo, InputStream, int, BlobWriteOption...)
+   */
+  void upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options)
+      throws IOException;
+
+  /**
+   * Reads bytes from an input stream and uploads those bytes to the blob using {@link #writer} and
+   * {@code bufferSize}. By default any MD5 and CRC32C values in the given {@code blobInfo} are
+   * ignored unless requested via the {@link BlobWriteOption#md5Match()} and {@link
+   * BlobWriteOption#crc32cMatch()} options.
+   *
+   * <p>{@link #upload(BlobInfo, InputStream, BlobWriteOption...)} )} invokes this method with a
+   * buffer size of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the
+   * upload performance but require more memory. This can cause an OutOfMemoryError or add
+   * significant garbage collection overhead. Smaller buffer sizes reduce memory consumption, that
+   * is noticeable when uploading many objects in parallel. Buffer sizes less than 256 KiB are
+   * treated as 256 KiB.
+   *
+   * @param blobInfo blob to create
+   * @param content input stream to read from
+   * @param bufferSize size of the buffer I/O operations
+   * @param options blob write options
+   * @throws IOException on I/O error
+   * @throws StorageException on failure
+   */
+  void upload(BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options)
+      throws IOException;
+
   /**
    * Returns the requested bucket or {@code null} if not found.
    *

google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java

@@ -48,6 +48,7 @@
 import com.google.cloud.ReadChannel;
 import com.google.cloud.RetryHelper.RetryHelperException;
 import com.google.cloud.Tuple;
+import com.google.cloud.WriteChannel;
 import com.google.cloud.storage.Acl.Entity;
 import com.google.cloud.storage.HmacKey.HmacKeyMetadata;
 import com.google.cloud.storage.spi.v1.StorageRpc;
@@ -66,12 +67,18 @@
 import com.google.common.io.BaseEncoding;
 import com.google.common.primitives.Ints;
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
 import java.net.URLEncoder;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumMap;
@@ -92,6 +99,9 @@ final class StorageImpl extends BaseService<StorageOptions> implements Storage {
 
   private static final String STORAGE_XML_URI_HOST_NAME = "storage.googleapis.com";
 
+  private static final int DEFAULT_BUFFER_SIZE = 15 * 1024 * 1024;
+  private static final int MIN_BUFFER_SIZE = 256 * 1024;
+
   private static final Function<Tuple<Storage, Boolean>, Boolean> DELETE_FUNCTION =
       new Function<Tuple<Storage, Boolean>, Boolean>() {
         @Override
@@ -204,6 +214,54 @@ public StorageObject call() {
     }
   }
 
+  @Override
+  public void upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException {
+    upload(blobInfo, path, DEFAULT_BUFFER_SIZE, options);
+  }
+
+  @Override
+  public void upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options)
+      throws IOException {
+    if (Files.isDirectory(path)) {
+      throw new StorageException(0, path + " is a directory");
+    }
+    try (InputStream input = Files.newInputStream(path)) {
+      upload(blobInfo, input, bufferSize, options);
+    }
+  }
+
+  @Override
+  public void upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options)
+      throws IOException {
+    upload(blobInfo, content, DEFAULT_BUFFER_SIZE, options);
+  }
+
+  @Override
+  public void upload(
+      BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options)
+      throws IOException {
+    try (WriteChannel writer = writer(blobInfo, options)) {
+      upload(Channels.newChannel(content), writer, bufferSize);
+    }
+  }
+
+  /*
+   * Uploads the given content to the storage using specified write channel and the given buffer
+   * size. This method does not close any channels.
+   */
+  private static void upload(ReadableByteChannel reader, WriteChannel writer, int bufferSize)
+      throws IOException {
+    bufferSize = Math.max(bufferSize, MIN_BUFFER_SIZE);
+    ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
+    writer.setChunkSize(bufferSize);
+
+    while (reader.read(buffer) >= 0) {
+      buffer.flip();
+      writer.write(buffer);
+      buffer.clear();
+    }
+  }
+
   @Override
   public Bucket get(String bucket, BucketGetOption... options) {
     final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb();