Skip to content

Commit

Permalink
Allow adding query params to GCS signed URLs (#7045)
Browse files Browse the repository at this point in the history 
* Allow adding arbitrary query params to signed URLs

TODO: Add unit tests to detect correct sorting (move out of integration
test).
TODO: Add support for V2 signing, along with tests.

* Move sorting check test to unit tests.

* Allow query params to be added to v2 signed urls

Includes tests.
  • Loading branch information
@houglum @frankyn
houglum authored and frankyn committed Dec 19, 2019
1 parent 513ba6b commit 035ae2a
Show file tree
Hide file tree
Showing 5 changed files with 348 additions and 59 deletions.
154 changes: 105 additions & 49 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java
View file
Expand Up @@ -16,23 +16,21 @@

package com.google.cloud.storage;

import static com.google.cloud.storage.SignedUrlEncodingHelper.Rfc3986UriEncode;
import static com.google.common.base.Preconditions.checkArgument;

import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.hash.Hashing;
import com.google.common.net.UrlEscapers;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.TreeMap;

/**
* Signature Info holds payload components of the string that requires signing.
Expand All @@ -46,12 +44,24 @@ public class SignatureInfo {
public static final char COMPONENT_SEPARATOR = '\n';
public static final String GOOG4_RSA_SHA256 = "GOOG4-RSA-SHA256";
public static final String SCOPE = "/auto/storage/goog4_request";
private static final List<String> RESERVED_PARAMS_LOWER =
ImmutableList.<String>of(
// V2:
"expires",
"googleaccessid",
// V4:
"x-goog-algorithm",
"x-goog-credential",
"x-goog-date",
"x-goog-expires",
"x-goog-signedheaders");

private final HttpMethod httpVerb;
private final String contentMd5;
private final String contentType;
private final long expiration;
private final Map<String, String> canonicalizedExtensionHeaders;
private final Map<String, String> queryParams;
private final URI canonicalizedResource;
private final Storage.SignUrlOption.SignatureVersion signatureVersion;
private final String accountEmail;
Expand All @@ -70,17 +80,16 @@ private SignatureInfo(Builder builder) {
this.accountEmail = builder.accountEmail;
this.timestamp = builder.timestamp;

ImmutableMap.Builder<String, String> headerBuilder =
new ImmutableMap.Builder<String, String>().putAll(builder.canonicalizedExtensionHeaders);
// The "host" header only needs to be present and signed if using V4.
if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion)
&& (!builder.canonicalizedExtensionHeaders.containsKey("host"))) {
canonicalizedExtensionHeaders =
new ImmutableMap.Builder<String, String>()
.putAll(builder.canonicalizedExtensionHeaders)
.put("host", "storage.googleapis.com")
.build();
} else {
canonicalizedExtensionHeaders = builder.canonicalizedExtensionHeaders;
headerBuilder.put("host", "storage.googleapis.com");
}
canonicalizedExtensionHeaders = headerBuilder.build();

queryParams = ImmutableMap.<String, String>copyOf(builder.queryParams);

Date date = new Date(timestamp);

Expand Down Expand Up @@ -123,7 +132,7 @@ private String constructV2UnsignedPayload() {
payload.append(COMPONENT_SEPARATOR);
payload.append(expiration).append(COMPONENT_SEPARATOR);

if (canonicalizedExtensionHeaders != null) {
if (canonicalizedExtensionHeaders.size() > 0) {
payload.append(
new CanonicalExtensionHeadersSerializer(Storage.SignUrlOption.SignatureVersion.V2)
.serialize(canonicalizedExtensionHeaders));
Expand Down Expand Up @@ -167,49 +176,80 @@ private String constructV4CanonicalRequestHash() {
.toString();
}

public String constructV4QueryString() {
ArrayListMultimap<String, String> paramMap = ArrayListMultimap.create();
/**
* Returns a TreeMap containing the user-supplied query parameters that do not have reserved keys.
*/
private TreeMap<String, String> getNonReservedUserQueryParams() {
TreeMap<String, String> sortedParamMap = new TreeMap<String, String>();

// Skip any instances of well-known required headers that might have been supplied by the
// caller.
for (Map.Entry<String, String> entry : queryParams.entrySet()) {
// Convert to (and check for the existence of) lowercase keys to prevent cases like a user
// supplying "x-goog-algorithm", in order to prevent the resulting query string from
// containing "x-goog-algorithm" and "X-Goog-Algorithm".
if (!RESERVED_PARAMS_LOWER.contains(entry.getKey().toLowerCase())) {
// URI encode user-supplied parameter, both the name and the value.
sortedParamMap.put(
Rfc3986UriEncode(entry.getKey(), true), Rfc3986UriEncode(entry.getValue(), true));
}
}

return sortedParamMap;
}

private String queryStringFromParamMap(Map<String, String> map) {
StringBuilder queryStringBuilder = new StringBuilder();

// TODO: Once we support supplying additional query params, we should remove any of the reserved
// ones that users might have added.
String sep = "";
for (Map.Entry<String, String> entry : map.entrySet()) {
queryStringBuilder.append(sep);
sep = "&";
queryStringBuilder.append(entry.getKey()).append('=').append(entry.getValue());
}

return queryStringBuilder.toString();
}

/**
* Returns a query string constructed from this object's stored query parameters, sorted in code
* point order. Note that these query parameters are not used when constructing the URL's
* signature. The returned value does not include the leading ? character, as this is not part of
* a query string.
*
* @return A URI query string. Returns an empty string if the user supplied no query parameters.
*/
public String constructV2QueryString() {
TreeMap<String, String> sortedParamMap = getNonReservedUserQueryParams();
// The "GoogleAccessId", "Expires", and "Signature" params are not included here.
return queryStringFromParamMap(sortedParamMap);
}

/**
* Returns a query string constructed from this object's stored query parameters, sorted in code
* point order so that the query string can be used in a V4 canonical request string. The returned
* value does not include the leading ? character, as this is not part of a query string.
*
* @see <a href= "https://cloud.google.com/storage/docs/authentication/canonical-requests">
* Canonical Requests</a>
*/
public String constructV4QueryString() {
TreeMap<String, String> sortedParamMap = getNonReservedUserQueryParams();

// Add in the reserved auth-specific query params.
paramMap.put(
"X-Goog-Algorithm", UrlEscapers.urlFormParameterEscaper().escape(GOOG4_RSA_SHA256));
paramMap.put(
"X-Goog-Credential",
UrlEscapers.urlFormParameterEscaper().escape(accountEmail + "/" + yearMonthDay + SCOPE));
paramMap.put("X-Goog-Date", UrlEscapers.urlFormParameterEscaper().escape(exactDate));
paramMap.put(
"X-Goog-Expires", UrlEscapers.urlFormParameterEscaper().escape(Long.toString(expiration)));
sortedParamMap.put("X-Goog-Algorithm", Rfc3986UriEncode(GOOG4_RSA_SHA256, true));
sortedParamMap.put(
"X-Goog-Credential", Rfc3986UriEncode(accountEmail + "/" + yearMonthDay + SCOPE, true));
sortedParamMap.put("X-Goog-Date", Rfc3986UriEncode(exactDate, true));
sortedParamMap.put("X-Goog-Expires", Rfc3986UriEncode(Long.toString(expiration), true));
StringBuilder signedHeadersBuilder =
new CanonicalExtensionHeadersSerializer(Storage.SignUrlOption.SignatureVersion.V4)
.serializeHeaderNames(canonicalizedExtensionHeaders);
paramMap.put(
"X-Goog-SignedHeaders",
UrlEscapers.urlFormParameterEscaper().escape(signedHeadersBuilder.toString()));

StringBuilder queryStringBuilder = new StringBuilder();
ArrayList<String> paramKeys = Lists.newArrayList(paramMap.keySet());
Collections.sort(paramKeys);
for (String key : paramKeys) {
List<String> valuesForCurrentKey = paramMap.get(key);
if (valuesForCurrentKey.size() > 1) {
// If there's more than 1 value for the given key, create a standalone list from the given
// view collection and sort it; params with multiple values must be sorted by value.
valuesForCurrentKey = Lists.newArrayList(valuesForCurrentKey);
Collections.sort(valuesForCurrentKey);
}
for (String value : valuesForCurrentKey) {
queryStringBuilder.append(key).append('=').append(value).append('&');
}
}
// Remove trailing '&' from last-added param.
if (queryStringBuilder.length() > 0) {
queryStringBuilder.setLength(queryStringBuilder.length() - 1);
}
sortedParamMap.put(
"X-Goog-SignedHeaders", Rfc3986UriEncode(signedHeadersBuilder.toString(), true));

return queryStringBuilder.toString();
// The "X-Goog-Signature" param is not included here.
return queryStringFromParamMap(sortedParamMap);
}

public HttpMethod getHttpVerb() {
Expand All @@ -232,6 +272,10 @@ public Map<String, String> getCanonicalizedExtensionHeaders() {
return canonicalizedExtensionHeaders;
}

public Map<String, String> getQueryParams() {
return queryParams;
}

public URI getCanonicalizedResource() {
return canonicalizedResource;
}
Expand All @@ -255,6 +299,7 @@ public static final class Builder {
private String contentType;
private final long expiration;
private Map<String, String> canonicalizedExtensionHeaders;
private Map<String, String> queryParams;
private final URI canonicalizedResource;
private Storage.SignUrlOption.SignatureVersion signatureVersion;
private String accountEmail;
Expand All @@ -280,6 +325,7 @@ public Builder(SignatureInfo signatureInfo) {
this.contentType = signatureInfo.contentType;
this.expiration = signatureInfo.expiration;
this.canonicalizedExtensionHeaders = signatureInfo.canonicalizedExtensionHeaders;
this.queryParams = signatureInfo.queryParams;
this.canonicalizedResource = signatureInfo.canonicalizedResource;
this.signatureVersion = signatureInfo.signatureVersion;
this.accountEmail = signatureInfo.accountEmail;
Expand All @@ -305,6 +351,12 @@ public Builder setCanonicalizedExtensionHeaders(
return this;
}

public Builder setCanonicalizedQueryParams(Map<String, String> queryParams) {
this.queryParams = queryParams;

return this;
}

public Builder setSignatureVersion(Storage.SignUrlOption.SignatureVersion signatureVersion) {
this.signatureVersion = signatureVersion;

Expand Down Expand Up @@ -340,6 +392,10 @@ public SignatureInfo build() {
canonicalizedExtensionHeaders = new HashMap<>();
}

if (queryParams == null) {
queryParams = new HashMap<>();
}

return new SignatureInfo(this);
}
}
Expand Down
20 changes: 19 additions & 1 deletion google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java
View file
Expand Up @@ -1037,7 +1037,8 @@ enum Option {
SIGNATURE_VERSION,
HOST_NAME,
PATH_STYLE,
VIRTUAL_HOSTED_STYLE
VIRTUAL_HOSTED_STYLE,
QUERY_PARAMS
}

enum SignatureVersion {
Expand Down Expand Up @@ -1158,6 +1159,23 @@ public static SignUrlOption withVirtualHostedStyle() {
public static SignUrlOption withPathStyle() {
return new SignUrlOption(Option.PATH_STYLE, "");
}

/**
* Use if the URL should contain additional query parameters.
*
* <p>Warning: For V2 Signed URLs, it is possible for query parameters to be altered after the
* URL has been signed, as the parameters are not used to compute the signature. The V4 signing
* method should be preferred when supplying additional query parameters, as the parameters
* cannot be added, removed, or otherwise altered after a V4 signature is generated.
*
* @see <a href="https://cloud.google.com/storage/docs/authentication/canonical-requests">
* Canonical Requests</a>
* @see <a href="https://cloud.google.com/storage/docs/access-control/signed-urls-v2">V2 Signing
* Process</a>
*/
public static SignUrlOption withQueryParams(Map<String, String> queryParams) {
return new SignUrlOption(Option.QUERY_PARAMS, queryParams);
}
}

/**
Expand Down
37 changes: 28 additions & 9 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java
View file
Expand Up @@ -699,13 +699,22 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio
if (isV4) {
BaseEncoding encoding = BaseEncoding.base16().lowerCase();
String signature = URLEncoder.encode(encoding.encode(signatureBytes), UTF_8.name());
stBuilder.append("?");
stBuilder.append(signatureInfo.constructV4QueryString());
stBuilder.append("&X-Goog-Signature=").append(signature);
String v4QueryString = signatureInfo.constructV4QueryString();

stBuilder.append('?');
if (!Strings.isNullOrEmpty(v4QueryString)) {
stBuilder.append(v4QueryString).append('&');
}
stBuilder.append("X-Goog-Signature=").append(signature);
} else {
BaseEncoding encoding = BaseEncoding.base64();
String signature = URLEncoder.encode(encoding.encode(signatureBytes), UTF_8.name());
stBuilder.append("?");
String v2QueryString = signatureInfo.constructV2QueryString();

stBuilder.append('?');
if (!Strings.isNullOrEmpty(v2QueryString)) {
stBuilder.append(v2QueryString).append('&');
}
stBuilder.append("GoogleAccessId=").append(credentials.getAccount());
stBuilder.append("&Expires=").append(expiration);
stBuilder.append("&Signature=").append(signature);
Expand Down Expand Up @@ -815,28 +824,38 @@ private SignatureInfo buildSignatureInfo(

signatureInfoBuilder.setTimestamp(getOptions().getClock().millisTime());

ImmutableMap.Builder<String, String> extHeaders = new ImmutableMap.Builder<String, String>();
ImmutableMap.Builder<String, String> extHeadersBuilder =
new ImmutableMap.Builder<String, String>();

boolean isV4 =
SignUrlOption.SignatureVersion.V4.equals(
optionMap.get(SignUrlOption.Option.SIGNATURE_VERSION));
if (isV4) { // We don't sign the host header for V2 signed URLs; only do this for V4.
// Add the host here first, allowing it to be overridden in the EXT_HEADERS option below.
if (optionMap.containsKey(SignUrlOption.Option.VIRTUAL_HOSTED_STYLE)) {
extHeaders.put(
extHeadersBuilder.put(
"host",
slashlessBucketNameFromBlobInfo(blobInfo) + "." + getBaseStorageHostName(optionMap));
} else if (optionMap.containsKey(SignUrlOption.Option.HOST_NAME)) {
extHeaders.put("host", getBaseStorageHostName(optionMap));
extHeadersBuilder.put("host", getBaseStorageHostName(optionMap));
}
}

if (optionMap.containsKey(SignUrlOption.Option.EXT_HEADERS)) {
extHeaders.putAll((Map<String, String>) optionMap.get(SignUrlOption.Option.EXT_HEADERS));
extHeadersBuilder.putAll(
(Map<String, String>) optionMap.get(SignUrlOption.Option.EXT_HEADERS));
}

ImmutableMap.Builder<String, String> queryParamsBuilder =
new ImmutableMap.Builder<String, String>();
if (optionMap.containsKey(SignUrlOption.Option.QUERY_PARAMS)) {
queryParamsBuilder.putAll(
(Map<String, String>) optionMap.get(SignUrlOption.Option.QUERY_PARAMS));
}

return signatureInfoBuilder
.setCanonicalizedExtensionHeaders((Map<String, String>) extHeaders.build())
.setCanonicalizedExtensionHeaders((Map<String, String>) extHeadersBuilder.build())
.setCanonicalizedQueryParams((Map<String, String>) queryParamsBuilder.build())
.build();
}

Expand Down

0 comments on commit 035ae2a

Please sign in to comment.