Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add requirePartitionFilter field in HivePartitioningOptions #803

Merged
merged 1 commit into from Oct 15, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Expand Up @@ -23,17 +23,20 @@
public final class HivePartitioningOptions {

private final String mode;
private final Boolean requirePartitionFilter;
private final String sourceUriPrefix;

public static final class Builder {

private String mode;
private Boolean requirePartitionFilter;
private String sourceUriPrefix;

private Builder() {}

private Builder(HivePartitioningOptions options) {
this.mode = options.mode;
this.requirePartitionFilter = options.requirePartitionFilter;
this.sourceUriPrefix = options.sourceUriPrefix;
}

Expand All @@ -50,6 +53,17 @@ public Builder setMode(String mode) {
return this;
}

/**
* [Optional] If set to true, queries over this table require a partition filter that can be
* used for partition elimination to be specified. Note that this field should only be true when
* creating a permanent external table or querying a temporary external table. Hive-partitioned
* loads with requirePartitionFilter explicitly set to true will fail.
*/
public Builder setRequirePartitionFilter(Boolean requirePartitionFilter) {
this.requirePartitionFilter = requirePartitionFilter;
return this;
}

/**
* [Optional] When hive partition detection is requested, a common prefix for all source uris
* should be supplied. The prefix must end immediately before the partition key encoding begins.
Expand All @@ -72,6 +86,7 @@ public HivePartitioningOptions build() {

private HivePartitioningOptions(Builder builder) {
this.mode = builder.mode;
this.requirePartitionFilter = builder.requirePartitionFilter;
this.sourceUriPrefix = builder.sourceUriPrefix;
}

Expand All @@ -80,6 +95,14 @@ public String getMode() {
return mode;
}

/**
* Returns true if a partition filter (that can be used for partition elimination) is required for
* queries over this table.
*/
public Boolean getRequirePartitionFilter() {
return requirePartitionFilter;
}

/* Returns the sourceUriPrefix of hive partitioning */
public String getSourceUriPrefix() {
return sourceUriPrefix;
Expand All @@ -99,6 +122,7 @@ public static Builder newBuilder() {
public String toString() {
return MoreObjects.toStringHelper(this)
.add("mode", mode)
.add("requirePartitionFilter", requirePartitionFilter)
.add("sourceUriPrefix", sourceUriPrefix)
.toString();
}
Expand All @@ -109,6 +133,8 @@ public boolean equals(Object obj) {
|| obj != null
&& obj.getClass().equals(HivePartitioningOptions.class)
&& Objects.equals(mode, ((HivePartitioningOptions) obj).getMode())
&& Objects.equals(
requirePartitionFilter, ((HivePartitioningOptions) obj).getRequirePartitionFilter())
&& Objects.equals(
sourceUriPrefix, ((HivePartitioningOptions) obj).getSourceUriPrefix());
}
Expand All @@ -122,6 +148,7 @@ com.google.api.services.bigquery.model.HivePartitioningOptions toPb() {
com.google.api.services.bigquery.model.HivePartitioningOptions options =
new com.google.api.services.bigquery.model.HivePartitioningOptions();
options.setMode(mode);
options.setRequirePartitionFilter(requirePartitionFilter);
options.setSourceUriPrefix(sourceUriPrefix);
return options;
}
Expand All @@ -132,6 +159,9 @@ static HivePartitioningOptions fromPb(
if (options.getMode() != null) {
builder.setMode(options.getMode());
}
if (options.getRequirePartitionFilter() != null) {
builder.setRequirePartitionFilter(options.getRequirePartitionFilter());
}
if (options.getSourceUriPrefix() != null) {
builder.setSourceUriPrefix(options.getSourceUriPrefix());
}
Expand Down
Expand Up @@ -24,9 +24,11 @@ public class HivePartitioningOptionsTest {

private static final String MODE = "STRING";
private static final String SOURCE_URI_PREFIX = "gs://bucket/path_to_table";
private static final Boolean REQUIRE_PARTITION_FILTER = true;
private static final HivePartitioningOptions HIVE_PARTITIONING_OPTIONS =
HivePartitioningOptions.newBuilder()
.setMode(MODE)
.setRequirePartitionFilter(REQUIRE_PARTITION_FILTER)
.setSourceUriPrefix(SOURCE_URI_PREFIX)
.build();

Expand All @@ -49,6 +51,8 @@ public void testToBuilderIncomplete() {
@Test
public void testBuilder() {
assertThat(HIVE_PARTITIONING_OPTIONS.getMode()).isEqualTo(MODE);
assertThat(HIVE_PARTITIONING_OPTIONS.getRequirePartitionFilter())
.isEqualTo(REQUIRE_PARTITION_FILTER);
assertThat(HIVE_PARTITIONING_OPTIONS.getSourceUriPrefix()).isEqualTo(SOURCE_URI_PREFIX);
}

Expand All @@ -62,6 +66,7 @@ public void testToAndFromPb() {
private void compareHivePartitioningOptions(
HivePartitioningOptions expected, HivePartitioningOptions value) {
assertThat(value.getMode()).isEqualTo(expected.getMode());
assertThat(value.getRequirePartitionFilter()).isEqualTo(expected.getRequirePartitionFilter());
assertThat(value.getSourceUriPrefix()).isEqualTo(expected.getSourceUriPrefix());
assertThat(value.toString()).isEqualTo(expected.toString());
assertThat(value.hashCode()).isEqualTo(expected.hashCode());
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Expand Up @@ -63,7 +63,7 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<github.global.server>github</github.global.server>
<site.installationModule>google-cloud-bigquery-parent</site.installationModule>
<google-api-services-bigquery.version>v2-rev20200916-1.30.10
<google-api-services-bigquery.version>v2-rev20200925-1.30.10
</google-api-services-bigquery.version>
</properties>

Expand Down