Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for Hive partitioning options when creating external tables #236

Merged
merged 1 commit into from Mar 20, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Expand Up @@ -135,6 +135,14 @@ public Builder setFormatOptions(FormatOptions formatOptions) {
/** Sets the table schema. */
public abstract Builder setSchema(Schema schema);

/** Sets the table Hive partitioning options. */
public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioningOptions) {
return setHivePartitioningOptionsInner(hivePartitioningOptions);
};

abstract Builder setHivePartitioningOptionsInner(
HivePartitioningOptions hivePartitioningOptions);

/** Creates an {@code ExternalTableDefinition} object. */
@Override
public abstract ExternalTableDefinition build();
Expand Down Expand Up @@ -212,6 +220,19 @@ public <F extends FormatOptions> F getFormatOptions() {
@Nullable
public abstract Boolean getAutodetect();

/**
* [Experimental] Returns the HivePartitioningOptions when the data layout follows Hive
* partitioning convention
*/
@SuppressWarnings("unchecked")
@Nullable
public HivePartitioningOptions getHivePartitioningOptions() {
return getHivePartitioningOptionsInner();
}

@Nullable
abstract HivePartitioningOptions getHivePartitioningOptionsInner();

/** Returns a builder for the {@code ExternalTableDefinition} object. */
public abstract Builder toBuilder();

Expand Down Expand Up @@ -257,6 +278,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
if (getAutodetect() != null) {
externalConfigurationPb.setAutodetect(getAutodetect());
}
if (getHivePartitioningOptions() != null) {
externalConfigurationPb.setHivePartitioningOptions(getHivePartitioningOptions().toPb());
}
return externalConfigurationPb;
}

Expand Down Expand Up @@ -405,6 +429,10 @@ static ExternalTableDefinition fromPb(Table tablePb) {
}
builder.setMaxBadRecords(externalDataConfiguration.getMaxBadRecords());
builder.setAutodetect(externalDataConfiguration.getAutodetect());
if (externalDataConfiguration.getHivePartitioningOptions() != null) {
builder.setHivePartitioningOptions(
HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions()));
}
}
return builder.build();
}
Expand Down Expand Up @@ -444,6 +472,10 @@ static ExternalTableDefinition fromExternalDataConfiguration(
if (externalDataConfiguration.getAutodetect() != null) {
builder.setAutodetect(externalDataConfiguration.getAutodetect());
}
if (externalDataConfiguration.getHivePartitioningOptions() != null) {
builder.setHivePartitioningOptions(
HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions()));
}
return builder.build();
}
}
Expand Up @@ -103,6 +103,16 @@ public String toString() {
.toString();
}

@Override
public boolean equals(Object obj) {
return obj == this
|| obj != null
&& obj.getClass().equals(HivePartitioningOptions.class)
&& Objects.equals(mode, ((HivePartitioningOptions) obj).getMode())
&& Objects.equals(
sourceUriPrefix, ((HivePartitioningOptions) obj).getSourceUriPrefix());
}

@Override
public int hashCode() {
return Objects.hash(mode, sourceUriPrefix);
Expand Down
Expand Up @@ -46,12 +46,18 @@ public class ExternalTableDefinitionTest {
private static final String COMPRESSION = "GZIP";
private static final Boolean AUTODETECT = true;
private static final CsvOptions CSV_OPTIONS = CsvOptions.newBuilder().build();
private static final HivePartitioningOptions HIVE_PARTITIONING_OPTIONS =
HivePartitioningOptions.newBuilder()
.setMode("AUTO")
.setSourceUriPrefix(SOURCE_URIS.get(0))
.build();
private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION =
ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS)
.setCompression(COMPRESSION)
.setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
.setMaxBadRecords(MAX_BAD_RECORDS)
.setAutodetect(AUTODETECT)
.setHivePartitioningOptions(HIVE_PARTITIONING_OPTIONS)
.build();

@Test
Expand Down Expand Up @@ -83,6 +89,7 @@ public void testBuilder() {
assertEquals(TABLE_SCHEMA, EXTERNAL_TABLE_DEFINITION.getSchema());
assertEquals(SOURCE_URIS, EXTERNAL_TABLE_DEFINITION.getSourceUris());
assertEquals(AUTODETECT, EXTERNAL_TABLE_DEFINITION.getAutodetect());
assertEquals(HIVE_PARTITIONING_OPTIONS, EXTERNAL_TABLE_DEFINITION.getHivePartitioningOptions());
}

@Test
Expand All @@ -107,5 +114,6 @@ private void compareExternalTableDefinition(
assertEquals(expected.getSourceUris(), value.getSourceUris());
assertEquals(expected.hashCode(), value.hashCode());
assertEquals(expected.getAutodetect(), value.getAutodetect());
assertEquals(expected.getHivePartitioningOptions(), value.getHivePartitioningOptions());
}
}