feat: add support for DecimalTargetTypes (#1345)

* feat: add support for DecimalTargetType Towards b/188414733 Implemented in LoadJobConfiguration * update test to verify that the destination table's decimal column is loaded as BIGNUMERIC type * add decimalTargetTypes support in ExternalTableDefinition.java * add decimalTargetTypes support in ExternalTableDefinition.java
googleapis · Jun 8, 2021 · ba528df · ba528df
1 parent 5f6307f
commit ba528df
Show file tree

Hide file tree

Showing 6 changed files with 114 additions and 24 deletions.
diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml
@@ -4,32 +4,12 @@
   <!-- TODO: REMOVE AFTER RELEASE -->
   <difference>
     <differenceType>7013</differenceType>
-    <className>com/google/cloud/bigquery/MaterializedViewDefinition</className>
-    <method>com.google.cloud.bigquery.Clustering getClustering()</method>
+    <className>com/google/cloud/bigquery/ExternalTableDefinition</className>
+    <method>com.google.common.collect.ImmutableList getDecimalTargetTypes()</method>
   </difference>
   <difference>
     <differenceType>7013</differenceType>
-    <className>com/google/cloud/bigquery/MaterializedViewDefinition</className>
-    <method>com.google.cloud.bigquery.RangePartitioning getRangePartitioning()</method>
-  </difference>
-  <difference>
-    <differenceType>7013</differenceType>
-    <className>com/google/cloud/bigquery/MaterializedViewDefinition</className>
-    <method>com.google.cloud.bigquery.TimePartitioning getTimePartitioning()</method>
-  </difference>
-  <difference>
-    <differenceType>7013</differenceType>
-    <className>com/google/cloud/bigquery/MaterializedViewDefinition$Builder</className>
-    <method>com.google.cloud.bigquery.MaterializedViewDefinition$Builder setClustering(com.google.cloud.bigquery.Clustering)</method>
-  </difference>
-  <difference>
-    <differenceType>7013</differenceType>
-    <className>com/google/cloud/bigquery/MaterializedViewDefinition$Builder</className>
-    <method>com.google.cloud.bigquery.MaterializedViewDefinition$Builder setRangePartitioning(com.google.cloud.bigquery.RangePartitioning)</method>
-  </difference>
-  <difference>
-    <differenceType>7013</differenceType>
-    <className>com/google/cloud/bigquery/MaterializedViewDefinition$Builder</className>
-    <method>com.google.cloud.bigquery.MaterializedViewDefinition$Builder setTimePartitioning(com.google.cloud.bigquery.TimePartitioning)</method>
+    <className>com/google/cloud/bigquery/ExternalTableDefinition$Builder</className>
+    <method> com.google.cloud.bigquery.ExternalTableDefinition$Builder setDecimalTargetTypes(java.util.List)</method>
   </difference>
 </differences>
diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java
@@ -93,6 +93,17 @@ public Builder setFormatOptions(FormatOptions formatOptions) {
       return setFormatOptionsInner(formatOptions);
     }
 
+    /**
+     * Defines the list of possible SQL data types to which the source decimal values are converted.
+     * This list and the precision and the scale parameters of the decimal field determine the
+     * target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in
+     * the specified list and if it supports the precision and the scale. STRING supports all
+     * precision and scale values.
+     *
+     * @param decimalTargetTypes decimalTargetType or {@code null} for none
+     */
+    public abstract Builder setDecimalTargetTypes(List<String> decimalTargetTypes);
+
     abstract Builder setFormatOptionsInner(FormatOptions formatOptions);
 
     /**
@@ -229,6 +240,9 @@ public <F extends FormatOptions> F getFormatOptions() {
   @Nullable
   abstract FormatOptions getFormatOptionsInner();
 
+  @Nullable
+  public abstract ImmutableList<String> getDecimalTargetTypes();
+
   /**
    * [Experimental] Returns whether automatic detection of schema and format options should be
    * performed.
@@ -283,6 +297,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
     if (getSourceUris() != null) {
       externalConfigurationPb.setSourceUris(getSourceUris());
     }
+    if (getDecimalTargetTypes() != null) {
+      externalConfigurationPb.setDecimalTargetTypes(getDecimalTargetTypes());
+    }
     if (getFormatOptions() != null && FormatOptions.CSV.equals(getFormatOptions().getType())) {
       externalConfigurationPb.setCsvOptions(((CsvOptions) getFormatOptions()).toPb());
     }
@@ -430,6 +447,10 @@ static ExternalTableDefinition fromPb(Table tablePb) {
       if (externalDataConfiguration.getSourceUris() != null) {
         builder.setSourceUris(ImmutableList.copyOf(externalDataConfiguration.getSourceUris()));
       }
+      if (externalDataConfiguration.getDecimalTargetTypes() != null) {
+        builder.setDecimalTargetTypes(
+            ImmutableList.copyOf(externalDataConfiguration.getDecimalTargetTypes()));
+      }
       if (externalDataConfiguration.getSourceFormat() != null) {
         builder.setFormatOptions(FormatOptions.of(externalDataConfiguration.getSourceFormat()));
       }
@@ -469,6 +490,9 @@ static ExternalTableDefinition fromExternalDataConfiguration(
     if (externalDataConfiguration.getSourceUris() != null) {
       builder.setSourceUris(externalDataConfiguration.getSourceUris());
     }
+    if (externalDataConfiguration.getDecimalTargetTypes() != null) {
+      builder.setDecimalTargetTypes(externalDataConfiguration.getDecimalTargetTypes());
+    }
     if (externalDataConfiguration.getSchema() != null) {
       builder.setSchema(Schema.fromPb(externalDataConfiguration.getSchema()));
     }

diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java
@@ -38,6 +38,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load
 
   private final List<String> sourceUris;
   private final TableId destinationTable;
+  private final List<String> decimalTargetTypes;
   private final EncryptionConfiguration destinationEncryptionConfiguration;
   private final JobInfo.CreateDisposition createDisposition;
   private final JobInfo.WriteDisposition writeDisposition;
@@ -61,6 +62,7 @@ public static final class Builder extends JobConfiguration.Builder<LoadJobConfig
 
     private List<String> sourceUris;
     private TableId destinationTable;
+    private List<String> decimalTargetTypes;
     private EncryptionConfiguration destinationEncryptionConfiguration;
     private JobInfo.CreateDisposition createDisposition;
     private JobInfo.WriteDisposition writeDisposition;
@@ -87,6 +89,7 @@ private Builder() {
     private Builder(LoadJobConfiguration loadConfiguration) {
       this();
       this.destinationTable = loadConfiguration.destinationTable;
+      this.decimalTargetTypes = loadConfiguration.decimalTargetTypes;
       this.createDisposition = loadConfiguration.createDisposition;
       this.writeDisposition = loadConfiguration.writeDisposition;
       this.formatOptions = loadConfiguration.formatOptions;
@@ -112,6 +115,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
       this();
       JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad();
       this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable());
+      if (loadConfigurationPb.getDecimalTargetTypes() != null) {
+        this.decimalTargetTypes = ImmutableList.copyOf(loadConfigurationPb.getDecimalTargetTypes());
+      }
       if (loadConfigurationPb.getCreateDisposition() != null) {
         this.createDisposition =
             JobInfo.CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition());
@@ -278,6 +284,20 @@ public Builder setSourceUris(List<String> sourceUris) {
       return this;
     }
 
+    /**
+     * Defines the list of possible SQL data types to which the source decimal values are converted.
+     * This list and the precision and the scale parameters of the decimal field determine the
+     * target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in
+     * the specified list and if it supports the precision and the scale. STRING supports all
+     * precision and scale values.
+     *
+     * @param decimalTargetTypes decimalTargetType or {@code null} for none
+     */
+    public Builder setDecimalTargetTypes(List<String> decimalTargetTypes) {
+      this.decimalTargetTypes = decimalTargetTypes;
+      return this;
+    }
+
     public Builder setAutodetect(Boolean autodetect) {
       this.autodetect = autodetect;
       return this;
@@ -341,6 +361,7 @@ private LoadJobConfiguration(Builder builder) {
     super(builder);
     this.sourceUris = builder.sourceUris;
     this.destinationTable = builder.destinationTable;
+    this.decimalTargetTypes = builder.decimalTargetTypes;
     this.createDisposition = builder.createDisposition;
     this.writeDisposition = builder.writeDisposition;
     this.formatOptions = builder.formatOptions;
@@ -430,6 +451,10 @@ public List<String> getSourceUris() {
     return sourceUris;
   }
 
+  public List<String> getDecimalTargetTypes() {
+    return decimalTargetTypes;
+  }
+
   public Boolean getAutodetect() {
     return autodetect;
   }
@@ -482,6 +507,7 @@ public Builder toBuilder() {
   ToStringHelper toStringHelper() {
     return super.toStringHelper()
         .add("destinationTable", destinationTable)
+        .add("decimalTargetTypes", decimalTargetTypes)
         .add("destinationEncryptionConfiguration", destinationEncryptionConfiguration)
         .add("createDisposition", createDisposition)
         .add("writeDisposition", writeDisposition)
@@ -568,6 +594,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
     if (sourceUris != null) {
       loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris));
     }
+    if (decimalTargetTypes != null) {
+      loadConfigurationPb.setDecimalTargetTypes(ImmutableList.copyOf(decimalTargetTypes));
+    }
     if (schemaUpdateOptions != null) {
       ImmutableList.Builder<String> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
       for (JobInfo.SchemaUpdateOption schemaUpdateOption : schemaUpdateOptions) {

diff --git a/...e-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java b/...e-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java
@@ -27,6 +27,8 @@
 public class ExternalTableDefinitionTest {
 
   private static final List<String> SOURCE_URIS = ImmutableList.of("uri1", "uri2");
+  private static final List<String> DECIMAL_TARGET_TYPES =
+      ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING");
   private static final Field FIELD_SCHEMA1 =
       Field.newBuilder("StringField", LegacySQLTypeName.STRING)
           .setMode(Field.Mode.NULLABLE)
@@ -56,6 +58,7 @@ public class ExternalTableDefinitionTest {
           .build();
   private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION =
       ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS)
+          .setDecimalTargetTypes(DECIMAL_TARGET_TYPES)
           .setCompression(COMPRESSION)
           .setConnectionId(CONNECTION_ID)
           .setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
@@ -111,6 +114,7 @@ public void testBuilder() {
     assertEquals(MAX_BAD_RECORDS, EXTERNAL_TABLE_DEFINITION.getMaxBadRecords());
     assertEquals(TABLE_SCHEMA, EXTERNAL_TABLE_DEFINITION.getSchema());
     assertEquals(SOURCE_URIS, EXTERNAL_TABLE_DEFINITION.getSourceUris());
+    assertEquals(DECIMAL_TARGET_TYPES, EXTERNAL_TABLE_DEFINITION.getDecimalTargetTypes());
     assertEquals(AUTODETECT, EXTERNAL_TABLE_DEFINITION.getAutodetect());
     assertEquals(HIVE_PARTITIONING_OPTIONS, EXTERNAL_TABLE_DEFINITION.getHivePartitioningOptions());
     assertNotEquals(EXTERNAL_TABLE_DEFINITION, TableDefinition.Type.EXTERNAL);
@@ -130,6 +134,7 @@ public void testToAndFromPb() {
   private void compareExternalTableDefinition(
       ExternalTableDefinition expected, ExternalTableDefinition value) {
     assertEquals(expected, value);
+    assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes());
     assertEquals(expected.getCompression(), value.getCompression());
     assertEquals(expected.getConnectionId(), value.getConnectionId());
     assertEquals(expected.getFormatOptions(), value.getFormatOptions());

diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java
@@ -50,6 +50,8 @@ public class LoadJobConfigurationTest {
           .setDescription("FieldDescription")
           .build();
   private static final List<String> SOURCE_URIS = ImmutableList.of("uri1", "uri2");
+  private static final List<String> DECIMAL_TARGET_TYPES =
+      ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING");
   private static final List<SchemaUpdateOption> SCHEMA_UPDATE_OPTIONS =
       ImmutableList.of(SchemaUpdateOption.ALLOW_FIELD_ADDITION);
   private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA);
@@ -76,6 +78,7 @@ public class LoadJobConfigurationTest {
           .build();
   private static final LoadJobConfiguration LOAD_CONFIGURATION_CSV =
       LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS)
+          .setDecimalTargetTypes(DECIMAL_TARGET_TYPES)
           .setCreateDisposition(CREATE_DISPOSITION)
           .setWriteDisposition(WRITE_DISPOSITION)
           .setFormatOptions(CSV_OPTIONS)
@@ -228,6 +231,7 @@ private void compareLoadJobConfiguration(
     assertEquals(expected.hashCode(), value.hashCode());
     assertEquals(expected.toString(), value.toString());
     assertEquals(expected.getDestinationTable(), value.getDestinationTable());
+    assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes());
     assertEquals(expected.getCreateDisposition(), value.getCreateDisposition());
     assertEquals(expected.getWriteDisposition(), value.getWriteDisposition());
     assertEquals(expected.getCsvOptions(), value.getCsvOptions());

diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java
@@ -2747,6 +2747,54 @@ public void testLoadJobWithRangePartitioning() throws InterruptedException {
     }
   }
 
+  @Test
+  public void testLoadJobWithDecimalTargetTypes() throws InterruptedException {
+    String tableName = "test_load_job_table_parquet_decimalTargetTypes";
+    TableId destinationTable = TableId.of(DATASET, tableName);
+    String sourceUri = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/numeric/numeric_38_12.parquet";
+    try {
+      LoadJobConfiguration configuration =
+          LoadJobConfiguration.newBuilder(destinationTable, sourceUri, FormatOptions.parquet())
+              .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
+              .setDecimalTargetTypes(ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"))
+              .build();
+      Job job = bigquery.create(JobInfo.of(configuration));
+      job = job.waitFor();
+      assertNull(job.getStatus().getError());
+      LoadJobConfiguration loadJobConfiguration = job.getConfiguration();
+      assertEquals(
+          ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"),
+          loadJobConfiguration.getDecimalTargetTypes());
+      Table remoteTable = bigquery.getTable(DATASET, tableName);
+      assertNotNull(remoteTable);
+      assertEquals(
+          remoteTable.getDefinition().getSchema().getFields().get(0).getType().toString(),
+          "BIGNUMERIC");
+    } finally {
+      bigquery.delete(destinationTable);
+    }
+  }
+
+  @Test
+  public void testExternalTableWithDecimalTargetTypes() throws InterruptedException {
+    String tableName = "test_create_external_table_parquet_decimalTargetTypes";
+    TableId destinationTable = TableId.of(DATASET, tableName);
+    String sourceUri = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/numeric/numeric_38_12.parquet";
+    ExternalTableDefinition externalTableDefinition =
+        ExternalTableDefinition.newBuilder(sourceUri, FormatOptions.parquet())
+            .setDecimalTargetTypes(ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"))
+            .build();
+    TableInfo tableInfo = TableInfo.of(destinationTable, externalTableDefinition);
+    Table createdTable = bigquery.create(tableInfo);
+    assertNotNull(createdTable);
+    Table remoteTable = bigquery.getTable(DATASET, tableName);
+    assertNotNull(remoteTable);
+    assertEquals(
+        remoteTable.getDefinition().getSchema().getFields().get(0).getType().toString(),
+        "BIGNUMERIC");
+    assertTrue(remoteTable.delete());
+  }
+
   @Test
   public void testQueryJobWithDryRun() throws InterruptedException, TimeoutException {
     String tableName = "test_query_job_table";