diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index d8b1ba813..2801782f4 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -4,32 +4,12 @@ 7013 - com/google/cloud/bigquery/MaterializedViewDefinition - com.google.cloud.bigquery.Clustering getClustering() + com/google/cloud/bigquery/ExternalTableDefinition + com.google.common.collect.ImmutableList getDecimalTargetTypes() 7013 - com/google/cloud/bigquery/MaterializedViewDefinition - com.google.cloud.bigquery.RangePartitioning getRangePartitioning() - - - 7013 - com/google/cloud/bigquery/MaterializedViewDefinition - com.google.cloud.bigquery.TimePartitioning getTimePartitioning() - - - 7013 - com/google/cloud/bigquery/MaterializedViewDefinition$Builder - com.google.cloud.bigquery.MaterializedViewDefinition$Builder setClustering(com.google.cloud.bigquery.Clustering) - - - 7013 - com/google/cloud/bigquery/MaterializedViewDefinition$Builder - com.google.cloud.bigquery.MaterializedViewDefinition$Builder setRangePartitioning(com.google.cloud.bigquery.RangePartitioning) - - - 7013 - com/google/cloud/bigquery/MaterializedViewDefinition$Builder - com.google.cloud.bigquery.MaterializedViewDefinition$Builder setTimePartitioning(com.google.cloud.bigquery.TimePartitioning) + com/google/cloud/bigquery/ExternalTableDefinition$Builder + com.google.cloud.bigquery.ExternalTableDefinition$Builder setDecimalTargetTypes(java.util.List) \ No newline at end of file diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index 1d2cfa293..2b68a7ff2 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -93,6 +93,17 @@ public Builder setFormatOptions(FormatOptions formatOptions) { return setFormatOptionsInner(formatOptions); } + /** + * Defines the list of possible SQL data types to which the source decimal values are converted. + * This list and the precision and the scale parameters of the decimal field determine the + * target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in + * the specified list and if it supports the precision and the scale. STRING supports all + * precision and scale values. + * + * @param decimalTargetTypes decimalTargetType or {@code null} for none + */ + public abstract Builder setDecimalTargetTypes(List decimalTargetTypes); + abstract Builder setFormatOptionsInner(FormatOptions formatOptions); /** @@ -229,6 +240,9 @@ public F getFormatOptions() { @Nullable abstract FormatOptions getFormatOptionsInner(); + @Nullable + public abstract ImmutableList getDecimalTargetTypes(); + /** * [Experimental] Returns whether automatic detection of schema and format options should be * performed. @@ -283,6 +297,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getSourceUris() != null) { externalConfigurationPb.setSourceUris(getSourceUris()); } + if (getDecimalTargetTypes() != null) { + externalConfigurationPb.setDecimalTargetTypes(getDecimalTargetTypes()); + } if (getFormatOptions() != null && FormatOptions.CSV.equals(getFormatOptions().getType())) { externalConfigurationPb.setCsvOptions(((CsvOptions) getFormatOptions()).toPb()); } @@ -430,6 +447,10 @@ static ExternalTableDefinition fromPb(Table tablePb) { if (externalDataConfiguration.getSourceUris() != null) { builder.setSourceUris(ImmutableList.copyOf(externalDataConfiguration.getSourceUris())); } + if (externalDataConfiguration.getDecimalTargetTypes() != null) { + builder.setDecimalTargetTypes( + ImmutableList.copyOf(externalDataConfiguration.getDecimalTargetTypes())); + } if (externalDataConfiguration.getSourceFormat() != null) { builder.setFormatOptions(FormatOptions.of(externalDataConfiguration.getSourceFormat())); } @@ -469,6 +490,9 @@ static ExternalTableDefinition fromExternalDataConfiguration( if (externalDataConfiguration.getSourceUris() != null) { builder.setSourceUris(externalDataConfiguration.getSourceUris()); } + if (externalDataConfiguration.getDecimalTargetTypes() != null) { + builder.setDecimalTargetTypes(externalDataConfiguration.getDecimalTargetTypes()); + } if (externalDataConfiguration.getSchema() != null) { builder.setSchema(Schema.fromPb(externalDataConfiguration.getSchema())); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index a317f1285..2f625acf5 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -38,6 +38,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load private final List sourceUris; private final TableId destinationTable; + private final List decimalTargetTypes; private final EncryptionConfiguration destinationEncryptionConfiguration; private final JobInfo.CreateDisposition createDisposition; private final JobInfo.WriteDisposition writeDisposition; @@ -61,6 +62,7 @@ public static final class Builder extends JobConfiguration.Builder sourceUris; private TableId destinationTable; + private List decimalTargetTypes; private EncryptionConfiguration destinationEncryptionConfiguration; private JobInfo.CreateDisposition createDisposition; private JobInfo.WriteDisposition writeDisposition; @@ -87,6 +89,7 @@ private Builder() { private Builder(LoadJobConfiguration loadConfiguration) { this(); this.destinationTable = loadConfiguration.destinationTable; + this.decimalTargetTypes = loadConfiguration.decimalTargetTypes; this.createDisposition = loadConfiguration.createDisposition; this.writeDisposition = loadConfiguration.writeDisposition; this.formatOptions = loadConfiguration.formatOptions; @@ -112,6 +115,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur this(); JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad(); this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable()); + if (loadConfigurationPb.getDecimalTargetTypes() != null) { + this.decimalTargetTypes = ImmutableList.copyOf(loadConfigurationPb.getDecimalTargetTypes()); + } if (loadConfigurationPb.getCreateDisposition() != null) { this.createDisposition = JobInfo.CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition()); @@ -278,6 +284,20 @@ public Builder setSourceUris(List sourceUris) { return this; } + /** + * Defines the list of possible SQL data types to which the source decimal values are converted. + * This list and the precision and the scale parameters of the decimal field determine the + * target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in + * the specified list and if it supports the precision and the scale. STRING supports all + * precision and scale values. + * + * @param decimalTargetTypes decimalTargetType or {@code null} for none + */ + public Builder setDecimalTargetTypes(List decimalTargetTypes) { + this.decimalTargetTypes = decimalTargetTypes; + return this; + } + public Builder setAutodetect(Boolean autodetect) { this.autodetect = autodetect; return this; @@ -341,6 +361,7 @@ private LoadJobConfiguration(Builder builder) { super(builder); this.sourceUris = builder.sourceUris; this.destinationTable = builder.destinationTable; + this.decimalTargetTypes = builder.decimalTargetTypes; this.createDisposition = builder.createDisposition; this.writeDisposition = builder.writeDisposition; this.formatOptions = builder.formatOptions; @@ -430,6 +451,10 @@ public List getSourceUris() { return sourceUris; } + public List getDecimalTargetTypes() { + return decimalTargetTypes; + } + public Boolean getAutodetect() { return autodetect; } @@ -482,6 +507,7 @@ public Builder toBuilder() { ToStringHelper toStringHelper() { return super.toStringHelper() .add("destinationTable", destinationTable) + .add("decimalTargetTypes", decimalTargetTypes) .add("destinationEncryptionConfiguration", destinationEncryptionConfiguration) .add("createDisposition", createDisposition) .add("writeDisposition", writeDisposition) @@ -568,6 +594,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (sourceUris != null) { loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris)); } + if (decimalTargetTypes != null) { + loadConfigurationPb.setDecimalTargetTypes(ImmutableList.copyOf(decimalTargetTypes)); + } if (schemaUpdateOptions != null) { ImmutableList.Builder schemaUpdateOptionsBuilder = new ImmutableList.Builder<>(); for (JobInfo.SchemaUpdateOption schemaUpdateOption : schemaUpdateOptions) { diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java index 23a095cb6..38dcd2714 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java @@ -27,6 +27,8 @@ public class ExternalTableDefinitionTest { private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); + private static final List DECIMAL_TARGET_TYPES = + ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"); private static final Field FIELD_SCHEMA1 = Field.newBuilder("StringField", LegacySQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) @@ -56,6 +58,7 @@ public class ExternalTableDefinitionTest { .build(); private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION = ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS) + .setDecimalTargetTypes(DECIMAL_TARGET_TYPES) .setCompression(COMPRESSION) .setConnectionId(CONNECTION_ID) .setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES) @@ -111,6 +114,7 @@ public void testBuilder() { assertEquals(MAX_BAD_RECORDS, EXTERNAL_TABLE_DEFINITION.getMaxBadRecords()); assertEquals(TABLE_SCHEMA, EXTERNAL_TABLE_DEFINITION.getSchema()); assertEquals(SOURCE_URIS, EXTERNAL_TABLE_DEFINITION.getSourceUris()); + assertEquals(DECIMAL_TARGET_TYPES, EXTERNAL_TABLE_DEFINITION.getDecimalTargetTypes()); assertEquals(AUTODETECT, EXTERNAL_TABLE_DEFINITION.getAutodetect()); assertEquals(HIVE_PARTITIONING_OPTIONS, EXTERNAL_TABLE_DEFINITION.getHivePartitioningOptions()); assertNotEquals(EXTERNAL_TABLE_DEFINITION, TableDefinition.Type.EXTERNAL); @@ -130,6 +134,7 @@ public void testToAndFromPb() { private void compareExternalTableDefinition( ExternalTableDefinition expected, ExternalTableDefinition value) { assertEquals(expected, value); + assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes()); assertEquals(expected.getCompression(), value.getCompression()); assertEquals(expected.getConnectionId(), value.getConnectionId()); assertEquals(expected.getFormatOptions(), value.getFormatOptions()); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java index 9f42d62b7..a2f164f8a 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java @@ -50,6 +50,8 @@ public class LoadJobConfigurationTest { .setDescription("FieldDescription") .build(); private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); + private static final List DECIMAL_TARGET_TYPES = + ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"); private static final List SCHEMA_UPDATE_OPTIONS = ImmutableList.of(SchemaUpdateOption.ALLOW_FIELD_ADDITION); private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA); @@ -76,6 +78,7 @@ public class LoadJobConfigurationTest { .build(); private static final LoadJobConfiguration LOAD_CONFIGURATION_CSV = LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS) + .setDecimalTargetTypes(DECIMAL_TARGET_TYPES) .setCreateDisposition(CREATE_DISPOSITION) .setWriteDisposition(WRITE_DISPOSITION) .setFormatOptions(CSV_OPTIONS) @@ -228,6 +231,7 @@ private void compareLoadJobConfiguration( assertEquals(expected.hashCode(), value.hashCode()); assertEquals(expected.toString(), value.toString()); assertEquals(expected.getDestinationTable(), value.getDestinationTable()); + assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes()); assertEquals(expected.getCreateDisposition(), value.getCreateDisposition()); assertEquals(expected.getWriteDisposition(), value.getWriteDisposition()); assertEquals(expected.getCsvOptions(), value.getCsvOptions()); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index eae807bf7..b151ad22c 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2747,6 +2747,54 @@ public void testLoadJobWithRangePartitioning() throws InterruptedException { } } + @Test + public void testLoadJobWithDecimalTargetTypes() throws InterruptedException { + String tableName = "test_load_job_table_parquet_decimalTargetTypes"; + TableId destinationTable = TableId.of(DATASET, tableName); + String sourceUri = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/numeric/numeric_38_12.parquet"; + try { + LoadJobConfiguration configuration = + LoadJobConfiguration.newBuilder(destinationTable, sourceUri, FormatOptions.parquet()) + .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) + .setDecimalTargetTypes(ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING")) + .build(); + Job job = bigquery.create(JobInfo.of(configuration)); + job = job.waitFor(); + assertNull(job.getStatus().getError()); + LoadJobConfiguration loadJobConfiguration = job.getConfiguration(); + assertEquals( + ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"), + loadJobConfiguration.getDecimalTargetTypes()); + Table remoteTable = bigquery.getTable(DATASET, tableName); + assertNotNull(remoteTable); + assertEquals( + remoteTable.getDefinition().getSchema().getFields().get(0).getType().toString(), + "BIGNUMERIC"); + } finally { + bigquery.delete(destinationTable); + } + } + + @Test + public void testExternalTableWithDecimalTargetTypes() throws InterruptedException { + String tableName = "test_create_external_table_parquet_decimalTargetTypes"; + TableId destinationTable = TableId.of(DATASET, tableName); + String sourceUri = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/numeric/numeric_38_12.parquet"; + ExternalTableDefinition externalTableDefinition = + ExternalTableDefinition.newBuilder(sourceUri, FormatOptions.parquet()) + .setDecimalTargetTypes(ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING")) + .build(); + TableInfo tableInfo = TableInfo.of(destinationTable, externalTableDefinition); + Table createdTable = bigquery.create(tableInfo); + assertNotNull(createdTable); + Table remoteTable = bigquery.getTable(DATASET, tableName); + assertNotNull(remoteTable); + assertEquals( + remoteTable.getDefinition().getSchema().getFields().get(0).getType().toString(), + "BIGNUMERIC"); + assertTrue(remoteTable.delete()); + } + @Test public void testQueryJobWithDryRun() throws InterruptedException, TimeoutException { String tableName = "test_query_job_table";