From 718ec2ae25150dceef8fb9bc3e2fcf6238fc2220 Mon Sep 17 00:00:00 2001 From: Stephanie Wang Date: Thu, 19 Nov 2020 15:58:07 -0500 Subject: [PATCH] feat: add BIGNUMERIC support (#703) Expose BIGNUMERIC type in the client cc: @mingyuzhong --- .../cloud/bigquery/LegacySQLTypeName.java | 5 + .../cloud/bigquery/QueryParameterValue.java | 12 +- .../cloud/bigquery/StandardSQLTypeName.java | 4 + .../cloud/bigquery/FieldValueListTest.java | 25 +++- .../bigquery/QueryJobConfigurationTest.java | 5 +- .../bigquery/QueryParameterValueTest.java | 43 +++++++ .../cloud/bigquery/it/ITBigQueryTest.java | 113 +++++++++++++++++- 7 files changed, 193 insertions(+), 14 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LegacySQLTypeName.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LegacySQLTypeName.java index e5a46f5e7..56d66cb1a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LegacySQLTypeName.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LegacySQLTypeName.java @@ -61,6 +61,11 @@ public LegacySQLTypeName apply(String constant) { */ public static final LegacySQLTypeName NUMERIC = type.createAndRegister("NUMERIC").setStandardType(StandardSQLTypeName.NUMERIC); + /** + * A decimal value with 76+ digits of precision (the 77th digit is partial) and 38 digits of scale + */ + public static final LegacySQLTypeName BIGNUMERIC = + type.createAndRegister("BIGNUMERIC").setStandardType(StandardSQLTypeName.BIGNUMERIC); /** A Boolean value (true or false). */ public static final LegacySQLTypeName BOOLEAN = type.createAndRegister("BOOLEAN").setStandardType(StandardSQLTypeName.BOOL); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java index e9c21674f..76e521d56 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java @@ -60,6 +60,7 @@ *
  • Double: StandardSQLTypeName.FLOAT64 *
  • Float: StandardSQLTypeName.FLOAT64 *
  • BigDecimal: StandardSQLTypeName.NUMERIC + *
  • BigNumeric: StandardSQLTypeName.BIGNUMERIC * * *

    No other types are supported through that entry point. The other types can be created by @@ -197,7 +198,10 @@ public Map getStructTypes() { @Nullable abstract Map getStructTypesInner(); - /** Creates a {@code QueryParameterValue} object with the given value and type. */ + /** + * Creates a {@code QueryParameterValue} object with the given value and type. Note: this does not + * support BigNumeric + */ public static QueryParameterValue of(T value, Class type) { return of(value, classToType(type)); } @@ -240,6 +244,11 @@ public static QueryParameterValue numeric(BigDecimal value) { return of(value, StandardSQLTypeName.NUMERIC); } + /** Creates a {@code QueryParameterValue} object with a type of BIGNUMERIC. */ + public static QueryParameterValue bigNumeric(BigDecimal value) { + return of(value, StandardSQLTypeName.BIGNUMERIC); + } + /** Creates a {@code QueryParameterValue} object with a type of STRING. */ public static QueryParameterValue string(String value) { return of(value, StandardSQLTypeName.STRING); @@ -363,6 +372,7 @@ private static String valueToStringOrNull(T value, StandardSQLTypeName type) } break; case NUMERIC: + case BIGNUMERIC: if (value instanceof BigDecimal) { return value.toString(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardSQLTypeName.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardSQLTypeName.java index aa156df76..d618b7656 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardSQLTypeName.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardSQLTypeName.java @@ -32,6 +32,10 @@ public enum StandardSQLTypeName { FLOAT64, /** A decimal value with 38 digits of precision and 9 digits of scale. */ NUMERIC, + /** + * A decimal value with 76+ digits of precision (the 77th digit is partial) and 38 digits of scale + */ + BIGNUMERIC, /** Variable-length character (Unicode) data. */ STRING, /** Variable-length binary data. */ diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueListTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueListTest.java index 779fd85a1..7d10a9750 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueListTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueListTest.java @@ -49,7 +49,8 @@ public class FieldValueListTest { LegacySQLTypeName.RECORD, Field.of("first", LegacySQLTypeName.FLOAT), Field.of("second", LegacySQLTypeName.TIMESTAMP)), - Field.of("tenth", LegacySQLTypeName.NUMERIC)); + Field.of("tenth", LegacySQLTypeName.NUMERIC), + Field.of("eleventh", LegacySQLTypeName.BIGNUMERIC)); private final Map integerPb = ImmutableMap.of("v", "1"); private final Map floatPb = ImmutableMap.of("v", "1.5"); @@ -62,6 +63,9 @@ public class FieldValueListTest { private final Map recordPb = ImmutableMap.of("f", ImmutableList.of(floatPb, timestampPb)); private final Map numericPb = ImmutableMap.of("v", "123456789.123456789"); + private final Map bigNumericPb = + ImmutableMap.of( + "v", "99999999999999999999999999999999999999.99999999999999999999999999999999999999"); private final FieldValue booleanFv = FieldValue.of(Attribute.PRIMITIVE, "false"); private final FieldValue integerFv = FieldValue.of(Attribute.PRIMITIVE, "1"); @@ -78,6 +82,10 @@ public class FieldValueListTest { FieldValueList.of( ImmutableList.of(floatFv, timestampFv), schema.get("ninth").getSubFields())); private final FieldValue numericFv = FieldValue.of(Attribute.PRIMITIVE, "123456789.123456789"); + private final FieldValue bigNumericFv = + FieldValue.of( + Attribute.PRIMITIVE, + "99999999999999999999999999999999999999.99999999999999999999999999999999999999"); private final List fieldValuesPb = ImmutableList.of( @@ -90,7 +98,8 @@ public class FieldValueListTest { nullPb, repeatedPb, recordPb, - numericPb); + numericPb, + bigNumericPb); private final FieldValueList fieldValues = FieldValueList.of( @@ -104,7 +113,8 @@ public class FieldValueListTest { nullFv, repeatedFv, recordFv, - numericFv), + numericFv, + bigNumericFv), schema); @Test @@ -116,7 +126,7 @@ public void testFromPb() { @Test public void testGetByIndex() { - assertEquals(10, fieldValues.size()); + assertEquals(11, fieldValues.size()); assertEquals(booleanFv, fieldValues.get(0)); assertEquals(integerFv, fieldValues.get(1)); assertEquals(floatFv, fieldValues.get(2)); @@ -133,11 +143,12 @@ public void testGetByIndex() { assertEquals(floatFv, fieldValues.get(8).getRecordValue().get(0)); assertEquals(timestampFv, fieldValues.get(8).getRecordValue().get(1)); assertEquals(numericFv, fieldValues.get(9)); + assertEquals(bigNumericFv, fieldValues.get(10)); } @Test public void testGetByName() { - assertEquals(10, fieldValues.size()); + assertEquals(11, fieldValues.size()); assertEquals(booleanFv, fieldValues.get("first")); assertEquals(integerFv, fieldValues.get("second")); assertEquals(floatFv, fieldValues.get("third")); @@ -154,6 +165,7 @@ public void testGetByName() { assertEquals(floatFv, fieldValues.get("ninth").getRecordValue().get("first")); assertEquals(timestampFv, fieldValues.get("ninth").getRecordValue().get("second")); assertEquals(numericFv, fieldValues.get("tenth")); + assertEquals(bigNumericFv, fieldValues.get("eleventh")); } @Test @@ -170,7 +182,8 @@ public void testNullSchema() { nullFv, repeatedFv, recordFv, - numericFv)); + numericFv, + bigNumericFv)); assertEquals(fieldValues, fieldValuesNoSchema); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java index 0e892b6e1..3cd418204 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java @@ -27,6 +27,7 @@ import com.google.cloud.bigquery.TimePartitioning.Type; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.math.BigDecimal; import java.util.List; import java.util.Map; import org.junit.Test; @@ -101,8 +102,10 @@ public class QueryJobConfigurationTest { QueryParameterValue.string("stringValue"); private static final QueryParameterValue TIMESTAMP_PARAMETER = QueryParameterValue.timestamp("2014-01-01 07:00:00.000000+00:00"); + private static final QueryParameterValue BIGNUMERIC_PARAMETER = + QueryParameterValue.bigNumeric(new BigDecimal(1 / 3)); private static final List POSITIONAL_PARAMETER = - ImmutableList.of(STRING_PARAMETER, TIMESTAMP_PARAMETER); + ImmutableList.of(STRING_PARAMETER, TIMESTAMP_PARAMETER, BIGNUMERIC_PARAMETER); private static final Map NAME_PARAMETER = ImmutableMap.of("string", STRING_PARAMETER, "timestamp", TIMESTAMP_PARAMETER); private static final QueryJobConfiguration QUERY_JOB_CONFIGURATION = diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java index 09421565c..b643ae580 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java @@ -141,6 +141,49 @@ public void testNumeric() { assertThat(value.getArrayValues()).isNull(); } + @Test + public void testBigNumeric() { + QueryParameterValue value = + QueryParameterValue.bigNumeric(new BigDecimal("0.33333333333333333333333333333333333333")); + QueryParameterValue value1 = + QueryParameterValue.bigNumeric(new BigDecimal("0.50000000000000000000000000000000000000")); + QueryParameterValue value2 = + QueryParameterValue.bigNumeric(new BigDecimal("0.00000000500000000000000000000000000000")); + QueryParameterValue value3 = + QueryParameterValue.bigNumeric(new BigDecimal("-0.00000000500000000000000000000000000000")); + QueryParameterValue value4 = + QueryParameterValue.bigNumeric( + new BigDecimal("0.33333333333333333333333333333333333333888888888888888")); + QueryParameterValue value5 = QueryParameterValue.bigNumeric(new BigDecimal("1e-38")); + QueryParameterValue value6 = QueryParameterValue.bigNumeric(new BigDecimal("-1e38")); + QueryParameterValue value7 = + QueryParameterValue.bigNumeric( + new BigDecimal( + "578960446186580977117854925043439539266.34992332820282019728792003956564819967")); + QueryParameterValue value8 = + QueryParameterValue.bigNumeric( + new BigDecimal( + "-578960446186580977117854925043439539266.34992332820282019728792003956564819968")); + + assertThat(value.getValue()).isEqualTo("0.33333333333333333333333333333333333333"); + assertThat(value1.getValue()).isEqualTo("0.50000000000000000000000000000000000000"); + assertThat(value2.getValue()).isEqualTo("5.00000000000000000000000000000E-9"); + assertThat(value3.getValue()).isEqualTo("-5.00000000000000000000000000000E-9"); + assertThat(value4.getValue()) + .isEqualTo("0.33333333333333333333333333333333333333888888888888888"); + assertThat(value5.getValue()).isEqualTo("1E-38"); + assertThat(value6.getValue()).isEqualTo("-1E+38"); + assertThat(value7.getValue()) + .isEqualTo( + "578960446186580977117854925043439539266.34992332820282019728792003956564819967"); + assertThat(value8.getValue()) + .isEqualTo( + "-578960446186580977117854925043439539266.34992332820282019728792003956564819968"); + assertThat(value.getType()).isEqualTo(StandardSQLTypeName.BIGNUMERIC); + assertThat(value.getArrayType()).isNull(); + assertThat(value.getArrayValues()).isNull(); + } + @Test public void testString() { QueryParameterValue value = QueryParameterValue.string("foo"); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index ab25534a7..3a3af4898 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -212,6 +212,31 @@ public class ITBigQueryTest { .setMode(Field.Mode.NULLABLE) .setDescription("NumericDescription") .build(); + private static final Field BIGNUMERIC_FIELD_SCHEMA = + Field.newBuilder("BigNumericField", LegacySQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumericDescription") + .build(); + private static final Field BIGNUMERIC_FIELD_SCHEMA1 = + Field.newBuilder("BigNumericField1", LegacySQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumeric1Description") + .build(); + private static final Field BIGNUMERIC_FIELD_SCHEMA2 = + Field.newBuilder("BigNumericField2", LegacySQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumeric2Description") + .build(); + private static final Field BIGNUMERIC_FIELD_SCHEMA3 = + Field.newBuilder("BigNumericField3", LegacySQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumeric3Description") + .build(); + private static final Field BIGNUMERIC_FIELD_SCHEMA4 = + Field.newBuilder("BigNumericField4", LegacySQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumeric4Description") + .build(); private static final Field STRING_FIELD_SCHEMA_WITH_POLICY = Field.newBuilder("StringFieldWithPolicy", LegacySQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) @@ -229,7 +254,12 @@ public class ITBigQueryTest { INTEGER_FIELD_SCHEMA, FLOAT_FIELD_SCHEMA, GEOGRAPHY_FIELD_SCHEMA, - NUMERIC_FIELD_SCHEMA); + NUMERIC_FIELD_SCHEMA, + BIGNUMERIC_FIELD_SCHEMA, + BIGNUMERIC_FIELD_SCHEMA1, + BIGNUMERIC_FIELD_SCHEMA2, + BIGNUMERIC_FIELD_SCHEMA3, + BIGNUMERIC_FIELD_SCHEMA4); private static final Field DDL_TIMESTAMP_FIELD_SCHEMA = Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) @@ -261,6 +291,7 @@ public class ITBigQueryTest { Field.newBuilder("deaths", LegacySQLTypeName.INTEGER) .setMode(Field.Mode.NULLABLE) .build()); + private static final Schema SIMPLE_SCHEMA = Schema.of(STRING_FIELD_SCHEMA); private static final Schema POLICY_SCHEMA = Schema.of(STRING_FIELD_SCHEMA, STRING_FIELD_SCHEMA_WITH_POLICY, INTEGER_FIELD_SCHEMA); @@ -275,6 +306,32 @@ public class ITBigQueryTest { Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) .setMode(Field.Mode.NULLABLE) .build()); + private static final Schema QUERY_RESULT_SCHEMA_BIGNUMERIC = + Schema.of( + Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("StringField", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("BigNumericField", LegacySQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("BigNumericField1", LegacySQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("BigNumericField2", LegacySQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("BigNumericField3", LegacySQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("BigNumericField4", LegacySQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .build()); private static final Schema VIEW_SCHEMA = Schema.of( Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) @@ -323,7 +380,12 @@ public class ITBigQueryTest { + " \"IntegerField\": \"3\"," + " \"FloatField\": \"1.2\"," + " \"GeographyField\": \"POINT(-122.35022 47.649154)\"," - + " \"NumericField\": \"123456.789012345\"" + + " \"NumericField\": \"123456.789012345\"," + + " \"BigNumericField\": \"0.33333333333333333333333333333333333333\"," + + " \"BigNumericField1\": \"1e-38\"," + + " \"BigNumericField2\": \"-1e38\"," + + " \"BigNumericField3\": \"578960446186580977117854925043439539266.34992332820282019728792003956564819967\"," + + " \"BigNumericField4\": \"-578960446186580977117854925043439539266.34992332820282019728792003956564819968\"" + "}\n" + "{" + " \"TimestampField\": \"2014-08-19 07:41:35.220 -05:00\"," @@ -345,7 +407,12 @@ public class ITBigQueryTest { + " \"IntegerField\": \"3\"," + " \"FloatField\": \"1.2\"," + " \"GeographyField\": \"POINT(-122.35022 47.649154)\"," - + " \"NumericField\": \"123456.789012345\"" + + " \"NumericField\": \"123456.789012345\"," + + " \"BigNumericField\": \"0.33333333333333333333333333333333333333\"," + + " \"BigNumericField1\": \"1e-38\"," + + " \"BigNumericField2\": \"-1e38\"," + + " \"BigNumericField3\": \"578960446186580977117854925043439539266.34992332820282019728792003956564819967\"," + + " \"BigNumericField4\": \"-578960446186580977117854925043439539266.34992332820282019728792003956564819968\"" + "}"; private static final String JSON_CONTENT_SIMPLE = "{" @@ -1932,14 +1999,15 @@ public void testScriptStatistics() throws InterruptedException { @Test public void testPositionalQueryParameters() throws InterruptedException { String query = - "SELECT TimestampField, StringField, BooleanField FROM " + "SELECT TimestampField, StringField, BooleanField, BigNumericField, BigNumericField1, BigNumericField2, BigNumericField3, BigNumericField4 FROM " + TABLE_ID.getTable() + " WHERE StringField = ?" + " AND TimestampField > ?" + " AND IntegerField IN UNNEST(?)" + " AND IntegerField < ?" + " AND FloatField > ?" - + " AND NumericField < ?"; + + " AND NumericField < ?" + + " AND BigNumericField = ?"; QueryParameterValue stringParameter = QueryParameterValue.string("stringValue"); QueryParameterValue timestampParameter = QueryParameterValue.timestamp("2014-01-01 07:00:00.000000+00:00"); @@ -1949,6 +2017,20 @@ public void testPositionalQueryParameters() throws InterruptedException { QueryParameterValue float64Parameter = QueryParameterValue.float64(0.5); QueryParameterValue numericParameter = QueryParameterValue.numeric(new BigDecimal("234567890.123456")); + QueryParameterValue bigNumericParameter = + QueryParameterValue.bigNumeric(new BigDecimal("0.33333333333333333333333333333333333333")); + QueryParameterValue bigNumericParameter1 = + QueryParameterValue.bigNumeric(new BigDecimal("1e-38")); + QueryParameterValue bigNumericParameter2 = + QueryParameterValue.bigNumeric(new BigDecimal("-1e38")); + QueryParameterValue bigNumericParameter3 = + QueryParameterValue.bigNumeric( + new BigDecimal( + "578960446186580977117854925043439539266.34992332820282019728792003956564819967")); + QueryParameterValue bigNumericParameter4 = + QueryParameterValue.bigNumeric( + new BigDecimal( + "-578960446186580977117854925043439539266.34992332820282019728792003956564819968")); QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query) .setDefaultDataset(DatasetId.of(DATASET)) @@ -1959,10 +2041,29 @@ public void testPositionalQueryParameters() throws InterruptedException { .addPositionalParameter(int64Parameter) .addPositionalParameter(float64Parameter) .addPositionalParameter(numericParameter) + .addPositionalParameter(bigNumericParameter) + .addPositionalParameter(bigNumericParameter1) + .addPositionalParameter(bigNumericParameter2) + .addPositionalParameter(bigNumericParameter3) + .addPositionalParameter(bigNumericParameter4) .build(); TableResult result = bigquery.query(config); - assertEquals(QUERY_RESULT_SCHEMA, result.getSchema()); + assertEquals(QUERY_RESULT_SCHEMA_BIGNUMERIC, result.getSchema()); assertEquals(2, Iterables.size(result.getValues())); + for (FieldValueList values : result.iterateAll()) { + assertEquals("1.40845209522E9", values.get(0).getValue()); + assertEquals("stringValue", values.get(1).getValue()); + assertEquals(false, values.get(2).getBooleanValue()); + assertEquals("0.33333333333333333333333333333333333333", values.get(3).getValue()); + assertEquals("0.00000000000000000000000000000000000001", values.get(4).getValue()); + assertEquals("-100000000000000000000000000000000000000", values.get(5).getValue()); + assertEquals( + "578960446186580977117854925043439539266.34992332820282019728792003956564819967", + values.get(6).getValue()); + assertEquals( + "-578960446186580977117854925043439539266.34992332820282019728792003956564819968", + values.get(7).getValue()); + } } @Test