From 82b556e08d19a4dd969bda53409276c6408a4126 Mon Sep 17 00:00:00 2001 From: JacobStocklass <35313153+JacobStocklass@users.noreply.github.com> Date: Tue, 30 Mar 2021 15:45:33 +0000 Subject: [PATCH] feat: BigDecimal and ByteString encoding (#971) * Porting over Big Decimal Byte String Encoder * Adding BigDecimal ByteString tests, currently not working * working on testing * Fixed Testing * Lint * Lint * Renamed methods for better clarity * Changing from BigNumeric to Numeric --- .../v1beta2/BigDecimalByteStringEncoder.java | 86 +++++++++++++++++++ .../BigDecimalByteStringEncoderTest.java | 62 +++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/BigDecimalByteStringEncoder.java create mode 100644 google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BigDecimalByteStringEncoderTest.java diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/BigDecimalByteStringEncoder.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/BigDecimalByteStringEncoder.java new file mode 100644 index 0000000000..6cdcad95dd --- /dev/null +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/BigDecimalByteStringEncoder.java @@ -0,0 +1,86 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This code was ported from ZetaSQL and can be found here: + * https://github.com/google/zetasql/blob/c55f967a5ae35b476437210c529691d8a73f5507/java/com/google/zetasql/Value.java + */ + +package com.google.cloud.bigquery.storage.v1beta2; + +import com.google.common.primitives.Bytes; +import com.google.protobuf.ByteString; +import java.math.BigDecimal; +import java.math.BigInteger; + +public class BigDecimalByteStringEncoder { + private static int NumericScale = 9; + private static final BigDecimal MAX_NUMERIC_VALUE = + new BigDecimal("99999999999999999999999999999.999999999"); + private static final BigDecimal MIN_NUMERIC_VALUE = + new BigDecimal("-99999999999999999999999999999.999999999"); + + public static ByteString encodeToNumericByteString(BigDecimal bigDecimal) { + ByteString byteString = + serializeBigDecimal( + bigDecimal, NumericScale, MAX_NUMERIC_VALUE, MIN_NUMERIC_VALUE, "ByteString"); + return byteString; + } + + public static BigDecimal decodeNumericByteString(ByteString byteString) { + BigDecimal bigDecimal = + deserializeBigDecimal( + byteString, NumericScale, MAX_NUMERIC_VALUE, MIN_NUMERIC_VALUE, "BigDecimal"); + return bigDecimal; + } + // Make these private and make public wrapper that internalizes these min/max/scale/type + private static BigDecimal deserializeBigDecimal( + ByteString serializedValue, + int scale, + BigDecimal maxValue, + BigDecimal minValue, + String typeName) { + byte[] bytes = serializedValue.toByteArray(); + // NUMERIC/BIGNUMERIC values are serialized as scaled integers in two's complement form in + // little endian order. BigInteger requires the same encoding but in big endian order, + // therefore we must reverse the bytes that come from the proto. + Bytes.reverse(bytes); + BigInteger scaledValue = new BigInteger(bytes); + BigDecimal decimalValue = new BigDecimal(scaledValue, scale); + if (decimalValue.compareTo(maxValue) > 0 || decimalValue.compareTo(minValue) < 0) { + throw new IllegalArgumentException(typeName + " overflow: " + decimalValue.toPlainString()); + } + return decimalValue; + } + /** Returns a numeric Value that equals to {@code v}. */ + private static ByteString serializeBigDecimal( + BigDecimal v, int scale, BigDecimal maxValue, BigDecimal minValue, String typeName) { + if (v.scale() > scale) { + throw new IllegalArgumentException( + typeName + " scale cannot exceed " + scale + ": " + v.toPlainString()); + } + if (v.compareTo(maxValue) > 0 || v.compareTo(minValue) < 0) { + throw new IllegalArgumentException(typeName + " overflow: " + v.toPlainString()); + } + byte[] bytes = v.setScale(scale).unscaledValue().toByteArray(); + // NUMERIC/BIGNUMERIC values are serialized as scaled integers in two's complement form in + // little endian + // order. BigInteger requires the same encoding but in big endian order, therefore we must + // reverse the bytes that come from the proto. + Bytes.reverse(bytes); + return ByteString.copyFrom(bytes); + } +} diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BigDecimalByteStringEncoderTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BigDecimalByteStringEncoderTest.java new file mode 100644 index 0000000000..f73a0e1549 --- /dev/null +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BigDecimalByteStringEncoderTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.storage.v1beta2; + +import com.google.protobuf.ByteString; +import java.math.BigDecimal; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BigDecimalByteStringEncoderTest { + @Test + public void testEncodeBigDecimalandEncodeByteString() { + BigDecimal testBD = new BigDecimal("0"); // expected result bd + ByteString testBS = + BigDecimalByteStringEncoder.encodeToNumericByteString(testBD); // convert expected to bs + BigDecimal resultBD = + BigDecimalByteStringEncoder.decodeNumericByteString(testBS); // convert bs to bd + Assert.assertEquals( + 0, resultBD.compareTo(testBD)); // ensure converted bd is equal to expected bd + + testBD = new BigDecimal("1.2"); + testBS = BigDecimalByteStringEncoder.encodeToNumericByteString(testBD); + resultBD = BigDecimalByteStringEncoder.decodeNumericByteString(testBS); + Assert.assertEquals( + 0, resultBD.compareTo(testBD)); // ensure converted bd is equal to expected bd + + testBD = new BigDecimal("-1.2"); + testBS = BigDecimalByteStringEncoder.encodeToNumericByteString(testBD); + resultBD = BigDecimalByteStringEncoder.decodeNumericByteString(testBS); + Assert.assertEquals( + 0, resultBD.compareTo(testBD)); // ensure converted bd is equal to expected bd + + testBD = new BigDecimal("99999999999999999999999999999.999999999"); + testBS = BigDecimalByteStringEncoder.encodeToNumericByteString(testBD); + resultBD = BigDecimalByteStringEncoder.decodeNumericByteString(testBS); + Assert.assertEquals( + 0, resultBD.compareTo(testBD)); // ensure converted bd is equal to expected bd + + testBD = new BigDecimal("-99999999999999999999999999999.999999999"); + testBS = BigDecimalByteStringEncoder.encodeToNumericByteString(testBD); + resultBD = BigDecimalByteStringEncoder.decodeNumericByteString(testBS); + Assert.assertEquals( + 0, resultBD.compareTo(testBD)); // ensure converted bd is equal to expected bd + } +}