Skip to content

Commit

Permalink
feat(bigquery): add support for bignumeric (#2779)
Browse files Browse the repository at this point in the history 
feat(bigquery): add support for bignumeric

This PR adds basic support for the BIGNUMERIC type in BigQuery.

This library has several cases where it tries to infer the appropriate BigQuery type from a native Go type. For big.Rat types, we continue the existing behavior of mapping them to NUMERIC, as the big.Rat doesn't
provide a general way of indicating desired precision or scale to determine whether BIGNUMERIC is a more
appropriate mapping.
  • Loading branch information
@shollyman
shollyman committed Nov 19, 2020
1 parent a781a3a commit ea3cde5
Show file tree
Hide file tree
Showing 7 changed files with 172 additions and 95 deletions.
28 changes: 15 additions & 13 deletions bigquery/integration_test.go
View file
Expand Up @@ -1434,6 +1434,7 @@ func TestIntegration_InsertAndReadNullable(t *testing.T) {
ctm := civil.Time{Hour: 15, Minute: 4, Second: 5, Nanosecond: 6000}
cdt := civil.DateTime{Date: testDate, Time: ctm}
rat := big.NewRat(33, 100)
rat2 := big.NewRat(66, 100)
geo := "POINT(-122.198939 47.669865)"

// Nil fields in the struct.
Expand All @@ -1455,20 +1456,21 @@ func TestIntegration_InsertAndReadNullable(t *testing.T) {

// Populate the struct with values.
testInsertAndReadNullable(t, testStructNullable{
String: NullString{"x", true},
Bytes: []byte{1, 2, 3},
Integer: NullInt64{1, true},
Float: NullFloat64{2.3, true},
Boolean: NullBool{true, true},
Timestamp: NullTimestamp{testTimestamp, true},
Date: NullDate{testDate, true},
Time: NullTime{ctm, true},
DateTime: NullDateTime{cdt, true},
Numeric: rat,
Geography: NullGeography{geo, true},
Record: &subNullable{X: NullInt64{4, true}},
String: NullString{"x", true},
Bytes: []byte{1, 2, 3},
Integer: NullInt64{1, true},
Float: NullFloat64{2.3, true},
Boolean: NullBool{true, true},
Timestamp: NullTimestamp{testTimestamp, true},
Date: NullDate{testDate, true},
Time: NullTime{ctm, true},
DateTime: NullDateTime{cdt, true},
Numeric: rat,
BigNumeric: rat2,
Geography: NullGeography{geo, true},
Record: &subNullable{X: NullInt64{4, true}},
},
[]Value{"x", []byte{1, 2, 3}, int64(1), 2.3, true, testTimestamp, testDate, ctm, cdt, rat, geo, []Value{int64(4)}})
[]Value{"x", []byte{1, 2, 3}, int64(1), 2.3, true, testTimestamp, testDate, ctm, cdt, rat, rat2, geo, []Value{int64(4)}})
}

func testInsertAndReadNullable(t *testing.T, ts testStructNullable, wantRow []Value) {
Expand Down
41 changes: 23 additions & 18 deletions bigquery/params.go
View file
Expand Up @@ -65,16 +65,17 @@ func (e invalidFieldNameError) Error() string {
var fieldCache = fields.NewCache(bqTagParser, nil, nil)

var (
int64ParamType = &bq.QueryParameterType{Type: "INT64"}
float64ParamType = &bq.QueryParameterType{Type: "FLOAT64"}
boolParamType = &bq.QueryParameterType{Type: "BOOL"}
stringParamType = &bq.QueryParameterType{Type: "STRING"}
bytesParamType = &bq.QueryParameterType{Type: "BYTES"}
dateParamType = &bq.QueryParameterType{Type: "DATE"}
timeParamType = &bq.QueryParameterType{Type: "TIME"}
dateTimeParamType = &bq.QueryParameterType{Type: "DATETIME"}
timestampParamType = &bq.QueryParameterType{Type: "TIMESTAMP"}
numericParamType = &bq.QueryParameterType{Type: "NUMERIC"}
int64ParamType = &bq.QueryParameterType{Type: "INT64"}
float64ParamType = &bq.QueryParameterType{Type: "FLOAT64"}
boolParamType = &bq.QueryParameterType{Type: "BOOL"}
stringParamType = &bq.QueryParameterType{Type: "STRING"}
bytesParamType = &bq.QueryParameterType{Type: "BYTES"}
dateParamType = &bq.QueryParameterType{Type: "DATE"}
timeParamType = &bq.QueryParameterType{Type: "TIME"}
dateTimeParamType = &bq.QueryParameterType{Type: "DATETIME"}
timestampParamType = &bq.QueryParameterType{Type: "TIMESTAMP"}
numericParamType = &bq.QueryParameterType{Type: "NUMERIC"}
bigNumericParamType = &bq.QueryParameterType{Type: "BIGNUMERIC"}
)

var (
Expand Down Expand Up @@ -233,6 +234,9 @@ func paramValue(v reflect.Value) (bq.QueryParameterValue, error) {
return res, nil

case typeOfRat:
// big.Rat types don't communicate scale or precision, so we cannot
// disambiguate between NUMERIC and BIGNUMERIC. For now, we'll continue
// to honor previous behavior and send as Numeric type.
res.Value = NumericString(v.Interface().(*big.Rat))
return res, nil
}
Expand Down Expand Up @@ -304,14 +308,15 @@ func bqToQueryParameter(q *bq.QueryParameter) (QueryParameter, error) {
}

var paramTypeToFieldType = map[string]FieldType{
int64ParamType.Type: IntegerFieldType,
float64ParamType.Type: FloatFieldType,
boolParamType.Type: BooleanFieldType,
stringParamType.Type: StringFieldType,
bytesParamType.Type: BytesFieldType,
dateParamType.Type: DateFieldType,
timeParamType.Type: TimeFieldType,
numericParamType.Type: NumericFieldType,
int64ParamType.Type: IntegerFieldType,
float64ParamType.Type: FloatFieldType,
boolParamType.Type: BooleanFieldType,
stringParamType.Type: StringFieldType,
bytesParamType.Type: BytesFieldType,
dateParamType.Type: DateFieldType,
timeParamType.Type: TimeFieldType,
numericParamType.Type: NumericFieldType,
bigNumericParamType.Type: BigNumericFieldType,
}

// Convert a parameter value from the service to a Go value. This is similar to, but
Expand Down
32 changes: 20 additions & 12 deletions bigquery/schema.go
View file
Expand Up @@ -182,23 +182,27 @@ const (
// GeographyFieldType is a string field type. Geography types represent a set of points
// on the Earth's surface, represented in Well Known Text (WKT) format.
GeographyFieldType FieldType = "GEOGRAPHY"
// BigNumericFieldType is a numeric field type that supports values of larger precision
// and scale than the NumericFieldType.
BigNumericFieldType FieldType = "BIGNUMERIC"
)

var (
errEmptyJSONSchema = errors.New("bigquery: empty JSON schema")
fieldTypes = map[FieldType]bool{
StringFieldType: true,
BytesFieldType: true,
IntegerFieldType: true,
FloatFieldType: true,
BooleanFieldType: true,
TimestampFieldType: true,
RecordFieldType: true,
DateFieldType: true,
TimeFieldType: true,
DateTimeFieldType: true,
NumericFieldType: true,
GeographyFieldType: true,
StringFieldType: true,
BytesFieldType: true,
IntegerFieldType: true,
FloatFieldType: true,
BooleanFieldType: true,
TimestampFieldType: true,
RecordFieldType: true,
DateFieldType: true,
TimeFieldType: true,
DateTimeFieldType: true,
NumericFieldType: true,
GeographyFieldType: true,
BigNumericFieldType: true,
}
// The API will accept alias names for the types based on the Standard SQL type names.
fieldAliases = map[FieldType]FieldType{
Expand Down Expand Up @@ -346,6 +350,10 @@ func inferFieldSchema(fieldName string, rt reflect.Type, nullable bool) (*FieldS
case typeOfDateTime:
return &FieldSchema{Required: true, Type: DateTimeFieldType}, nil
case typeOfRat:
// We automatically infer big.Rat values as NUMERIC as we cannot
// determine precision/scale from the type. Users who want the
// larger precision of BIGNUMERIC need to manipulate the inferred
// schema.
return &FieldSchema{Required: !nullable, Type: NumericFieldType}, nil
}
if ft := nullableFieldType(rt); ft != "" {
Expand Down
6 changes: 4 additions & 2 deletions bigquery/schema_test.go
View file
Expand Up @@ -1041,7 +1041,8 @@ func TestSchemaFromJSON(t *testing.T) {
{"name":"flat_date","type":"DATE","mode":"NULLABLE","description":"Flat required DATE"},
{"name":"flat_time","type":"TIME","mode":"REQUIRED","description":"Flat nullable TIME"},
{"name":"flat_datetime","type":"DATETIME","mode":"NULLABLE","description":"Flat required DATETIME"},
{"name":"flat_numeric","type":"NUMERIC","mode":"REQUIRED","description":"Flat nullable NUMERIC"},
{"name":"flat_numeric","type":"NUMERIC","mode":"REQUIRED","description":"Flat required NUMERIC"},
{"name":"flat_bignumeric","type":"BIGNUMERIC","mode":"NULLABLE","description":"Flat nullable BIGNUMERIC"},
{"name":"flat_geography","type":"GEOGRAPHY","mode":"REQUIRED","description":"Flat required GEOGRAPHY"},
{"name":"aliased_integer","type":"INT64","mode":"REQUIRED","description":"Aliased required integer"},
{"name":"aliased_boolean","type":"BOOL","mode":"NULLABLE","description":"Aliased nullable boolean"},
Expand All @@ -1058,7 +1059,8 @@ func TestSchemaFromJSON(t *testing.T) {
fieldSchema("Flat required DATE", "flat_date", "DATE", false, false, nil),
fieldSchema("Flat nullable TIME", "flat_time", "TIME", false, true, nil),
fieldSchema("Flat required DATETIME", "flat_datetime", "DATETIME", false, false, nil),
fieldSchema("Flat nullable NUMERIC", "flat_numeric", "NUMERIC", false, true, nil),
fieldSchema("Flat required NUMERIC", "flat_numeric", "NUMERIC", false, true, nil),
fieldSchema("Flat nullable BIGNUMERIC", "flat_bignumeric", "BIGNUMERIC", false, false, nil),
fieldSchema("Flat required GEOGRAPHY", "flat_geography", "GEOGRAPHY", false, true, nil),
fieldSchema("Aliased required integer", "aliased_integer", "INTEGER", false, true, nil),
fieldSchema("Aliased nullable boolean", "aliased_boolean", "BOOLEAN", false, false, nil),
Expand Down
34 changes: 33 additions & 1 deletion bigquery/value.go
View file
Expand Up @@ -407,6 +407,13 @@ func determineSetFunc(ftype reflect.Type, stype FieldType) setFunc {
return setNull(v, x, func() interface{} { return x.(*big.Rat) })
}
}

case BigNumericFieldType:
if ftype == typeOfRat {
return func(v reflect.Value, x interface{}) error {
return setNull(v, x, func() interface{} { return x.(*big.Rat) })
}
}
}
return nil
}
Expand Down Expand Up @@ -692,7 +699,7 @@ func structFieldToUploadValue(vfield reflect.Value, schemaField *FieldSchema) (i
}

func toUploadValue(val interface{}, fs *FieldSchema) interface{} {
if fs.Type == TimeFieldType || fs.Type == DateTimeFieldType || fs.Type == NumericFieldType {
if fs.Type == TimeFieldType || fs.Type == DateTimeFieldType || fs.Type == NumericFieldType || fs.Type == BigNumericFieldType {
return toUploadValueReflect(reflect.ValueOf(val), fs)
}
return val
Expand Down Expand Up @@ -721,6 +728,13 @@ func toUploadValueReflect(v reflect.Value, fs *FieldSchema) interface{} {
return formatUploadValue(v, fs, func(v reflect.Value) string {
return NumericString(v.Interface().(*big.Rat))
})
case BigNumericFieldType:
if r, ok := v.Interface().(*big.Rat); ok && r == nil {
return nil
}
return formatUploadValue(v, fs, func(v reflect.Value) string {
return BigNumericString(v.Interface().(*big.Rat))
})
default:
if !fs.Repeated || v.Len() > 0 {
return v.Interface()
Expand Down Expand Up @@ -786,6 +800,12 @@ const (

// NumericScaleDigits is the maximum number of digits after the decimal point in a NUMERIC value.
NumericScaleDigits = 9

// BigNumericPrecisionDigits is the maximum number of full digits in a BIGNUMERIC value.
BigNumericPrecisionDigits = 76

// BigNumericScaleDigits is the maximum number of full digits in a BIGNUMERIC value.
BigNumericScaleDigits = 38
)

// NumericString returns a string representing a *big.Rat in a format compatible
Expand All @@ -795,6 +815,12 @@ func NumericString(r *big.Rat) string {
return r.FloatString(NumericScaleDigits)
}

// BigNumericString returns a string representing a *big.Rat in a format compatible with BigQuery
// SQL. It returns a floating point literal with 38 digits after the decimal point.
func BigNumericString(r *big.Rat) string {
return r.FloatString(BigNumericScaleDigits)
}

// convertRows converts a series of TableRows into a series of Value slices.
// schema is used to interpret the data from rows; its length must match the
// length of each row.
Expand Down Expand Up @@ -913,6 +939,12 @@ func convertBasicType(val string, typ FieldType) (Value, error) {
return nil, fmt.Errorf("bigquery: invalid NUMERIC value %q", val)
}
return Value(r), nil
case BigNumericFieldType:
r, ok := (&big.Rat{}).SetString(val)
if !ok {
return nil, fmt.Errorf("bigquery: invalid BIGNUMERIC value %q", val)
}
return Value(r), nil
case GeographyFieldType:
return val, nil
default:
Expand Down

0 comments on commit ea3cde5

Please sign in to comment.