feat(bigquery): add support for bignumeric

bigquery/integration_test.go

@@ -1434,6 +1434,7 @@ func TestIntegration_InsertAndReadNullable(t *testing.T) {
 	ctm := civil.Time{Hour: 15, Minute: 4, Second: 5, Nanosecond: 6000}
 	cdt := civil.DateTime{Date: testDate, Time: ctm}
 	rat := big.NewRat(33, 100)
+	rat2 := big.NewRat(66, 100)
 	geo := "POINT(-122.198939 47.669865)"
 
 	// Nil fields in the struct.
@@ -1455,20 +1456,21 @@ func TestIntegration_InsertAndReadNullable(t *testing.T) {
 
 	// Populate the struct with values.
 	testInsertAndReadNullable(t, testStructNullable{
-		String:    NullString{"x", true},
-		Bytes:     []byte{1, 2, 3},
-		Integer:   NullInt64{1, true},
-		Float:     NullFloat64{2.3, true},
-		Boolean:   NullBool{true, true},
-		Timestamp: NullTimestamp{testTimestamp, true},
-		Date:      NullDate{testDate, true},
-		Time:      NullTime{ctm, true},
-		DateTime:  NullDateTime{cdt, true},
-		Numeric:   rat,
-		Geography: NullGeography{geo, true},
-		Record:    &subNullable{X: NullInt64{4, true}},
+		String:     NullString{"x", true},
+		Bytes:      []byte{1, 2, 3},
+		Integer:    NullInt64{1, true},
+		Float:      NullFloat64{2.3, true},
+		Boolean:    NullBool{true, true},
+		Timestamp:  NullTimestamp{testTimestamp, true},
+		Date:       NullDate{testDate, true},
+		Time:       NullTime{ctm, true},
+		DateTime:   NullDateTime{cdt, true},
+		Numeric:    rat,
+		BigNumeric: rat2,
+		Geography:  NullGeography{geo, true},
+		Record:     &subNullable{X: NullInt64{4, true}},
 	},
-		[]Value{"x", []byte{1, 2, 3}, int64(1), 2.3, true, testTimestamp, testDate, ctm, cdt, rat, geo, []Value{int64(4)}})
+		[]Value{"x", []byte{1, 2, 3}, int64(1), 2.3, true, testTimestamp, testDate, ctm, cdt, rat, rat2, geo, []Value{int64(4)}})
 }
 
 func testInsertAndReadNullable(t *testing.T, ts testStructNullable, wantRow []Value) {

bigquery/params.go

@@ -65,16 +65,17 @@ func (e invalidFieldNameError) Error() string {
 var fieldCache = fields.NewCache(bqTagParser, nil, nil)
 
 var (
-	int64ParamType     = &bq.QueryParameterType{Type: "INT64"}
-	float64ParamType   = &bq.QueryParameterType{Type: "FLOAT64"}
-	boolParamType      = &bq.QueryParameterType{Type: "BOOL"}
-	stringParamType    = &bq.QueryParameterType{Type: "STRING"}
-	bytesParamType     = &bq.QueryParameterType{Type: "BYTES"}
-	dateParamType      = &bq.QueryParameterType{Type: "DATE"}
-	timeParamType      = &bq.QueryParameterType{Type: "TIME"}
-	dateTimeParamType  = &bq.QueryParameterType{Type: "DATETIME"}
-	timestampParamType = &bq.QueryParameterType{Type: "TIMESTAMP"}
-	numericParamType   = &bq.QueryParameterType{Type: "NUMERIC"}
+	int64ParamType      = &bq.QueryParameterType{Type: "INT64"}
+	float64ParamType    = &bq.QueryParameterType{Type: "FLOAT64"}
+	boolParamType       = &bq.QueryParameterType{Type: "BOOL"}
+	stringParamType     = &bq.QueryParameterType{Type: "STRING"}
+	bytesParamType      = &bq.QueryParameterType{Type: "BYTES"}
+	dateParamType       = &bq.QueryParameterType{Type: "DATE"}
+	timeParamType       = &bq.QueryParameterType{Type: "TIME"}
+	dateTimeParamType   = &bq.QueryParameterType{Type: "DATETIME"}
+	timestampParamType  = &bq.QueryParameterType{Type: "TIMESTAMP"}
+	numericParamType    = &bq.QueryParameterType{Type: "NUMERIC"}
+	bigNumericParamType = &bq.QueryParameterType{Type: "BIGNUMERIC"}
 )
 
 var (
@@ -233,6 +234,9 @@ func paramValue(v reflect.Value) (bq.QueryParameterValue, error) {
 		return res, nil
 
 	case typeOfRat:
+		// big.Rat types don't communicate scale or precision, so we cannot
+		// disambiguate between NUMERIC and BIGNUMERIC.  For now, we'll continue
+		// to honor previous behavior and send as Numeric type.
 		res.Value = NumericString(v.Interface().(*big.Rat))
 		return res, nil
 	}
@@ -304,14 +308,15 @@ func bqToQueryParameter(q *bq.QueryParameter) (QueryParameter, error) {
 }
 
 var paramTypeToFieldType = map[string]FieldType{
-	int64ParamType.Type:   IntegerFieldType,
-	float64ParamType.Type: FloatFieldType,
-	boolParamType.Type:    BooleanFieldType,
-	stringParamType.Type:  StringFieldType,
-	bytesParamType.Type:   BytesFieldType,
-	dateParamType.Type:    DateFieldType,
-	timeParamType.Type:    TimeFieldType,
-	numericParamType.Type: NumericFieldType,
+	int64ParamType.Type:      IntegerFieldType,
+	float64ParamType.Type:    FloatFieldType,
+	boolParamType.Type:       BooleanFieldType,
+	stringParamType.Type:     StringFieldType,
+	bytesParamType.Type:      BytesFieldType,
+	dateParamType.Type:       DateFieldType,
+	timeParamType.Type:       TimeFieldType,
+	numericParamType.Type:    NumericFieldType,
+	bigNumericParamType.Type: BigNumericFieldType,
 }
 
 // Convert a parameter value from the service to a Go value. This is similar to, but

bigquery/schema.go

@@ -182,23 +182,27 @@ const (
 	// GeographyFieldType is a string field type.  Geography types represent a set of points
 	// on the Earth's surface, represented in Well Known Text (WKT) format.
 	GeographyFieldType FieldType = "GEOGRAPHY"
+	// BigNumericFieldType is a numeric field type that supports values of larger precision
+	// and scale than the NumericFieldType.
+	BigNumericFieldType FieldType = "BIGNUMERIC"
 )
 
 var (
 	errEmptyJSONSchema = errors.New("bigquery: empty JSON schema")
 	fieldTypes         = map[FieldType]bool{
-		StringFieldType:    true,
-		BytesFieldType:     true,
-		IntegerFieldType:   true,
-		FloatFieldType:     true,
-		BooleanFieldType:   true,
-		TimestampFieldType: true,
-		RecordFieldType:    true,
-		DateFieldType:      true,
-		TimeFieldType:      true,
-		DateTimeFieldType:  true,
-		NumericFieldType:   true,
-		GeographyFieldType: true,
+		StringFieldType:     true,
+		BytesFieldType:      true,
+		IntegerFieldType:    true,
+		FloatFieldType:      true,
+		BooleanFieldType:    true,
+		TimestampFieldType:  true,
+		RecordFieldType:     true,
+		DateFieldType:       true,
+		TimeFieldType:       true,
+		DateTimeFieldType:   true,
+		NumericFieldType:    true,
+		GeographyFieldType:  true,
+		BigNumericFieldType: true,
 	}
 	// The API will accept alias names for the types based on the Standard SQL type names.
 	fieldAliases = map[FieldType]FieldType{
@@ -346,6 +350,10 @@ func inferFieldSchema(fieldName string, rt reflect.Type, nullable bool) (*FieldS
 	case typeOfDateTime:
 		return &FieldSchema{Required: true, Type: DateTimeFieldType}, nil
 	case typeOfRat:
+		// We automatically infer big.Rat values as NUMERIC as we cannot
+		// determine precision/scale from the type.  Users who want the
+		// larger precision of BIGNUMERIC need to manipulate the inferred
+		// schema.
 		return &FieldSchema{Required: !nullable, Type: NumericFieldType}, nil
 	}
 	if ft := nullableFieldType(rt); ft != "" {

bigquery/schema_test.go

@@ -1041,7 +1041,8 @@ func TestSchemaFromJSON(t *testing.T) {
 	{"name":"flat_date","type":"DATE","mode":"NULLABLE","description":"Flat required DATE"},
 	{"name":"flat_time","type":"TIME","mode":"REQUIRED","description":"Flat nullable TIME"},
 	{"name":"flat_datetime","type":"DATETIME","mode":"NULLABLE","description":"Flat required DATETIME"},
-	{"name":"flat_numeric","type":"NUMERIC","mode":"REQUIRED","description":"Flat nullable NUMERIC"},
+	{"name":"flat_numeric","type":"NUMERIC","mode":"REQUIRED","description":"Flat required NUMERIC"},
+	{"name":"flat_bignumeric","type":"BIGNUMERIC","mode":"NULLABLE","description":"Flat nullable BIGNUMERIC"},
 	{"name":"flat_geography","type":"GEOGRAPHY","mode":"REQUIRED","description":"Flat required GEOGRAPHY"},
 	{"name":"aliased_integer","type":"INT64","mode":"REQUIRED","description":"Aliased required integer"},
 	{"name":"aliased_boolean","type":"BOOL","mode":"NULLABLE","description":"Aliased nullable boolean"},
@@ -1058,7 +1059,8 @@ func TestSchemaFromJSON(t *testing.T) {
 				fieldSchema("Flat required DATE", "flat_date", "DATE", false, false, nil),
 				fieldSchema("Flat nullable TIME", "flat_time", "TIME", false, true, nil),
 				fieldSchema("Flat required DATETIME", "flat_datetime", "DATETIME", false, false, nil),
-				fieldSchema("Flat nullable NUMERIC", "flat_numeric", "NUMERIC", false, true, nil),
+				fieldSchema("Flat required NUMERIC", "flat_numeric", "NUMERIC", false, true, nil),
+				fieldSchema("Flat nullable BIGNUMERIC", "flat_bignumeric", "BIGNUMERIC", false, false, nil),
 				fieldSchema("Flat required GEOGRAPHY", "flat_geography", "GEOGRAPHY", false, true, nil),
 				fieldSchema("Aliased required integer", "aliased_integer", "INTEGER", false, true, nil),
 				fieldSchema("Aliased nullable boolean", "aliased_boolean", "BOOLEAN", false, false, nil),

bigquery/value.go

@@ -407,6 +407,13 @@ func determineSetFunc(ftype reflect.Type, stype FieldType) setFunc {
 				return setNull(v, x, func() interface{} { return x.(*big.Rat) })
 			}
 		}
+
+	case BigNumericFieldType:
+		if ftype == typeOfRat {
+			return func(v reflect.Value, x interface{}) error {
+				return setNull(v, x, func() interface{} { return x.(*big.Rat) })
+			}
+		}
 	}
 	return nil
 }
@@ -692,7 +699,7 @@ func structFieldToUploadValue(vfield reflect.Value, schemaField *FieldSchema) (i
 }
 
 func toUploadValue(val interface{}, fs *FieldSchema) interface{} {
-	if fs.Type == TimeFieldType || fs.Type == DateTimeFieldType || fs.Type == NumericFieldType {
+	if fs.Type == TimeFieldType || fs.Type == DateTimeFieldType || fs.Type == NumericFieldType || fs.Type == BigNumericFieldType {
 		return toUploadValueReflect(reflect.ValueOf(val), fs)
 	}
 	return val
@@ -721,6 +728,13 @@ func toUploadValueReflect(v reflect.Value, fs *FieldSchema) interface{} {
 		return formatUploadValue(v, fs, func(v reflect.Value) string {
 			return NumericString(v.Interface().(*big.Rat))
 		})
+	case BigNumericFieldType:
+		if r, ok := v.Interface().(*big.Rat); ok && r == nil {
+			return nil
+		}
+		return formatUploadValue(v, fs, func(v reflect.Value) string {
+			return BigNumericString(v.Interface().(*big.Rat))
+		})
 	default:
 		if !fs.Repeated || v.Len() > 0 {
 			return v.Interface()
@@ -786,6 +800,14 @@ const (
 
 	// NumericScaleDigits is the maximum number of digits after the decimal point in a NUMERIC value.
 	NumericScaleDigits = 9
+
+	// BigNumericPrecisionDigits is the maximum number of full digits in a BIGNUMERIC value.
+	// BigNumeric does support a partial digit beyond this.
+	BigNumericPrecisionDigits = 76
+
+	// BigNumericScaleDigits is the maximum number of full digits in a BIGNUMERIC value.
+	// BigNumeric does support a partial 39th digit.
+	BigNumericScaleDigits = 38
 )
 
 // NumericString returns a string representing a *big.Rat in a format compatible
@@ -795,6 +817,12 @@ func NumericString(r *big.Rat) string {
 	return r.FloatString(NumericScaleDigits)
 }
 
+// BigNumericString returns a string representing a *big.Rat in a format compatible with BigQuery
+// SQL.  It returns a floating point literal with 38 digits after the decimal point.
+func BigNumericString(r *big.Rat) string {
+	return r.FloatString(BigNumericScaleDigits)
+}
+
 // convertRows converts a series of TableRows into a series of Value slices.
 // schema is used to interpret the data from rows; its length must match the
 // length of each row.
@@ -913,6 +941,12 @@ func convertBasicType(val string, typ FieldType) (Value, error) {
 			return nil, fmt.Errorf("bigquery: invalid NUMERIC value %q", val)
 		}
 		return Value(r), nil
+	case BigNumericFieldType:
+		r, ok := (&big.Rat{}).SetString(val)
+		if !ok {
+			return nil, fmt.Errorf("bigquery: invalid BIGNUMERIC value %q", val)
+		}
+		return Value(r), nil
 	case GeographyFieldType:
 		return val, nil
 	default: