Skip to content

Commit

Permalink
feat(bigquery): support nullable params and geography params (#4225)
Browse files Browse the repository at this point in the history 
* feat(bigquery): support nullable params, geography params

Details:
* Support the passing of Null<type> values for query parameters.

* Add support for GEOGRAPHY param type, which was added after the initial
  GEOGRAPHY type support launched.
  • Loading branch information
@shollyman
shollyman committed Jun 17, 2021
1 parent 0232e6f commit 43755d3
Show file tree
Hide file tree
Showing 2 changed files with 240 additions and 52 deletions.
138 changes: 120 additions & 18 deletions bigquery/params.go
View file
Expand Up @@ -76,6 +76,7 @@ var (
timestampParamType = &bq.QueryParameterType{Type: "TIMESTAMP"}
numericParamType = &bq.QueryParameterType{Type: "NUMERIC"}
bigNumericParamType = &bq.QueryParameterType{Type: "BIGNUMERIC"}
geographyParamType = &bq.QueryParameterType{Type: "GEOGRAPHY"}
)

var (
Expand Down Expand Up @@ -108,16 +109,19 @@ type QueryParameter struct {
// Arrays and slices of the above.
// Structs of the above. Only the exported fields are used.
//
// BigQuery does not support params of type GEOGRAPHY. For users wishing
// to parameterize Geography values, use string parameters and cast in the
// SQL query, e.g. `SELECT ST_GeogFromText(@string_param) as geo`
// For scalar values, you can supply the Null types within this library
// to send the appropriate NULL values (e.g. NullInt64, NullString, etc).
//
// When a QueryParameter is returned inside a QueryConfig from a call to
// Job.Config:
// Integers are of type int64.
// Floating-point values are of type float64.
// Arrays are of type []interface{}, regardless of the array element type.
// Structs are of type map[string]interface{}.
//
// When valid (non-null) Null types are sent, they come back as the Go types indicated
// above. Null strings will report in query statistics as a valid empty
// string.
Value interface{}
}

Expand All @@ -132,7 +136,7 @@ func (p QueryParameter) toBQ() (*bq.QueryParameter, error) {
}
return &bq.QueryParameter{
Name: p.Name,
ParameterValue: &pv,
ParameterValue: pv,
ParameterType: pt,
}, nil
}
Expand All @@ -142,16 +146,26 @@ func paramType(t reflect.Type) (*bq.QueryParameterType, error) {
return nil, errors.New("bigquery: nil parameter")
}
switch t {
case typeOfDate:
case typeOfDate, typeOfNullDate:
return dateParamType, nil
case typeOfTime:
case typeOfTime, typeOfNullTime:
return timeParamType, nil
case typeOfDateTime:
case typeOfDateTime, typeOfNullDateTime:
return dateTimeParamType, nil
case typeOfGoTime:
case typeOfGoTime, typeOfNullTimestamp:
return timestampParamType, nil
case typeOfRat:
return numericParamType, nil
case typeOfNullBool:
return boolParamType, nil
case typeOfNullFloat64:
return float64ParamType, nil
case typeOfNullInt64:
return int64ParamType, nil
case typeOfNullString:
return stringParamType, nil
case typeOfNullGeography:
return geographyParamType, nil
}
switch t.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint8, reflect.Uint16, reflect.Uint32:
Expand Down Expand Up @@ -207,17 +221,64 @@ func paramType(t reflect.Type) (*bq.QueryParameterType, error) {
return nil, fmt.Errorf("bigquery: Go type %s cannot be represented as a parameter type", t)
}

func paramValue(v reflect.Value) (bq.QueryParameterValue, error) {
var res bq.QueryParameterValue
func paramValue(v reflect.Value) (*bq.QueryParameterValue, error) {
res := &bq.QueryParameterValue{}
if !v.IsValid() {
return res, errors.New("bigquery: nil parameter")
}
t := v.Type()
switch t {

// Handle all the custom null types as a group first, as they all have the same logic when invalid.
case typeOfNullInt64,
typeOfNullString,
typeOfNullGeography,
typeOfNullFloat64,
typeOfNullBool,
typeOfNullTimestamp,
typeOfNullDate,
typeOfNullTime,
typeOfNullDateTime:
// Shared: If the Null type isn't valid, we have no value to send.
// However, the backend requires us to send the QueryParameterValue with
// the fields empty.
if !v.FieldByName("Valid").Bool() {
// Ensure we don't send a default value by using NullFields in the JSON
// serialization.
res.NullFields = append(res.NullFields, "Value")
return res, nil
}
// For cases where the Null type is valid, populate the scalar value as needed.
switch t {
case typeOfNullInt64:
res.Value = fmt.Sprint(v.FieldByName("Int64").Interface())
case typeOfNullString:
res.Value = fmt.Sprint(v.FieldByName("StringVal").Interface())
case typeOfNullGeography:
res.Value = fmt.Sprint(v.FieldByName("GeographyVal").Interface())
case typeOfNullFloat64:
res.Value = fmt.Sprint(v.FieldByName("Float64").Interface())
case typeOfNullBool:
res.Value = fmt.Sprint(v.FieldByName("Bool").Interface())
case typeOfNullTimestamp:
res.Value = v.FieldByName("Timestamp").Interface().(time.Time).Format(timestampFormat)
case typeOfNullDate:
res.Value = v.FieldByName("Date").Interface().(civil.Date).String()
case typeOfNullTime:
res.Value = CivilTimeString(v.FieldByName("Time").Interface().(civil.Time))
case typeOfNullDateTime:
res.Value = CivilDateTimeString(v.FieldByName("DateTime").Interface().(civil.DateTime))
}
// We expect to produce a value in all these cases, so force send if the result is the empty
// string.
if res.Value == "" {
res.ForceSendFields = append(res.ForceSendFields, "Value")
}
return res, nil

case typeOfDate:
res.Value = v.Interface().(civil.Date).String()
return res, nil

case typeOfTime:
// civil.Time has nanosecond resolution, but BigQuery TIME only microsecond.
// (If we send nanoseconds, then when we try to read the result we get "query job
Expand Down Expand Up @@ -253,11 +314,11 @@ func paramValue(v reflect.Value) (bq.QueryParameterValue, error) {
for i := 0; i < v.Len(); i++ {
val, err := paramValue(v.Index(i))
if err != nil {
return bq.QueryParameterValue{}, err
return nil, err
}
vals = append(vals, &val)
vals = append(vals, val)
}
return bq.QueryParameterValue{ArrayValues: vals}, nil
return &bq.QueryParameterValue{ArrayValues: vals}, nil

case reflect.Ptr:
if t.Elem().Kind() != reflect.Struct {
Expand All @@ -274,16 +335,16 @@ func paramValue(v reflect.Value) (bq.QueryParameterValue, error) {
case reflect.Struct:
fields, err := fieldCache.Fields(t)
if err != nil {
return bq.QueryParameterValue{}, err
return nil, err
}
res.StructValues = map[string]bq.QueryParameterValue{}
for _, f := range fields {
fv := v.FieldByIndex(f.Index)
fp, err := paramValue(fv)
if err != nil {
return bq.QueryParameterValue{}, err
return nil, err
}
res.StructValues[f.Name] = fp
res.StructValues[f.Name] = *fp
}
return res, nil
}
Expand Down Expand Up @@ -317,10 +378,12 @@ var paramTypeToFieldType = map[string]FieldType{
timeParamType.Type: TimeFieldType,
numericParamType.Type: NumericFieldType,
bigNumericParamType.Type: BigNumericFieldType,
geographyParamType.Type: GeographyFieldType,
}

// Convert a parameter value from the service to a Go value. This is similar to, but
// not quite the same as, converting data values.
// not quite the same as, converting data values. Namely, rather than returning nil
// directly, we wrap them in the appropriate Null types (NullInt64, etc).
func convertParamValue(qval *bq.QueryParameterValue, qtype *bq.QueryParameterType) (interface{}, error) {
switch qtype.Type {
case "ARRAY":
Expand All @@ -334,14 +397,53 @@ func convertParamValue(qval *bq.QueryParameterValue, qtype *bq.QueryParameterTyp
}
return convertParamStruct(qval.StructValues, qtype.StructTypes)
case "TIMESTAMP":
if isNullScalar(qval) {
return NullTimestamp{Valid: false}, nil
}
return time.Parse(timestampFormat, qval.Value)
case "DATETIME":
if isNullScalar(qval) {
return NullDateTime{Valid: false}, nil
}
return parseCivilDateTime(qval.Value)
default:
if isNullScalar(qval) {
switch qtype.Type {
case "INT64":
return NullInt64{Valid: false}, nil
case "STRING":
return NullString{Valid: false}, nil
case "FLOAT64":
return NullFloat64{Valid: false}, nil
case "BOOL":
return NullBool{Valid: false}, nil
case "DATE":
return NullDate{Valid: false}, nil
case "TIME":
return NullTime{Valid: false}, nil
case "GEOGRAPHY":
return NullGeography{Valid: false}, nil
}

}
return convertBasicType(qval.Value, paramTypeToFieldType[qtype.Type])
}
}

// isNullScalar determines if the input is meant to represent a null scalar
// value.
func isNullScalar(qval *bq.QueryParameterValue) bool {
if qval == nil {
return true
}
for _, v := range qval.NullFields {
if v == "Value" {
return true
}
}
return false
}

// convertParamArray converts a query parameter array value to a Go value. It
// always returns a []interface{}.
func convertParamArray(elVals []*bq.QueryParameterValue, elType *bq.QueryParameterType) ([]interface{}, error) {
Expand Down

0 comments on commit 43755d3

Please sign in to comment.