feat(internal/gapicgen): only update relevant gapic files

.github/workflows/release-submodule.yaml

@@ -71,7 +71,7 @@ jobs:
            path: ${{ matrix.package }}
            token: ${{ secrets.FORKING_TOKEN }}
            fork: true
-           release-type: go
+           release-type: go-yoshi
            bump-minor-pre-major: true
            package-name: ${{ matrix.package }}
            monorepo-tags: true
@@ -104,7 +104,7 @@ jobs:
           path: ${{ matrix.package }}
           changelog-path: CHANGES.md
           token: ${{ secrets.GITHUB_TOKEN }}
-          release-type: go
+          release-type: go-yoshi
           monorepo-tags: true
           package-name: ${{ matrix.package }}
           command: github-release

bigquery/CHANGES.md

@@ -1,5 +1,15 @@
 # Changes
 
+## [1.18.0](https://www.github.com/googleapis/google-cloud-go/compare/bigquery/v1.17.0...bigquery/v1.18.0) (2021-05-06)
+
+
+### Features
+
+* **bigquery/storage:** new JSON type through BigQuery Write ([9029071](https://www.github.com/googleapis/google-cloud-go/commit/90290710158cf63de918c2d790df48f55a23adc5))
+* **bigquery:** augment retry predicate to support additional errors ([#4046](https://www.github.com/googleapis/google-cloud-go/issues/4046)) ([d4af6f7](https://www.github.com/googleapis/google-cloud-go/commit/d4af6f7707b3c5ee12cde53c7485a9b743034119))
+* **bigquery:** expose ParquetOptions for loads and external tables ([#4016](https://www.github.com/googleapis/google-cloud-go/issues/4016)) ([f9c4ccb](https://www.github.com/googleapis/google-cloud-go/commit/f9c4ccb6efb271c421edf3f67d5249b1cfb0ecb2))
+* **bigquery:** support mutable clustering configuration ([#3950](https://www.github.com/googleapis/google-cloud-go/issues/3950)) ([0ab30da](https://www.github.com/googleapis/google-cloud-go/commit/0ab30dadc43ae85354dc12a4130ecfcc56273882))
+
 ## [1.17.0](https://www.github.com/googleapis/google-cloud-go/compare/bigquery/v1.15.0...bigquery/v1.17.0) (2021-04-08)
 
 

bigquery/bigquery.go

@@ -19,6 +19,8 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
+	"strings"
 	"time"
 
 	"cloud.google.com/go/internal"
@@ -169,6 +171,9 @@ func retryableError(err error) bool {
 	if err == nil {
 		return false
 	}
+	if err == io.ErrUnexpectedEOF {
+		return true
+	}
 	// Special case due to http2: https://github.com/googleapis/google-cloud-go/issues/1793
 	// Due to Go's default being higher for streams-per-connection than is accepted by the
 	// BQ backend, it's possible to get streams refused immediately after a connection is
@@ -177,13 +182,32 @@ func retryableError(err error) bool {
 	if err.Error() == "http2: stream closed" {
 		return true
 	}
-	e, ok := err.(*googleapi.Error)
-	if !ok {
-		return false
+
+	switch e := err.(type) {
+	case *googleapi.Error:
+		// We received a structured error from backend.
+		var reason string
+		if len(e.Errors) > 0 {
+			reason = e.Errors[0].Reason
+		}
+		if e.Code == http.StatusServiceUnavailable || e.Code == http.StatusBadGateway || reason == "backendError" || reason == "rateLimitExceeded" {
+			return true
+		}
+	case *url.Error:
+		retryable := []string{"connection refused", "connection reset"}
+		for _, s := range retryable {
+			if strings.Contains(e.Error(), s) {
+				return true
+			}
+		}
+	case interface{ Temporary() bool }:
+		if e.Temporary() {
+			return true
+		}
 	}
-	var reason string
-	if len(e.Errors) > 0 {
-		reason = e.Errors[0].Reason
+	// Unwrap is only supported in go1.13.x+
+	if e, ok := err.(interface{ Unwrap() error }); ok {
+		return retryableError(e.Unwrap())
 	}
-	return e.Code == http.StatusServiceUnavailable || e.Code == http.StatusBadGateway || reason == "backendError" || reason == "rateLimitExceeded"
+	return false
 }

bigquery/bigquery_test.go

@@ -16,9 +16,12 @@ package bigquery
 
 import (
 	"errors"
+	"io"
 	"net/http"
+	"net/url"
 	"testing"
 
+	"golang.org/x/xerrors"
 	"google.golang.org/api/googleapi"
 )
 
@@ -38,6 +41,11 @@ func TestRetryableErrors(t *testing.T) {
 			errors.New("http2: stream closed"),
 			true,
 		},
+		{
+			"io ErrUnexpectedEOF",
+			io.ErrUnexpectedEOF,
+			true,
+		},
 		{
 			"unavailable",
 			&googleapi.Error{
@@ -46,6 +54,32 @@ func TestRetryableErrors(t *testing.T) {
 			},
 			true,
 		},
+		{
+			"url connection error",
+			&url.Error{Op: "blah", URL: "blah", Err: errors.New("connection refused")},
+			true,
+		},
+		{
+			"url other error",
+			&url.Error{Op: "blah", URL: "blah", Err: errors.New("blah")},
+			false,
+		},
+		{
+			"wrapped retryable",
+			xerrors.Errorf("test of wrapped retryable: %w", &googleapi.Error{
+				Code:    http.StatusServiceUnavailable,
+				Message: "foo",
+				Errors: []googleapi.ErrorItem{
+					{Reason: "backendError", Message: "foo"},
+				},
+			}),
+			true,
+		},
+		{
+			"wrapped non-retryable",
+			xerrors.Errorf("test of wrapped retryable: %w", errors.New("blah")),
+			false,
+		},
 		{
 			// not retried per https://google.aip.dev/194
 			"internal error",

bigquery/external.go

@@ -90,7 +90,7 @@ type ExternalDataConfig struct {
 	// when reading data.
 	MaxBadRecords int64
 
-	// Additional options for CSV, GoogleSheets and Bigtable formats.
+	// Additional options for CSV, GoogleSheets, Bigtable, and Parquet formats.
 	Options ExternalDataConfigOptions
 
 	// HivePartitioningOptions allows use of Hive partitioning based on the
@@ -139,6 +139,8 @@ func bqToExternalDataConfig(q *bq.ExternalDataConfiguration) (*ExternalDataConfi
 		if err != nil {
 			return nil, err
 		}
+	case q.ParquetOptions != nil:
+		e.Options = bqToParquetOptions(q.ParquetOptions)
 	}
 	return e, nil
 }
@@ -416,6 +418,36 @@ func bqToBigtableColumn(q *bq.BigtableColumn) (*BigtableColumn, error) {
 	return b, nil
 }
 
+// ParquetOptions are additional options for Parquet external data sources.
+type ParquetOptions struct {
+	// EnumAsString indicates whether to infer Parquet ENUM logical type as
+	// STRING instead of BYTES by default.
+	EnumAsString bool
+
+	// EnableListInference indicates whether to use schema inference
+	// specifically for Parquet LIST logical type.
+	EnableListInference bool
+}
+
+func (o *ParquetOptions) populateExternalDataConfig(c *bq.ExternalDataConfiguration) {
+	if o != nil {
+		c.ParquetOptions = &bq.ParquetOptions{
+			EnumAsString:        o.EnumAsString,
+			EnableListInference: o.EnableListInference,
+		}
+	}
+}
+
+func bqToParquetOptions(q *bq.ParquetOptions) *ParquetOptions {
+	if q == nil {
+		return nil
+	}
+	return &ParquetOptions{
+		EnumAsString:        q.EnumAsString,
+		EnableListInference: q.EnableListInference,
+	}
+}
+
 // HivePartitioningMode is used in conjunction with HivePartitioningOptions.
 type HivePartitioningMode string
 

bigquery/external_test.go

@@ -80,6 +80,13 @@ func TestExternalDataConfig(t *testing.T) {
 				},
 			},
 		},
+		{
+			SourceFormat: Parquet,
+			Options: &ParquetOptions{
+				EnumAsString:        true,
+				EnableListInference: true,
+			},
+		},
 	} {
 		q := want.toBQ()
 		got, err := bqToExternalDataConfig(&q)

bigquery/file.go

@@ -74,6 +74,9 @@ type FileConfig struct {
 
 	// Additional options for CSV files.
 	CSVOptions
+
+	// Additional options for Parquet files.
+	ParquetOptions *ParquetOptions
 }
 
 func (fc *FileConfig) populateLoadConfig(conf *bq.JobConfigurationLoad) {
@@ -89,6 +92,12 @@ func (fc *FileConfig) populateLoadConfig(conf *bq.JobConfigurationLoad) {
 	if fc.Schema != nil {
 		conf.Schema = fc.Schema.toBQ()
 	}
+	if fc.ParquetOptions != nil {
+		conf.ParquetOptions = &bq.ParquetOptions{
+			EnumAsString:        fc.ParquetOptions.EnumAsString,
+			EnableListInference: fc.ParquetOptions.EnableListInference,
+		}
+	}
 	conf.Quote = fc.quote()
 }
 
@@ -122,6 +131,12 @@ func (fc *FileConfig) populateExternalDataConfig(conf *bq.ExternalDataConfigurat
 	if format == CSV {
 		fc.CSVOptions.populateExternalDataConfig(conf)
 	}
+	if fc.ParquetOptions != nil {
+		conf.ParquetOptions = &bq.ParquetOptions{
+			EnumAsString:        fc.ParquetOptions.EnumAsString,
+			EnableListInference: fc.ParquetOptions.EnableListInference,
+		}
+	}
 }
 
 // Encoding specifies the character encoding of data to be loaded into BigQuery.