feat(storage): add net.ErrClosed to default retry (#5384)

Co-authored-by: Chris Cotter <cjcotter@google.com>

storage/invoke.go

@@ -17,11 +17,13 @@ package storage
 import (
 	"context"
 	"io"
+	"net"
 	"net/url"
 	"strings"
 
 	"cloud.google.com/go/internal"
 	gax "github.com/googleapis/gax-go/v2"
+	"golang.org/x/xerrors"
 	"google.golang.org/api/googleapi"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -59,16 +61,22 @@ func shouldRetry(err error) bool {
 	if err == nil {
 		return false
 	}
-	if err == io.ErrUnexpectedEOF {
+	if xerrors.Is(err, io.ErrUnexpectedEOF) {
 		return true
 	}
+
 	switch e := err.(type) {
+	case *net.OpError:
+		if strings.Contains(e.Error(), "use of closed network connection") {
+			// TODO: check against net.ErrClosed (go 1.16+) instead of string
+			return true
+		}
 	case *googleapi.Error:
 		// Retry on 408, 429, and 5xx, according to
 		// https://cloud.google.com/storage/docs/exponential-backoff.
 		return e.Code == 408 || e.Code == 429 || (e.Code >= 500 && e.Code < 600)
 	case *url.Error:
-		// Retry socket-level errors ECONNREFUSED and ENETUNREACH (from syscall).
+		// Retry socket-level errors ECONNREFUSED and ECONNRESET (from syscall).
 		// Unfortunately the error type is unexported, so we resort to string
 		// matching.
 		retriable := []string{"connection refused", "connection reset"}

storage/invoke_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"errors"
 	"io"
+	"net"
 	"net/url"
 	"testing"
 
@@ -263,6 +264,12 @@ func TestShouldRetry(t *testing.T) {
 			inputErr:    status.Error(codes.PermissionDenied, "non-retryable gRPC error"),
 			shouldRetry: false,
 		},
+		{
+			desc: "wrapped ErrClosed text",
+			// TODO: check directly against wrapped net.ErrClosed (go 1.16+)
+			inputErr:    &net.OpError{Op: "write", Err: errors.New("use of closed network connection")},
+			shouldRetry: true,
+		},
 	} {
 		t.Run(test.desc, func(s *testing.T) {
 			got := shouldRetry(test.inputErr)