Reduce the default TCP timeout to fix #3700. (#3794) (#3814)

* Reduce the default TCP timeout to fix #3700.

* Fix lint comments.

* Don't modify http.DefaultTransport but make a copy.

pkg/activator/util/transports.go

@@ -14,9 +14,12 @@ limitations under the License.
 package util
 
 import (
+	"net"
 	"net/http"
+	"time"
 
 	h2cutil "github.com/knative/serving/pkg/http/h2c"
+	"github.com/knative/serving/pkg/network"
 )
 
 // RoundTripperFunc implementation roundtrips a request.
@@ -27,8 +30,8 @@ func (rt RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
 	return rt(r)
 }
 
-// NewHTTPTransport will use the appropriate transport for the request's HTTP protocol version
-func NewHTTPTransport(v1 http.RoundTripper, v2 http.RoundTripper) http.RoundTripper {
+// NewAutoTransport will use the appropriate transport for the request's HTTP protocol version
+func NewAutoTransport(v1 http.RoundTripper, v2 http.RoundTripper) http.RoundTripper {
 	return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
 		t := v1
 		if r.ProtoMajor == 2 {
@@ -39,5 +42,15 @@ func NewHTTPTransport(v1 http.RoundTripper, v2 http.RoundTripper) http.RoundTrip
 	})
 }
 
+func newHTTPTransport(connTimeout time.Duration) http.RoundTripper {
+	transport := *http.DefaultTransport.(*http.Transport)
+	transport.DialContext = (&net.Dialer{
+		Timeout:   connTimeout,
+		KeepAlive: 30 * time.Second,
+		DualStack: true,
+	}).DialContext
+	return &transport
+}
+
 // AutoTransport uses h2c for HTTP2 requests and falls back to `http.DefaultTransport` for all others
-var AutoTransport = NewHTTPTransport(http.DefaultTransport, h2cutil.DefaultTransport)
+var AutoTransport = NewAutoTransport(newHTTPTransport(network.DefaultConnTimeout), h2cutil.DefaultTransport)

pkg/activator/util/transports_test.go

@@ -49,7 +49,7 @@ func TestHTTPRoundTripper(t *testing.T) {
 		})
 	}
 
-	rt := NewHTTPTransport(frt("v1"), frt("v2"))
+	rt := NewAutoTransport(frt("v1"), frt("v2"))
 
 	examples := []struct {
 		label      string

pkg/http/h2c/h2c.go

@@ -19,6 +19,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/knative/serving/pkg/network"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
 )
@@ -46,11 +47,10 @@ var DefaultTransport http.RoundTripper = &http2.Transport{
 	AllowHTTP: true,
 	DialTLS: func(netw, addr string, cfg *tls.Config) (net.Conn, error) {
 		d := &net.Dialer{
-			Timeout:   30 * time.Second,
+			Timeout:   network.DefaultConnTimeout,
 			KeepAlive: 30 * time.Second,
 			DualStack: true,
 		}
-
 		return d.Dial(netw, addr)
 	},
 }

pkg/network/network.go

@@ -19,6 +19,7 @@ package network
 import (
 	"net"
 	"strings"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 )
@@ -55,6 +56,19 @@ const (
 	// Knative service's DNS name.
 	DomainTemplateKey = "domainTemplate"
 
+	// DefaultConnTimeout specifies a short default connection timeout
+	// to avoid hitting the issue fixed in
+	// https://github.com/kubernetes/kubernetes/pull/72534 but only
+	// avalailable after Kubernetes 1.14.
+	//
+	// Our connections are usually between pods in the same cluster
+	// like activator <-> queue-proxy, or even between containers
+	// within the same pod queue-proxy <-> user-container, so a
+	// smaller connect timeout would be justifiable.
+	//
+	// We should consider exposing this as a configuration.
+	DefaultConnTimeout = 200 * time.Millisecond
+
 	// DefaultDomainTemplate is the default golang template to use when
 	// constructing the Knative Route's Domain(host)
 	DefaultDomainTemplate = "{{.Name}}.{{.Namespace}}.{{.Domain}}"