Merge branch 'master' into metrics-v3-cluster-webhook

cmd/admin-handlers.go

@@ -1576,7 +1576,8 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 
 // NetperfHandler - perform mesh style network throughput test
 func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -1597,6 +1598,15 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 	ctx = lkctx.Context()
 	defer nsLock.Unlock(lkctx)
 
+	// Freeze all incoming S3 API calls before running speedtest.
+	globalNotificationSys.ServiceFreeze(ctx, true)
+
+	// Unfreeze as soon as request context is canceled or when the function returns.
+	go func() {
+		<-ctx.Done()
+		globalNotificationSys.ServiceFreeze(ctx, false)
+	}()
+
 	durationStr := r.Form.Get(peerRESTDuration)
 	duration, err := time.ParseDuration(durationStr)
 	if err != nil {
@@ -1622,7 +1632,8 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 // increasing concurrency and stopping when we have reached the limits on the
 // system.
 func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -1692,8 +1703,11 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
 
-	// unfreeze all incoming S3 API calls after speedtest.
-	defer globalNotificationSys.ServiceFreeze(ctx, false)
+	// Unfreeze as soon as request context is canceled or when the function returns.
+	go func() {
+		<-ctx.Done()
+		globalNotificationSys.ServiceFreeze(ctx, false)
+	}()
 
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
 	defer keepAliveTicker.Stop()
@@ -1793,7 +1807,8 @@ func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo,
 
 // DriveSpeedtestHandler - reports throughput of drives available in the cluster
 func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -1803,8 +1818,11 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
 
-	// unfreeze all incoming S3 API calls after speedtest.
-	defer globalNotificationSys.ServiceFreeze(ctx, false)
+	// Unfreeze as soon as request context is canceled or when the function returns.
+	go func() {
+		<-ctx.Done()
+		globalNotificationSys.ServiceFreeze(ctx, false)
+	}()
 
 	serial := r.Form.Get("serial") == "true"
 	blockSizeStr := r.Form.Get("blocksize")

cmd/metrics-v3-types.go

@@ -232,10 +232,13 @@ func (m *MetricValues) Set(name MetricName, value float64, labels ...string) {
 	if !ok {
 		v = make([]metricValue, 0, 1)
 	}
-	m.values[name] = append(v, metricValue{
-		Labels: labelMap,
-		Value:  value,
-	})
+	// If valid non zero value set the metrics
+	if value > 0 {
+		m.values[name] = append(v, metricValue{
+			Labels: labelMap,
+			Value:  value,
+		})
+	}
 }
 
 // SetHistogram - sets values for the given MetricName using the provided
@@ -274,7 +277,10 @@ func (m *MetricValues) SetHistogram(name MetricName, hist *prometheus.HistogramV
 			}
 		}
 		labels = append(labels, extraLabels...)
-		m.Set(name, metric.Value, labels...)
+		// If valid non zero value set the metrics
+		if metric.Value > 0 {
+			m.Set(name, metric.Value, labels...)
+		}
 	}
 }
 

cmd/sftp-server.go

@@ -40,6 +40,85 @@ func (s *sftpLogger) Info(tag xsftp.LogType, msg string) {
 	logger.Info(msg)
 }
 
+const (
+	kexAlgoDH1SHA1                = "diffie-hellman-group1-sha1"
+	kexAlgoDH14SHA1               = "diffie-hellman-group14-sha1"
+	kexAlgoDH14SHA256             = "diffie-hellman-group14-sha256"
+	kexAlgoDH16SHA512             = "diffie-hellman-group16-sha512"
+	kexAlgoECDH256                = "ecdh-sha2-nistp256"
+	kexAlgoECDH384                = "ecdh-sha2-nistp384"
+	kexAlgoECDH521                = "ecdh-sha2-nistp521"
+	kexAlgoCurve25519SHA256LibSSH = "curve25519-sha256@libssh.org"
+	kexAlgoCurve25519SHA256       = "curve25519-sha256"
+
+	chacha20Poly1305ID = "chacha20-poly1305@openssh.com"
+	gcm256CipherID     = "aes256-gcm@openssh.com"
+	aes128cbcID        = "aes128-cbc"
+	tripledescbcID     = "3des-cbc"
+)
+
+// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=46
+// preferredKexAlgos specifies the default preference for key-exchange
+// algorithms in preference order. The diffie-hellman-group16-sha512 algorithm
+// is disabled by default because it is a bit slower than the others.
+var preferredKexAlgos = []string{
+	kexAlgoCurve25519SHA256, kexAlgoCurve25519SHA256LibSSH,
+	kexAlgoECDH256, kexAlgoECDH384, kexAlgoECDH521,
+	kexAlgoDH14SHA256, kexAlgoDH14SHA1,
+}
+
+// supportedKexAlgos specifies the supported key-exchange algorithms in
+// preference order.
+// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=44
+var supportedKexAlgos = []string{
+	kexAlgoCurve25519SHA256, kexAlgoCurve25519SHA256LibSSH,
+	// P384 and P521 are not constant-time yet, but since we don't
+	// reuse ephemeral keys, using them for ECDH should be OK.
+	kexAlgoECDH256, kexAlgoECDH384, kexAlgoECDH521,
+	kexAlgoDH14SHA256, kexAlgoDH16SHA512, kexAlgoDH14SHA1,
+	kexAlgoDH1SHA1,
+}
+
+// supportedPubKeyAuthAlgos specifies the supported client public key
+// authentication algorithms. Note that this doesn't include certificate types
+// since those use the underlying algorithm. This list is sent to the client if
+// it supports the server-sig-algs extension. Order is irrelevant.
+// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=142
+var supportedPubKeyAuthAlgos = []string{
+	ssh.KeyAlgoED25519,
+	ssh.KeyAlgoSKED25519, ssh.KeyAlgoSKECDSA256,
+	ssh.KeyAlgoECDSA256, ssh.KeyAlgoECDSA384, ssh.KeyAlgoECDSA521,
+	ssh.KeyAlgoRSASHA256, ssh.KeyAlgoRSASHA512, ssh.KeyAlgoRSA,
+	ssh.KeyAlgoDSA,
+}
+
+// supportedCiphers lists ciphers we support but might not recommend.
+// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=28
+var supportedCiphers = []string{
+	"aes128-ctr", "aes192-ctr", "aes256-ctr",
+	"aes128-gcm@openssh.com", gcm256CipherID,
+	chacha20Poly1305ID,
+	"arcfour256", "arcfour128", "arcfour",
+	aes128cbcID,
+	tripledescbcID,
+}
+
+// preferredCiphers specifies the default preference for ciphers.
+// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=37
+var preferredCiphers = []string{
+	"aes128-gcm@openssh.com", gcm256CipherID,
+	chacha20Poly1305ID,
+	"aes128-ctr", "aes192-ctr", "aes256-ctr",
+}
+
+// supportedMACs specifies a default set of MAC algorithms in preference order.
+// This is based on RFC 4253, section 6.4, but with hmac-md5 variants removed
+// because they have reached the end of their useful life.
+// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=85
+var supportedMACs = []string{
+	"hmac-sha2-256-etm@openssh.com", "hmac-sha2-512-etm@openssh.com", "hmac-sha2-256", "hmac-sha2-512", "hmac-sha1", "hmac-sha1-96",
+}
+
 func (s *sftpLogger) Error(tag xsftp.LogType, err error) {
 	switch tag {
 	case xsftp.AcceptNetworkError:
@@ -53,13 +132,41 @@ func (s *sftpLogger) Error(tag xsftp.LogType, err error) {
 	}
 }
 
+func filterAlgos(arg string, want []string, allowed []string) []string {
+	var filteredAlgos []string
+	found := false
+	for _, algo := range want {
+		if len(algo) == 0 {
+			continue
+		}
+		for _, allowedAlgo := range allowed {
+			algo := strings.ToLower(strings.TrimSpace(algo))
+			if algo == allowedAlgo {
+				filteredAlgos = append(filteredAlgos, algo)
+				found = true
+				break
+			}
+		}
+		if !found {
+			logger.Fatal(fmt.Errorf("unknown algorithm %q passed to --sftp=%s\nValid algorithms: %v", algo, arg, strings.Join(allowed, ", ")), "unable to start SFTP server")
+		}
+	}
+	if len(filteredAlgos) == 0 {
+		logger.Fatal(fmt.Errorf("no valid algorithms passed to --sftp=%s\nValid algorithms: %v", arg, strings.Join(allowed, ", ")), "unable to start SFTP server")
+	}
+	return filteredAlgos
+}
+
 func startSFTPServer(args []string) {
 	var (
 		port          int
 		publicIP      string
 		sshPrivateKey string
 	)
-
+	allowPubKeys := supportedPubKeyAuthAlgos
+	allowKexAlgos := preferredKexAlgos
+	allowCiphers := preferredCiphers
+	allowMACs := supportedMACs
 	var err error
 	for _, arg := range args {
 		tokens := strings.SplitN(arg, "=", 2)
@@ -82,6 +189,14 @@ func startSFTPServer(args []string) {
 			publicIP = host
 		case "ssh-private-key":
 			sshPrivateKey = tokens[1]
+		case "pub-key-algos":
+			allowPubKeys = filterAlgos(arg, strings.Split(tokens[1], ","), supportedPubKeyAuthAlgos)
+		case "kex-algos":
+			allowKexAlgos = filterAlgos(arg, strings.Split(tokens[1], ","), supportedKexAlgos)
+		case "cipher-algos":
+			allowCiphers = filterAlgos(arg, strings.Split(tokens[1], ","), supportedCiphers)
+		case "mac-algos":
+			allowCiphers = filterAlgos(arg, strings.Split(tokens[1], ","), supportedMACs)
 		}
 	}
 
@@ -106,6 +221,12 @@ func startSFTPServer(args []string) {
 	// An SSH server is represented by a ServerConfig, which holds
 	// certificate details and handles authentication of ServerConns.
 	sshConfig := &ssh.ServerConfig{
+		Config: ssh.Config{
+			KeyExchanges: allowKexAlgos,
+			Ciphers:      allowCiphers,
+			MACs:         allowMACs,
+		},
+		PublicKeyAuthAlgorithms: allowPubKeys,
 		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
 			if globalIAMSys.LDAPConfig.Enabled() {
 				sa, _, err := globalIAMSys.getServiceAccount(context.Background(), c.User())

docs/ftp/README.md

@@ -167,3 +167,78 @@ Unlike SFTP server, FTP server is insecure by default. To operate under TLS mode
 > certs from the server certificate chain, this is mainly to add simplicity of setup. However if you wish to terminate
 > TLS certificates via a different domain for your FTP servers you may choose the above command line options.
 
+
+### Custom Algorithms (SFTP)
+
+Custom algorithms can be specified via command line parameters.
+Algorithms are comma separated. 
+Note that valid values does not in all cases represent default values.
+
+`--sftp=pub-key-algos=...` specifies the supported client public key
+authentication algorithms. Note that this doesn't include certificate types
+since those use the underlying algorithm. This list is sent to the client if
+it supports the server-sig-algs extension. Order is irrelevant.
+
+Valid values
+```
+ssh-ed25519
+sk-ssh-ed25519@openssh.com
+sk-ecdsa-sha2-nistp256@openssh.com
+ecdsa-sha2-nistp256
+ecdsa-sha2-nistp384
+ecdsa-sha2-nistp521
+rsa-sha2-256
+rsa-sha2-512
+ssh-rsa
+ssh-dss
+```
+
+`--sftp=kex-algos=...` specifies the supported key-exchange algorithms in preference order.
+
+Valid values: 
+
+```
+curve25519-sha256
+curve25519-sha256@libssh.org
+ecdh-sha2-nistp256
+ecdh-sha2-nistp384
+ecdh-sha2-nistp521
+diffie-hellman-group14-sha256
+diffie-hellman-group16-sha512
+diffie-hellman-group14-sha1
+diffie-hellman-group1-sha1
+```
+
+`--sftp=cipher-algos=...` specifies the allowed cipher algorithms. 
+If unspecified then a sensible default is used.
+
+Valid values: 
+```
+aes128-ctr
+aes192-ctr
+aes256-ctr
+aes128-gcm@openssh.com
+aes256-gcm@openssh.com
+chacha20-poly1305@openssh.com
+arcfour256
+arcfour128
+arcfour
+aes128-cbc
+3des-cbc
+```
+
+`--sftp=mac-algos=...` specifies a default set of MAC algorithms in preference order.
+This is based on RFC 4253, section 6.4, but with hmac-md5 variants removed because they have 
+reached the end of their useful life.
+
+Valid values: 
+
+```
+hmac-sha2-256-etm@openssh.com
+hmac-sha2-512-etm@openssh.com
+hmac-sha2-256
+hmac-sha2-512
+hmac-sha1
+hmac-sha1-96
+```
+

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2024-04-18T19-09-19Z
+  image: quay.io/minio/minio:RELEASE.2024-04-28T17-53-50Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

internal/grid/grid_test.go

@@ -470,7 +470,7 @@ func testStreamCancel(t *testing.T, local, remote *Manager) {
 			Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr {
 				<-ctx.Done()
 				serverCanceled <- struct{}{}
-				t.Log(GetCaller(ctx).Name, "Server Context canceled")
+				fmt.Println(GetCaller(ctx).Name, "Server Context canceled")
 				return nil
 			},
 			OutCapacity: 1,
@@ -480,7 +480,7 @@ func testStreamCancel(t *testing.T, local, remote *Manager) {
 			Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr {
 				<-ctx.Done()
 				serverCanceled <- struct{}{}
-				t.Log(GetCaller(ctx).Name, "Server Context canceled")
+				fmt.Println(GetCaller(ctx).Name, "Server Context canceled")
 				return nil
 			},
 			OutCapacity: 1,