Merge branch 'master' into metrics-v3-cluster-webhook

cmd/admin-handlers.go

@@ -50,6 +50,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/madmin-go/v3/estream"
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/handlers"
@@ -1627,6 +1628,47 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
+func isAllowedRWAccess(r *http.Request, cred auth.Credentials, bucketName string) (rd, wr bool) {
+	owner := cred.AccessKey == globalActiveCred.AccessKey
+
+	// Set prefix value for "s3:prefix" policy conditionals.
+	r.Header.Set("prefix", "")
+
+	// Set delimiter value for "s3:delimiter" policy conditionals.
+	r.Header.Set("delimiter", SlashSeparator)
+
+	isAllowedAccess := func(bucketName string) (rd, wr bool) {
+		if globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.GetObjectAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			ObjectName:      "",
+			Claims:          cred.Claims,
+		}) {
+			rd = true
+		}
+
+		if globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.PutObjectAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			ObjectName:      "",
+			Claims:          cred.Claims,
+		}) {
+			wr = true
+		}
+
+		return rd, wr
+	}
+	return isAllowedAccess(bucketName)
+}
+
 // ObjectSpeedTestHandler - reports maximum speed of a cluster by performing PUT and
 // GET operations on the server, supports auto tuning by default by automatically
 // increasing concurrency and stopping when we have reached the limits on the
@@ -1635,11 +1677,24 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 	ctx, cancel := context.WithCancel(r.Context())
 	defer cancel()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
+	objectAPI, creds := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
+	if !globalAPIConfig.permitRootAccess() {
+		rd, wr := isAllowedRWAccess(r, creds, globalObjectPerfBucket)
+		if !rd || !wr {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{
+				Code: "XMinioSpeedtestInsufficientPermissions",
+				Message: fmt.Sprintf("%s does not have read and write access to '%s' bucket", creds.AccessKey,
+					globalObjectPerfBucket),
+				StatusCode: http.StatusForbidden,
+			}), r.URL)
+			return
+		}
+	}
+
 	sizeStr := r.Form.Get(peerRESTSize)
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
@@ -1648,6 +1703,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 	autotune := r.Form.Get("autotune") == "true"
 	noClear := r.Form.Get("noclear") == "true"
 	enableSha256 := r.Form.Get("enableSha256") == "true"
+	enableMultipart := r.Form.Get("enableMultipart") == "true"
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -1721,6 +1777,8 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 		storageClass:     storageClass,
 		bucketName:       customBucket,
 		enableSha256:     enableSha256,
+		enableMultipart:  enableMultipart,
+		creds:            creds,
 	})
 	var prevResult madmin.SpeedTestResult
 	for {

cmd/peer-rest-client.go

@@ -715,6 +715,7 @@ func (client *peerRESTClient) SpeedTest(ctx context.Context, opts speedTestOpts)
 	values.Set(peerRESTStorageClass, opts.storageClass)
 	values.Set(peerRESTBucket, opts.bucketName)
 	values.Set(peerRESTEnableSha256, strconv.FormatBool(opts.enableSha256))
+	values.Set(peerRESTEnableMultipart, strconv.FormatBool(opts.enableMultipart))
 
 	respBody, err := client.callWithContext(context.Background(), peerRESTMethodSpeedTest, values, nil, -1)
 	if err != nil {

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v38" // Convert RPC calls
+	peerRESTVersion       = "v39" // add more flags to speedtest API
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -38,31 +38,33 @@ const (
 )
 
 const (
-	peerRESTBucket         = "bucket"
-	peerRESTBuckets        = "buckets"
-	peerRESTUser           = "user"
-	peerRESTGroup          = "group"
-	peerRESTUserTemp       = "user-temp"
-	peerRESTPolicy         = "policy"
-	peerRESTUserOrGroup    = "user-or-group"
-	peerRESTUserType       = "user-type"
-	peerRESTIsGroup        = "is-group"
-	peerRESTSignal         = "signal"
-	peerRESTSubSys         = "sub-sys"
-	peerRESTProfiler       = "profiler"
-	peerRESTSize           = "size"
-	peerRESTConcurrent     = "concurrent"
-	peerRESTDuration       = "duration"
-	peerRESTStorageClass   = "storage-class"
-	peerRESTEnableSha256   = "enableSha256"
-	peerRESTMetricsTypes   = "types"
-	peerRESTDisk           = "disk"
-	peerRESTHost           = "host"
-	peerRESTJobID          = "job-id"
-	peerRESTDepID          = "depID"
-	peerRESTStartRebalance = "start-rebalance"
-	peerRESTMetrics        = "metrics"
-	peerRESTDryRun         = "dry-run"
+	peerRESTBucket          = "bucket"
+	peerRESTBuckets         = "buckets"
+	peerRESTUser            = "user"
+	peerRESTGroup           = "group"
+	peerRESTUserTemp        = "user-temp"
+	peerRESTPolicy          = "policy"
+	peerRESTUserOrGroup     = "user-or-group"
+	peerRESTUserType        = "user-type"
+	peerRESTIsGroup         = "is-group"
+	peerRESTSignal          = "signal"
+	peerRESTSubSys          = "sub-sys"
+	peerRESTProfiler        = "profiler"
+	peerRESTSize            = "size"
+	peerRESTConcurrent      = "concurrent"
+	peerRESTDuration        = "duration"
+	peerRESTStorageClass    = "storage-class"
+	peerRESTEnableSha256    = "enableSha256"
+	peerRESTEnableMultipart = "enableMultipart"
+	peerRESTAccessKey       = "access-key"
+	peerRESTMetricsTypes    = "types"
+	peerRESTDisk            = "disk"
+	peerRESTHost            = "host"
+	peerRESTJobID           = "job-id"
+	peerRESTDepID           = "depID"
+	peerRESTStartRebalance  = "start-rebalance"
+	peerRESTMetrics         = "metrics"
+	peerRESTDryRun          = "dry-run"
 
 	peerRESTURL         = "url"
 	peerRESTSha256Sum   = "sha256sum"

cmd/peer-rest-server.go

@@ -1059,6 +1059,13 @@ func (s *peerRESTServer) SpeedTestHandler(w http.ResponseWriter, r *http.Request
 	storageClass := r.Form.Get(peerRESTStorageClass)
 	bucketName := r.Form.Get(peerRESTBucket)
 	enableSha256 := r.Form.Get(peerRESTEnableSha256) == "true"
+	enableMultipart := r.Form.Get(peerRESTEnableMultipart) == "true"
+
+	u, ok := globalIAMSys.GetUser(r.Context(), r.Form.Get(peerRESTAccessKey))
+	if !ok {
+		s.writeErrorResponse(w, errAuthentication)
+		return
+	}
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -1078,12 +1085,14 @@ func (s *peerRESTServer) SpeedTestHandler(w http.ResponseWriter, r *http.Request
 	done := keepHTTPResponseAlive(w)
 
 	result, err := selfSpeedTest(r.Context(), speedTestOpts{
-		objectSize:   size,
-		concurrency:  concurrent,
-		duration:     duration,
-		storageClass: storageClass,
-		bucketName:   bucketName,
-		enableSha256: enableSha256,
+		objectSize:      size,
+		concurrency:     concurrent,
+		duration:        duration,
+		storageClass:    storageClass,
+		bucketName:      bucketName,
+		enableSha256:    enableSha256,
+		enableMultipart: enableMultipart,
+		creds:           u.Credentials,
 	})
 	if err != nil {
 		result.Error = err.Error()

cmd/perf-tests.go

@@ -33,6 +33,7 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/pkg/v2/randreader"
@@ -72,7 +73,7 @@ func (f *firstByteRecorder) Read(p []byte) (n int, err error) {
 }
 
 // Runs the speedtest on local MinIO process.
-func selfSpeedTest(ctx context.Context, opts speedTestOpts) (SpeedTestResult, error) {
+func selfSpeedTest(ctx context.Context, opts speedTestOpts) (res SpeedTestResult, err error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return SpeedTestResult{}, errServerNotInitialized
@@ -96,7 +97,24 @@ func selfSpeedTest(ctx context.Context, opts speedTestOpts) (SpeedTestResult, er
 	popts := minio.PutObjectOptions{
 		UserMetadata:         userMetadata,
 		DisableContentSha256: !opts.enableSha256,
-		DisableMultipart:     true,
+		DisableMultipart:     !opts.enableMultipart,
+	}
+
+	clnt := globalMinioClient
+	if !globalAPIConfig.permitRootAccess() {
+		region := globalSite.Region
+		if region == "" {
+			region = "us-east-1"
+		}
+		clnt, err = minio.New(globalLocalNodeName, &minio.Options{
+			Creds:     credentials.NewStaticV4(opts.creds.AccessKey, opts.creds.SecretKey, opts.creds.SessionToken),
+			Secure:    globalIsTLS,
+			Transport: globalRemoteTargetTransport,
+			Region:    region,
+		})
+		if err != nil {
+			return res, err
+		}
 	}
 
 	var mu sync.Mutex
@@ -109,7 +127,7 @@ func selfSpeedTest(ctx context.Context, opts speedTestOpts) (SpeedTestResult, er
 				t := time.Now()
 				reader := newRandomReader(opts.objectSize)
 				tmpObjName := pathJoin(objNamePrefix, fmt.Sprintf("%d/%d", i, objCountPerThread[i]))
-				info, err := globalMinioClient.PutObject(uploadsCtx, opts.bucketName, tmpObjName, reader, int64(opts.objectSize), popts)
+				info, err := clnt.PutObject(uploadsCtx, opts.bucketName, tmpObjName, reader, int64(opts.objectSize), popts)
 				if err != nil {
 					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
 						errOnce.Do(func() {
@@ -150,7 +168,7 @@ func selfSpeedTest(ctx context.Context, opts speedTestOpts) (SpeedTestResult, er
 	var downloadTTFB madmin.TimeDurations
 	wg.Add(opts.concurrency)
 
-	c := minio.Core{Client: globalMinioClient}
+	c := minio.Core{Client: clnt}
 	for i := 0; i < opts.concurrency; i++ {
 		go func(i int) {
 			defer wg.Done()

cmd/post-policy_test.go

@@ -210,18 +210,23 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
 	// Test cases for signature-V2.
 	testCasesV2 := []struct {
 		expectedStatus int
-		accessKey      string
 		secretKey      string
+		formData       map[string]string
 	}{
-		{http.StatusForbidden, "invalidaccesskey", credentials.SecretKey},
-		{http.StatusForbidden, credentials.AccessKey, "invalidsecretkey"},
-		{http.StatusNoContent, credentials.AccessKey, credentials.SecretKey},
+		{http.StatusForbidden, credentials.SecretKey, map[string]string{"AWSAccessKeyId": "invalidaccesskey"}},
+		{http.StatusForbidden, "invalidsecretkey", map[string]string{"AWSAccessKeyId": credentials.AccessKey}},
+		{http.StatusNoContent, credentials.SecretKey, map[string]string{"AWSAccessKeyId": credentials.AccessKey}},
+		{http.StatusForbidden, credentials.SecretKey, map[string]string{"Awsaccesskeyid": "invalidaccesskey"}},
+		{http.StatusForbidden, "invalidsecretkey", map[string]string{"Awsaccesskeyid": credentials.AccessKey}},
+		{http.StatusNoContent, credentials.SecretKey, map[string]string{"Awsaccesskeyid": credentials.AccessKey}},
+		// Forbidden with key not in policy.conditions for signed requests V2.
+		{http.StatusForbidden, credentials.SecretKey, map[string]string{"Awsaccesskeyid": credentials.AccessKey, "AnotherKey": "AnotherContent"}},
 	}
 
 	for i, test := range testCasesV2 {
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
-		req, perr := newPostRequestV2("", bucketName, "testobject", test.accessKey, test.secretKey)
+		req, perr := newPostRequestV2("", bucketName, "testobject", test.secretKey, test.formData)
 		if perr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: <ERROR> %v", i+1, instanceType, perr)
 		}
@@ -593,7 +598,7 @@ func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, l
 	return signature
 }
 
-func newPostRequestV2(endPoint, bucketName, objectName string, accessKey, secretKey string) (*http.Request, error) {
+func newPostRequestV2(endPoint, bucketName, objectName string, secretKey string, formInputData map[string]string) (*http.Request, error) {
 	// Expire the request five minutes from now.
 	expirationTime := UTCNow().Add(time.Minute * 5)
 	// Create a new post policy.
@@ -605,12 +610,14 @@ func newPostRequestV2(endPoint, bucketName, objectName string, accessKey, secret
 	signature := calculateSignatureV2(encodedPolicy, secretKey)
 
 	formData := map[string]string{
-		"AWSAccessKeyId":              accessKey,
-		"bucket":                      bucketName,
-		"key":                         objectName + "/${filename}",
-		"policy":                      encodedPolicy,
-		"signature":                   signature,
-		"X-Amz-Ignore-AWSAccessKeyId": "",
+		"bucket":    bucketName,
+		"key":       objectName + "/${filename}",
+		"policy":    encodedPolicy,
+		"signature": signature,
+	}
+
+	for key, value := range formInputData {
+		formData[key] = value
 	}
 
 	// Create the multipart form.

cmd/postpolicyform.go

@@ -347,10 +347,16 @@ func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) erro
 		}
 		delete(checkHeader, formCanonicalName)
 	}
-	// For SignV2 - Signature field will be ignored
-	// Policy is generated from Signature with other fields, so it should be ignored
+	// For SignV2 - Signature/AWSAccessKeyId field will be ignored.
 	if _, ok := formValues[xhttp.AmzSignatureV2]; ok {
 		delete(checkHeader, xhttp.AmzSignatureV2)
+		for k := range checkHeader {
+			// case-insensitivity for AWSAccessKeyId
+			if strings.EqualFold(k, xhttp.AmzAccessKeyID) {
+				delete(checkHeader, k)
+				break
+			}
+		}
 	}
 
 	if len(checkHeader) != 0 {