storage/s3: Load endpoint_url from AWS config file if profile has b…

…een specified

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/lanrat/extsort v1.0.0
 	github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae
 	github.com/urfave/cli/v2 v2.11.2
+	gopkg.in/ini.v1 v1.67.0
 	gotest.tools/v3 v3.0.2
 )
 

go.sum

@@ -109,6 +109,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

storage/s3.go

@@ -22,13 +22,15 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface"
+	"gopkg.in/ini.v1"
 
 	"github.com/peak/s5cmd/v2/log"
 	"github.com/peak/s5cmd/v2/storage/url"
@@ -49,6 +51,9 @@ const (
 
 	// the key of the object metadata which is used to handle retry decision on NoSuchUpload error
 	metadataKeyRetryID = "s5cmd-upload-retry-id"
+
+	// The endpoint_url key in the profile configuration (AWS config file)
+	endpointUrlKey = "endpoint_url"
 )
 
 // Re-used AWS sessions dramatically improve performance.
@@ -1056,6 +1061,24 @@ func (sc *SessionCache) newSession(ctx context.Context, opts Options) (*session.
 		)
 	}
 
+	if opts.Profile != "" && opts.Endpoint == "" {
+		awsConfigFilePath, awsConfigFileEnvExists := os.LookupEnv("AWS_CONFIG_FILE")
+		if !awsConfigFileEnvExists {
+			awsConfigFilePath = defaults.SharedConfigFilename()
+		}
+
+		if _, awsConfigFileErr := os.Stat(awsConfigFilePath); awsConfigFileErr == nil {
+			awsProfileConfig, err := ini.Load(awsConfigFilePath)
+			if err != nil {
+				return nil, err
+			}
+
+			if awsProfileConfig.HasSection(opts.Profile) && awsProfileConfig.Section(opts.Profile).HasKey(endpointUrlKey) {
+				opts.Endpoint = awsProfileConfig.Section(opts.Profile).Key(endpointUrlKey).String()
+			}
+		}
+	}
+
 	endpointURL, err := parseEndpoint(opts.Endpoint)
 	if err != nil {
 		return nil, err

storage/s3_test.go

@@ -215,6 +215,65 @@ aws_secret_access_key = p2_profile_access_key`
 	}
 }
 
+func TestNewSessionWithProfileFromFileAndEndpointInAwsProfile(t *testing.T) {
+	// create a temporary config file
+	configFile, err := os.CreateTemp("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(configFile.Name())
+
+	profiles := `[p1]
+endpoint_url = https://some-custom-s3-endpoint/`
+
+	_, err = configFile.Write([]byte(profiles))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Set the AWS_CONFIG_FILE location to temporary created config file
+	t.Setenv("AWS_CONFIG_FILE", configFile.Name())
+
+	testcases := []struct {
+		name               string
+		fileName           string
+		profileName        string
+		expAccessKeyID     string
+		expSecretAccessKey string
+	}{
+		{
+			name:        "use a non-default profile with endpoint configured",
+			profileName: "p1",
+		},
+		{
+			name:        "use a non-default profile without endpoint configured",
+			profileName: "p2",
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			globalSessionCache.clear()
+			sess, err := globalSessionCache.newSession(context.Background(), Options{
+				Profile:        tc.profileName,
+				CredentialFile: tc.fileName,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if tc.profileName == "p1" {
+				if *sess.Config.Endpoint != "https://some-custom-s3-endpoint/" {
+					t.Errorf("Expected endpoint to match endpoint configured in AWS config file")
+				}
+			} else if tc.profileName == "p2" {
+				if *sess.Config.Endpoint != "" {
+					t.Errorf("Expected endpoint to be empty (Wrong: '%s')", *sess.Config.Endpoint)
+				}
+			}
+		})
+	}
+}
+
 func TestS3ListURL(t *testing.T) {
 	url, err := url.New("s3://bucket/key")
 	if err != nil {