Fix region handling

README.md

@@ -35,17 +35,16 @@ Configuration
 To connect to Amazon S3, WAL-G requires that these variables be set:
 
 * `WALE_S3_PREFIX` (eg. `s3://bucket/path/to/folder`)
-* `AWS_REGION`(eg. `us-west-2`)
-* `AWS_ACCESS_KEY_ID`
-* `AWS_SECRET_ACCESS_KEY`
+
+WAL-G determines AWS credentials [like other AWS tools](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#config-settings-and-precedence). You can set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (optionally with `AWS_SECURITY_TOKEN`), or `~/.aws/credentials` (optionally with `AWS_PROFILE`), or you can set nothing to automatically fetch credentials from the EC2 metadata service.
 
 WAL-G uses [the usual PostgreSQL environment variables](https://www.postgresql.org/docs/current/static/libpq-envars.html) to configure its connection, especially including `PGHOST`, `PGPORT`, `PGUSER`, and `PGPASSWORD`/`PGPASSFILE`/`~/.pgpass`.
 
 **Optional**
 
-Required if using AWS STS:
+WAL-G can automatically determine the S3 bucket's region using `s3:GetBucketLocation`, but if you wish to avoid this API call or forbid it from the applicable IAM policy, specify:
 
-* `AWS_SESSION_TOKEN`
+* `AWS_REGION`(eg. `us-west-2`)
 
 Concurrency values can be configured using:
 

cmd/wal-g/main.go

@@ -3,14 +3,15 @@ package main
 import (
 	"flag"
 	"fmt"
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/wal-g/wal-g"
 	"log"
 	"os"
 	"path/filepath"
 	"regexp"
 	"runtime/pprof"
 	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/wal-g/wal-g"
 )
 
 var profile bool

upload.go

@@ -3,6 +3,14 @@ package walg
 import (
 	"archive/tar"
 	"fmt"
+	"io"
+	"log"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -13,13 +21,6 @@ import (
 	"github.com/jackc/pgx"
 	"github.com/pierrec/lz4"
 	"github.com/pkg/errors"
-	"io"
-	"log"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
 )
 
 // MAXRETRIES is the maximum number of retries for upload.
@@ -28,23 +29,29 @@ var MAXRETRIES = 7
 // MAXBACKOFF is the maxmimum backoff time in seconds for upload.
 var MAXBACKOFF = float64(32)
 
-// Checks that the following environment variables are set:
-// WALE_S3_PREFIX
-// AWS_REGION
-func checkVar(n map[string]string) error {
-	u := &UnsetEnvVarError{
-		names: make([]string, 0, 2),
+// Given an S3 bucket name, attempt to determine its region
+func findS3BucketRegion(bucket string, config *aws.Config) (string, error) {
+	input := s3.GetBucketLocationInput{
+		Bucket: aws.String(bucket),
 	}
-	for i, val := range n {
-		if val == "" {
-			u.names = append(u.names, i)
-		}
+
+	sess, err := session.NewSession(config.WithRegion("us-east-1"))
+	if err != nil {
+		return "", err
 	}
-	if len(u.names) != 0 {
-		return u
+
+	output, err := s3.New(sess).GetBucketLocation(&input)
+	if err != nil {
+		return "", err
 	}
 
-	return nil
+	if output.LocationConstraint == nil {
+		// buckets in "US Standard", a.k.a. us-east-1, are returned as a nil region
+		return "us-east-1", nil
+	} else {
+		// all other regions are strings
+		return *output.LocationConstraint, nil
+	}
 }
 
 // Configure connects to S3 and creates an uploader. It makes sure
@@ -53,39 +60,42 @@ func checkVar(n map[string]string) error {
 //
 // Requires these environment variables to be set:
 // WALE_S3_PREFIX
-// AWS_REGION
-// AWS_ACCESS_KEY_ID
-// AWS_SECRET_ACCESS_KEY
 //
 // Able to configure the upload part size in the S3 uploader.
 func Configure() (*TarUploader, *Prefix, error) {
-	chk := make(map[string]string)
-	chk["WALE_S3_PREFIX"] = os.Getenv("WALE_S3_PREFIX")
-	chk["AWS_REGION"] = os.Getenv("AWS_REGION")
-	preErr := checkVar(chk)
-	if preErr != nil {
-		return nil, nil, preErr
+	wale_s3_prefix := os.Getenv("WALE_S3_PREFIX")
+	if wale_s3_prefix == "" {
+		return nil, nil, &UnsetEnvVarError{names: []string{"WALE_S3_PREFIX"}}
 	}
 
-	u, err := url.Parse(chk["WALE_S3_PREFIX"])
+	u, err := url.Parse(wale_s3_prefix)
 	if err != nil {
-		return nil, nil, errors.Wrapf(err, "Configure: failed to parse url '%s'", chk["WALE_S3_PREFIX"])
+		return nil, nil, errors.Wrapf(err, "Configure: failed to parse url '%s'", wale_s3_prefix)
 	}
 
 	bucket := u.Host
 	server := u.Path[1:]
-	region := chk["AWS_REGION"]
-
-	pre := &Prefix{
-		Bucket: aws.String(bucket),
-		Server: aws.String(server),
-	}
 
 	config := defaults.Get().Config
+
 	if _, err := config.Credentials.Get(); err != nil {
 		return nil, nil, errors.Wrapf(err, "Configure: failed to get AWS credentials; please specify AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY")
 	}
 
+	region := os.Getenv("AWS_REGION")
+	if region == "" {
+		region, err = findS3BucketRegion(bucket, config)
+		if err != nil {
+			return nil, nil, errors.Wrapf(err, "Configure: AWS_REGION is not set and s3:GetBucketLocation failed")
+		}
+	}
+	config = config.WithRegion(region)
+
+	pre := &Prefix{
+		Bucket: aws.String(bucket),
+		Server: aws.String(server),
+	}
+
 	sess, err := session.NewSession(config)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "Configure: failed to create new session")