feat: add aws credentials & session token config support

* allow aws multiple buckets to be configured directly in s3 yaml field (now an objects array)
* add another minio instance to multiple-minio-buckets testing env
* update all yaml example config files

config/config.go

@@ -40,12 +40,15 @@ type S3BucketConfig struct {
 
 // AWSConfig stores the DynamoDB table and S3 Bucket configuration
 type AWSConfig struct {
-	DynamoDBTable string         `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."`
-	S3            S3BucketConfig `group:"S3 Options" yaml:"s3"`
-	Endpoint      string         `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."`
-	Region        string         `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."`
-	APPRoleArn    string         `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."`
-	ExternalID    string         `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."`
+	AccessKey       string           `long:"aws-access-key" env:"AWS_ACCESS_KEY_ID" yaml:"access-key" description:"AWS account access key."`
+	SecretAccessKey string           `long:"aws-secret-access-key" env:"AWS_SECRET_ACCESS_KEY" yaml:"secret-access-key" description:"AWS secret account access key."`
+	SessionToken    string           `long:"aws-session-token" env:"AWS_SESSION_TOKEN" yaml:"session-token" description:"AWS session token."`
+	DynamoDBTable   string           `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."`
+	S3              []S3BucketConfig `group:"S3 Options" yaml:"s3"`
+	Endpoint        string           `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."`
+	Region          string           `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."`
+	APPRoleArn      string           `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."`
+	ExternalID      string           `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."`
 }
 
 // TFEConfig stores the Terraform Enterprise configuration
@@ -124,13 +127,17 @@ func (c *Config) LoadConfigFromYaml() *Config {
 func initDefaultConfig() Config {
 	var c Config
 	var awsInitialConfig AWSConfig
+	var s3InitialConfig S3BucketConfig
 	var tfeInitialConfig TFEConfig
 	var gcpInitialConfig GCPConfig
 	var gitlabInitialConfig GitlabConfig
 
 	parseStructFlagsAndEnv(&awsInitialConfig)
 	c.AWS = append(c.AWS, awsInitialConfig)
 
+	parseStructFlagsAndEnv(&s3InitialConfig)
+	c.AWS[0].S3 = append(c.AWS[0].S3, s3InitialConfig)
+
 	parseStructFlagsAndEnv(&tfeInitialConfig)
 	c.TFE = append(c.TFE, tfeInitialConfig)
 

config/config_test.go

@@ -24,13 +24,15 @@ func TestLoadConfigFromYaml(t *testing.T) {
 		},
 		AWS: []AWSConfig{
 			{
-				DynamoDBTable: "terraboard-dynamodb",
-				S3: S3BucketConfig{
+				AccessKey:       "root",
+				SecretAccessKey: "mypassword",
+				DynamoDBTable:   "terraboard-dynamodb",
+				S3: []S3BucketConfig{{
 					Bucket:         "terraboard-bucket",
 					KeyPrefix:      "test/",
 					FileExtension:  []string{".tfstate"},
 					ForcePathStyle: true,
-				},
+				}},
 			},
 		},
 		TFE: []TFEConfig{

config/config_test.yml

@@ -11,12 +11,14 @@ database:
   no-sync: true
 
 aws:
-  - dynamodb-table: terraboard-dynamodb
+  - access-key: root
+    secret-access-key: mypassword
+    dynamodb-table: terraboard-dynamodb
     s3:
-      bucket: terraboard-bucket
-      key-prefix: test/
-      file-extension: [.tfstate]
-      force-path-style: true
+      - bucket: terraboard-bucket
+        key-prefix: test/
+        file-extension: [.tfstate]
+        force-path-style: true
 
 tfe:
   - address: https://tfe.example.com

example.yml

@@ -16,11 +16,13 @@ provider:
   no-versioning: false
 
 aws:
-  dynamodb-table: terraboard
-  s3:
-    bucket: terraboard
-    key-prefix:
-    file-extension: .tfstate
+  - access-key: root
+    secret-access-key: mypassword
+    dynamodb-table: terraboard
+    s3:
+      - bucket: terraboard
+        key-prefix:
+        file-extension: .tfstate
 
 web:
   port: 9090

state/aws.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	aws_sdk "github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/dynamodb"
@@ -34,38 +35,46 @@ type AWS struct {
 func NewAWS(c *config.Config) []*AWS {
 	var awsInstances []*AWS
 	for _, aws := range c.AWS {
-		if aws.S3.Bucket != "" {
-			sess := session.Must(session.NewSession())
-			awsConfig := aws_sdk.NewConfig()
-			if len(aws.APPRoleArn) > 0 {
-				log.Debugf("Using %s role", aws.APPRoleArn)
-				creds := stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) {
-					if aws.ExternalID != "" {
-						p.ExternalID = aws_sdk.String(aws.ExternalID)
+		for _, bucket := range aws.S3 {
+			if bucket.Bucket != "" {
+				sess := session.Must(session.NewSession())
+				awsConfig := aws_sdk.NewConfig()
+				var creds *credentials.Credentials
+				if len(aws.APPRoleArn) > 0 {
+					log.Debugf("Using %s role", aws.APPRoleArn)
+					creds = stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) {
+						if aws.ExternalID != "" {
+							p.ExternalID = aws_sdk.String(aws.ExternalID)
+						}
+					})
+				} else {
+					if aws.AccessKey == "" || aws.SecretAccessKey == "" {
+						log.Fatal("Missing AccessKey or SecretAccessKey for AWS provider. Please check your configuration and retry")
 					}
-				})
+					creds = credentials.NewStaticCredentials(aws.AccessKey, aws.SecretAccessKey, aws.SessionToken)
+				}
 				awsConfig.WithCredentials(creds)
-			}
 
-			if e := aws.Endpoint; e != "" {
-				awsConfig.WithEndpoint(e)
-			}
-			if e := aws.Region; e != "" {
-				awsConfig.WithRegion(e)
-			}
-			awsConfig.S3ForcePathStyle = &aws.S3.ForcePathStyle
-
-			instance := &AWS{
-				svc:           s3.New(sess, awsConfig),
-				bucket:        aws.S3.Bucket,
-				keyPrefix:     aws.S3.KeyPrefix,
-				fileExtension: aws.S3.FileExtension,
-				dynamoSvc:     dynamodb.New(sess, awsConfig),
-				dynamoTable:   aws.DynamoDBTable,
-				noLocks:       c.Provider.NoLocks,
-				noVersioning:  c.Provider.NoVersioning,
+				if e := aws.Endpoint; e != "" {
+					awsConfig.WithEndpoint(e)
+				}
+				if e := aws.Region; e != "" {
+					awsConfig.WithRegion(e)
+				}
+				awsConfig.S3ForcePathStyle = &bucket.ForcePathStyle
+
+				instance := &AWS{
+					svc:           s3.New(sess, awsConfig),
+					bucket:        bucket.Bucket,
+					keyPrefix:     bucket.KeyPrefix,
+					fileExtension: bucket.FileExtension,
+					dynamoSvc:     dynamodb.New(sess, awsConfig),
+					dynamoTable:   aws.DynamoDBTable,
+					noLocks:       c.Provider.NoLocks,
+					noVersioning:  c.Provider.NoVersioning,
+				}
+				awsInstances = append(awsInstances, instance)
 			}
-			awsInstances = append(awsInstances, instance)
 		}
 	}