From dae5a1ed5b8e950c9a87ff77765f4417d87ef53d Mon Sep 17 00:00:00 2001 From: Hugo Bollon Date: Mon, 5 Jul 2021 15:28:16 +0200 Subject: [PATCH] feat: add aws credentials & session token config support * allow aws multiple buckets to be configured directly in s3 yaml field (now an objects array) * add another minio instance to multiple-minio-buckets testing env * update all yaml example config files --- config/config.go | 19 ++++-- config/config_test.go | 8 ++- config/config_test.yml | 12 ++-- example.yml | 12 ++-- state/aws.go | 65 +++++++++++-------- test/data/test-bucket/.gitkeep | 0 .../test-bucket/terraform_0.12.28.tfstate | 36 ---------- .../data/test-bucket/terraform_0.13.5.tfstate | 36 ---------- .../data/test-bucket/terraform_0.14.8.tfstate | 37 ----------- .../data/test-bucket/terraform_0.15.5.tfstate | 37 ----------- test/data/test-bucket/terraform_1.0.0.tfstate | 37 ----------- .../terraform_1.0.0_bucket2.tfstate | 37 ----------- test/multiple-minio-buckets/config.yml | 29 ++++++--- .../multiple-minio-buckets/docker-compose.yml | 18 ++++- test/single-minio-bucket/docker-compose.yml | 2 +- 15 files changed, 104 insertions(+), 281 deletions(-) delete mode 100644 test/data/test-bucket/.gitkeep delete mode 100644 test/data/test-bucket/terraform_0.12.28.tfstate delete mode 100644 test/data/test-bucket/terraform_0.13.5.tfstate delete mode 100644 test/data/test-bucket/terraform_0.14.8.tfstate delete mode 100644 test/data/test-bucket/terraform_0.15.5.tfstate delete mode 100644 test/data/test-bucket/terraform_1.0.0.tfstate delete mode 100644 test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate diff --git a/config/config.go b/config/config.go index cc557427..ec47f387 100644 --- a/config/config.go +++ b/config/config.go @@ -40,12 +40,15 @@ type S3BucketConfig struct { // AWSConfig stores the DynamoDB table and S3 Bucket configuration type AWSConfig struct { - DynamoDBTable string `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."` - S3 S3BucketConfig `group:"S3 Options" yaml:"s3"` - Endpoint string `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."` - Region string `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."` - APPRoleArn string `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."` - ExternalID string `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."` + AccessKey string `long:"aws-access-key" env:"AWS_ACCESS_KEY_ID" yaml:"access-key" description:"AWS account access key."` + SecretAccessKey string `long:"aws-secret-access-key" env:"AWS_SECRET_ACCESS_KEY" yaml:"secret-access-key" description:"AWS secret account access key."` + SessionToken string `long:"aws-session-token" env:"AWS_SESSION_TOKEN" yaml:"session-token" description:"AWS session token."` + DynamoDBTable string `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."` + S3 []S3BucketConfig `group:"S3 Options" yaml:"s3"` + Endpoint string `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."` + Region string `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."` + APPRoleArn string `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."` + ExternalID string `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."` } // TFEConfig stores the Terraform Enterprise configuration @@ -124,6 +127,7 @@ func (c *Config) LoadConfigFromYaml() *Config { func initDefaultConfig() Config { var c Config var awsInitialConfig AWSConfig + var s3InitialConfig S3BucketConfig var tfeInitialConfig TFEConfig var gcpInitialConfig GCPConfig var gitlabInitialConfig GitlabConfig @@ -131,6 +135,9 @@ func initDefaultConfig() Config { parseStructFlagsAndEnv(&awsInitialConfig) c.AWS = append(c.AWS, awsInitialConfig) + parseStructFlagsAndEnv(&s3InitialConfig) + c.AWS[0].S3 = append(c.AWS[0].S3, s3InitialConfig) + parseStructFlagsAndEnv(&tfeInitialConfig) c.TFE = append(c.TFE, tfeInitialConfig) diff --git a/config/config_test.go b/config/config_test.go index fadda18d..3e2081c9 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -24,13 +24,15 @@ func TestLoadConfigFromYaml(t *testing.T) { }, AWS: []AWSConfig{ { - DynamoDBTable: "terraboard-dynamodb", - S3: S3BucketConfig{ + AccessKey: "root", + SecretAccessKey: "mypassword", + DynamoDBTable: "terraboard-dynamodb", + S3: []S3BucketConfig{{ Bucket: "terraboard-bucket", KeyPrefix: "test/", FileExtension: []string{".tfstate"}, ForcePathStyle: true, - }, + }}, }, }, TFE: []TFEConfig{ diff --git a/config/config_test.yml b/config/config_test.yml index 4f7d7d32..6f4610e5 100644 --- a/config/config_test.yml +++ b/config/config_test.yml @@ -11,12 +11,14 @@ database: no-sync: true aws: - - dynamodb-table: terraboard-dynamodb + - access-key: root + secret-access-key: mypassword + dynamodb-table: terraboard-dynamodb s3: - bucket: terraboard-bucket - key-prefix: test/ - file-extension: [.tfstate] - force-path-style: true + - bucket: terraboard-bucket + key-prefix: test/ + file-extension: [.tfstate] + force-path-style: true tfe: - address: https://tfe.example.com diff --git a/example.yml b/example.yml index d7d7d5fb..79346f60 100644 --- a/example.yml +++ b/example.yml @@ -16,11 +16,13 @@ provider: no-versioning: false aws: - dynamodb-table: terraboard - s3: - bucket: terraboard - key-prefix: - file-extension: .tfstate + - access-key: root + secret-access-key: mypassword + dynamodb-table: terraboard + s3: + - bucket: terraboard + key-prefix: + file-extension: .tfstate web: port: 9090 diff --git a/state/aws.go b/state/aws.go index 9c3181df..5a9e8108 100644 --- a/state/aws.go +++ b/state/aws.go @@ -8,6 +8,7 @@ import ( "time" aws_sdk "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/dynamodb" @@ -34,38 +35,46 @@ type AWS struct { func NewAWS(c *config.Config) []*AWS { var awsInstances []*AWS for _, aws := range c.AWS { - if aws.S3.Bucket != "" { - sess := session.Must(session.NewSession()) - awsConfig := aws_sdk.NewConfig() - if len(aws.APPRoleArn) > 0 { - log.Debugf("Using %s role", aws.APPRoleArn) - creds := stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) { - if aws.ExternalID != "" { - p.ExternalID = aws_sdk.String(aws.ExternalID) + for _, bucket := range aws.S3 { + if bucket.Bucket != "" { + sess := session.Must(session.NewSession()) + awsConfig := aws_sdk.NewConfig() + var creds *credentials.Credentials + if len(aws.APPRoleArn) > 0 { + log.Debugf("Using %s role", aws.APPRoleArn) + creds = stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) { + if aws.ExternalID != "" { + p.ExternalID = aws_sdk.String(aws.ExternalID) + } + }) + } else { + if aws.AccessKey == "" || aws.SecretAccessKey == "" { + log.Fatal("Missing AccessKey or SecretAccessKey for AWS provider. Please check your configuration and retry") } - }) + creds = credentials.NewStaticCredentials(aws.AccessKey, aws.SecretAccessKey, aws.SessionToken) + } awsConfig.WithCredentials(creds) - } - if e := aws.Endpoint; e != "" { - awsConfig.WithEndpoint(e) - } - if e := aws.Region; e != "" { - awsConfig.WithRegion(e) - } - awsConfig.S3ForcePathStyle = &aws.S3.ForcePathStyle - - instance := &AWS{ - svc: s3.New(sess, awsConfig), - bucket: aws.S3.Bucket, - keyPrefix: aws.S3.KeyPrefix, - fileExtension: aws.S3.FileExtension, - dynamoSvc: dynamodb.New(sess, awsConfig), - dynamoTable: aws.DynamoDBTable, - noLocks: c.Provider.NoLocks, - noVersioning: c.Provider.NoVersioning, + if e := aws.Endpoint; e != "" { + awsConfig.WithEndpoint(e) + } + if e := aws.Region; e != "" { + awsConfig.WithRegion(e) + } + awsConfig.S3ForcePathStyle = &bucket.ForcePathStyle + + instance := &AWS{ + svc: s3.New(sess, awsConfig), + bucket: bucket.Bucket, + keyPrefix: bucket.KeyPrefix, + fileExtension: bucket.FileExtension, + dynamoSvc: dynamodb.New(sess, awsConfig), + dynamoTable: aws.DynamoDBTable, + noLocks: c.Provider.NoLocks, + noVersioning: c.Provider.NoVersioning, + } + awsInstances = append(awsInstances, instance) } - awsInstances = append(awsInstances, instance) } } diff --git a/test/data/test-bucket/.gitkeep b/test/data/test-bucket/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/test/data/test-bucket/terraform_0.12.28.tfstate b/test/data/test-bucket/terraform_0.12.28.tfstate deleted file mode 100644 index 9590a256..00000000 --- a/test/data/test-bucket/terraform_0.12.28.tfstate +++ /dev/null @@ -1,36 +0,0 @@ -{ - "version": 4, - "terraform_version": "0.12.28", - "serial": 2, - "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00", - "outputs": { - "content": { - "value": "foo!", - "type": "string" - } - }, - "resources": [ - { - "mode": "managed", - "type": "local_file", - "name": "foo", - "provider": "provider.local", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content": "foo!", - "content_base64": null, - "directory_permission": "0777", - "file_permission": "0777", - "filename": "./foo.bar", - "id": "4bf3e335199107182c6f7638efaad377acc7f452", - "sensitive_content": null, - "source": null - }, - "private": "bnVsbA==" - } - ] - } - ] -} diff --git a/test/data/test-bucket/terraform_0.13.5.tfstate b/test/data/test-bucket/terraform_0.13.5.tfstate deleted file mode 100644 index 8325cf53..00000000 --- a/test/data/test-bucket/terraform_0.13.5.tfstate +++ /dev/null @@ -1,36 +0,0 @@ -{ - "version": 4, - "terraform_version": "0.13.5", - "serial": 3, - "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00", - "outputs": { - "content": { - "value": "foo!", - "type": "string" - } - }, - "resources": [ - { - "mode": "managed", - "type": "local_file", - "name": "foo", - "provider": "provider[\"registry.terraform.io/hashicorp/local\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content": "foo!", - "content_base64": null, - "directory_permission": "0777", - "file_permission": "0777", - "filename": "./foo.bar", - "id": "4bf3e335199107182c6f7638efaad377acc7f452", - "sensitive_content": null, - "source": null - }, - "private": "bnVsbA==" - } - ] - } - ] -} diff --git a/test/data/test-bucket/terraform_0.14.8.tfstate b/test/data/test-bucket/terraform_0.14.8.tfstate deleted file mode 100644 index efa47929..00000000 --- a/test/data/test-bucket/terraform_0.14.8.tfstate +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version": 4, - "terraform_version": "0.14.8", - "serial": 3, - "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00", - "outputs": { - "content": { - "value": "foo!", - "type": "string" - } - }, - "resources": [ - { - "mode": "managed", - "type": "local_file", - "name": "foo", - "provider": "provider[\"registry.terraform.io/hashicorp/local\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content": "foo!", - "content_base64": null, - "directory_permission": "0777", - "file_permission": "0777", - "filename": "./foo.bar", - "id": "4bf3e335199107182c6f7638efaad377acc7f452", - "sensitive_content": null, - "source": null - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - } - ] -} diff --git a/test/data/test-bucket/terraform_0.15.5.tfstate b/test/data/test-bucket/terraform_0.15.5.tfstate deleted file mode 100644 index f8751ad9..00000000 --- a/test/data/test-bucket/terraform_0.15.5.tfstate +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version": 4, - "terraform_version": "0.15.5", - "serial": 3, - "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00", - "outputs": { - "content": { - "value": "foo!", - "type": "string" - } - }, - "resources": [ - { - "mode": "managed", - "type": "local_file", - "name": "foo", - "provider": "provider[\"registry.terraform.io/hashicorp/local\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content": "foo!", - "content_base64": null, - "directory_permission": "0777", - "file_permission": "0777", - "filename": "./foo.bar", - "id": "4bf3e335199107182c6f7638efaad377acc7f452", - "sensitive_content": null, - "source": null - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - } - ] -} diff --git a/test/data/test-bucket/terraform_1.0.0.tfstate b/test/data/test-bucket/terraform_1.0.0.tfstate deleted file mode 100644 index a3835bd3..00000000 --- a/test/data/test-bucket/terraform_1.0.0.tfstate +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version": 4, - "terraform_version": "1.0.0", - "serial": 3, - "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00", - "outputs": { - "content": { - "value": "foo!", - "type": "string" - } - }, - "resources": [ - { - "mode": "managed", - "type": "local_file", - "name": "foo", - "provider": "provider[\"registry.terraform.io/hashicorp/local\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content": "foo!", - "content_base64": null, - "directory_permission": "0777", - "file_permission": "0777", - "filename": "./foo.bar", - "id": "4bf3e335199107182c6f7638efaad377acc7f452", - "sensitive_content": null, - "source": null - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - } - ] -} diff --git a/test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate b/test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate deleted file mode 100644 index a3835bd3..00000000 --- a/test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version": 4, - "terraform_version": "1.0.0", - "serial": 3, - "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00", - "outputs": { - "content": { - "value": "foo!", - "type": "string" - } - }, - "resources": [ - { - "mode": "managed", - "type": "local_file", - "name": "foo", - "provider": "provider[\"registry.terraform.io/hashicorp/local\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content": "foo!", - "content_base64": null, - "directory_permission": "0777", - "file_permission": "0777", - "filename": "./foo.bar", - "id": "4bf3e335199107182c6f7638efaad377acc7f452", - "sensitive_content": null, - "source": null - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - } - ] -} diff --git a/test/multiple-minio-buckets/config.yml b/test/multiple-minio-buckets/config.yml index 5999c435..acb12c42 100644 --- a/test/multiple-minio-buckets/config.yml +++ b/test/multiple-minio-buckets/config.yml @@ -10,17 +10,26 @@ provider: no-versioning: true aws: - - endpoint: http://minio:9000/ + - access-key: root + secret-access-key: mypassword + endpoint: http://minio:9000/ region: eu-west-1 s3: - bucket: test-bucket - force-path-style: true - file-extension: - - .tfstate - - endpoint: http://minio:9000/ + - bucket: test-bucket + force-path-style: true + file-extension: + - .tfstate + - bucket: test-bucket2 + force-path-style: true + file-extension: + - .tfstate + + - access-key: admin + secret-access-key: password + endpoint: http://minio-2:9000/ region: eu-west-1 s3: - bucket: test-bucket2 - force-path-style: true - file-extension: - - .tfstate + - bucket: test-bucket + force-path-style: true + file-extension: + - .tfstate diff --git a/test/multiple-minio-buckets/docker-compose.yml b/test/multiple-minio-buckets/docker-compose.yml index 51b4d2e3..ba62c425 100644 --- a/test/multiple-minio-buckets/docker-compose.yml +++ b/test/multiple-minio-buckets/docker-compose.yml @@ -6,8 +6,6 @@ services: context: ../../ dockerfile: ./Dockerfile environment: - AWS_ACCESS_KEY_ID: root - AWS_SECRET_ACCESS_KEY: mypassword DB_SSLMODE: disable CONFIG_FILE: config/config.yml GODEBUG: netdns=go @@ -33,7 +31,21 @@ services: ports: - "9200:9000" volumes: - - ../data:/data + - ../data/minio-1:/data + command: server /data + + minio-2: + image: minio/minio:latest + environment: + MINIO_ROOT_USER: admin + MINIO_ROOT_PASSWORD: password + user: "${UID}:${GID}" + expose: + - "9000" + ports: + - "9300:9000" + volumes: + - ../data/minio-2:/data command: server /data db: diff --git a/test/single-minio-bucket/docker-compose.yml b/test/single-minio-bucket/docker-compose.yml index f8785745..3a366cd1 100755 --- a/test/single-minio-bucket/docker-compose.yml +++ b/test/single-minio-bucket/docker-compose.yml @@ -39,7 +39,7 @@ services: ports: - "9200:9000" volumes: - - ../data:/data + - ../data/minio-1:/data command: server /data db: