feat!: multiple buckets / providers support (#48) (#177)

* feat: multi-provider support (WIP) * feat: env variables/flags available again for single provider configuration * feat: add some custom yaml unmarshalling rules to add default values where go-flags ones aren't applicable * docs: update readme with multiple buckets/providers config advices * feat: parallelized db refreshing and add mutex security + code review fixes from #177 * feat: add aws credentials & session token config support * allow aws multiple buckets to be configured directly in s3 yaml field (now an objects array) * add another minio instance to multiple-minio-buckets testing env * update all yaml example config files * docs: update readme with new aws configuration fields
camptocamp · Jul 20, 2021 · e44ebce · e44ebce
1 parent ad88878
commit e44ebce
Show file tree

Hide file tree

Showing 26 changed files with 513 additions and 308 deletions.
diff --git a/README.md b/README.md
@@ -41,6 +41,7 @@
     - [AWS S3 (state) + DynamoDB (lock)](#aws-s3-state--dynamodb-lock)
     - [Terraform Cloud](#terraform-cloud)
 - [Configuration](#configuration)
+  - [Multiple buckets/providers](#multiple-bucketsproviders)
   - [Available parameters](#available-parameters)
     - [Application Options](#application-options)
     - [General Provider Options](#general-provider-options)
@@ -90,6 +91,9 @@ It currently supports several remote state backend providers:
 - [Terraform Cloud (remote)](https://www.terraform.io/docs/backends/types/remote.html)
 - [GitLab](https://docs.gitlab.com/ee/user/infrastructure/terraform_state.html)
 
+With the upcoming **v1.2.0** update, Terraboard will be now able to handle multiple buckets/providers configuration! 🥳
+Check *configuration* section for more details. 
+
 ### Overview
 
 The overview presents all the state files in the S3 bucket, by most recent
@@ -143,12 +147,50 @@ Data resiliency is not paramount though as this dataset can be rebuilt upon your
 
 Terraboard currently supports configuration in three different ways:
 
-1. Environment variables
-2. CLI parameters
-3. Configuration file (YAML). A configuration file example can be found in the root directory of this repository.
+1. Environment variables **(only usable for single provider configuration)**
+2. CLI parameters **(only usable for single provider configuration)**
+3. Configuration file (YAML). A configuration file example can be found in the root directory of this repository and in the `test/` subdirectory.
+
+**Important: all flags/environment variables related to the providers settings aren't compatible with multi-provider configuration! Instead, you must use the YAML config file to be able to configure multiples buckets/providers.**
 
 The precedence of configurations is as described below.
 
+### Multiple buckets/providers
+
+In order for Terraboard to import states from multiples buckets or even providers, you must use the YAML configuration method:
+
+- Set the `CONFIG_FILE` environment variable or the `-c`/`--config-file` flag to point to a valid YAML config file.
+- In the YAML file, specify your desired providers configuration. For example with two MinIO buckets (using the AWS provider with compatible mode):
+
+```yaml
+# Needed since MinIO doesn't support versioning or locking
+provider:
+  no-locks: true
+  no-versioning: true
+
+aws:
+  - endpoint: http://minio:9000/
+    region: eu-west-1
+    s3:
+      - bucket: test-bucket
+        force-path-style: true
+        file-extension: 
+          - .tfstate
+
+  - endpoint: http://minio:9000/
+    region: eu-west-1
+    s3:
+      - bucket: test-bucket2
+        force-path-style: true
+        file-extension: 
+          - .tfstate
+```
+
+In the case of AWS, don't forget to set the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables.
+
+That's it! Terraboard will now fetch these two buckets on DB refresh. You can also mix providers like AWS and Gitlab or anything else.
+You can find a ready-to-use Docker example with two *MinIO* buckets in the `test/multiple-minio-buckets/` sub-folder. 
+
 ### Available parameters
 
 #### Application Options
@@ -202,6 +244,15 @@ The precedence of configurations is as described below.
 
 #### AWS (and S3 compatible providers) Options
 
+- `--aws-access-key` <default: *$AWS_ACCESS_KEY_ID*> AWS account access key.
+  - Env: *AWS_ACCESS_KEY_ID*
+  - Yaml: *aws.access-key*
+- `--aws-secret-access-key` <default: *$AWS_SECRET_ACCESS_KEY*> AWS secret account access key.
+  - Env: *AWS_SECRET_ACCESS_KEY*
+  - Yaml: *aws.secret-access-key*
+- `--aws-session-token` <default: *$AWS_SESSION_TOKEN*> AWS session token.
+  - Env: *AWS_SESSION_TOKEN*
+  - Yaml: *aws.session-token*
 - `--dynamodb-table` <default: *$AWS_DYNAMODB_TABLE*> AWS DynamoDB table for locks.
   - Env: *AWS_DYNAMODB_TABLE*
   - Yaml: *aws.dynamodb-table*

diff --git a/api/api.go b/api/api.go
@@ -148,15 +148,21 @@ func StateCompare(w http.ResponseWriter, r *http.Request, d *db.Database) {
 }
 
 // GetLocks returns information on locked States
-func GetLocks(w http.ResponseWriter, _ *http.Request, sp state.Provider) {
+func GetLocks(w http.ResponseWriter, _ *http.Request, sps []state.Provider) {
 	w.Header().Set("Access-Control-Allow-Origin", "*")
-	locks, err := sp.GetLocks()
-	if err != nil {
-		JSONError(w, "Failed to get locks", err)
-		return
+	allLocks := make(map[string]state.LockInfo)
+	for _, sp := range sps {
+		locks, err := sp.GetLocks()
+		if err != nil {
+			JSONError(w, "Failed to get locks on a provider", err)
+			return
+		}
+		for k, v := range locks {
+			allLocks[k] = v
+		}
 	}
 
-	j, err := json.Marshal(locks)
+	j, err := json.Marshal(allLocks)
 	if err != nil {
 		JSONError(w, "Failed to marshal locks", err)
 		return

diff --git a/config/config.go b/config/config.go
@@ -40,12 +40,15 @@ type S3BucketConfig struct {
 
 // AWSConfig stores the DynamoDB table and S3 Bucket configuration
 type AWSConfig struct {
-	DynamoDBTable string         `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."`
-	S3            S3BucketConfig `group:"S3 Options" yaml:"s3"`
-	Endpoint      string         `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."`
-	Region        string         `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."`
-	APPRoleArn    string         `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."`
-	ExternalID    string         `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."`
+	AccessKey       string           `long:"aws-access-key" env:"AWS_ACCESS_KEY_ID" yaml:"access-key" description:"AWS account access key."`
+	SecretAccessKey string           `long:"aws-secret-access-key" env:"AWS_SECRET_ACCESS_KEY" yaml:"secret-access-key" description:"AWS secret account access key."`
+	SessionToken    string           `long:"aws-session-token" env:"AWS_SESSION_TOKEN" yaml:"session-token" description:"AWS session token."`
+	DynamoDBTable   string           `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."`
+	S3              []S3BucketConfig `group:"S3 Options" yaml:"s3"`
+	Endpoint        string           `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."`
+	Region          string           `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."`
+	APPRoleArn      string           `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."`
+	ExternalID      string           `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."`
 }
 
 // TFEConfig stores the Terraform Enterprise configuration
@@ -92,13 +95,13 @@ type Config struct {
 
 	DB DBConfig `group:"Database Options" yaml:"database"`
 
-	AWS AWSConfig `group:"AWS Options" yaml:"aws"`
+	AWS []AWSConfig `group:"AWS Options" yaml:"aws"`
 
-	TFE TFEConfig `group:"Terraform Enterprise Options" yaml:"tfe"`
+	TFE []TFEConfig `group:"Terraform Enterprise Options" yaml:"tfe"`
 
-	GCP GCPConfig `group:"Google Cloud Platform Options" yaml:"gcp"`
+	GCP []GCPConfig `group:"Google Cloud Platform Options" yaml:"gcp"`
 
-	Gitlab GitlabConfig `group:"GitLab Options" yaml:"gitlab"`
+	Gitlab []GitlabConfig `group:"GitLab Options" yaml:"gitlab"`
 
 	Web WebConfig `group:"Web" yaml:"web"`
 }
@@ -119,14 +122,48 @@ func (c *Config) LoadConfigFromYaml() *Config {
 	return c
 }
 
-// LoadConfig loads the config from flags & environment
-func LoadConfig(version string) *Config {
+// Initialize Config with one obj per providers arrays
+// to allow usage of flags / env variables on single provider configuration
+func initDefaultConfig() Config {
 	var c Config
-	parser := flags.NewParser(&c, flags.Default)
+	var awsInitialConfig AWSConfig
+	var s3InitialConfig S3BucketConfig
+	var tfeInitialConfig TFEConfig
+	var gcpInitialConfig GCPConfig
+	var gitlabInitialConfig GitlabConfig
+
+	parseStructFlagsAndEnv(&awsInitialConfig)
+	c.AWS = append(c.AWS, awsInitialConfig)
+
+	parseStructFlagsAndEnv(&s3InitialConfig)
+	c.AWS[0].S3 = append(c.AWS[0].S3, s3InitialConfig)
+
+	parseStructFlagsAndEnv(&tfeInitialConfig)
+	c.TFE = append(c.TFE, tfeInitialConfig)
+
+	parseStructFlagsAndEnv(&gcpInitialConfig)
+	c.GCP = append(c.GCP, gcpInitialConfig)
+
+	parseStructFlagsAndEnv(&gitlabInitialConfig)
+	c.Gitlab = append(c.Gitlab, gitlabInitialConfig)
+
+	return c
+}
+
+// Parse flags and env variables to given struct using go-flags
+// parser
+func parseStructFlagsAndEnv(obj interface{}) {
+	parser := flags.NewParser(obj, flags.Default)
 	if _, err := parser.Parse(); err != nil {
 		fmt.Printf("Failed to parse flags: %s", err)
 		os.Exit(1)
 	}
+}
+
+// LoadConfig loads the config from flags & environment
+func LoadConfig(version string) *Config {
+	c := initDefaultConfig()
+	parseStructFlagsAndEnv(&c)
 
 	if c.ConfigFilePath != "" {
 		if _, err := os.Stat(c.ConfigFilePath); err == nil {

diff --git a/config/config_test.go b/config/config_test.go
@@ -22,27 +22,37 @@ func TestLoadConfigFromYaml(t *testing.T) {
 			Name:     "terraboard-db",
 			NoSync:   true,
 		},
-		AWS: AWSConfig{
-			DynamoDBTable: "terraboard-dynamodb",
-			S3: S3BucketConfig{
-				Bucket:         "terraboard-bucket",
-				KeyPrefix:      "test/",
-				FileExtension:  []string{".tfstate"},
-				ForcePathStyle: true,
+		AWS: []AWSConfig{
+			{
+				AccessKey:       "root",
+				SecretAccessKey: "mypassword",
+				DynamoDBTable:   "terraboard-dynamodb",
+				S3: []S3BucketConfig{{
+					Bucket:         "terraboard-bucket",
+					KeyPrefix:      "test/",
+					FileExtension:  []string{".tfstate"},
+					ForcePathStyle: true,
+				}},
 			},
 		},
-		TFE: TFEConfig{
-			Address:      "https://tfe.example.com",
-			Token:        "foo",
-			Organization: "bar",
+		TFE: []TFEConfig{
+			{
+				Address:      "https://tfe.example.com",
+				Token:        "foo",
+				Organization: "bar",
+			},
 		},
-		GCP: GCPConfig{
-			GCSBuckets: []string{"my-bucket-1", "my-bucket-2"},
-			GCPSAKey:   "/path/to/key",
+		GCP: []GCPConfig{
+			{
+				GCSBuckets: []string{"my-bucket-1", "my-bucket-2"},
+				GCPSAKey:   "/path/to/key",
+			},
 		},
-		Gitlab: GitlabConfig{
-			Address: "https://gitlab.example.com",
-			Token:   "foo",
+		Gitlab: []GitlabConfig{
+			{
+				Address: "https://gitlab.example.com",
+				Token:   "foo",
+			},
 		},
 		Web: WebConfig{
 			Port:      39090,

diff --git a/config/config_test.yml b/config/config_test.yml
@@ -11,27 +11,29 @@ database:
   no-sync: true
 
 aws:
-  dynamodb-table: terraboard-dynamodb
-  s3:
-    bucket: terraboard-bucket
-    key-prefix: test/
-    file-extension: [.tfstate]
-    force-path-style: true
+  - access-key: root
+    secret-access-key: mypassword
+    dynamodb-table: terraboard-dynamodb
+    s3:
+      - bucket: terraboard-bucket
+        key-prefix: test/
+        file-extension: [.tfstate]
+        force-path-style: true
 
 tfe:
-  address: https://tfe.example.com
-  token: foo
-  organization: bar
+  - address: https://tfe.example.com
+    token: foo
+    organization: bar
 
 gcp:
-  gcs-bucket:
-    - my-bucket-1
-    - my-bucket-2
-  gcp-sa-key-path: /path/to/key
+  - gcs-bucket:
+      - my-bucket-1
+      - my-bucket-2
+    gcp-sa-key-path: /path/to/key
 
 gitlab:
-  address: https://gitlab.example.com
-  token: foo
+  - address: https://gitlab.example.com
+    token: foo
 
 web:
   port: 39090

diff --git a/config/yaml.go b/config/yaml.go
@@ -0,0 +1,33 @@
+package config
+
+/*********************************************
+ * Custom UnmarshalYAML used to define some struct fields
+ * default values where go-flags ones aren't applicable
+ * (and so makes them optional)
+ *********************************************/
+
+func (s *S3BucketConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawS3BucketConfig S3BucketConfig
+	raw := rawS3BucketConfig{
+		FileExtension: []string{".tfstate"},
+	}
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*s = S3BucketConfig(raw)
+	return nil
+}
+
+func (s *GitlabConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawGitlabConfig GitlabConfig
+	raw := rawGitlabConfig{
+		Address: "https://gitlab.com",
+	}
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*s = GitlabConfig(raw)
+	return nil
+}
diff --git a/db/db.go b/db/db.go
@@ -7,6 +7,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/camptocamp/terraboard/config"
 	"github.com/camptocamp/terraboard/internal/terraform/addrs"
@@ -25,6 +26,7 @@ import (
 // Database is a wrapping structure to *gorm.DB
 type Database struct {
 	*gorm.DB
+	lock sync.Mutex
 }
 
 var pageSize = 20
@@ -78,7 +80,7 @@ func Init(config config.DBConfig, debug bool) *Database {
 		db.Config.Logger.LogMode(logger.Info)
 	}
 
-	d := &Database{db}
+	d := &Database{DB: db}
 	if err = d.MigrateLineage(); err != nil {
 		log.Fatalf("Lineage migration failed: %v\n", err)
 	}
@@ -120,11 +122,14 @@ func (db *Database) stateS3toDB(sf *statefile.File, path string, versionID strin
 	// Check if the associated lineage is already present in lineages table
 	// If so, it recovers its ID otherwise it inserts it at the same time as the state
 	var lineage types.Lineage
+	db.lock.Lock()
 	err = db.FirstOrCreate(&lineage, types.Lineage{Value: sf.Lineage}).Error
 	if err != nil || lineage.ID == 0 {
-		log.Error("Unknown error in stateS3toDB during lineage finding")
+		log.WithField("error", err).
+			Error("Unknown error in stateS3toDB during lineage finding", err)
 		return types.State{}, err
 	}
+	db.lock.Unlock()
 
 	st = types.State{
 		Path:      path,
@@ -264,10 +269,12 @@ func (db *Database) UpdateState(st types.State) error {
 // InsertVersion inserts an AWS S3 Version in the Database
 func (db *Database) InsertVersion(version *state.Version) error {
 	var v types.Version
+	db.lock.Lock()
 	db.FirstOrCreate(&v, types.Version{
 		VersionID:    version.ID,
 		LastModified: version.LastModified,
 	})
+	db.lock.Unlock()
 	return nil
 }