From 154786054dee0d38b07791506aeff1327ca13e99 Mon Sep 17 00:00:00 2001
From: Hugo Bollon <hugo.bollon@gmail.com>
Date: Wed, 23 Jun 2021 15:08:31 +0200
Subject: [PATCH 1/7] feat: multi-provider support (WIP)

---
 api/api.go             |  18 +++++---
 config/config.go       |   8 ++--
 config/config_test.go  |  42 ++++++++++-------
 config/config_test.yml |  30 ++++++------
 main.go                | 101 +++++++++++++++++++++--------------------
 state/aws.go           |  65 ++++++++++++++------------
 state/gcp.go           |  49 +++++++++++---------
 state/gitlab.go        |  11 +++--
 state/state.go         |  49 ++++++++++++++------
 state/tfe.go           |  35 ++++++++------
 test/config.yml        |  26 +++++++++++
 11 files changed, 262 insertions(+), 172 deletions(-)
 create mode 100644 test/config.yml

diff --git a/api/api.go b/api/api.go
index 8af3e42e..1ff3ec50 100644
--- a/api/api.go
+++ b/api/api.go
@@ -147,15 +147,21 @@ func StateCompare(w http.ResponseWriter, r *http.Request, d *db.Database) {
 }
 
 // GetLocks returns information on locked States
-func GetLocks(w http.ResponseWriter, _ *http.Request, sp state.Provider) {
+func GetLocks(w http.ResponseWriter, _ *http.Request, sps []state.Provider) {
 	w.Header().Set("Access-Control-Allow-Origin", "*")
-	locks, err := sp.GetLocks()
-	if err != nil {
-		JSONError(w, "Failed to get locks", err)
-		return
+	allLocks := make(map[string]state.LockInfo)
+	for _, sp := range sps {
+		locks, err := sp.GetLocks()
+		if err != nil {
+			JSONError(w, "Failed to get locks on a provider", err)
+			return
+		}
+		for k, v := range locks {
+			allLocks[k] = v
+		}
 	}
 
-	j, err := json.Marshal(locks)
+	j, err := json.Marshal(allLocks)
 	if err != nil {
 		JSONError(w, "Failed to marshal locks", err)
 		return
diff --git a/config/config.go b/config/config.go
index 6184b119..349c5cf7 100644
--- a/config/config.go
+++ b/config/config.go
@@ -92,13 +92,13 @@ type Config struct {
 
 	DB DBConfig `group:"Database Options" yaml:"database"`
 
-	AWS AWSConfig `group:"AWS Options" yaml:"aws"`
+	AWS []AWSConfig `group:"AWS Options" yaml:"aws"`
 
-	TFE TFEConfig `group:"Terraform Enterprise Options" yaml:"tfe"`
+	TFE []TFEConfig `group:"Terraform Enterprise Options" yaml:"tfe"`
 
-	GCP GCPConfig `group:"Google Cloud Platform Options" yaml:"gcp"`
+	GCP []GCPConfig `group:"Google Cloud Platform Options" yaml:"gcp"`
 
-	Gitlab GitlabConfig `group:"GitLab Options" yaml:"gitlab"`
+	Gitlab []GitlabConfig `group:"GitLab Options" yaml:"gitlab"`
 
 	Web WebConfig `group:"Web" yaml:"web"`
 }
diff --git a/config/config_test.go b/config/config_test.go
index 05aa4579..fadda18d 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -22,27 +22,35 @@ func TestLoadConfigFromYaml(t *testing.T) {
 			Name:     "terraboard-db",
 			NoSync:   true,
 		},
-		AWS: AWSConfig{
-			DynamoDBTable: "terraboard-dynamodb",
-			S3: S3BucketConfig{
-				Bucket:         "terraboard-bucket",
-				KeyPrefix:      "test/",
-				FileExtension:  []string{".tfstate"},
-				ForcePathStyle: true,
+		AWS: []AWSConfig{
+			{
+				DynamoDBTable: "terraboard-dynamodb",
+				S3: S3BucketConfig{
+					Bucket:         "terraboard-bucket",
+					KeyPrefix:      "test/",
+					FileExtension:  []string{".tfstate"},
+					ForcePathStyle: true,
+				},
 			},
 		},
-		TFE: TFEConfig{
-			Address:      "https://tfe.example.com",
-			Token:        "foo",
-			Organization: "bar",
+		TFE: []TFEConfig{
+			{
+				Address:      "https://tfe.example.com",
+				Token:        "foo",
+				Organization: "bar",
+			},
 		},
-		GCP: GCPConfig{
-			GCSBuckets: []string{"my-bucket-1", "my-bucket-2"},
-			GCPSAKey:   "/path/to/key",
+		GCP: []GCPConfig{
+			{
+				GCSBuckets: []string{"my-bucket-1", "my-bucket-2"},
+				GCPSAKey:   "/path/to/key",
+			},
 		},
-		Gitlab: GitlabConfig{
-			Address: "https://gitlab.example.com",
-			Token:   "foo",
+		Gitlab: []GitlabConfig{
+			{
+				Address: "https://gitlab.example.com",
+				Token:   "foo",
+			},
 		},
 		Web: WebConfig{
 			Port:      39090,
diff --git a/config/config_test.yml b/config/config_test.yml
index 4fac8d9f..4f7d7d32 100644
--- a/config/config_test.yml
+++ b/config/config_test.yml
@@ -11,27 +11,27 @@ database:
   no-sync: true
 
 aws:
-  dynamodb-table: terraboard-dynamodb
-  s3:
-    bucket: terraboard-bucket
-    key-prefix: test/
-    file-extension: [.tfstate]
-    force-path-style: true
+  - dynamodb-table: terraboard-dynamodb
+    s3:
+      bucket: terraboard-bucket
+      key-prefix: test/
+      file-extension: [.tfstate]
+      force-path-style: true
 
 tfe:
-  address: https://tfe.example.com
-  token: foo
-  organization: bar
+  - address: https://tfe.example.com
+    token: foo
+    organization: bar
 
 gcp:
-  gcs-bucket:
-    - my-bucket-1
-    - my-bucket-2
-  gcp-sa-key-path: /path/to/key
+  - gcs-bucket:
+      - my-bucket-1
+      - my-bucket-2
+    gcp-sa-key-path: /path/to/key
 
 gitlab:
-  address: https://gitlab.example.com
-  token: foo
+  - address: https://gitlab.example.com
+    token: foo
 
 web:
   port: 39090
diff --git a/main.go b/main.go
index 6019093d..cdb6c20e 100644
--- a/main.go
+++ b/main.go
@@ -46,9 +46,9 @@ func handleWithDB(apiF func(w http.ResponseWriter, r *http.Request,
 }
 
 func handleWithStateProvider(apiF func(w http.ResponseWriter, r *http.Request,
-	sp state.Provider), sp state.Provider) func(http.ResponseWriter, *http.Request) {
+	sps []state.Provider), sps []state.Provider) func(http.ResponseWriter, *http.Request) {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		apiF(w, r, sp)
+		apiF(w, r, sps)
 	})
 }
 
@@ -65,55 +65,60 @@ func isKnownStateVersion(statesVersions map[string][]string, versionID, path str
 
 // Refresh the DB
 // This should be the only direct bridge between the state provider and the DB
-func refreshDB(syncInterval uint16, d *db.Database, sp state.Provider) {
+func refreshDB(syncInterval uint16, d *db.Database, sps []state.Provider) {
 	interval := time.Duration(syncInterval) * time.Minute
+	log.Debugf("Providers: %+v\n", sps)
 	for {
 		log.Infof("Refreshing DB")
-		states, err := sp.GetStates()
-		if err != nil {
-			log.WithFields(log.Fields{
-				"error": err,
-			}).Error("Failed to retrieve states. Retrying in 1 minute.")
-			time.Sleep(interval)
-			continue
-		}
+		log.Debugf("Total providers: %d\n", len(sps))
+		for i, sp := range sps {
+			log.Debugf("Fetching provider %d/%d\n", i+1, len(sps))
+			states, err := sp.GetStates()
+			if err != nil {
+				log.WithFields(log.Fields{
+					"error": err,
+				}).Error("Failed to retrieve states. Retrying in 1 minute.")
+				time.Sleep(interval)
+				continue
+			}
 
-		statesVersions := d.ListStatesVersions()
-		for _, st := range states {
-			versions, _ := sp.GetVersions(st)
-			for k, v := range versions {
-				if _, ok := statesVersions[v.ID]; ok {
-					log.WithFields(log.Fields{
-						"version_id": v.ID,
-					}).Debug("Version is already in the database, skipping")
-				} else {
-					if err := d.InsertVersion(&versions[k]); err != nil {
-						log.Error(err.Error())
+			statesVersions := d.ListStatesVersions()
+			for _, st := range states {
+				versions, _ := sp.GetVersions(st)
+				for k, v := range versions {
+					if _, ok := statesVersions[v.ID]; ok {
+						log.WithFields(log.Fields{
+							"version_id": v.ID,
+						}).Debug("Version is already in the database, skipping")
+					} else {
+						if err := d.InsertVersion(&versions[k]); err != nil {
+							log.Error(err.Error())
+						}
 					}
-				}
 
-				if isKnownStateVersion(statesVersions, v.ID, st) {
-					log.WithFields(log.Fields{
-						"path":       st,
-						"version_id": v.ID,
-					}).Debug("State is already in the database, skipping")
-					continue
-				}
-				state, err := sp.GetState(st, v.ID)
-				if err != nil {
-					log.WithFields(log.Fields{
-						"path":       st,
-						"version_id": v.ID,
-						"error":      err,
-					}).Error("Failed to fetch state from bucket")
-					continue
-				}
-				if err = d.InsertState(st, v.ID, state); err != nil {
-					log.WithFields(log.Fields{
-						"path":       st,
-						"version_id": v.ID,
-						"error":      err,
-					}).Error("Failed to insert state in the database")
+					if isKnownStateVersion(statesVersions, v.ID, st) {
+						log.WithFields(log.Fields{
+							"path":       st,
+							"version_id": v.ID,
+						}).Debug("State is already in the database, skipping")
+						continue
+					}
+					state, err := sp.GetState(st, v.ID)
+					if err != nil {
+						log.WithFields(log.Fields{
+							"path":       st,
+							"version_id": v.ID,
+							"error":      err,
+						}).Error("Failed to fetch state from bucket")
+						continue
+					}
+					if err = d.InsertState(st, v.ID, state); err != nil {
+						log.WithFields(log.Fields{
+							"path":       st,
+							"version_id": v.ID,
+							"error":      err,
+						}).Error("Failed to insert state in the database")
+					}
 				}
 			}
 		}
@@ -155,7 +160,7 @@ func main() {
 	}
 
 	// Set up the state provider
-	sp, err := state.Configure(c)
+	sps, err := state.Configure(c)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -168,7 +173,7 @@ func main() {
 	if c.DB.NoSync {
 		log.Infof("Not syncing database, as requested.")
 	} else {
-		go refreshDB(c.DB.SyncInterval, database, sp)
+		go refreshDB(c.DB.SyncInterval, database, sps)
 	}
 	defer database.Close()
 
@@ -189,7 +194,7 @@ func main() {
 	http.HandleFunc(util.GetFullPath("api/state/"), handleWithDB(api.GetState, database))
 	http.HandleFunc(util.GetFullPath("api/state/activity/"), handleWithDB(api.GetStateActivity, database))
 	http.HandleFunc(util.GetFullPath("api/state/compare/"), handleWithDB(api.StateCompare, database))
-	http.HandleFunc(util.GetFullPath("api/locks"), handleWithStateProvider(api.GetLocks, sp))
+	http.HandleFunc(util.GetFullPath("api/locks"), handleWithStateProvider(api.GetLocks, sps))
 	http.HandleFunc(util.GetFullPath("api/search/attribute"), handleWithDB(api.SearchAttribute, database))
 	http.HandleFunc(util.GetFullPath("api/resource/types"), handleWithDB(api.ListResourceTypes, database))
 	http.HandleFunc(util.GetFullPath("api/resource/types/count"), handleWithDB(api.ListResourceTypesWithCount, database))
diff --git a/state/aws.go b/state/aws.go
index f54a202b..69b88c4f 100644
--- a/state/aws.go
+++ b/state/aws.go
@@ -31,39 +31,44 @@ type AWS struct {
 }
 
 // NewAWS creates an AWS object
-func NewAWS(c *config.Config) AWS {
-	sess := session.Must(session.NewSession())
-
-	awsConfig := aws_sdk.NewConfig()
+func NewAWS(c *config.Config) []*AWS {
+	var awsInstances []*AWS
+	for _, aws := range c.AWS {
+		sess := session.Must(session.NewSession())
+		awsConfig := aws_sdk.NewConfig()
+		if len(aws.APPRoleArn) > 0 {
+			log.Debugf("Using %s role", aws.APPRoleArn)
+			creds := stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) {
+				if aws.ExternalID != "" {
+					p.ExternalID = aws_sdk.String(aws.ExternalID)
+				}
+			})
+			awsConfig.WithCredentials(creds)
+		}
 
-	if len(c.AWS.APPRoleArn) > 0 {
-		log.Debugf("Using %s role", c.AWS.APPRoleArn)
-		creds := stscreds.NewCredentials(sess, c.AWS.APPRoleArn, func(p *stscreds.AssumeRoleProvider) {
-			if c.AWS.ExternalID != "" {
-				p.ExternalID = aws_sdk.String(c.AWS.ExternalID)
-			}
-		})
-		awsConfig.WithCredentials(creds)
+		if e := aws.Endpoint; e != "" {
+			awsConfig.WithEndpoint(e)
+		}
+		if e := aws.Region; e != "" {
+			awsConfig.WithRegion(e)
+		}
+		awsConfig.S3ForcePathStyle = &aws.S3.ForcePathStyle
+
+		instance := &AWS{
+			svc:           s3.New(sess, awsConfig),
+			bucket:        aws.S3.Bucket,
+			keyPrefix:     aws.S3.KeyPrefix,
+			fileExtension: aws.S3.FileExtension,
+			dynamoSvc:     dynamodb.New(sess, awsConfig),
+			dynamoTable:   aws.DynamoDBTable,
+			noLocks:       c.Provider.NoLocks,
+			noVersioning:  c.Provider.NoVersioning,
+		}
+		log.Debugf("Instance: %+v\n", *instance)
+		awsInstances = append(awsInstances, instance)
 	}
 
-	if e := c.AWS.Endpoint; e != "" {
-		awsConfig.WithEndpoint(e)
-	}
-	if e := c.AWS.Region; e != "" {
-		awsConfig.WithRegion(e)
-	}
-	awsConfig.S3ForcePathStyle = &c.AWS.S3.ForcePathStyle
-
-	return AWS{
-		svc:           s3.New(sess, awsConfig),
-		bucket:        c.AWS.S3.Bucket,
-		keyPrefix:     c.AWS.S3.KeyPrefix,
-		fileExtension: c.AWS.S3.FileExtension,
-		dynamoSvc:     dynamodb.New(sess, awsConfig),
-		dynamoTable:   c.AWS.DynamoDBTable,
-		noLocks:       c.Provider.NoLocks,
-		noVersioning:  c.Provider.NoVersioning,
-	}
+	return awsInstances
 }
 
 // GetLocks returns a map of locks by State path
diff --git a/state/gcp.go b/state/gcp.go
index 31b50ba6..a77e5c90 100644
--- a/state/gcp.go
+++ b/state/gcp.go
@@ -24,33 +24,40 @@ type GCP struct {
 }
 
 // NewGCP creates an GCP object
-func NewGCP(c *config.Config) (*GCP, error) {
+func NewGCP(c *config.Config) ([]*GCP, error) {
 	ctx := context.Background()
+
 	var client *storage.Client
+	var gcpInstances []*GCP
 	var err error
-	if c.GCP.GCPSAKey != "" {
-		log.WithFields(log.Fields{
-			"path": c.GCP.GCPSAKey,
-		}).Info("Authenticating using service account key")
-		opt := option.WithCredentialsFile(c.GCP.GCPSAKey)
-		client, err = storage.NewClient(ctx, opt) // Use service account key
-	} else {
-		client, err = storage.NewClient(ctx) // Use base credentials
-	}
+	for _, gcp := range c.GCP {
+		if gcp.GCPSAKey != "" {
+			log.WithFields(log.Fields{
+				"path": gcp.GCPSAKey,
+			}).Info("Authenticating using service account key")
+			opt := option.WithCredentialsFile(gcp.GCPSAKey)
+			client, err = storage.NewClient(ctx, opt) // Use service account key
+		} else {
+			client, err = storage.NewClient(ctx) // Use base credentials
+		}
 
-	if err != nil {
-		log.Fatalf("Failed to create client: %v", err)
-		return nil, err
-	}
+		if err != nil {
+			log.Fatalf("Failed to create client: %v", err)
+			return nil, err
+		}
 
-	log.WithFields(log.Fields{
-		"buckets": c.GCP.GCSBuckets,
-	}).Info("Client successfully created")
+		instance := &GCP{
+			svc:     client,
+			buckets: gcp.GCSBuckets,
+		}
+		gcpInstances = append(gcpInstances, instance)
+
+		log.WithFields(log.Fields{
+			"buckets": gcp.GCSBuckets,
+		}).Info("Client successfully created")
+	}
 
-	return &GCP{
-		svc:     client,
-		buckets: c.GCP.GCSBuckets,
-	}, nil
+	return gcpInstances, nil
 }
 
 // GetLocks returns a map of locks by State path
diff --git a/state/gitlab.go b/state/gitlab.go
index f027768e..79f86595 100644
--- a/state/gitlab.go
+++ b/state/gitlab.go
@@ -18,10 +18,15 @@ type Gitlab struct {
 }
 
 // NewGitlab creates a new Gitlab object
-func NewGitlab(c *config.Config) *Gitlab {
-	return &Gitlab{
-		Client: gitlab.NewClient(c.Gitlab.Address, c.Gitlab.Token),
+func NewGitlab(c *config.Config) []*Gitlab {
+	var gitlabInstances []*Gitlab
+	for _, gl := range c.Gitlab {
+		instance := &Gitlab{
+			Client: gitlab.NewClient(gl.Address, gl.Token),
+		}
+		gitlabInstances = append(gitlabInstances, instance)
 	}
+	return gitlabInstances
 }
 
 // GetLocks returns a map of locks by State path
diff --git a/state/state.go b/state/state.go
index 6abcddcd..ef4198f7 100644
--- a/state/state.go
+++ b/state/state.go
@@ -40,23 +40,46 @@ type Provider interface {
 }
 
 // Configure the state provider
-func Configure(c *config.Config) (Provider, error) {
-	if len(c.TFE.Token) > 0 {
-		log.Info("Using Terraform Enterprise as the state/locks provider")
-		return NewTFE(c)
+func Configure(c *config.Config) ([]Provider, error) {
+	log.Infof("%+v\n", *c)
+	var providers []Provider
+	if len(c.TFE) > 0 {
+		log.Info("Using Terraform Enterprise as state/locks provider")
+		objs, err := NewTFE(c)
+		if err != nil {
+			return []Provider{}, err
+		}
+		for _, tfeObj := range objs {
+			providers = append(providers, tfeObj)
+		}
 	}
 
-	if c.GCP.GCSBuckets != nil {
-		log.Info("Using Google Cloud as the state/locks provider")
-		return NewGCP(c)
+	if len(c.GCP) > 0 {
+		log.Info("Using Google Cloud as state/locks provider")
+		objs, err := NewGCP(c)
+		if err != nil {
+			return []Provider{}, err
+		}
+		for _, gcpObj := range objs {
+			providers = append(providers, gcpObj)
+		}
 	}
 
-	if len(c.Gitlab.Token) > 0 {
-		log.Info("Using Gitab as the state/locks provider")
-		return NewGitlab(c), nil
+	if len(c.Gitlab) > 0 {
+		log.Info("Using Gitab as state/locks provider")
+		for _, glObj := range NewGitlab(c) {
+			providers = append(providers, glObj)
+		}
 	}
 
-	log.Info("Using AWS (S3+DynamoDB) as the state/locks provider")
-	provider := NewAWS(c)
-	return &provider, nil
+	if len(c.AWS) > 0 {
+		log.Info("Using AWS (S3+DynamoDB) as state/locks provider")
+		for _, awsObj := range NewAWS(c) {
+			log.Infof("AWS: %+v\n", *awsObj)
+			providers = append(providers, awsObj)
+		}
+	}
+
+	log.Infof("%+v\n", providers)
+	return providers, nil
 }
diff --git a/state/tfe.go b/state/tfe.go
index 301322b8..020ac16d 100644
--- a/state/tfe.go
+++ b/state/tfe.go
@@ -19,24 +19,29 @@ type TFE struct {
 }
 
 // NewTFE creates a new TFE object
-func NewTFE(c *config.Config) (*TFE, error) {
-	config := &tfe.Config{
-		Address: c.TFE.Address,
-		Token:   c.TFE.Token,
-	}
+func NewTFE(c *config.Config) ([]*TFE, error) {
+	var tfeInstances []*TFE
+	for _, tfeObj := range c.TFE {
+		config := &tfe.Config{
+			Address: tfeObj.Address,
+			Token:   tfeObj.Token,
+		}
 
-	client, err := tfe.NewClient(config)
-	if err != nil {
-		return nil, err
-	}
+		client, err := tfe.NewClient(config)
+		if err != nil {
+			return nil, err
+		}
 
-	ctx := context.Background()
+		ctx := context.Background()
+		instance := &TFE{
+			Client: client,
+			org:    tfeObj.Organization,
+			ctx:    &ctx,
+		}
+		tfeInstances = append(tfeInstances, instance)
+	}
 
-	return &TFE{
-		Client: client,
-		org:    c.TFE.Organization,
-		ctx:    &ctx,
-	}, nil
+	return tfeInstances, nil
 }
 
 // GetLocks returns a map of locks by State path
diff --git a/test/config.yml b/test/config.yml
new file mode 100644
index 00000000..0c4656de
--- /dev/null
+++ b/test/config.yml
@@ -0,0 +1,26 @@
+log:
+  level: debug
+  format: plain
+
+database:
+  password: mypassword
+
+provider:
+  no-locks: true
+  no-versioning: true
+
+aws:
+  - endpoint: http://minio:9000/
+    region: eu-west-1
+    s3:
+      bucket: test-bucket
+      force-path-style: true
+      file-extension: 
+        - .tfstate
+  - endpoint: http://minio:9000/
+    region: eu-west-1
+    s3:
+      bucket: test
+      force-path-style: true
+      file-extension: 
+        - .tfstate

From 21a29a71099e53d3460d949f9e5c0831f7bea13b Mon Sep 17 00:00:00 2001
From: Hugo Bollon <hugo.bollon@gmail.com>
Date: Thu, 24 Jun 2021 11:35:56 +0200
Subject: [PATCH 2/7] feat: env variables/flags available again for single
 provider configuration

---
 config/config.go        | 36 ++++++++++++++++++++++--
 state/aws.go            | 62 +++++++++++++++++++++--------------------
 state/gcp.go            | 44 +++++++++++++++--------------
 state/gitlab.go         |  8 ++++--
 state/state.go          | 37 ++++++++++++++----------
 state/tfe.go            | 30 ++++++++++----------
 test/docker-compose.yml | 21 ++++++++++++++
 7 files changed, 152 insertions(+), 86 deletions(-)

diff --git a/config/config.go b/config/config.go
index 349c5cf7..cc557427 100644
--- a/config/config.go
+++ b/config/config.go
@@ -119,14 +119,44 @@ func (c *Config) LoadConfigFromYaml() *Config {
 	return c
 }
 
-// LoadConfig loads the config from flags & environment
-func LoadConfig(version string) *Config {
+// Initialize Config with one obj per providers arrays
+// to allow usage of flags / env variables on single provider configuration
+func initDefaultConfig() Config {
 	var c Config
-	parser := flags.NewParser(&c, flags.Default)
+	var awsInitialConfig AWSConfig
+	var tfeInitialConfig TFEConfig
+	var gcpInitialConfig GCPConfig
+	var gitlabInitialConfig GitlabConfig
+
+	parseStructFlagsAndEnv(&awsInitialConfig)
+	c.AWS = append(c.AWS, awsInitialConfig)
+
+	parseStructFlagsAndEnv(&tfeInitialConfig)
+	c.TFE = append(c.TFE, tfeInitialConfig)
+
+	parseStructFlagsAndEnv(&gcpInitialConfig)
+	c.GCP = append(c.GCP, gcpInitialConfig)
+
+	parseStructFlagsAndEnv(&gitlabInitialConfig)
+	c.Gitlab = append(c.Gitlab, gitlabInitialConfig)
+
+	return c
+}
+
+// Parse flags and env variables to given struct using go-flags
+// parser
+func parseStructFlagsAndEnv(obj interface{}) {
+	parser := flags.NewParser(obj, flags.Default)
 	if _, err := parser.Parse(); err != nil {
 		fmt.Printf("Failed to parse flags: %s", err)
 		os.Exit(1)
 	}
+}
+
+// LoadConfig loads the config from flags & environment
+func LoadConfig(version string) *Config {
+	c := initDefaultConfig()
+	parseStructFlagsAndEnv(&c)
 
 	if c.ConfigFilePath != "" {
 		if _, err := os.Stat(c.ConfigFilePath); err == nil {
diff --git a/state/aws.go b/state/aws.go
index 69b88c4f..7d2d35ec 100644
--- a/state/aws.go
+++ b/state/aws.go
@@ -34,38 +34,40 @@ type AWS struct {
 func NewAWS(c *config.Config) []*AWS {
 	var awsInstances []*AWS
 	for _, aws := range c.AWS {
-		sess := session.Must(session.NewSession())
-		awsConfig := aws_sdk.NewConfig()
-		if len(aws.APPRoleArn) > 0 {
-			log.Debugf("Using %s role", aws.APPRoleArn)
-			creds := stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) {
-				if aws.ExternalID != "" {
-					p.ExternalID = aws_sdk.String(aws.ExternalID)
-				}
-			})
-			awsConfig.WithCredentials(creds)
-		}
+		if aws.S3.Bucket != "" {
+			sess := session.Must(session.NewSession())
+			awsConfig := aws_sdk.NewConfig()
+			if len(aws.APPRoleArn) > 0 {
+				log.Debugf("Using %s role", aws.APPRoleArn)
+				creds := stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) {
+					if aws.ExternalID != "" {
+						p.ExternalID = aws_sdk.String(aws.ExternalID)
+					}
+				})
+				awsConfig.WithCredentials(creds)
+			}
 
-		if e := aws.Endpoint; e != "" {
-			awsConfig.WithEndpoint(e)
-		}
-		if e := aws.Region; e != "" {
-			awsConfig.WithRegion(e)
-		}
-		awsConfig.S3ForcePathStyle = &aws.S3.ForcePathStyle
-
-		instance := &AWS{
-			svc:           s3.New(sess, awsConfig),
-			bucket:        aws.S3.Bucket,
-			keyPrefix:     aws.S3.KeyPrefix,
-			fileExtension: aws.S3.FileExtension,
-			dynamoSvc:     dynamodb.New(sess, awsConfig),
-			dynamoTable:   aws.DynamoDBTable,
-			noLocks:       c.Provider.NoLocks,
-			noVersioning:  c.Provider.NoVersioning,
+			if e := aws.Endpoint; e != "" {
+				awsConfig.WithEndpoint(e)
+			}
+			if e := aws.Region; e != "" {
+				awsConfig.WithRegion(e)
+			}
+			awsConfig.S3ForcePathStyle = &aws.S3.ForcePathStyle
+
+			instance := &AWS{
+				svc:           s3.New(sess, awsConfig),
+				bucket:        aws.S3.Bucket,
+				keyPrefix:     aws.S3.KeyPrefix,
+				fileExtension: aws.S3.FileExtension,
+				dynamoSvc:     dynamodb.New(sess, awsConfig),
+				dynamoTable:   aws.DynamoDBTable,
+				noLocks:       c.Provider.NoLocks,
+				noVersioning:  c.Provider.NoVersioning,
+			}
+			log.Debugf("Instance: %+v\n", *instance)
+			awsInstances = append(awsInstances, instance)
 		}
-		log.Debugf("Instance: %+v\n", *instance)
-		awsInstances = append(awsInstances, instance)
 	}
 
 	return awsInstances
diff --git a/state/gcp.go b/state/gcp.go
index a77e5c90..13e8cf7f 100644
--- a/state/gcp.go
+++ b/state/gcp.go
@@ -31,30 +31,32 @@ func NewGCP(c *config.Config) ([]*GCP, error) {
 	var gcpInstances []*GCP
 	var err error
 	for _, gcp := range c.GCP {
-		if gcp.GCPSAKey != "" {
-			log.WithFields(log.Fields{
-				"path": gcp.GCPSAKey,
-			}).Info("Authenticating using service account key")
-			opt := option.WithCredentialsFile(gcp.GCPSAKey)
-			client, err = storage.NewClient(ctx, opt) // Use service account key
-		} else {
-			client, err = storage.NewClient(ctx) // Use base credentials
-		}
+		if gcp.GCSBuckets != nil {
+			if gcp.GCPSAKey != "" {
+				log.WithFields(log.Fields{
+					"path": gcp.GCPSAKey,
+				}).Info("Authenticating using service account key")
+				opt := option.WithCredentialsFile(gcp.GCPSAKey)
+				client, err = storage.NewClient(ctx, opt) // Use service account key
+			} else {
+				client, err = storage.NewClient(ctx) // Use base credentials
+			}
 
-		if err != nil {
-			log.Fatalf("Failed to create client: %v", err)
-			return nil, err
-		}
+			if err != nil {
+				log.Fatalf("Failed to create client: %v", err)
+				return nil, err
+			}
 
-		instance := &GCP{
-			svc:     client,
-			buckets: gcp.GCSBuckets,
-		}
-		gcpInstances = append(gcpInstances, instance)
+			instance := &GCP{
+				svc:     client,
+				buckets: gcp.GCSBuckets,
+			}
+			gcpInstances = append(gcpInstances, instance)
 
-		log.WithFields(log.Fields{
-			"buckets": gcp.GCSBuckets,
-		}).Info("Client successfully created")
+			log.WithFields(log.Fields{
+				"buckets": gcp.GCSBuckets,
+			}).Info("Client successfully created")
+		}
 	}
 
 	return gcpInstances, nil
diff --git a/state/gitlab.go b/state/gitlab.go
index 79f86595..6f48cf2e 100644
--- a/state/gitlab.go
+++ b/state/gitlab.go
@@ -21,10 +21,12 @@ type Gitlab struct {
 func NewGitlab(c *config.Config) []*Gitlab {
 	var gitlabInstances []*Gitlab
 	for _, gl := range c.Gitlab {
-		instance := &Gitlab{
-			Client: gitlab.NewClient(gl.Address, gl.Token),
+		if gl.Token != "" {
+			instance := &Gitlab{
+				Client: gitlab.NewClient(gl.Address, gl.Token),
+			}
+			gitlabInstances = append(gitlabInstances, instance)
 		}
-		gitlabInstances = append(gitlabInstances, instance)
 	}
 	return gitlabInstances
 }
diff --git a/state/state.go b/state/state.go
index ef4198f7..7407abfa 100644
--- a/state/state.go
+++ b/state/state.go
@@ -41,45 +41,52 @@ type Provider interface {
 
 // Configure the state provider
 func Configure(c *config.Config) ([]Provider, error) {
-	log.Infof("%+v\n", *c)
 	var providers []Provider
 	if len(c.TFE) > 0 {
-		log.Info("Using Terraform Enterprise as state/locks provider")
 		objs, err := NewTFE(c)
 		if err != nil {
 			return []Provider{}, err
 		}
-		for _, tfeObj := range objs {
-			providers = append(providers, tfeObj)
+		if len(objs) > 0 {
+			log.Info("Using Terraform Enterprise as state/locks provider")
+			for _, tfeObj := range objs {
+				providers = append(providers, tfeObj)
+			}
 		}
 	}
 
 	if len(c.GCP) > 0 {
-		log.Info("Using Google Cloud as state/locks provider")
 		objs, err := NewGCP(c)
 		if err != nil {
 			return []Provider{}, err
 		}
-		for _, gcpObj := range objs {
-			providers = append(providers, gcpObj)
+		if len(objs) > 0 {
+			log.Info("Using Google Cloud as state/locks provider")
+			for _, gcpObj := range objs {
+				providers = append(providers, gcpObj)
+			}
 		}
 	}
 
 	if len(c.Gitlab) > 0 {
-		log.Info("Using Gitab as state/locks provider")
-		for _, glObj := range NewGitlab(c) {
-			providers = append(providers, glObj)
+		objs := NewGitlab(c)
+		if len(objs) > 0 {
+			log.Info("Using Gitab as state/locks provider")
+			for _, glObj := range objs {
+				providers = append(providers, glObj)
+			}
 		}
 	}
 
 	if len(c.AWS) > 0 {
-		log.Info("Using AWS (S3+DynamoDB) as state/locks provider")
-		for _, awsObj := range NewAWS(c) {
-			log.Infof("AWS: %+v\n", *awsObj)
-			providers = append(providers, awsObj)
+		objs := NewAWS(c)
+		if len(objs) > 0 {
+			log.Info("Using AWS (S3+DynamoDB) as state/locks provider")
+			for _, awsObj := range objs {
+				providers = append(providers, awsObj)
+			}
 		}
 	}
 
-	log.Infof("%+v\n", providers)
 	return providers, nil
 }
diff --git a/state/tfe.go b/state/tfe.go
index 020ac16d..ed227ce1 100644
--- a/state/tfe.go
+++ b/state/tfe.go
@@ -22,23 +22,25 @@ type TFE struct {
 func NewTFE(c *config.Config) ([]*TFE, error) {
 	var tfeInstances []*TFE
 	for _, tfeObj := range c.TFE {
-		config := &tfe.Config{
-			Address: tfeObj.Address,
-			Token:   tfeObj.Token,
-		}
+		if tfeObj.Token != "" {
+			config := &tfe.Config{
+				Address: tfeObj.Address,
+				Token:   tfeObj.Token,
+			}
 
-		client, err := tfe.NewClient(config)
-		if err != nil {
-			return nil, err
-		}
+			client, err := tfe.NewClient(config)
+			if err != nil {
+				return nil, err
+			}
 
-		ctx := context.Background()
-		instance := &TFE{
-			Client: client,
-			org:    tfeObj.Organization,
-			ctx:    &ctx,
+			ctx := context.Background()
+			instance := &TFE{
+				Client: client,
+				org:    tfeObj.Organization,
+				ctx:    &ctx,
+			}
+			tfeInstances = append(tfeInstances, instance)
 		}
-		tfeInstances = append(tfeInstances, instance)
 	}
 
 	return tfeInstances, nil
diff --git a/test/docker-compose.yml b/test/docker-compose.yml
index 4f187dd2..a1d50bdf 100755
--- a/test/docker-compose.yml
+++ b/test/docker-compose.yml
@@ -28,6 +28,27 @@ services:
     ports:
       - "8080:8080"
 
+  # terraboard-dev-multi-providers:
+  #   build:
+  #     context: ../
+  #     dockerfile: ./Dockerfile
+  #   environment:
+  #     AWS_ACCESS_KEY_ID: root
+  #     AWS_SECRET_ACCESS_KEY: mypassword
+  #     DB_SSLMODE: disable
+  #     CONFIG_FILE: config/config.yml
+  #     GODEBUG: netdns=go
+  #   depends_on:
+  #     db:
+  #       condition: service_healthy
+  #     minio:
+  #       condition: service_started
+  #   volumes:
+  #     - ../static:/static:ro
+  #     - ./config.yml:/config/config.yml:ro
+  #   ports:
+  #     - "8081:8081"
+
   minio:
     image: minio/minio:latest
     environment:

From 83f0727b1032dccddbe2d41ffa8a175d47ceebba Mon Sep 17 00:00:00 2001
From: Hugo Bollon <hugo.bollon@gmail.com>
Date: Thu, 24 Jun 2021 14:45:01 +0200
Subject: [PATCH 3/7] feat: add some custom yaml unmarshalling rules to add
 default values where go-flags ones aren't applicable

---
 config/yaml.go          | 33 +++++++++++++++++++++++++++++++++
 state/aws.go            |  1 -
 test/docker-compose.yml |  2 +-
 3 files changed, 34 insertions(+), 2 deletions(-)
 create mode 100644 config/yaml.go

diff --git a/config/yaml.go b/config/yaml.go
new file mode 100644
index 00000000..f5e8cc3a
--- /dev/null
+++ b/config/yaml.go
@@ -0,0 +1,33 @@
+package config
+
+/*********************************************
+ * Custom UnmarshalYAML used to define some struct fields
+ * default values where go-flags ones aren't applicable
+ * (and so makes them optional)
+ *********************************************/
+
+func (s *S3BucketConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawS3BucketConfig S3BucketConfig
+	raw := rawS3BucketConfig{
+		FileExtension: []string{".tfstate"},
+	}
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*s = S3BucketConfig(raw)
+	return nil
+}
+
+func (s *GitlabConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawGitlabConfig GitlabConfig
+	raw := GitlabConfig{
+		Address: "https://gitlab.com",
+	}
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*s = GitlabConfig(raw)
+	return nil
+}
diff --git a/state/aws.go b/state/aws.go
index 7d2d35ec..9c3181df 100644
--- a/state/aws.go
+++ b/state/aws.go
@@ -65,7 +65,6 @@ func NewAWS(c *config.Config) []*AWS {
 				noLocks:       c.Provider.NoLocks,
 				noVersioning:  c.Provider.NoVersioning,
 			}
-			log.Debugf("Instance: %+v\n", *instance)
 			awsInstances = append(awsInstances, instance)
 		}
 	}
diff --git a/test/docker-compose.yml b/test/docker-compose.yml
index a1d50bdf..37c59d00 100755
--- a/test/docker-compose.yml
+++ b/test/docker-compose.yml
@@ -47,7 +47,7 @@ services:
   #     - ../static:/static:ro
   #     - ./config.yml:/config/config.yml:ro
   #   ports:
-  #     - "8081:8081"
+  #     - "8081:8080"
 
   minio:
     image: minio/minio:latest

From f719332aaff12449b9a75c4328f08e171038a516 Mon Sep 17 00:00:00 2001
From: Hugo Bollon <hugo.bollon@gmail.com>
Date: Thu, 24 Jun 2021 14:47:07 +0200
Subject: [PATCH 4/7] docs: update readme with multiple buckets/providers
 config advices

---
 README.md                                     | 45 +++++++++++++++++--
 config/yaml.go                                |  2 +-
 example.yml                                   |  4 +-
 main.go                                       |  6 +--
 test/.gitignore                               |  3 +-
 test/config.yml                               |  2 +-
 .../terraform_1.0.0_bucket2.tfstate           | 37 +++++++++++++++
 7 files changed, 88 insertions(+), 11 deletions(-)
 create mode 100644 test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate

diff --git a/README.md b/README.md
index 52846c7e..956032ac 100644
--- a/README.md
+++ b/README.md
@@ -41,6 +41,7 @@
     - [AWS S3 (state) + DynamoDB (lock)](#aws-s3-state--dynamodb-lock)
     - [Terraform Cloud](#terraform-cloud)
 - [Configuration](#configuration)
+  - [Multiple buckets/providers](#multiple-bucketsproviders)
   - [Available parameters](#available-parameters)
     - [Application Options](#application-options)
     - [General Provider Options](#general-provider-options)
@@ -89,6 +90,9 @@ It currently supports several remote state backend providers:
 - [Terraform Cloud (remote)](https://www.terraform.io/docs/backends/types/remote.html)
 - [GitLab](https://docs.gitlab.com/ee/user/infrastructure/terraform_state.html)
 
+With the upcoming **v1.2.0** update, Terraboard will be now able to handle multiple buckets/providers configuration! 🥳
+Check *configuration* section for more details. 
+
 ### Overview
 
 The overview presents all the state files in the S3 bucket, by most recent
@@ -142,12 +146,47 @@ Data resiliency is not paramount though as this dataset can be rebuilt upon your
 
 Terraboard currently supports configuration in three different ways:
 
-1. Environment variables
-2. CLI parameters
-3. Configuration file (YAML). A configuration file example can be found in the root directory of this repository.
+1. Environment variables **(only usable for mono provider configuration)**
+2. CLI parameters **(only usable for mono provider configuration)**
+3. Configuration file (YAML). A configuration file example can be found in the root directory of this repository and in the `test/` subdirectory.
+
+**Important: all flags/environment variables related to the providers settings aren't compatible with multi-provider configuration! For that, you must use the Yaml config file to be able to configure multiples buckets/providers.**
 
 The precedence of configurations is as described below.
 
+### Multiple buckets/providers
+
+In order to be able to link to Terraboard multiples buckets or even providers, you must use the Yaml configuration method.
+
+- Set the environment variable **CONFIG_FILE** or the flag **-c** / **--config-file** to a valid Yaml config file.
+
+- In the Yaml file, specify your desired providers configuration. For example with two S3 buckets:
+
+```yaml
+aws:
+  - endpoint: http://minio:9000/
+    region: eu-west-1
+    s3:
+      bucket: test-bucket
+      force-path-style: true
+      file-extension: 
+        - .tfstate
+
+  - endpoint: http://minio:9000/
+    region: eu-west-1
+    s3:
+      bucket: test-bucket2
+      force-path-style: true
+      file-extension: 
+        - .tfstate
+
+```
+
+In the case of AWS, don't forget to set **AWS_ACCESS_KEY_ID** and **AWS_SECRET_ACCESS_KEY** environment variables.
+
+That's it! Terraboard will now fetch these two buckets on DB refresh. You can also mix providers like AWS and Gitlab or anything else.
+You can find a ready-to-use docker example with two *MinIO* buckets in the `test/` sub-folder (just swipe the two terraboard services in the docker-compose file). 
+
 ### Available parameters
 
 #### Application Options
diff --git a/config/yaml.go b/config/yaml.go
index f5e8cc3a..8e6f453f 100644
--- a/config/yaml.go
+++ b/config/yaml.go
@@ -21,7 +21,7 @@ func (s *S3BucketConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
 
 func (s *GitlabConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawGitlabConfig GitlabConfig
-	raw := GitlabConfig{
+	raw := rawGitlabConfig{
 		Address: "https://gitlab.com",
 	}
 	if err := unmarshal(&raw); err != nil {
diff --git a/example.yml b/example.yml
index 75bbca38..d7d7d5fb 100644
--- a/example.yml
+++ b/example.yml
@@ -12,8 +12,8 @@ database:
   sync-interval: 5
 
 provider:
-  no-locks: "true"
-  no-versioning: "false"
+  no-locks: true
+  no-versioning: false
 
 aws:
   dynamodb-table: terraboard
diff --git a/main.go b/main.go
index cdb6c20e..245f21bc 100644
--- a/main.go
+++ b/main.go
@@ -45,7 +45,7 @@ func handleWithDB(apiF func(w http.ResponseWriter, r *http.Request,
 	})
 }
 
-func handleWithStateProvider(apiF func(w http.ResponseWriter, r *http.Request,
+func handleWithStateProviders(apiF func(w http.ResponseWriter, r *http.Request,
 	sps []state.Provider), sps []state.Provider) func(http.ResponseWriter, *http.Request) {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		apiF(w, r, sps)
@@ -64,7 +64,7 @@ func isKnownStateVersion(statesVersions map[string][]string, versionID, path str
 }
 
 // Refresh the DB
-// This should be the only direct bridge between the state provider and the DB
+// This should be the only direct bridge between the state providers and the DB
 func refreshDB(syncInterval uint16, d *db.Database, sps []state.Provider) {
 	interval := time.Duration(syncInterval) * time.Minute
 	log.Debugf("Providers: %+v\n", sps)
@@ -194,7 +194,7 @@ func main() {
 	http.HandleFunc(util.GetFullPath("api/state/"), handleWithDB(api.GetState, database))
 	http.HandleFunc(util.GetFullPath("api/state/activity/"), handleWithDB(api.GetStateActivity, database))
 	http.HandleFunc(util.GetFullPath("api/state/compare/"), handleWithDB(api.StateCompare, database))
-	http.HandleFunc(util.GetFullPath("api/locks"), handleWithStateProvider(api.GetLocks, sps))
+	http.HandleFunc(util.GetFullPath("api/locks"), handleWithStateProviders(api.GetLocks, sps))
 	http.HandleFunc(util.GetFullPath("api/search/attribute"), handleWithDB(api.SearchAttribute, database))
 	http.HandleFunc(util.GetFullPath("api/resource/types"), handleWithDB(api.ListResourceTypes, database))
 	http.HandleFunc(util.GetFullPath("api/resource/types/count"), handleWithDB(api.ListResourceTypesWithCount, database))
diff --git a/test/.gitignore b/test/.gitignore
index 24b61d13..acb9efef 100755
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -1,2 +1,3 @@
 data/*
-!data/test-bucket
\ No newline at end of file
+!data/test-bucket
+!data/test-bucket2
\ No newline at end of file
diff --git a/test/config.yml b/test/config.yml
index 0c4656de..5999c435 100644
--- a/test/config.yml
+++ b/test/config.yml
@@ -20,7 +20,7 @@ aws:
   - endpoint: http://minio:9000/
     region: eu-west-1
     s3:
-      bucket: test
+      bucket: test-bucket2
       force-path-style: true
       file-extension: 
         - .tfstate
diff --git a/test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate b/test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate
new file mode 100644
index 00000000..a3835bd3
--- /dev/null
+++ b/test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate
@@ -0,0 +1,37 @@
+{
+  "version": 4,
+  "terraform_version": "1.0.0",
+  "serial": 3,
+  "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00",
+  "outputs": {
+    "content": {
+      "value": "foo!",
+      "type": "string"
+    }
+  },
+  "resources": [
+    {
+      "mode": "managed",
+      "type": "local_file",
+      "name": "foo",
+      "provider": "provider[\"registry.terraform.io/hashicorp/local\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "content": "foo!",
+            "content_base64": null,
+            "directory_permission": "0777",
+            "file_permission": "0777",
+            "filename": "./foo.bar",
+            "id": "4bf3e335199107182c6f7638efaad377acc7f452",
+            "sensitive_content": null,
+            "source": null
+          },
+          "sensitive_attributes": [],
+          "private": "bnVsbA=="
+        }
+      ]
+    }
+  ]
+}

From 65e6f3969838c5de799e0c6dc4451e4bbf056317 Mon Sep 17 00:00:00 2001
From: Hugo Bollon <hugo.bollon@gmail.com>
Date: Fri, 25 Jun 2021 10:34:07 +0200
Subject: [PATCH 5/7] feat: parallelized db refreshing and add mutex security +
 code review fixes from #177

---
 README.md                                     | 19 ++--
 db/db.go                                      | 11 ++-
 main.go                                       | 96 +++++++++----------
 test/{ => multiple-minio-buckets}/Makefile    |  1 +
 test/{ => multiple-minio-buckets}/config.yml  |  0
 .../multiple-minio-buckets/docker-compose.yml | 64 +++++++++++++
 test/single-minio-bucket/Makefile             | 11 +++
 .../docker-compose.yml                        | 27 +-----
 8 files changed, 146 insertions(+), 83 deletions(-)
 rename test/{ => multiple-minio-buckets}/Makefile (99%)
 rename test/{ => multiple-minio-buckets}/config.yml (100%)
 create mode 100644 test/multiple-minio-buckets/docker-compose.yml
 create mode 100644 test/single-minio-bucket/Makefile
 rename test/{ => single-minio-bucket}/docker-compose.yml (70%)

diff --git a/README.md b/README.md
index 956032ac..1f3e979d 100644
--- a/README.md
+++ b/README.md
@@ -150,19 +150,23 @@ Terraboard currently supports configuration in three different ways:
 2. CLI parameters **(only usable for mono provider configuration)**
 3. Configuration file (YAML). A configuration file example can be found in the root directory of this repository and in the `test/` subdirectory.
 
-**Important: all flags/environment variables related to the providers settings aren't compatible with multi-provider configuration! For that, you must use the Yaml config file to be able to configure multiples buckets/providers.**
+**Important: all flags/environment variables related to the providers settings aren't compatible with multi-provider configuration! Instead, you must use the YAML config file to be able to configure multiples buckets/providers.**
 
 The precedence of configurations is as described below.
 
 ### Multiple buckets/providers
 
-In order to be able to link to Terraboard multiples buckets or even providers, you must use the Yaml configuration method.
+In order for Terraboard to import states from multiples buckets or even providers, you must use the YAML configuration method:
 
-- Set the environment variable **CONFIG_FILE** or the flag **-c** / **--config-file** to a valid Yaml config file.
-
-- In the Yaml file, specify your desired providers configuration. For example with two S3 buckets:
+- Set the `CONFIG_FILE` environment variable or the `-c`/`--config-file` flag to point to a valid YAML config file.
+- In the YAML file, specify your desired providers configuration. For example with two MinIO buckets (using the AWS provider with compatible mode):
 
 ```yaml
+# Needed since MinIO doesn't support versioning or locking
+provider:
+  no-locks: true
+  no-versioning: true
+
 aws:
   - endpoint: http://minio:9000/
     region: eu-west-1
@@ -179,13 +183,12 @@ aws:
       force-path-style: true
       file-extension: 
         - .tfstate
-
 ```
 
-In the case of AWS, don't forget to set **AWS_ACCESS_KEY_ID** and **AWS_SECRET_ACCESS_KEY** environment variables.
+In the case of AWS, don't forget to set the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables.
 
 That's it! Terraboard will now fetch these two buckets on DB refresh. You can also mix providers like AWS and Gitlab or anything else.
-You can find a ready-to-use docker example with two *MinIO* buckets in the `test/` sub-folder (just swipe the two terraboard services in the docker-compose file). 
+You can find a ready-to-use Docker example with two *MinIO* buckets in the `test/multiple-minio-buckets/` sub-folder. 
 
 ### Available parameters
 
diff --git a/db/db.go b/db/db.go
index 0c2f86f3..2262d506 100644
--- a/db/db.go
+++ b/db/db.go
@@ -7,6 +7,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/camptocamp/terraboard/config"
 	"github.com/camptocamp/terraboard/state"
@@ -25,6 +26,7 @@ import (
 // Database is a wrapping structure to *gorm.DB
 type Database struct {
 	*gorm.DB
+	lock sync.Mutex
 }
 
 var pageSize = 20
@@ -79,7 +81,7 @@ func Init(config config.DBConfig, debug bool) *Database {
 		db.Config.Logger.LogMode(logger.Info)
 	}
 
-	d := &Database{db}
+	d := &Database{DB: db}
 	if err = d.MigrateLineage(); err != nil {
 		log.Fatalf("Lineage migration failed: %v\n", err)
 	}
@@ -124,11 +126,14 @@ func (db *Database) stateS3toDB(sf *statefile.File, path string, versionID strin
 	// Check if the associated lineage is already present in lineages table
 	// If so, it recovers its ID otherwise it inserts it at the same time as the state
 	var lineage types.Lineage
+	db.lock.Lock()
 	err = db.FirstOrCreate(&lineage, types.Lineage{Value: sf.Lineage}).Error
 	if err != nil || lineage.ID == 0 {
-		log.Error("Unknown error in stateS3toDB during lineage finding")
+		log.WithField("error", err).
+			Error("Unknown error in stateS3toDB during lineage finding", err)
 		return types.State{}, err
 	}
+	db.lock.Unlock()
 
 	st = types.State{
 		Path:      path,
@@ -242,10 +247,12 @@ func (db *Database) UpdateState(st types.State) error {
 // InsertVersion inserts an AWS S3 Version in the Database
 func (db *Database) InsertVersion(version *state.Version) error {
 	var v types.Version
+	db.lock.Lock()
 	db.FirstOrCreate(&v, types.Version{
 		VersionID:    version.ID,
 		LastModified: version.LastModified,
 	})
+	db.lock.Unlock()
 	return nil
 }
 
diff --git a/main.go b/main.go
index 245f21bc..1e96b2fc 100644
--- a/main.go
+++ b/main.go
@@ -65,60 +65,55 @@ func isKnownStateVersion(statesVersions map[string][]string, versionID, path str
 
 // Refresh the DB
 // This should be the only direct bridge between the state providers and the DB
-func refreshDB(syncInterval uint16, d *db.Database, sps []state.Provider) {
+func refreshDB(syncInterval uint16, d *db.Database, sp state.Provider) {
 	interval := time.Duration(syncInterval) * time.Minute
-	log.Debugf("Providers: %+v\n", sps)
 	for {
 		log.Infof("Refreshing DB")
-		log.Debugf("Total providers: %d\n", len(sps))
-		for i, sp := range sps {
-			log.Debugf("Fetching provider %d/%d\n", i+1, len(sps))
-			states, err := sp.GetStates()
-			if err != nil {
-				log.WithFields(log.Fields{
-					"error": err,
-				}).Error("Failed to retrieve states. Retrying in 1 minute.")
-				time.Sleep(interval)
-				continue
-			}
+		states, err := sp.GetStates()
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+			}).Error("Failed to retrieve states. Retrying in 1 minute.")
+			time.Sleep(interval)
+			continue
+		}
 
-			statesVersions := d.ListStatesVersions()
-			for _, st := range states {
-				versions, _ := sp.GetVersions(st)
-				for k, v := range versions {
-					if _, ok := statesVersions[v.ID]; ok {
-						log.WithFields(log.Fields{
-							"version_id": v.ID,
-						}).Debug("Version is already in the database, skipping")
-					} else {
-						if err := d.InsertVersion(&versions[k]); err != nil {
-							log.Error(err.Error())
-						}
+		statesVersions := d.ListStatesVersions()
+		for _, st := range states {
+			versions, _ := sp.GetVersions(st)
+			for k, v := range versions {
+				if _, ok := statesVersions[v.ID]; ok {
+					log.WithFields(log.Fields{
+						"version_id": v.ID,
+					}).Debug("Version is already in the database, skipping")
+				} else {
+					if err := d.InsertVersion(&versions[k]); err != nil {
+						log.Error(err.Error())
 					}
+				}
 
-					if isKnownStateVersion(statesVersions, v.ID, st) {
-						log.WithFields(log.Fields{
-							"path":       st,
-							"version_id": v.ID,
-						}).Debug("State is already in the database, skipping")
-						continue
-					}
-					state, err := sp.GetState(st, v.ID)
-					if err != nil {
-						log.WithFields(log.Fields{
-							"path":       st,
-							"version_id": v.ID,
-							"error":      err,
-						}).Error("Failed to fetch state from bucket")
-						continue
-					}
-					if err = d.InsertState(st, v.ID, state); err != nil {
-						log.WithFields(log.Fields{
-							"path":       st,
-							"version_id": v.ID,
-							"error":      err,
-						}).Error("Failed to insert state in the database")
-					}
+				if isKnownStateVersion(statesVersions, v.ID, st) {
+					log.WithFields(log.Fields{
+						"path":       st,
+						"version_id": v.ID,
+					}).Debug("State is already in the database, skipping")
+					continue
+				}
+				state, err := sp.GetState(st, v.ID)
+				if err != nil {
+					log.WithFields(log.Fields{
+						"path":       st,
+						"version_id": v.ID,
+						"error":      err,
+					}).Error("Failed to fetch state from bucket")
+					continue
+				}
+				if err = d.InsertState(st, v.ID, state); err != nil {
+					log.WithFields(log.Fields{
+						"path":       st,
+						"version_id": v.ID,
+						"error":      err,
+					}).Error("Failed to insert state in the database")
 				}
 			}
 		}
@@ -173,7 +168,10 @@ func main() {
 	if c.DB.NoSync {
 		log.Infof("Not syncing database, as requested.")
 	} else {
-		go refreshDB(c.DB.SyncInterval, database, sps)
+		log.Debugf("Total providers: %d\n", len(sps))
+		for _, sp := range sps {
+			go refreshDB(c.DB.SyncInterval, database, sp)
+		}
 	}
 	defer database.Close()
 
diff --git a/test/Makefile b/test/multiple-minio-buckets/Makefile
similarity index 99%
rename from test/Makefile
rename to test/multiple-minio-buckets/Makefile
index b59d9d4d..5e30c909 100644
--- a/test/Makefile
+++ b/test/multiple-minio-buckets/Makefile
@@ -1,3 +1,4 @@
+
 UID=$(shell id -u)
 GID=$(shell id -g)
 
diff --git a/test/config.yml b/test/multiple-minio-buckets/config.yml
similarity index 100%
rename from test/config.yml
rename to test/multiple-minio-buckets/config.yml
diff --git a/test/multiple-minio-buckets/docker-compose.yml b/test/multiple-minio-buckets/docker-compose.yml
new file mode 100644
index 00000000..51b4d2e3
--- /dev/null
+++ b/test/multiple-minio-buckets/docker-compose.yml
@@ -0,0 +1,64 @@
+---
+version: "3.8"
+services:
+  terraboard-dev:
+    build:
+      context: ../../
+      dockerfile: ./Dockerfile
+    environment:
+      AWS_ACCESS_KEY_ID: root
+      AWS_SECRET_ACCESS_KEY: mypassword
+      DB_SSLMODE: disable
+      CONFIG_FILE: config/config.yml
+      GODEBUG: netdns=go
+    depends_on:
+      db:
+        condition: service_healthy
+      minio:
+        condition: service_started
+    volumes:
+      - ../../static:/static:ro
+      - ./config.yml:/config/config.yml:ro
+    ports:
+      - "8080:8080"
+
+  minio:
+    image: minio/minio:latest
+    environment:
+      MINIO_ROOT_USER: root
+      MINIO_ROOT_PASSWORD: mypassword
+    user: "${UID}:${GID}"
+    expose: 
+      - "9000"
+    ports:
+      - "9200:9000"
+    volumes:
+      - ../data:/data
+    command: server /data
+
+  db:
+    image: postgres:9.5
+    environment:
+      POSTGRES_USER: gorm
+      POSTGRES_PASSWORD: mypassword
+      POSTGRES_DB: gorm
+    volumes:
+      - tb-data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  
+  pgadmin:
+    container_name: pgadmin4_container
+    image: dpage/pgadmin4
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin@admin.com
+      PGADMIN_DEFAULT_PASSWORD: root
+    ports:
+      - "5050:80"
+
+volumes:
+  tb-data: {}
diff --git a/test/single-minio-bucket/Makefile b/test/single-minio-bucket/Makefile
new file mode 100644
index 00000000..3edb92d0
--- /dev/null
+++ b/test/single-minio-bucket/Makefile
@@ -0,0 +1,11 @@
+
+UID=$(shell id -u)
+GID=$(shell id -g)
+
+build:
+	UID="${UID}" GID="${GID}" docker-compose build 
+
+test:
+	UID="${UID}" GID="${GID}" docker-compose up -d
+
+all: build test
\ No newline at end of file
diff --git a/test/docker-compose.yml b/test/single-minio-bucket/docker-compose.yml
similarity index 70%
rename from test/docker-compose.yml
rename to test/single-minio-bucket/docker-compose.yml
index 37c59d00..f8785745 100755
--- a/test/docker-compose.yml
+++ b/test/single-minio-bucket/docker-compose.yml
@@ -3,7 +3,7 @@ version: "3.8"
 services:
   terraboard-dev:
     build:
-      context: ../
+      context: ../../
       dockerfile: ./Dockerfile
     environment:
       AWS_ACCESS_KEY_ID: root
@@ -24,31 +24,10 @@ services:
       minio:
         condition: service_started
     volumes:
-      - ../static:/static:ro
+      - ../../static:/static:ro
     ports:
       - "8080:8080"
 
-  # terraboard-dev-multi-providers:
-  #   build:
-  #     context: ../
-  #     dockerfile: ./Dockerfile
-  #   environment:
-  #     AWS_ACCESS_KEY_ID: root
-  #     AWS_SECRET_ACCESS_KEY: mypassword
-  #     DB_SSLMODE: disable
-  #     CONFIG_FILE: config/config.yml
-  #     GODEBUG: netdns=go
-  #   depends_on:
-  #     db:
-  #       condition: service_healthy
-  #     minio:
-  #       condition: service_started
-  #   volumes:
-  #     - ../static:/static:ro
-  #     - ./config.yml:/config/config.yml:ro
-  #   ports:
-  #     - "8081:8080"
-
   minio:
     image: minio/minio:latest
     environment:
@@ -60,7 +39,7 @@ services:
     ports:
       - "9200:9000"
     volumes:
-      - ./data:/data
+      - ../data:/data
     command: server /data
 
   db:

From dae5a1ed5b8e950c9a87ff77765f4417d87ef53d Mon Sep 17 00:00:00 2001
From: Hugo Bollon <hugo.bollon@gmail.com>
Date: Mon, 5 Jul 2021 15:28:16 +0200
Subject: [PATCH 6/7] feat: add aws credentials & session token config support
 * allow aws multiple buckets to be configured directly in s3 yaml field (now
 an objects array) * add another minio instance to multiple-minio-buckets
 testing env * update all yaml example config files

---
 config/config.go                              | 19 ++++--
 config/config_test.go                         |  8 ++-
 config/config_test.yml                        | 12 ++--
 example.yml                                   | 12 ++--
 state/aws.go                                  | 65 +++++++++++--------
 test/data/test-bucket/.gitkeep                |  0
 .../test-bucket/terraform_0.12.28.tfstate     | 36 ----------
 .../data/test-bucket/terraform_0.13.5.tfstate | 36 ----------
 .../data/test-bucket/terraform_0.14.8.tfstate | 37 -----------
 .../data/test-bucket/terraform_0.15.5.tfstate | 37 -----------
 test/data/test-bucket/terraform_1.0.0.tfstate | 37 -----------
 .../terraform_1.0.0_bucket2.tfstate           | 37 -----------
 test/multiple-minio-buckets/config.yml        | 29 ++++++---
 .../multiple-minio-buckets/docker-compose.yml | 18 ++++-
 test/single-minio-bucket/docker-compose.yml   |  2 +-
 15 files changed, 104 insertions(+), 281 deletions(-)
 delete mode 100644 test/data/test-bucket/.gitkeep
 delete mode 100644 test/data/test-bucket/terraform_0.12.28.tfstate
 delete mode 100644 test/data/test-bucket/terraform_0.13.5.tfstate
 delete mode 100644 test/data/test-bucket/terraform_0.14.8.tfstate
 delete mode 100644 test/data/test-bucket/terraform_0.15.5.tfstate
 delete mode 100644 test/data/test-bucket/terraform_1.0.0.tfstate
 delete mode 100644 test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate

diff --git a/config/config.go b/config/config.go
index cc557427..ec47f387 100644
--- a/config/config.go
+++ b/config/config.go
@@ -40,12 +40,15 @@ type S3BucketConfig struct {
 
 // AWSConfig stores the DynamoDB table and S3 Bucket configuration
 type AWSConfig struct {
-	DynamoDBTable string         `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."`
-	S3            S3BucketConfig `group:"S3 Options" yaml:"s3"`
-	Endpoint      string         `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."`
-	Region        string         `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."`
-	APPRoleArn    string         `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."`
-	ExternalID    string         `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."`
+	AccessKey       string           `long:"aws-access-key" env:"AWS_ACCESS_KEY_ID" yaml:"access-key" description:"AWS account access key."`
+	SecretAccessKey string           `long:"aws-secret-access-key" env:"AWS_SECRET_ACCESS_KEY" yaml:"secret-access-key" description:"AWS secret account access key."`
+	SessionToken    string           `long:"aws-session-token" env:"AWS_SESSION_TOKEN" yaml:"session-token" description:"AWS session token."`
+	DynamoDBTable   string           `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."`
+	S3              []S3BucketConfig `group:"S3 Options" yaml:"s3"`
+	Endpoint        string           `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."`
+	Region          string           `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."`
+	APPRoleArn      string           `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."`
+	ExternalID      string           `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."`
 }
 
 // TFEConfig stores the Terraform Enterprise configuration
@@ -124,6 +127,7 @@ func (c *Config) LoadConfigFromYaml() *Config {
 func initDefaultConfig() Config {
 	var c Config
 	var awsInitialConfig AWSConfig
+	var s3InitialConfig S3BucketConfig
 	var tfeInitialConfig TFEConfig
 	var gcpInitialConfig GCPConfig
 	var gitlabInitialConfig GitlabConfig
@@ -131,6 +135,9 @@ func initDefaultConfig() Config {
 	parseStructFlagsAndEnv(&awsInitialConfig)
 	c.AWS = append(c.AWS, awsInitialConfig)
 
+	parseStructFlagsAndEnv(&s3InitialConfig)
+	c.AWS[0].S3 = append(c.AWS[0].S3, s3InitialConfig)
+
 	parseStructFlagsAndEnv(&tfeInitialConfig)
 	c.TFE = append(c.TFE, tfeInitialConfig)
 
diff --git a/config/config_test.go b/config/config_test.go
index fadda18d..3e2081c9 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -24,13 +24,15 @@ func TestLoadConfigFromYaml(t *testing.T) {
 		},
 		AWS: []AWSConfig{
 			{
-				DynamoDBTable: "terraboard-dynamodb",
-				S3: S3BucketConfig{
+				AccessKey:       "root",
+				SecretAccessKey: "mypassword",
+				DynamoDBTable:   "terraboard-dynamodb",
+				S3: []S3BucketConfig{{
 					Bucket:         "terraboard-bucket",
 					KeyPrefix:      "test/",
 					FileExtension:  []string{".tfstate"},
 					ForcePathStyle: true,
-				},
+				}},
 			},
 		},
 		TFE: []TFEConfig{
diff --git a/config/config_test.yml b/config/config_test.yml
index 4f7d7d32..6f4610e5 100644
--- a/config/config_test.yml
+++ b/config/config_test.yml
@@ -11,12 +11,14 @@ database:
   no-sync: true
 
 aws:
-  - dynamodb-table: terraboard-dynamodb
+  - access-key: root
+    secret-access-key: mypassword
+    dynamodb-table: terraboard-dynamodb
     s3:
-      bucket: terraboard-bucket
-      key-prefix: test/
-      file-extension: [.tfstate]
-      force-path-style: true
+      - bucket: terraboard-bucket
+        key-prefix: test/
+        file-extension: [.tfstate]
+        force-path-style: true
 
 tfe:
   - address: https://tfe.example.com
diff --git a/example.yml b/example.yml
index d7d7d5fb..79346f60 100644
--- a/example.yml
+++ b/example.yml
@@ -16,11 +16,13 @@ provider:
   no-versioning: false
 
 aws:
-  dynamodb-table: terraboard
-  s3:
-    bucket: terraboard
-    key-prefix:
-    file-extension: .tfstate
+  - access-key: root
+    secret-access-key: mypassword
+    dynamodb-table: terraboard
+    s3:
+      - bucket: terraboard
+        key-prefix:
+        file-extension: .tfstate
 
 web:
   port: 9090
diff --git a/state/aws.go b/state/aws.go
index 9c3181df..5a9e8108 100644
--- a/state/aws.go
+++ b/state/aws.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	aws_sdk "github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/dynamodb"
@@ -34,38 +35,46 @@ type AWS struct {
 func NewAWS(c *config.Config) []*AWS {
 	var awsInstances []*AWS
 	for _, aws := range c.AWS {
-		if aws.S3.Bucket != "" {
-			sess := session.Must(session.NewSession())
-			awsConfig := aws_sdk.NewConfig()
-			if len(aws.APPRoleArn) > 0 {
-				log.Debugf("Using %s role", aws.APPRoleArn)
-				creds := stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) {
-					if aws.ExternalID != "" {
-						p.ExternalID = aws_sdk.String(aws.ExternalID)
+		for _, bucket := range aws.S3 {
+			if bucket.Bucket != "" {
+				sess := session.Must(session.NewSession())
+				awsConfig := aws_sdk.NewConfig()
+				var creds *credentials.Credentials
+				if len(aws.APPRoleArn) > 0 {
+					log.Debugf("Using %s role", aws.APPRoleArn)
+					creds = stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) {
+						if aws.ExternalID != "" {
+							p.ExternalID = aws_sdk.String(aws.ExternalID)
+						}
+					})
+				} else {
+					if aws.AccessKey == "" || aws.SecretAccessKey == "" {
+						log.Fatal("Missing AccessKey or SecretAccessKey for AWS provider. Please check your configuration and retry")
 					}
-				})
+					creds = credentials.NewStaticCredentials(aws.AccessKey, aws.SecretAccessKey, aws.SessionToken)
+				}
 				awsConfig.WithCredentials(creds)
-			}
 
-			if e := aws.Endpoint; e != "" {
-				awsConfig.WithEndpoint(e)
-			}
-			if e := aws.Region; e != "" {
-				awsConfig.WithRegion(e)
-			}
-			awsConfig.S3ForcePathStyle = &aws.S3.ForcePathStyle
-
-			instance := &AWS{
-				svc:           s3.New(sess, awsConfig),
-				bucket:        aws.S3.Bucket,
-				keyPrefix:     aws.S3.KeyPrefix,
-				fileExtension: aws.S3.FileExtension,
-				dynamoSvc:     dynamodb.New(sess, awsConfig),
-				dynamoTable:   aws.DynamoDBTable,
-				noLocks:       c.Provider.NoLocks,
-				noVersioning:  c.Provider.NoVersioning,
+				if e := aws.Endpoint; e != "" {
+					awsConfig.WithEndpoint(e)
+				}
+				if e := aws.Region; e != "" {
+					awsConfig.WithRegion(e)
+				}
+				awsConfig.S3ForcePathStyle = &bucket.ForcePathStyle
+
+				instance := &AWS{
+					svc:           s3.New(sess, awsConfig),
+					bucket:        bucket.Bucket,
+					keyPrefix:     bucket.KeyPrefix,
+					fileExtension: bucket.FileExtension,
+					dynamoSvc:     dynamodb.New(sess, awsConfig),
+					dynamoTable:   aws.DynamoDBTable,
+					noLocks:       c.Provider.NoLocks,
+					noVersioning:  c.Provider.NoVersioning,
+				}
+				awsInstances = append(awsInstances, instance)
 			}
-			awsInstances = append(awsInstances, instance)
 		}
 	}
 
diff --git a/test/data/test-bucket/.gitkeep b/test/data/test-bucket/.gitkeep
deleted file mode 100644
index e69de29b..00000000
diff --git a/test/data/test-bucket/terraform_0.12.28.tfstate b/test/data/test-bucket/terraform_0.12.28.tfstate
deleted file mode 100644
index 9590a256..00000000
--- a/test/data/test-bucket/terraform_0.12.28.tfstate
+++ /dev/null
@@ -1,36 +0,0 @@
-{
-  "version": 4,
-  "terraform_version": "0.12.28",
-  "serial": 2,
-  "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00",
-  "outputs": {
-    "content": {
-      "value": "foo!",
-      "type": "string"
-    }
-  },
-  "resources": [
-    {
-      "mode": "managed",
-      "type": "local_file",
-      "name": "foo",
-      "provider": "provider.local",
-      "instances": [
-        {
-          "schema_version": 0,
-          "attributes": {
-            "content": "foo!",
-            "content_base64": null,
-            "directory_permission": "0777",
-            "file_permission": "0777",
-            "filename": "./foo.bar",
-            "id": "4bf3e335199107182c6f7638efaad377acc7f452",
-            "sensitive_content": null,
-            "source": null
-          },
-          "private": "bnVsbA=="
-        }
-      ]
-    }
-  ]
-}
diff --git a/test/data/test-bucket/terraform_0.13.5.tfstate b/test/data/test-bucket/terraform_0.13.5.tfstate
deleted file mode 100644
index 8325cf53..00000000
--- a/test/data/test-bucket/terraform_0.13.5.tfstate
+++ /dev/null
@@ -1,36 +0,0 @@
-{
-  "version": 4,
-  "terraform_version": "0.13.5",
-  "serial": 3,
-  "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00",
-  "outputs": {
-    "content": {
-      "value": "foo!",
-      "type": "string"
-    }
-  },
-  "resources": [
-    {
-      "mode": "managed",
-      "type": "local_file",
-      "name": "foo",
-      "provider": "provider[\"registry.terraform.io/hashicorp/local\"]",
-      "instances": [
-        {
-          "schema_version": 0,
-          "attributes": {
-            "content": "foo!",
-            "content_base64": null,
-            "directory_permission": "0777",
-            "file_permission": "0777",
-            "filename": "./foo.bar",
-            "id": "4bf3e335199107182c6f7638efaad377acc7f452",
-            "sensitive_content": null,
-            "source": null
-          },
-          "private": "bnVsbA=="
-        }
-      ]
-    }
-  ]
-}
diff --git a/test/data/test-bucket/terraform_0.14.8.tfstate b/test/data/test-bucket/terraform_0.14.8.tfstate
deleted file mode 100644
index efa47929..00000000
--- a/test/data/test-bucket/terraform_0.14.8.tfstate
+++ /dev/null
@@ -1,37 +0,0 @@
-{
-  "version": 4,
-  "terraform_version": "0.14.8",
-  "serial": 3,
-  "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00",
-  "outputs": {
-    "content": {
-      "value": "foo!",
-      "type": "string"
-    }
-  },
-  "resources": [
-    {
-      "mode": "managed",
-      "type": "local_file",
-      "name": "foo",
-      "provider": "provider[\"registry.terraform.io/hashicorp/local\"]",
-      "instances": [
-        {
-          "schema_version": 0,
-          "attributes": {
-            "content": "foo!",
-            "content_base64": null,
-            "directory_permission": "0777",
-            "file_permission": "0777",
-            "filename": "./foo.bar",
-            "id": "4bf3e335199107182c6f7638efaad377acc7f452",
-            "sensitive_content": null,
-            "source": null
-          },
-          "sensitive_attributes": [],
-          "private": "bnVsbA=="
-        }
-      ]
-    }
-  ]
-}
diff --git a/test/data/test-bucket/terraform_0.15.5.tfstate b/test/data/test-bucket/terraform_0.15.5.tfstate
deleted file mode 100644
index f8751ad9..00000000
--- a/test/data/test-bucket/terraform_0.15.5.tfstate
+++ /dev/null
@@ -1,37 +0,0 @@
-{
-  "version": 4,
-  "terraform_version": "0.15.5",
-  "serial": 3,
-  "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00",
-  "outputs": {
-    "content": {
-      "value": "foo!",
-      "type": "string"
-    }
-  },
-  "resources": [
-    {
-      "mode": "managed",
-      "type": "local_file",
-      "name": "foo",
-      "provider": "provider[\"registry.terraform.io/hashicorp/local\"]",
-      "instances": [
-        {
-          "schema_version": 0,
-          "attributes": {
-            "content": "foo!",
-            "content_base64": null,
-            "directory_permission": "0777",
-            "file_permission": "0777",
-            "filename": "./foo.bar",
-            "id": "4bf3e335199107182c6f7638efaad377acc7f452",
-            "sensitive_content": null,
-            "source": null
-          },
-          "sensitive_attributes": [],
-          "private": "bnVsbA=="
-        }
-      ]
-    }
-  ]
-}
diff --git a/test/data/test-bucket/terraform_1.0.0.tfstate b/test/data/test-bucket/terraform_1.0.0.tfstate
deleted file mode 100644
index a3835bd3..00000000
--- a/test/data/test-bucket/terraform_1.0.0.tfstate
+++ /dev/null
@@ -1,37 +0,0 @@
-{
-  "version": 4,
-  "terraform_version": "1.0.0",
-  "serial": 3,
-  "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00",
-  "outputs": {
-    "content": {
-      "value": "foo!",
-      "type": "string"
-    }
-  },
-  "resources": [
-    {
-      "mode": "managed",
-      "type": "local_file",
-      "name": "foo",
-      "provider": "provider[\"registry.terraform.io/hashicorp/local\"]",
-      "instances": [
-        {
-          "schema_version": 0,
-          "attributes": {
-            "content": "foo!",
-            "content_base64": null,
-            "directory_permission": "0777",
-            "file_permission": "0777",
-            "filename": "./foo.bar",
-            "id": "4bf3e335199107182c6f7638efaad377acc7f452",
-            "sensitive_content": null,
-            "source": null
-          },
-          "sensitive_attributes": [],
-          "private": "bnVsbA=="
-        }
-      ]
-    }
-  ]
-}
diff --git a/test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate b/test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate
deleted file mode 100644
index a3835bd3..00000000
--- a/test/data/test-bucket2/terraform_1.0.0_bucket2.tfstate
+++ /dev/null
@@ -1,37 +0,0 @@
-{
-  "version": 4,
-  "terraform_version": "1.0.0",
-  "serial": 3,
-  "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00",
-  "outputs": {
-    "content": {
-      "value": "foo!",
-      "type": "string"
-    }
-  },
-  "resources": [
-    {
-      "mode": "managed",
-      "type": "local_file",
-      "name": "foo",
-      "provider": "provider[\"registry.terraform.io/hashicorp/local\"]",
-      "instances": [
-        {
-          "schema_version": 0,
-          "attributes": {
-            "content": "foo!",
-            "content_base64": null,
-            "directory_permission": "0777",
-            "file_permission": "0777",
-            "filename": "./foo.bar",
-            "id": "4bf3e335199107182c6f7638efaad377acc7f452",
-            "sensitive_content": null,
-            "source": null
-          },
-          "sensitive_attributes": [],
-          "private": "bnVsbA=="
-        }
-      ]
-    }
-  ]
-}
diff --git a/test/multiple-minio-buckets/config.yml b/test/multiple-minio-buckets/config.yml
index 5999c435..acb12c42 100644
--- a/test/multiple-minio-buckets/config.yml
+++ b/test/multiple-minio-buckets/config.yml
@@ -10,17 +10,26 @@ provider:
   no-versioning: true
 
 aws:
-  - endpoint: http://minio:9000/
+  - access-key: root
+    secret-access-key: mypassword
+    endpoint: http://minio:9000/
     region: eu-west-1
     s3:
-      bucket: test-bucket
-      force-path-style: true
-      file-extension: 
-        - .tfstate
-  - endpoint: http://minio:9000/
+      - bucket: test-bucket
+        force-path-style: true
+        file-extension: 
+          - .tfstate
+      - bucket: test-bucket2
+        force-path-style: true
+        file-extension: 
+          - .tfstate
+
+  - access-key: admin
+    secret-access-key: password
+    endpoint: http://minio-2:9000/
     region: eu-west-1
     s3:
-      bucket: test-bucket2
-      force-path-style: true
-      file-extension: 
-        - .tfstate
+      - bucket: test-bucket
+        force-path-style: true
+        file-extension: 
+          - .tfstate
diff --git a/test/multiple-minio-buckets/docker-compose.yml b/test/multiple-minio-buckets/docker-compose.yml
index 51b4d2e3..ba62c425 100644
--- a/test/multiple-minio-buckets/docker-compose.yml
+++ b/test/multiple-minio-buckets/docker-compose.yml
@@ -6,8 +6,6 @@ services:
       context: ../../
       dockerfile: ./Dockerfile
     environment:
-      AWS_ACCESS_KEY_ID: root
-      AWS_SECRET_ACCESS_KEY: mypassword
       DB_SSLMODE: disable
       CONFIG_FILE: config/config.yml
       GODEBUG: netdns=go
@@ -33,7 +31,21 @@ services:
     ports:
       - "9200:9000"
     volumes:
-      - ../data:/data
+      - ../data/minio-1:/data
+    command: server /data
+
+  minio-2:
+    image: minio/minio:latest
+    environment:
+      MINIO_ROOT_USER: admin
+      MINIO_ROOT_PASSWORD: password
+    user: "${UID}:${GID}"
+    expose: 
+      - "9000"
+    ports:
+      - "9300:9000"
+    volumes:
+      - ../data/minio-2:/data
     command: server /data
 
   db:
diff --git a/test/single-minio-bucket/docker-compose.yml b/test/single-minio-bucket/docker-compose.yml
index f8785745..3a366cd1 100755
--- a/test/single-minio-bucket/docker-compose.yml
+++ b/test/single-minio-bucket/docker-compose.yml
@@ -39,7 +39,7 @@ services:
     ports:
       - "9200:9000"
     volumes:
-      - ../data:/data
+      - ../data/minio-1:/data
     command: server /data
 
   db:

From f3a13f878087f174c4be68fb50f5eb57d8d4cc23 Mon Sep 17 00:00:00 2001
From: Hugo Bollon <hugo.bollon@gmail.com>
Date: Mon, 5 Jul 2021 16:08:50 +0200
Subject: [PATCH 7/7] docs: update readme with new aws configuration fields

---
 README.md       | 29 +++++++++++------
 state/aws.go    | 86 +++++++++++++++++++++++++++----------------------
 state/gcp.go    | 63 ++++++++++++++++++++++--------------
 state/gitlab.go | 25 ++++++++++----
 state/state.go  |  8 ++---
 state/tfe.go    | 51 ++++++++++++++++++-----------
 6 files changed, 158 insertions(+), 104 deletions(-)

diff --git a/README.md b/README.md
index 1f3e979d..43b0a9b1 100644
--- a/README.md
+++ b/README.md
@@ -146,8 +146,8 @@ Data resiliency is not paramount though as this dataset can be rebuilt upon your
 
 Terraboard currently supports configuration in three different ways:
 
-1. Environment variables **(only usable for mono provider configuration)**
-2. CLI parameters **(only usable for mono provider configuration)**
+1. Environment variables **(only usable for single provider configuration)**
+2. CLI parameters **(only usable for single provider configuration)**
 3. Configuration file (YAML). A configuration file example can be found in the root directory of this repository and in the `test/` subdirectory.
 
 **Important: all flags/environment variables related to the providers settings aren't compatible with multi-provider configuration! Instead, you must use the YAML config file to be able to configure multiples buckets/providers.**
@@ -171,18 +171,18 @@ aws:
   - endpoint: http://minio:9000/
     region: eu-west-1
     s3:
-      bucket: test-bucket
-      force-path-style: true
-      file-extension: 
-        - .tfstate
+      - bucket: test-bucket
+        force-path-style: true
+        file-extension: 
+          - .tfstate
 
   - endpoint: http://minio:9000/
     region: eu-west-1
     s3:
-      bucket: test-bucket2
-      force-path-style: true
-      file-extension: 
-        - .tfstate
+      - bucket: test-bucket2
+        force-path-style: true
+        file-extension: 
+          - .tfstate
 ```
 
 In the case of AWS, don't forget to set the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables.
@@ -243,6 +243,15 @@ You can find a ready-to-use Docker example with two *MinIO* buckets in the `test
 
 #### AWS (and S3 compatible providers) Options
 
+- `--aws-access-key` <default: *$AWS_ACCESS_KEY_ID*> AWS account access key.
+  - Env: *AWS_ACCESS_KEY_ID*
+  - Yaml: *aws.access-key*
+- `--aws-secret-access-key` <default: *$AWS_SECRET_ACCESS_KEY*> AWS secret account access key.
+  - Env: *AWS_SECRET_ACCESS_KEY*
+  - Yaml: *aws.secret-access-key*
+- `--aws-session-token` <default: *$AWS_SESSION_TOKEN*> AWS session token.
+  - Env: *AWS_SESSION_TOKEN*
+  - Yaml: *aws.session-token*
 - `--dynamodb-table` <default: *$AWS_DYNAMODB_TABLE*> AWS DynamoDB table for locks.
   - Env: *AWS_DYNAMODB_TABLE*
   - Yaml: *aws.dynamodb-table*
diff --git a/state/aws.go b/state/aws.go
index 5a9e8108..a7bc69f1 100644
--- a/state/aws.go
+++ b/state/aws.go
@@ -32,48 +32,56 @@ type AWS struct {
 }
 
 // NewAWS creates an AWS object
-func NewAWS(c *config.Config) []*AWS {
+func NewAWS(aws config.AWSConfig, bucket config.S3BucketConfig, noLocks, noVersioning bool) *AWS {
+	if bucket.Bucket == "" {
+		return nil
+	}
+
+	sess := session.Must(session.NewSession())
+	awsConfig := aws_sdk.NewConfig()
+	var creds *credentials.Credentials
+	if len(aws.APPRoleArn) > 0 {
+		log.Debugf("Using %s role", aws.APPRoleArn)
+		creds = stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) {
+			if aws.ExternalID != "" {
+				p.ExternalID = aws_sdk.String(aws.ExternalID)
+			}
+		})
+	} else {
+		if aws.AccessKey == "" || aws.SecretAccessKey == "" {
+			log.Fatal("Missing AccessKey or SecretAccessKey for AWS provider. Please check your configuration and retry")
+		}
+		creds = credentials.NewStaticCredentials(aws.AccessKey, aws.SecretAccessKey, aws.SessionToken)
+	}
+	awsConfig.WithCredentials(creds)
+
+	if e := aws.Endpoint; e != "" {
+		awsConfig.WithEndpoint(e)
+	}
+	if e := aws.Region; e != "" {
+		awsConfig.WithRegion(e)
+	}
+	awsConfig.S3ForcePathStyle = &bucket.ForcePathStyle
+
+	return &AWS{
+		svc:           s3.New(sess, awsConfig),
+		bucket:        bucket.Bucket,
+		keyPrefix:     bucket.KeyPrefix,
+		fileExtension: bucket.FileExtension,
+		dynamoSvc:     dynamodb.New(sess, awsConfig),
+		dynamoTable:   aws.DynamoDBTable,
+		noLocks:       noLocks,
+		noVersioning:  noVersioning,
+	}
+}
+
+// NewAWSCollection instantiate all needed AWS objects configurated by the user and return a slice
+func NewAWSCollection(c *config.Config) []*AWS {
 	var awsInstances []*AWS
 	for _, aws := range c.AWS {
 		for _, bucket := range aws.S3 {
-			if bucket.Bucket != "" {
-				sess := session.Must(session.NewSession())
-				awsConfig := aws_sdk.NewConfig()
-				var creds *credentials.Credentials
-				if len(aws.APPRoleArn) > 0 {
-					log.Debugf("Using %s role", aws.APPRoleArn)
-					creds = stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) {
-						if aws.ExternalID != "" {
-							p.ExternalID = aws_sdk.String(aws.ExternalID)
-						}
-					})
-				} else {
-					if aws.AccessKey == "" || aws.SecretAccessKey == "" {
-						log.Fatal("Missing AccessKey or SecretAccessKey for AWS provider. Please check your configuration and retry")
-					}
-					creds = credentials.NewStaticCredentials(aws.AccessKey, aws.SecretAccessKey, aws.SessionToken)
-				}
-				awsConfig.WithCredentials(creds)
-
-				if e := aws.Endpoint; e != "" {
-					awsConfig.WithEndpoint(e)
-				}
-				if e := aws.Region; e != "" {
-					awsConfig.WithRegion(e)
-				}
-				awsConfig.S3ForcePathStyle = &bucket.ForcePathStyle
-
-				instance := &AWS{
-					svc:           s3.New(sess, awsConfig),
-					bucket:        bucket.Bucket,
-					keyPrefix:     bucket.KeyPrefix,
-					fileExtension: bucket.FileExtension,
-					dynamoSvc:     dynamodb.New(sess, awsConfig),
-					dynamoTable:   aws.DynamoDBTable,
-					noLocks:       c.Provider.NoLocks,
-					noVersioning:  c.Provider.NoVersioning,
-				}
-				awsInstances = append(awsInstances, instance)
+			if awsInstance := NewAWS(aws, bucket, c.Provider.NoLocks, c.Provider.NoVersioning); awsInstance != nil {
+				awsInstances = append(awsInstances, awsInstance)
 			}
 		}
 	}
diff --git a/state/gcp.go b/state/gcp.go
index 13e8cf7f..3d969498 100644
--- a/state/gcp.go
+++ b/state/gcp.go
@@ -24,39 +24,52 @@ type GCP struct {
 }
 
 // NewGCP creates an GCP object
-func NewGCP(c *config.Config) ([]*GCP, error) {
+func NewGCP(gcp config.GCPConfig) (*GCP, error) {
 	ctx := context.Background()
 
 	var client *storage.Client
-	var gcpInstances []*GCP
+	var gcpInstance *GCP
 	var err error
-	for _, gcp := range c.GCP {
-		if gcp.GCSBuckets != nil {
-			if gcp.GCPSAKey != "" {
-				log.WithFields(log.Fields{
-					"path": gcp.GCPSAKey,
-				}).Info("Authenticating using service account key")
-				opt := option.WithCredentialsFile(gcp.GCPSAKey)
-				client, err = storage.NewClient(ctx, opt) // Use service account key
-			} else {
-				client, err = storage.NewClient(ctx) // Use base credentials
-			}
+	if gcp.GCSBuckets == nil || len(gcp.GCSBuckets) == 0 {
+		return nil, nil
+	}
 
-			if err != nil {
-				log.Fatalf("Failed to create client: %v", err)
-				return nil, err
-			}
+	if gcp.GCPSAKey != "" {
+		log.WithFields(log.Fields{
+			"path": gcp.GCPSAKey,
+		}).Info("Authenticating using service account key")
+		opt := option.WithCredentialsFile(gcp.GCPSAKey)
+		client, err = storage.NewClient(ctx, opt) // Use service account key
+	} else {
+		client, err = storage.NewClient(ctx) // Use base credentials
+	}
 
-			instance := &GCP{
-				svc:     client,
-				buckets: gcp.GCSBuckets,
-			}
-			gcpInstances = append(gcpInstances, instance)
+	if err != nil {
+		log.Fatalf("Failed to create client: %v", err)
+		return nil, err
+	}
+
+	gcpInstance = &GCP{
+		svc:     client,
+		buckets: gcp.GCSBuckets,
+	}
+
+	log.WithFields(log.Fields{
+		"buckets": gcp.GCSBuckets,
+	}).Info("Client successfully created")
 
-			log.WithFields(log.Fields{
-				"buckets": gcp.GCSBuckets,
-			}).Info("Client successfully created")
+	return gcpInstance, nil
+}
+
+// NewGCPCollection instantiate all needed GCP objects configurated by the user and return a slice
+func NewGCPCollection(c *config.Config) ([]*GCP, error) {
+	var gcpInstances []*GCP
+	for _, gcp := range c.GCP {
+		gcpInstance, err := NewGCP(gcp)
+		if err != nil || gcpInstance == nil {
+			return nil, err
 		}
+		gcpInstances = append(gcpInstances, gcpInstance)
 	}
 
 	return gcpInstances, nil
diff --git a/state/gitlab.go b/state/gitlab.go
index 6f48cf2e..50cf2a9f 100644
--- a/state/gitlab.go
+++ b/state/gitlab.go
@@ -18,16 +18,27 @@ type Gitlab struct {
 }
 
 // NewGitlab creates a new Gitlab object
-func NewGitlab(c *config.Config) []*Gitlab {
+func NewGitlab(gl config.GitlabConfig) *Gitlab {
+	var instance *Gitlab
+	if gl.Token == "" {
+		return nil
+	}
+
+	instance = &Gitlab{
+		Client: gitlab.NewClient(gl.Address, gl.Token),
+	}
+	return instance
+}
+
+// NewGitlabCollection instantiate all needed Gitlab objects configurated by the user and return a slice
+func NewGitlabCollection(c *config.Config) []*Gitlab {
 	var gitlabInstances []*Gitlab
-	for _, gl := range c.Gitlab {
-		if gl.Token != "" {
-			instance := &Gitlab{
-				Client: gitlab.NewClient(gl.Address, gl.Token),
-			}
-			gitlabInstances = append(gitlabInstances, instance)
+	for _, gitlab := range c.Gitlab {
+		if glInstance := NewGitlab(gitlab); glInstance != nil {
+			gitlabInstances = append(gitlabInstances, glInstance)
 		}
 	}
+
 	return gitlabInstances
 }
 
diff --git a/state/state.go b/state/state.go
index 7407abfa..8fe99da2 100644
--- a/state/state.go
+++ b/state/state.go
@@ -43,7 +43,7 @@ type Provider interface {
 func Configure(c *config.Config) ([]Provider, error) {
 	var providers []Provider
 	if len(c.TFE) > 0 {
-		objs, err := NewTFE(c)
+		objs, err := NewTFECollection(c)
 		if err != nil {
 			return []Provider{}, err
 		}
@@ -56,7 +56,7 @@ func Configure(c *config.Config) ([]Provider, error) {
 	}
 
 	if len(c.GCP) > 0 {
-		objs, err := NewGCP(c)
+		objs, err := NewGCPCollection(c)
 		if err != nil {
 			return []Provider{}, err
 		}
@@ -69,7 +69,7 @@ func Configure(c *config.Config) ([]Provider, error) {
 	}
 
 	if len(c.Gitlab) > 0 {
-		objs := NewGitlab(c)
+		objs := NewGitlabCollection(c)
 		if len(objs) > 0 {
 			log.Info("Using Gitab as state/locks provider")
 			for _, glObj := range objs {
@@ -79,7 +79,7 @@ func Configure(c *config.Config) ([]Provider, error) {
 	}
 
 	if len(c.AWS) > 0 {
-		objs := NewAWS(c)
+		objs := NewAWSCollection(c)
 		if len(objs) > 0 {
 			log.Info("Using AWS (S3+DynamoDB) as state/locks provider")
 			for _, awsObj := range objs {
diff --git a/state/tfe.go b/state/tfe.go
index ed227ce1..0de2dcc9 100644
--- a/state/tfe.go
+++ b/state/tfe.go
@@ -19,28 +19,41 @@ type TFE struct {
 }
 
 // NewTFE creates a new TFE object
-func NewTFE(c *config.Config) ([]*TFE, error) {
-	var tfeInstances []*TFE
-	for _, tfeObj := range c.TFE {
-		if tfeObj.Token != "" {
-			config := &tfe.Config{
-				Address: tfeObj.Address,
-				Token:   tfeObj.Token,
-			}
+func NewTFE(tfeObj config.TFEConfig) (*TFE, error) {
+	var tfeInstance *TFE
+	if tfeObj.Token == "" {
+		return nil, nil
+	}
 
-			client, err := tfe.NewClient(config)
-			if err != nil {
-				return nil, err
-			}
+	config := &tfe.Config{
+		Address: tfeObj.Address,
+		Token:   tfeObj.Token,
+	}
 
-			ctx := context.Background()
-			instance := &TFE{
-				Client: client,
-				org:    tfeObj.Organization,
-				ctx:    &ctx,
-			}
-			tfeInstances = append(tfeInstances, instance)
+	client, err := tfe.NewClient(config)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx := context.Background()
+	tfeInstance = &TFE{
+		Client: client,
+		org:    tfeObj.Organization,
+		ctx:    &ctx,
+	}
+
+	return tfeInstance, nil
+}
+
+// NewTFECollection instantiate all needed GCP objects configurated by the user and return a slice
+func NewTFECollection(c *config.Config) ([]*TFE, error) {
+	var tfeInstances []*TFE
+	for _, tfe := range c.TFE {
+		tfeInstance, err := NewTFE(tfe)
+		if err != nil || tfeInstance == nil {
+			return nil, err
 		}
+		tfeInstances = append(tfeInstances, tfeInstance)
 	}
 
 	return tfeInstances, nil