diff --git a/README.md b/README.md index 543931da..6f57da19 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ - [AWS S3 (state) + DynamoDB (lock)](#aws-s3-state--dynamodb-lock) - [Terraform Cloud](#terraform-cloud) - [Configuration](#configuration) + - [Multiple buckets/providers](#multiple-bucketsproviders) - [Available parameters](#available-parameters) - [Application Options](#application-options) - [General Provider Options](#general-provider-options) @@ -90,6 +91,9 @@ It currently supports several remote state backend providers: - [Terraform Cloud (remote)](https://www.terraform.io/docs/backends/types/remote.html) - [GitLab](https://docs.gitlab.com/ee/user/infrastructure/terraform_state.html) +With the upcoming **v1.2.0** update, Terraboard will be now able to handle multiple buckets/providers configuration! 🥳 +Check *configuration* section for more details. + ### Overview The overview presents all the state files in the S3 bucket, by most recent @@ -143,12 +147,50 @@ Data resiliency is not paramount though as this dataset can be rebuilt upon your Terraboard currently supports configuration in three different ways: -1. Environment variables -2. CLI parameters -3. Configuration file (YAML). A configuration file example can be found in the root directory of this repository. +1. Environment variables **(only usable for single provider configuration)** +2. CLI parameters **(only usable for single provider configuration)** +3. Configuration file (YAML). A configuration file example can be found in the root directory of this repository and in the `test/` subdirectory. + +**Important: all flags/environment variables related to the providers settings aren't compatible with multi-provider configuration! Instead, you must use the YAML config file to be able to configure multiples buckets/providers.** The precedence of configurations is as described below. +### Multiple buckets/providers + +In order for Terraboard to import states from multiples buckets or even providers, you must use the YAML configuration method: + +- Set the `CONFIG_FILE` environment variable or the `-c`/`--config-file` flag to point to a valid YAML config file. +- In the YAML file, specify your desired providers configuration. For example with two MinIO buckets (using the AWS provider with compatible mode): + +```yaml +# Needed since MinIO doesn't support versioning or locking +provider: + no-locks: true + no-versioning: true + +aws: + - endpoint: http://minio:9000/ + region: eu-west-1 + s3: + - bucket: test-bucket + force-path-style: true + file-extension: + - .tfstate + + - endpoint: http://minio:9000/ + region: eu-west-1 + s3: + - bucket: test-bucket2 + force-path-style: true + file-extension: + - .tfstate +``` + +In the case of AWS, don't forget to set the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables. + +That's it! Terraboard will now fetch these two buckets on DB refresh. You can also mix providers like AWS and Gitlab or anything else. +You can find a ready-to-use Docker example with two *MinIO* buckets in the `test/multiple-minio-buckets/` sub-folder. + ### Available parameters #### Application Options @@ -202,6 +244,15 @@ The precedence of configurations is as described below. #### AWS (and S3 compatible providers) Options +- `--aws-access-key` AWS account access key. + - Env: *AWS_ACCESS_KEY_ID* + - Yaml: *aws.access-key* +- `--aws-secret-access-key` AWS secret account access key. + - Env: *AWS_SECRET_ACCESS_KEY* + - Yaml: *aws.secret-access-key* +- `--aws-session-token` AWS session token. + - Env: *AWS_SESSION_TOKEN* + - Yaml: *aws.session-token* - `--dynamodb-table` AWS DynamoDB table for locks. - Env: *AWS_DYNAMODB_TABLE* - Yaml: *aws.dynamodb-table* diff --git a/api/api.go b/api/api.go index a2515024..aa0c9e94 100644 --- a/api/api.go +++ b/api/api.go @@ -148,15 +148,21 @@ func StateCompare(w http.ResponseWriter, r *http.Request, d *db.Database) { } // GetLocks returns information on locked States -func GetLocks(w http.ResponseWriter, _ *http.Request, sp state.Provider) { +func GetLocks(w http.ResponseWriter, _ *http.Request, sps []state.Provider) { w.Header().Set("Access-Control-Allow-Origin", "*") - locks, err := sp.GetLocks() - if err != nil { - JSONError(w, "Failed to get locks", err) - return + allLocks := make(map[string]state.LockInfo) + for _, sp := range sps { + locks, err := sp.GetLocks() + if err != nil { + JSONError(w, "Failed to get locks on a provider", err) + return + } + for k, v := range locks { + allLocks[k] = v + } } - j, err := json.Marshal(locks) + j, err := json.Marshal(allLocks) if err != nil { JSONError(w, "Failed to marshal locks", err) return diff --git a/config/config.go b/config/config.go index 6184b119..ec47f387 100644 --- a/config/config.go +++ b/config/config.go @@ -40,12 +40,15 @@ type S3BucketConfig struct { // AWSConfig stores the DynamoDB table and S3 Bucket configuration type AWSConfig struct { - DynamoDBTable string `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."` - S3 S3BucketConfig `group:"S3 Options" yaml:"s3"` - Endpoint string `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."` - Region string `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."` - APPRoleArn string `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."` - ExternalID string `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."` + AccessKey string `long:"aws-access-key" env:"AWS_ACCESS_KEY_ID" yaml:"access-key" description:"AWS account access key."` + SecretAccessKey string `long:"aws-secret-access-key" env:"AWS_SECRET_ACCESS_KEY" yaml:"secret-access-key" description:"AWS secret account access key."` + SessionToken string `long:"aws-session-token" env:"AWS_SESSION_TOKEN" yaml:"session-token" description:"AWS session token."` + DynamoDBTable string `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."` + S3 []S3BucketConfig `group:"S3 Options" yaml:"s3"` + Endpoint string `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."` + Region string `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."` + APPRoleArn string `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."` + ExternalID string `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."` } // TFEConfig stores the Terraform Enterprise configuration @@ -92,13 +95,13 @@ type Config struct { DB DBConfig `group:"Database Options" yaml:"database"` - AWS AWSConfig `group:"AWS Options" yaml:"aws"` + AWS []AWSConfig `group:"AWS Options" yaml:"aws"` - TFE TFEConfig `group:"Terraform Enterprise Options" yaml:"tfe"` + TFE []TFEConfig `group:"Terraform Enterprise Options" yaml:"tfe"` - GCP GCPConfig `group:"Google Cloud Platform Options" yaml:"gcp"` + GCP []GCPConfig `group:"Google Cloud Platform Options" yaml:"gcp"` - Gitlab GitlabConfig `group:"GitLab Options" yaml:"gitlab"` + Gitlab []GitlabConfig `group:"GitLab Options" yaml:"gitlab"` Web WebConfig `group:"Web" yaml:"web"` } @@ -119,14 +122,48 @@ func (c *Config) LoadConfigFromYaml() *Config { return c } -// LoadConfig loads the config from flags & environment -func LoadConfig(version string) *Config { +// Initialize Config with one obj per providers arrays +// to allow usage of flags / env variables on single provider configuration +func initDefaultConfig() Config { var c Config - parser := flags.NewParser(&c, flags.Default) + var awsInitialConfig AWSConfig + var s3InitialConfig S3BucketConfig + var tfeInitialConfig TFEConfig + var gcpInitialConfig GCPConfig + var gitlabInitialConfig GitlabConfig + + parseStructFlagsAndEnv(&awsInitialConfig) + c.AWS = append(c.AWS, awsInitialConfig) + + parseStructFlagsAndEnv(&s3InitialConfig) + c.AWS[0].S3 = append(c.AWS[0].S3, s3InitialConfig) + + parseStructFlagsAndEnv(&tfeInitialConfig) + c.TFE = append(c.TFE, tfeInitialConfig) + + parseStructFlagsAndEnv(&gcpInitialConfig) + c.GCP = append(c.GCP, gcpInitialConfig) + + parseStructFlagsAndEnv(&gitlabInitialConfig) + c.Gitlab = append(c.Gitlab, gitlabInitialConfig) + + return c +} + +// Parse flags and env variables to given struct using go-flags +// parser +func parseStructFlagsAndEnv(obj interface{}) { + parser := flags.NewParser(obj, flags.Default) if _, err := parser.Parse(); err != nil { fmt.Printf("Failed to parse flags: %s", err) os.Exit(1) } +} + +// LoadConfig loads the config from flags & environment +func LoadConfig(version string) *Config { + c := initDefaultConfig() + parseStructFlagsAndEnv(&c) if c.ConfigFilePath != "" { if _, err := os.Stat(c.ConfigFilePath); err == nil { diff --git a/config/config_test.go b/config/config_test.go index 05aa4579..3e2081c9 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -22,27 +22,37 @@ func TestLoadConfigFromYaml(t *testing.T) { Name: "terraboard-db", NoSync: true, }, - AWS: AWSConfig{ - DynamoDBTable: "terraboard-dynamodb", - S3: S3BucketConfig{ - Bucket: "terraboard-bucket", - KeyPrefix: "test/", - FileExtension: []string{".tfstate"}, - ForcePathStyle: true, + AWS: []AWSConfig{ + { + AccessKey: "root", + SecretAccessKey: "mypassword", + DynamoDBTable: "terraboard-dynamodb", + S3: []S3BucketConfig{{ + Bucket: "terraboard-bucket", + KeyPrefix: "test/", + FileExtension: []string{".tfstate"}, + ForcePathStyle: true, + }}, }, }, - TFE: TFEConfig{ - Address: "https://tfe.example.com", - Token: "foo", - Organization: "bar", + TFE: []TFEConfig{ + { + Address: "https://tfe.example.com", + Token: "foo", + Organization: "bar", + }, }, - GCP: GCPConfig{ - GCSBuckets: []string{"my-bucket-1", "my-bucket-2"}, - GCPSAKey: "/path/to/key", + GCP: []GCPConfig{ + { + GCSBuckets: []string{"my-bucket-1", "my-bucket-2"}, + GCPSAKey: "/path/to/key", + }, }, - Gitlab: GitlabConfig{ - Address: "https://gitlab.example.com", - Token: "foo", + Gitlab: []GitlabConfig{ + { + Address: "https://gitlab.example.com", + Token: "foo", + }, }, Web: WebConfig{ Port: 39090, diff --git a/config/config_test.yml b/config/config_test.yml index 4fac8d9f..6f4610e5 100644 --- a/config/config_test.yml +++ b/config/config_test.yml @@ -11,27 +11,29 @@ database: no-sync: true aws: - dynamodb-table: terraboard-dynamodb - s3: - bucket: terraboard-bucket - key-prefix: test/ - file-extension: [.tfstate] - force-path-style: true + - access-key: root + secret-access-key: mypassword + dynamodb-table: terraboard-dynamodb + s3: + - bucket: terraboard-bucket + key-prefix: test/ + file-extension: [.tfstate] + force-path-style: true tfe: - address: https://tfe.example.com - token: foo - organization: bar + - address: https://tfe.example.com + token: foo + organization: bar gcp: - gcs-bucket: - - my-bucket-1 - - my-bucket-2 - gcp-sa-key-path: /path/to/key + - gcs-bucket: + - my-bucket-1 + - my-bucket-2 + gcp-sa-key-path: /path/to/key gitlab: - address: https://gitlab.example.com - token: foo + - address: https://gitlab.example.com + token: foo web: port: 39090 diff --git a/config/yaml.go b/config/yaml.go new file mode 100644 index 00000000..8e6f453f --- /dev/null +++ b/config/yaml.go @@ -0,0 +1,33 @@ +package config + +/********************************************* + * Custom UnmarshalYAML used to define some struct fields + * default values where go-flags ones aren't applicable + * (and so makes them optional) + *********************************************/ + +func (s *S3BucketConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawS3BucketConfig S3BucketConfig + raw := rawS3BucketConfig{ + FileExtension: []string{".tfstate"}, + } + if err := unmarshal(&raw); err != nil { + return err + } + + *s = S3BucketConfig(raw) + return nil +} + +func (s *GitlabConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawGitlabConfig GitlabConfig + raw := rawGitlabConfig{ + Address: "https://gitlab.com", + } + if err := unmarshal(&raw); err != nil { + return err + } + + *s = GitlabConfig(raw) + return nil +} diff --git a/db/db.go b/db/db.go index 009c60fc..8f167456 100644 --- a/db/db.go +++ b/db/db.go @@ -7,6 +7,7 @@ import ( "net/url" "strconv" "strings" + "sync" "github.com/camptocamp/terraboard/config" "github.com/camptocamp/terraboard/internal/terraform/addrs" @@ -25,6 +26,7 @@ import ( // Database is a wrapping structure to *gorm.DB type Database struct { *gorm.DB + lock sync.Mutex } var pageSize = 20 @@ -78,7 +80,7 @@ func Init(config config.DBConfig, debug bool) *Database { db.Config.Logger.LogMode(logger.Info) } - d := &Database{db} + d := &Database{DB: db} if err = d.MigrateLineage(); err != nil { log.Fatalf("Lineage migration failed: %v\n", err) } @@ -120,11 +122,14 @@ func (db *Database) stateS3toDB(sf *statefile.File, path string, versionID strin // Check if the associated lineage is already present in lineages table // If so, it recovers its ID otherwise it inserts it at the same time as the state var lineage types.Lineage + db.lock.Lock() err = db.FirstOrCreate(&lineage, types.Lineage{Value: sf.Lineage}).Error if err != nil || lineage.ID == 0 { - log.Error("Unknown error in stateS3toDB during lineage finding") + log.WithField("error", err). + Error("Unknown error in stateS3toDB during lineage finding", err) return types.State{}, err } + db.lock.Unlock() st = types.State{ Path: path, @@ -264,10 +269,12 @@ func (db *Database) UpdateState(st types.State) error { // InsertVersion inserts an AWS S3 Version in the Database func (db *Database) InsertVersion(version *state.Version) error { var v types.Version + db.lock.Lock() db.FirstOrCreate(&v, types.Version{ VersionID: version.ID, LastModified: version.LastModified, }) + db.lock.Unlock() return nil } diff --git a/example.yml b/example.yml index 75bbca38..79346f60 100644 --- a/example.yml +++ b/example.yml @@ -12,15 +12,17 @@ database: sync-interval: 5 provider: - no-locks: "true" - no-versioning: "false" + no-locks: true + no-versioning: false aws: - dynamodb-table: terraboard - s3: - bucket: terraboard - key-prefix: - file-extension: .tfstate + - access-key: root + secret-access-key: mypassword + dynamodb-table: terraboard + s3: + - bucket: terraboard + key-prefix: + file-extension: .tfstate web: port: 9090 diff --git a/main.go b/main.go index dbc74dae..b999f1f3 100644 --- a/main.go +++ b/main.go @@ -45,10 +45,10 @@ func handleWithDB(apiF func(w http.ResponseWriter, r *http.Request, }) } -func handleWithStateProvider(apiF func(w http.ResponseWriter, r *http.Request, - sp state.Provider), sp state.Provider) func(http.ResponseWriter, *http.Request) { +func handleWithStateProviders(apiF func(w http.ResponseWriter, r *http.Request, + sps []state.Provider), sps []state.Provider) func(http.ResponseWriter, *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - apiF(w, r, sp) + apiF(w, r, sps) }) } @@ -64,7 +64,7 @@ func isKnownStateVersion(statesVersions map[string][]string, versionID, path str } // Refresh the DB -// This should be the only direct bridge between the state provider and the DB +// This should be the only direct bridge between the state providers and the DB func refreshDB(syncInterval uint16, d *db.Database, sp state.Provider) { interval := time.Duration(syncInterval) * time.Minute for { @@ -155,7 +155,7 @@ func main() { } // Set up the state provider - sp, err := state.Configure(c) + sps, err := state.Configure(c) if err != nil { log.Fatal(err) } @@ -168,7 +168,10 @@ func main() { if c.DB.NoSync { log.Infof("Not syncing database, as requested.") } else { - go refreshDB(c.DB.SyncInterval, database, sp) + log.Debugf("Total providers: %d\n", len(sps)) + for _, sp := range sps { + go refreshDB(c.DB.SyncInterval, database, sp) + } } defer database.Close() @@ -189,7 +192,7 @@ func main() { http.HandleFunc(util.GetFullPath("api/state/"), handleWithDB(api.GetState, database)) http.HandleFunc(util.GetFullPath("api/state/activity/"), handleWithDB(api.GetStateActivity, database)) http.HandleFunc(util.GetFullPath("api/state/compare/"), handleWithDB(api.StateCompare, database)) - http.HandleFunc(util.GetFullPath("api/locks"), handleWithStateProvider(api.GetLocks, sp)) + http.HandleFunc(util.GetFullPath("api/locks"), handleWithStateProviders(api.GetLocks, sps)) http.HandleFunc(util.GetFullPath("api/search/attribute"), handleWithDB(api.SearchAttribute, database)) http.HandleFunc(util.GetFullPath("api/resource/types"), handleWithDB(api.ListResourceTypes, database)) http.HandleFunc(util.GetFullPath("api/resource/types/count"), handleWithDB(api.ListResourceTypesWithCount, database)) diff --git a/state/aws.go b/state/aws.go index 4a75e334..672e89cc 100644 --- a/state/aws.go +++ b/state/aws.go @@ -8,6 +8,7 @@ import ( "time" aws_sdk "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/dynamodb" @@ -31,41 +32,63 @@ type AWS struct { } // NewAWS creates an AWS object -func NewAWS(c *config.Config) AWS { - sess := session.Must(session.NewSession()) +func NewAWS(aws config.AWSConfig, bucket config.S3BucketConfig, noLocks, noVersioning bool) *AWS { + if bucket.Bucket == "" { + return nil + } + sess := session.Must(session.NewSession()) awsConfig := aws_sdk.NewConfig() - - if len(c.AWS.APPRoleArn) > 0 { - log.Debugf("Using %s role", c.AWS.APPRoleArn) - creds := stscreds.NewCredentials(sess, c.AWS.APPRoleArn, func(p *stscreds.AssumeRoleProvider) { - if c.AWS.ExternalID != "" { - p.ExternalID = aws_sdk.String(c.AWS.ExternalID) + var creds *credentials.Credentials + if len(aws.APPRoleArn) > 0 { + log.Debugf("Using %s role", aws.APPRoleArn) + creds = stscreds.NewCredentials(sess, aws.APPRoleArn, func(p *stscreds.AssumeRoleProvider) { + if aws.ExternalID != "" { + p.ExternalID = aws_sdk.String(aws.ExternalID) } }) - awsConfig.WithCredentials(creds) + } else { + if aws.AccessKey == "" || aws.SecretAccessKey == "" { + log.Fatal("Missing AccessKey or SecretAccessKey for AWS provider. Please check your configuration and retry") + } + creds = credentials.NewStaticCredentials(aws.AccessKey, aws.SecretAccessKey, aws.SessionToken) } + awsConfig.WithCredentials(creds) - if e := c.AWS.Endpoint; e != "" { + if e := aws.Endpoint; e != "" { awsConfig.WithEndpoint(e) } - if e := c.AWS.Region; e != "" { + if e := aws.Region; e != "" { awsConfig.WithRegion(e) } - awsConfig.S3ForcePathStyle = &c.AWS.S3.ForcePathStyle + awsConfig.S3ForcePathStyle = &bucket.ForcePathStyle - return AWS{ + return &AWS{ svc: s3.New(sess, awsConfig), - bucket: c.AWS.S3.Bucket, - keyPrefix: c.AWS.S3.KeyPrefix, - fileExtension: c.AWS.S3.FileExtension, + bucket: bucket.Bucket, + keyPrefix: bucket.KeyPrefix, + fileExtension: bucket.FileExtension, dynamoSvc: dynamodb.New(sess, awsConfig), - dynamoTable: c.AWS.DynamoDBTable, - noLocks: c.Provider.NoLocks, - noVersioning: c.Provider.NoVersioning, + dynamoTable: aws.DynamoDBTable, + noLocks: noLocks, + noVersioning: noVersioning, } } +// NewAWSCollection instantiate all needed AWS objects configurated by the user and return a slice +func NewAWSCollection(c *config.Config) []*AWS { + var awsInstances []*AWS + for _, aws := range c.AWS { + for _, bucket := range aws.S3 { + if awsInstance := NewAWS(aws, bucket, c.Provider.NoLocks, c.Provider.NoVersioning); awsInstance != nil { + awsInstances = append(awsInstances, awsInstance) + } + } + } + + return awsInstances +} + // GetLocks returns a map of locks by State path func (a *AWS) GetLocks() (locks map[string]LockInfo, err error) { if a.noLocks { diff --git a/state/gcp.go b/state/gcp.go index c77f88df..f43afe51 100644 --- a/state/gcp.go +++ b/state/gcp.go @@ -24,15 +24,21 @@ type GCP struct { } // NewGCP creates an GCP object -func NewGCP(c *config.Config) (*GCP, error) { +func NewGCP(gcp config.GCPConfig) (*GCP, error) { ctx := context.Background() + var client *storage.Client + var gcpInstance *GCP var err error - if c.GCP.GCPSAKey != "" { + if gcp.GCSBuckets == nil || len(gcp.GCSBuckets) == 0 { + return nil, nil + } + + if gcp.GCPSAKey != "" { log.WithFields(log.Fields{ - "path": c.GCP.GCPSAKey, + "path": gcp.GCPSAKey, }).Info("Authenticating using service account key") - opt := option.WithCredentialsFile(c.GCP.GCPSAKey) + opt := option.WithCredentialsFile(gcp.GCPSAKey) client, err = storage.NewClient(ctx, opt) // Use service account key } else { client, err = storage.NewClient(ctx) // Use base credentials @@ -43,14 +49,30 @@ func NewGCP(c *config.Config) (*GCP, error) { return nil, err } + gcpInstance = &GCP{ + svc: client, + buckets: gcp.GCSBuckets, + } + log.WithFields(log.Fields{ - "buckets": c.GCP.GCSBuckets, + "buckets": gcp.GCSBuckets, }).Info("Client successfully created") - return &GCP{ - svc: client, - buckets: c.GCP.GCSBuckets, - }, nil + return gcpInstance, nil +} + +// NewGCPCollection instantiate all needed GCP objects configurated by the user and return a slice +func NewGCPCollection(c *config.Config) ([]*GCP, error) { + var gcpInstances []*GCP + for _, gcp := range c.GCP { + gcpInstance, err := NewGCP(gcp) + if err != nil || gcpInstance == nil { + return nil, err + } + gcpInstances = append(gcpInstances, gcpInstance) + } + + return gcpInstances, nil } // GetLocks returns a map of locks by State path diff --git a/state/gitlab.go b/state/gitlab.go index 129bb39b..507f3f69 100644 --- a/state/gitlab.go +++ b/state/gitlab.go @@ -18,10 +18,28 @@ type Gitlab struct { } // NewGitlab creates a new Gitlab object -func NewGitlab(c *config.Config) *Gitlab { - return &Gitlab{ - Client: gitlab.NewClient(c.Gitlab.Address, c.Gitlab.Token), +func NewGitlab(gl config.GitlabConfig) *Gitlab { + var instance *Gitlab + if gl.Token == "" { + return nil } + + instance = &Gitlab{ + Client: gitlab.NewClient(gl.Address, gl.Token), + } + return instance +} + +// NewGitlabCollection instantiate all needed Gitlab objects configurated by the user and return a slice +func NewGitlabCollection(c *config.Config) []*Gitlab { + var gitlabInstances []*Gitlab + for _, gitlab := range c.Gitlab { + if glInstance := NewGitlab(gitlab); glInstance != nil { + gitlabInstances = append(gitlabInstances, glInstance) + } + } + + return gitlabInstances } // GetLocks returns a map of locks by State path diff --git a/state/state.go b/state/state.go index a55e7958..9e9b8f1e 100644 --- a/state/state.go +++ b/state/state.go @@ -40,23 +40,53 @@ type Provider interface { } // Configure the state provider -func Configure(c *config.Config) (Provider, error) { - if len(c.TFE.Token) > 0 { - log.Info("Using Terraform Enterprise as the state/locks provider") - return NewTFE(c) +func Configure(c *config.Config) ([]Provider, error) { + var providers []Provider + if len(c.TFE) > 0 { + objs, err := NewTFECollection(c) + if err != nil { + return []Provider{}, err + } + if len(objs) > 0 { + log.Info("Using Terraform Enterprise as state/locks provider") + for _, tfeObj := range objs { + providers = append(providers, tfeObj) + } + } } - if c.GCP.GCSBuckets != nil { - log.Info("Using Google Cloud as the state/locks provider") - return NewGCP(c) + if len(c.GCP) > 0 { + objs, err := NewGCPCollection(c) + if err != nil { + return []Provider{}, err + } + if len(objs) > 0 { + log.Info("Using Google Cloud as state/locks provider") + for _, gcpObj := range objs { + providers = append(providers, gcpObj) + } + } } - if len(c.Gitlab.Token) > 0 { - log.Info("Using Gitab as the state/locks provider") - return NewGitlab(c), nil + if len(c.Gitlab) > 0 { + objs := NewGitlabCollection(c) + if len(objs) > 0 { + log.Info("Using Gitab as state/locks provider") + for _, glObj := range objs { + providers = append(providers, glObj) + } + } } - log.Info("Using AWS (S3+DynamoDB) as the state/locks provider") - provider := NewAWS(c) - return &provider, nil + if len(c.AWS) > 0 { + objs := NewAWSCollection(c) + if len(objs) > 0 { + log.Info("Using AWS (S3+DynamoDB) as state/locks provider") + for _, awsObj := range objs { + providers = append(providers, awsObj) + } + } + } + + return providers, nil } diff --git a/state/tfe.go b/state/tfe.go index 2be85830..2a13b7d2 100644 --- a/state/tfe.go +++ b/state/tfe.go @@ -19,10 +19,15 @@ type TFE struct { } // NewTFE creates a new TFE object -func NewTFE(c *config.Config) (*TFE, error) { +func NewTFE(tfeObj config.TFEConfig) (*TFE, error) { + var tfeInstance *TFE + if tfeObj.Token == "" { + return nil, nil + } + config := &tfe.Config{ - Address: c.TFE.Address, - Token: c.TFE.Token, + Address: tfeObj.Address, + Token: tfeObj.Token, } client, err := tfe.NewClient(config) @@ -31,12 +36,27 @@ func NewTFE(c *config.Config) (*TFE, error) { } ctx := context.Background() - - return &TFE{ + tfeInstance = &TFE{ Client: client, - org: c.TFE.Organization, + org: tfeObj.Organization, ctx: &ctx, - }, nil + } + + return tfeInstance, nil +} + +// NewTFECollection instantiate all needed GCP objects configurated by the user and return a slice +func NewTFECollection(c *config.Config) ([]*TFE, error) { + var tfeInstances []*TFE + for _, tfe := range c.TFE { + tfeInstance, err := NewTFE(tfe) + if err != nil || tfeInstance == nil { + return nil, err + } + tfeInstances = append(tfeInstances, tfeInstance) + } + + return tfeInstances, nil } // GetLocks returns a map of locks by State path diff --git a/test/.gitignore b/test/.gitignore index 24b61d13..acb9efef 100755 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,2 +1,3 @@ data/* -!data/test-bucket \ No newline at end of file +!data/test-bucket +!data/test-bucket2 \ No newline at end of file diff --git a/test/data/test-bucket/.gitkeep b/test/data/test-bucket/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/test/data/test-bucket/terraform_0.12.28.tfstate b/test/data/test-bucket/terraform_0.12.28.tfstate deleted file mode 100644 index 9590a256..00000000 --- a/test/data/test-bucket/terraform_0.12.28.tfstate +++ /dev/null @@ -1,36 +0,0 @@ -{ - "version": 4, - "terraform_version": "0.12.28", - "serial": 2, - "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00", - "outputs": { - "content": { - "value": "foo!", - "type": "string" - } - }, - "resources": [ - { - "mode": "managed", - "type": "local_file", - "name": "foo", - "provider": "provider.local", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content": "foo!", - "content_base64": null, - "directory_permission": "0777", - "file_permission": "0777", - "filename": "./foo.bar", - "id": "4bf3e335199107182c6f7638efaad377acc7f452", - "sensitive_content": null, - "source": null - }, - "private": "bnVsbA==" - } - ] - } - ] -} diff --git a/test/data/test-bucket/terraform_0.13.5.tfstate b/test/data/test-bucket/terraform_0.13.5.tfstate deleted file mode 100644 index 8325cf53..00000000 --- a/test/data/test-bucket/terraform_0.13.5.tfstate +++ /dev/null @@ -1,36 +0,0 @@ -{ - "version": 4, - "terraform_version": "0.13.5", - "serial": 3, - "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00", - "outputs": { - "content": { - "value": "foo!", - "type": "string" - } - }, - "resources": [ - { - "mode": "managed", - "type": "local_file", - "name": "foo", - "provider": "provider[\"registry.terraform.io/hashicorp/local\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content": "foo!", - "content_base64": null, - "directory_permission": "0777", - "file_permission": "0777", - "filename": "./foo.bar", - "id": "4bf3e335199107182c6f7638efaad377acc7f452", - "sensitive_content": null, - "source": null - }, - "private": "bnVsbA==" - } - ] - } - ] -} diff --git a/test/data/test-bucket/terraform_0.14.8.tfstate b/test/data/test-bucket/terraform_0.14.8.tfstate deleted file mode 100644 index efa47929..00000000 --- a/test/data/test-bucket/terraform_0.14.8.tfstate +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version": 4, - "terraform_version": "0.14.8", - "serial": 3, - "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00", - "outputs": { - "content": { - "value": "foo!", - "type": "string" - } - }, - "resources": [ - { - "mode": "managed", - "type": "local_file", - "name": "foo", - "provider": "provider[\"registry.terraform.io/hashicorp/local\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content": "foo!", - "content_base64": null, - "directory_permission": "0777", - "file_permission": "0777", - "filename": "./foo.bar", - "id": "4bf3e335199107182c6f7638efaad377acc7f452", - "sensitive_content": null, - "source": null - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - } - ] -} diff --git a/test/data/test-bucket/terraform_0.15.5.tfstate b/test/data/test-bucket/terraform_0.15.5.tfstate deleted file mode 100644 index f8751ad9..00000000 --- a/test/data/test-bucket/terraform_0.15.5.tfstate +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version": 4, - "terraform_version": "0.15.5", - "serial": 3, - "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00", - "outputs": { - "content": { - "value": "foo!", - "type": "string" - } - }, - "resources": [ - { - "mode": "managed", - "type": "local_file", - "name": "foo", - "provider": "provider[\"registry.terraform.io/hashicorp/local\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content": "foo!", - "content_base64": null, - "directory_permission": "0777", - "file_permission": "0777", - "filename": "./foo.bar", - "id": "4bf3e335199107182c6f7638efaad377acc7f452", - "sensitive_content": null, - "source": null - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - } - ] -} diff --git a/test/data/test-bucket/terraform_1.0.0.tfstate b/test/data/test-bucket/terraform_1.0.0.tfstate deleted file mode 100644 index a3835bd3..00000000 --- a/test/data/test-bucket/terraform_1.0.0.tfstate +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version": 4, - "terraform_version": "1.0.0", - "serial": 3, - "lineage": "b7d2aa5a-812a-2d1f-e5e6-835215928e00", - "outputs": { - "content": { - "value": "foo!", - "type": "string" - } - }, - "resources": [ - { - "mode": "managed", - "type": "local_file", - "name": "foo", - "provider": "provider[\"registry.terraform.io/hashicorp/local\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "content": "foo!", - "content_base64": null, - "directory_permission": "0777", - "file_permission": "0777", - "filename": "./foo.bar", - "id": "4bf3e335199107182c6f7638efaad377acc7f452", - "sensitive_content": null, - "source": null - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - } - ] -} diff --git a/test/Makefile b/test/multiple-minio-buckets/Makefile similarity index 99% rename from test/Makefile rename to test/multiple-minio-buckets/Makefile index b59d9d4d..5e30c909 100644 --- a/test/Makefile +++ b/test/multiple-minio-buckets/Makefile @@ -1,3 +1,4 @@ + UID=$(shell id -u) GID=$(shell id -g) diff --git a/test/multiple-minio-buckets/config.yml b/test/multiple-minio-buckets/config.yml new file mode 100644 index 00000000..acb12c42 --- /dev/null +++ b/test/multiple-minio-buckets/config.yml @@ -0,0 +1,35 @@ +log: + level: debug + format: plain + +database: + password: mypassword + +provider: + no-locks: true + no-versioning: true + +aws: + - access-key: root + secret-access-key: mypassword + endpoint: http://minio:9000/ + region: eu-west-1 + s3: + - bucket: test-bucket + force-path-style: true + file-extension: + - .tfstate + - bucket: test-bucket2 + force-path-style: true + file-extension: + - .tfstate + + - access-key: admin + secret-access-key: password + endpoint: http://minio-2:9000/ + region: eu-west-1 + s3: + - bucket: test-bucket + force-path-style: true + file-extension: + - .tfstate diff --git a/test/multiple-minio-buckets/docker-compose.yml b/test/multiple-minio-buckets/docker-compose.yml new file mode 100644 index 00000000..ba62c425 --- /dev/null +++ b/test/multiple-minio-buckets/docker-compose.yml @@ -0,0 +1,76 @@ +--- +version: "3.8" +services: + terraboard-dev: + build: + context: ../../ + dockerfile: ./Dockerfile + environment: + DB_SSLMODE: disable + CONFIG_FILE: config/config.yml + GODEBUG: netdns=go + depends_on: + db: + condition: service_healthy + minio: + condition: service_started + volumes: + - ../../static:/static:ro + - ./config.yml:/config/config.yml:ro + ports: + - "8080:8080" + + minio: + image: minio/minio:latest + environment: + MINIO_ROOT_USER: root + MINIO_ROOT_PASSWORD: mypassword + user: "${UID}:${GID}" + expose: + - "9000" + ports: + - "9200:9000" + volumes: + - ../data/minio-1:/data + command: server /data + + minio-2: + image: minio/minio:latest + environment: + MINIO_ROOT_USER: admin + MINIO_ROOT_PASSWORD: password + user: "${UID}:${GID}" + expose: + - "9000" + ports: + - "9300:9000" + volumes: + - ../data/minio-2:/data + command: server /data + + db: + image: postgres:9.5 + environment: + POSTGRES_USER: gorm + POSTGRES_PASSWORD: mypassword + POSTGRES_DB: gorm + volumes: + - tb-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + pgadmin: + container_name: pgadmin4_container + image: dpage/pgadmin4 + restart: always + environment: + PGADMIN_DEFAULT_EMAIL: admin@admin.com + PGADMIN_DEFAULT_PASSWORD: root + ports: + - "5050:80" + +volumes: + tb-data: {} diff --git a/test/single-minio-bucket/Makefile b/test/single-minio-bucket/Makefile new file mode 100644 index 00000000..3edb92d0 --- /dev/null +++ b/test/single-minio-bucket/Makefile @@ -0,0 +1,11 @@ + +UID=$(shell id -u) +GID=$(shell id -g) + +build: + UID="${UID}" GID="${GID}" docker-compose build + +test: + UID="${UID}" GID="${GID}" docker-compose up -d + +all: build test \ No newline at end of file diff --git a/test/docker-compose.yml b/test/single-minio-bucket/docker-compose.yml similarity index 94% rename from test/docker-compose.yml rename to test/single-minio-bucket/docker-compose.yml index 4f187dd2..3a366cd1 100755 --- a/test/docker-compose.yml +++ b/test/single-minio-bucket/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.8" services: terraboard-dev: build: - context: ../ + context: ../../ dockerfile: ./Dockerfile environment: AWS_ACCESS_KEY_ID: root @@ -24,7 +24,7 @@ services: minio: condition: service_started volumes: - - ../static:/static:ro + - ../../static:/static:ro ports: - "8080:8080" @@ -39,7 +39,7 @@ services: ports: - "9200:9000" volumes: - - ./data:/data + - ../data/minio-1:/data command: server /data db: