From 6d44ecbf5765e33b2cfcdbef1dce9059fe7dc94a Mon Sep 17 00:00:00 2001 From: Hugo Bollon Date: Wed, 16 Jun 2021 13:47:14 +0200 Subject: [PATCH] feat: add test environment with proper docker-compose using MinIO + add compatibility for it to AWS provider (#165) * feat: docker-compose for test environment using MinIO * feat: update AWS provider to be compatible with MinIO (and others S3 compatible ones) * feat: replace AWS_OTHER_COMPATIBLE_PROVIDER by two distinct generic parameters --- config/config.go | 8 +++++ example.yml | 4 +++ state/aws.go | 21 +++++++++++- test/.gitignore | 2 ++ test/data/test-bucket/.gitkeep | 0 test/docker-compose.yml | 62 ++++++++++++++++++++++++++++++++++ 6 files changed, 96 insertions(+), 1 deletion(-) create mode 100755 test/.gitignore create mode 100644 test/data/test-bucket/.gitkeep create mode 100755 test/docker-compose.yml diff --git a/config/config.go b/config/config.go index 685e24fa..6184b119 100644 --- a/config/config.go +++ b/config/config.go @@ -74,12 +74,20 @@ type WebConfig struct { LogoutURL string `long:"logout-url" env:"TERRABOARD_LOGOUT_URL" yaml:"logout-url" description:"Logout URL."` } +// ProviderConfig stores genral provider parameters +type ProviderConfig struct { + NoVersioning bool `long:"no-versioning" env:"TERRABOARD_NO_VERSIONING" yaml:"no-versioning" description:"Disable versioning support from Terraboard (useful for S3 compatible providers like MinIO)"` + NoLocks bool `long:"no-locks" env:"TERRABOARD_NO_LOCKS" yaml:"no-locks" description:"Disable locks support from Terraboard (useful for S3 compatible providers like MinIO)"` +} + // Config stores the handler's configuration and UI interface parameters type Config struct { Version bool `short:"V" long:"version" description:"Display version."` ConfigFilePath string `short:"c" long:"config-file" env:"CONFIG_FILE" description:"Config File path"` + Provider ProviderConfig `group:"General Provider Options" yaml:"provider"` + Log LogConfig `group:"Logging Options" yaml:"log"` DB DBConfig `group:"Database Options" yaml:"database"` diff --git a/example.yml b/example.yml index 4935c8b9..75bbca38 100644 --- a/example.yml +++ b/example.yml @@ -11,6 +11,10 @@ database: no-sync: false sync-interval: 5 +provider: + no-locks: "true" + no-versioning: "false" + aws: dynamodb-table: terraboard s3: diff --git a/state/aws.go b/state/aws.go index b0f90dd5..f54a202b 100644 --- a/state/aws.go +++ b/state/aws.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "strings" + "time" aws_sdk "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" @@ -25,6 +26,8 @@ type AWS struct { dynamoTable string keyPrefix string fileExtension []string + noLocks bool + noVersioning bool } // NewAWS creates an AWS object @@ -58,11 +61,18 @@ func NewAWS(c *config.Config) AWS { fileExtension: c.AWS.S3.FileExtension, dynamoSvc: dynamodb.New(sess, awsConfig), dynamoTable: c.AWS.DynamoDBTable, + noLocks: c.Provider.NoLocks, + noVersioning: c.Provider.NoVersioning, } } // GetLocks returns a map of locks by State path func (a *AWS) GetLocks() (locks map[string]LockInfo, err error) { + if a.noLocks { + locks = make(map[string]LockInfo) + return + } + if a.dynamoTable == "" { err = fmt.Errorf("no dynamoDB table provided, not getting locks") return @@ -94,6 +104,7 @@ func (a *AWS) GetLocks() (locks map[string]LockInfo, err error) { locks[strings.TrimPrefix(info.Path, infoPrefix)] = info } } + return } @@ -138,7 +149,7 @@ func (a *AWS) GetState(st, versionID string) (sf *statefile.File, err error) { Bucket: aws_sdk.String(a.bucket), Key: aws_sdk.String(st), } - if versionID != "" { + if versionID != "" && !a.noVersioning { input.VersionId = &versionID } result, err := a.svc.GetObjectWithContext(context.Background(), input) @@ -168,6 +179,14 @@ func (a *AWS) GetState(st, versionID string) (sf *statefile.File, err error) { // GetVersions returns a slice of Version objects func (a *AWS) GetVersions(state string) (versions []Version, err error) { versions = []Version{} + if a.noVersioning { + versions = append(versions, Version{ + ID: "1", + LastModified: time.Now(), + }) + return + } + result, err := a.svc.ListObjectVersions(&s3.ListObjectVersionsInput{ Bucket: aws_sdk.String(a.bucket), Prefix: aws_sdk.String(state), diff --git a/test/.gitignore b/test/.gitignore new file mode 100755 index 00000000..24b61d13 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,2 @@ +data/* +!data/test-bucket \ No newline at end of file diff --git a/test/data/test-bucket/.gitkeep b/test/data/test-bucket/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/test/docker-compose.yml b/test/docker-compose.yml new file mode 100755 index 00000000..6b8492db --- /dev/null +++ b/test/docker-compose.yml @@ -0,0 +1,62 @@ +--- +version: "3" +services: + terraboard-dev: + build: + context: ../ + dockerfile: ./Dockerfile + environment: + AWS_ACCESS_KEY_ID: root + AWS_SECRET_ACCESS_KEY: mypassword + AWS_BUCKET: test-bucket + AWS_REGION: eu-west-1 + AWS_ENDPOINT: http://minio:9000/ + AWS_FORCE_PATH_STYLE: "true" + TERRABOARD_LOG_LEVEL: debug + TERRABOARD_NO_LOCKS: "true" + TERRABOARD_NO_VERSIONING: "true" + DB_PASSWORD: mypassword + DB_SSLMODE: disable + GODEBUG: netdns=go + depends_on: + - "db" + - "minio" + volumes: + - ../static:/static:ro + ports: + - "8080:8080" + + minio: + image: minio/minio:latest + environment: + MINIO_ROOT_USER: root + MINIO_ROOT_PASSWORD: mypassword + expose: + - "9000" + ports: + - "9200:9000" + volumes: + - ./data:/data + command: server /data + + db: + image: postgres:9.5 + environment: + POSTGRES_USER: gorm + POSTGRES_PASSWORD: mypassword + POSTGRES_DB: gorm + volumes: + - tb-data:/var/lib/postgresql/data + + pgadmin: + container_name: pgadmin4_container + image: dpage/pgadmin4 + restart: always + environment: + PGADMIN_DEFAULT_EMAIL: admin@admin.com + PGADMIN_DEFAULT_PASSWORD: root + ports: + - "5050:80" + +volumes: + tb-data: {} \ No newline at end of file