PLAT-342: insconfig (#9)

* update deps

* dump reporter config, hide secrets

* use insconfig in metricreplicator cmd

* update metricreplicator config

* update packed template

README.md

@@ -18,7 +18,15 @@ Config sets host for prometheus, webdav auth options, time periods from which to
 
 ### Run metric replicator
 ```
-go run cmd/metricreplicator/main.go --cfg=cmd/metricreplicator/config.yml
+export REPORT_WEBDAV_HOST=https://webdav.yandex.ru
+export REPORT_WEBDAV_USERNAME=fspecter
+export REPORT_WEBDAV_PASSWORD=awkward20
+export REPORT_WEBDAV_DIRECTORY=fake102 
+
+export REPORT_GIT_BRANCH=master
+export REPORT_GIT_HASH=977022b
+
+go run cmd/metricreplicator/main.go --config=cmd/metricreplicator/config.yml
 ```
 
 Use `--rm=false` option if you want to save created file locally. Option is `true` by default.

cmd/metricreplicator/config.yml

@@ -3,7 +3,7 @@ quantiles:
   - "0.8"
   - "0.95"
   - "0.99"
-tmp_directory: "/tmp/metricreplicator"
+tmpdir: "/tmp/metricreplicator"
 prometheus:
   host: "http://localhost:9090"
 groups:
@@ -12,11 +12,17 @@ groups:
 #      - name: "latency"
 #        value: "50ms"
     ranges:
-      - start_time: 1589292280
+      - starttime: 1589292280
         interval: "3m"
         props:
           - name: "network_size"
             value: "17"
 webdav:
+  host: "localhost"
+  username: "replicator"
+  password: "replicator"
+  directory: ""
   timeout: "1m"
-commit_hash: "fake500"
+git:
+  branch: "master"
+  hash: ""

cmd/metricreplicator/main.go

@@ -2,65 +2,36 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
+	"github.com/insolar/insconfig"
 	"log"
 
-	"github.com/spf13/pflag"
-	"github.com/spf13/viper"
-
 	"github.com/insolar/consensus-reports/pkg/metricreplicator"
 	"github.com/insolar/consensus-reports/pkg/middleware"
 	"github.com/insolar/consensus-reports/pkg/replicator"
 )
 
 func main() {
-	cfgPath := pflag.String("cfg", "", "Path to cfg file")
-	removeAfter := pflag.Bool("rm", true, "Option to remove tmp dir after work")
-	pflag.Parse()
 
-	if *cfgPath == "" {
-		log.Fatalln("empty path to cfg file")
+	removeAfter := flag.Bool("rm", true, "Option to remove tmp dir after work")
+	cfg := middleware.Config{}
+	params := insconfig.Params{
+		EnvPrefix:       "report",
+		FileNotRequired: true,
+		ConfigPathGetter: &insconfig.FlagPathGetter{
+			GoFlags: flag.CommandLine,
+		},
 	}
+	insConfigurator := insconfig.New(params)
+	err := insConfigurator.Load(&cfg)
+	checkError(err)
 
-	// insConfig := insconfig.New(insconfig.Params{
-	// 	EnvPrefix:        "reports_webdav",
-	// 	ConfigPathGetter: &insconfig.PFlagPathGetter{PFlags: pflag.CommandLine},
-	// })
-	// var cfg middleware.Config
-	// if err := insConfig.Load(&cfg); err != nil {
-	// 	log.Fatalf("failed to load config: %v", err)
-	// }
-
-	vp := viper.New()
-	vp.SetConfigFile(*cfgPath)
-
-	vp.SetEnvPrefix("reports_webdav") // will be uppercased automatically
-	if err := vp.BindEnv("host"); err != nil {
-		log.Fatalf("failed to get webdav host: %v", err)
-	}
-	if err := vp.BindEnv("password"); err != nil {
-		log.Fatalf("failed to get webdav password: %v", err)
-	}
-	if err := vp.BindEnv("username"); err != nil {
-		log.Fatalf("failed to get webdav username: %v", err)
-	}
+	err = cfg.Validate()
+	checkError(err)
 
-	if err := vp.ReadInConfig(); err != nil {
-		log.Fatalf("failed to read config: %v", err)
-	}
-
-	var cfg middleware.Config
-	if err := vp.Unmarshal(&cfg); err != nil {
-		log.Fatalf("failed to unmarshal config: %v", err)
-	}
-
-	cfg.WebDav.Host = vp.GetString("host")
-	cfg.WebDav.Username = vp.GetString("username")
-	cfg.WebDav.Password = vp.GetString("password")
-
-	if err := cfg.Validate(); err != nil {
-		log.Fatalf("failed to validate config: %v\ncfg: %+v", err, cfg)
-	}
+	err = insconfig.NewYamlDumper(cfg).DumpTo(log.Writer())
+	checkError(err)
 
 	repl, err := metricreplicator.New(cfg.Prometheus.Host, cfg.TmpDir)
 	if err != nil {
@@ -107,3 +78,9 @@ func Run(repl replicator.Replicator, cfg middleware.Config, removeAfter bool) er
 	}
 	return nil
 }
+
+func checkError(err error) {
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

cmd/report/main.go

@@ -31,6 +31,9 @@ func main() {
 	err := insConfigurator.Load(&cfg)
 	checkError(err)
 
+	err = insconfig.NewYamlDumper(cfg).DumpTo(log.Writer())
+	checkError(err)
+
 	client := report.CreateWebdavClient(cfg)
 
 	if serveAddress != nil && *serveAddress != "" {

go.mod

@@ -4,14 +4,13 @@ go 1.14
 
 require (
 	github.com/go-playground/universal-translator v0.17.0 // indirect
-	github.com/insolar/insconfig v0.0.0-20200513150834-977022bc1445
+	github.com/insolar/insconfig v0.0.0-20200617131009-dd635d1fc9e9
 	github.com/leodido/go-urn v1.2.0 // indirect
-	github.com/markbates/pkger v0.15.1
+	github.com/markbates/pkger v0.17.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.6.0
 	github.com/prometheus/common v0.9.1
-	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.7.0
+	github.com/spf13/viper v1.7.0 // indirect
 	github.com/stretchr/testify v1.5.1
 	github.com/studio-b12/gowebdav v0.0.0-20200303150724-9380631c29a1
 	go.uber.org/zap v1.10.0

go.sum

@@ -115,8 +115,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/insolar/insconfig v0.0.0-20200513150834-977022bc1445 h1:UyynDBSMeJdIXJqY+9LXhg8bZRorRXaigmDbDkK30ZA=
-github.com/insolar/insconfig v0.0.0-20200513150834-977022bc1445/go.mod h1:zB9bawcGFKS4hwXcu6yH4rYH06EZnGlFfhl0+KMKM9k=
+github.com/insolar/insconfig v0.0.0-20200617131009-dd635d1fc9e9 h1:/fkjgehGsZxavp52GYnnns/rS1Mg3YIazANtxzFJkU4=
+github.com/insolar/insconfig v0.0.0-20200617131009-dd635d1fc9e9/go.mod h1:zB9bawcGFKS4hwXcu6yH4rYH06EZnGlFfhl0+KMKM9k=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
@@ -138,8 +138,8 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/markbates/pkger v0.15.1 h1:3MPelV53RnGSW07izx5xGxl4e/sdRD6zqseIk0rMASY=
-github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
+github.com/markbates/pkger v0.17.0 h1:RFfyBPufP2V6cddUyyEVSHBpaAnM1WzaMNyqomeT+iY=
+github.com/markbates/pkger v0.17.0/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=

pkg/middleware/config.go

@@ -1,12 +1,10 @@
 package middleware
 
 import (
-	"io/ioutil"
-	"time"
-
+	"github.com/insolar/insconfig"
 	"github.com/pkg/errors"
 	"gopkg.in/go-playground/validator.v9"
-	"gopkg.in/yaml.v2"
+	"time"
 
 	"github.com/insolar/consensus-reports/pkg/replicator"
 )
@@ -17,15 +15,15 @@ type PropertyConfig struct {
 }
 
 type RangeConfig struct {
-	StartTime  int64            `mapstructure:"start_time" validate:"required"`
+	StartTime  int64            `mapstructure:"starttime" validate:"required"`
 	Interval   time.Duration    `mapstructure:"interval" validate:"required"`
 	Properties []PropertyConfig `mapstructure:"props" validate:"min=1,dive,required"`
 }
 
 type WebDavConfig struct {
 	Host      string        `mapstructure:"host" validate:"required"`
 	Username  string        `mapstructure:"username" validate:"required"`
-	Password  string        `mapstructure:"password" validate:"required"`
+	Password  string        `mapstructure:"password" validate:"required" insconfigsecret:""`
 	Timeout   time.Duration `mapstructure:"timeout" validate:"required"`
 	Directory string        `directory:"host"`
 }
@@ -42,22 +40,36 @@ type PrometheusConfig struct {
 
 type Config struct {
 	Quantiles  []string         `mapstructure:"quantiles" validate:"min=1,dive,required"`
-	TmpDir     string           `mapstructure:"tmp_directory" validate:"required"`
+	TmpDir     string           `mapstructure:"tmpdir" validate:"required"`
 	Prometheus PrometheusConfig `mapstructure:"prometheus" validate:"required"`
 	Groups     []GroupConfig    `mapstructure:"groups" validate:"min=1,dive,required"`
 	WebDav     WebDavConfig     `mapstructure:"webdav" validate:"required"`
-	Commit     string           `mapstructure:"commit_hash" validate:"required"`
+	Git    struct {
+		Branch string
+		Hash   string
+	}
+}
+
+type pathGetter struct {
+	path string
+}
+func (g *pathGetter) GetConfigPath() string {
+	return g.path
 }
 
 func NewConfig(cfgPath string) (Config, error) {
-	rawData, err := ioutil.ReadFile(cfgPath)
-	if err != nil {
-		return Config{}, errors.Wrap(err, "failed to read cfg file")
+	cfg := Config{}
+	params := insconfig.Params{
+		EnvPrefix:       "metricreplicator",
+		FileNotRequired: true,
+		ConfigPathGetter: &pathGetter{
+			path: cfgPath,
+		},
 	}
-
-	var cfg Config
-	if err := yaml.Unmarshal(rawData, &cfg); err != nil {
-		return Config{}, errors.Wrap(err, "failed to unmarshal cfg file")
+	insConfigurator := insconfig.New(params)
+	err := insConfigurator.Load(&cfg)
+	if err != nil {
+		return Config{}, errors.Wrap(err, "failed to load config")
 	}
 
 	return cfg, nil
@@ -76,7 +88,7 @@ func (cfg Config) LoaderConfig() replicator.LoaderConfig {
 		URL:           cfg.WebDav.Host,
 		User:          cfg.WebDav.Username,
 		Password:      cfg.WebDav.Password,
-		RemoteDirName: cfg.Commit,
+		RemoteDirName: cfg.Git.Hash,
 		Timeout:       cfg.WebDav.Timeout,
 	}
 }

pkg/middleware/config_test.go

@@ -38,7 +38,10 @@ func TestConfig_Validate(t *testing.T) {
 				Password: "pwd",
 				Timeout:  time.Minute,
 			},
-			Commit: "hash",
+			Git: struct {
+				Branch string
+				Hash   string
+			}{"master", "hash"},
 		}
 		err := cfg.Validate()
 		require.NoError(t, err)
@@ -54,7 +57,10 @@ func TestConfig_Validate(t *testing.T) {
 				Username: "user",
 				Password: "pwd",
 			},
-			Commit: "hash",
+			Git: struct {
+				Branch string
+				Hash   string
+			}{"master", "hash"},
 		}
 		err := cfg.Validate()
 		require.Error(t, err)
@@ -82,7 +88,10 @@ func TestConfig_Validate(t *testing.T) {
 				Host:     "test",
 				Username: "user",
 			},
-			Commit: "hash",
+			Git: struct {
+				Branch string
+				Hash   string
+			}{"master", "hash"},
 		}
 		err := cfg.Validate()
 		require.Error(t, err)
@@ -110,7 +119,7 @@ func TestNewConfig(t *testing.T) {
 		cfgPath := "fake_config.yml"
 		_, err := NewConfig(cfgPath)
 		require.Error(t, err)
-		require.Contains(t, err.Error(), "failed to read cfg file")
+		require.Contains(t, err.Error(), "failed to load config")
 	})
 }