Cleanup config parsing and database migration errors

The long and the short of this is there were multiple errors with parsing configs and running migrations. What you need to know: There is a patched vendor file (shame on me, I know). However, being the nice netizen that I am, I made a PR upstream golang-migrate/migrate#36 The other stuff is mostly cleanup around how viper serializes config files and some bad quoting on my part in originsrv. Signed-off-by: Elliott Davis <elliott@excellent.io>
chuckleheads · May 22, 2018 · 70943cf · 70943cf
1 parent ad48acd
commit 70943cf
Show file tree

Hide file tree

Showing 45 changed files with 395 additions and 294 deletions.
diff --git a/.studiorc b/.studiorc
@@ -49,6 +49,14 @@ start-hurtlocker() {
   start-sessionsrv
 }
 
+stop-hurtlocker() {
+  stop-rabbitmq
+  stop-datastore
+  stop-originsrv
+  stop-agent
+  stop-sessionsrv
+}
+
 start-rabbitmq() {
   hab svc load core/rabbitmq
 }

diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/components/datastore/plan.sh b/components/datastore/plan.sh
@@ -6,6 +6,10 @@ pkg_upstream_url="https://github.com/chuckleheads/hurtlocker"
 pkg_license=('Apache-2.0')
 pkg_maintainer="The Habitat Maintainers <humans@habitat.sh>"
 pkg_deps=(core/cockroach)
+pkg_exports=(
+  [port]=port
+)
+pkg_exposes=(port)
 
 do_build() {
   return 0

diff --git a/components/originsrv/cmd/migrate.go b/components/originsrv/cmd/migrate.go
@@ -13,13 +13,13 @@ var migrateCmd = &cobra.Command{
 	Short: "Run the database migrations for originsrv",
 	Run: func(cmd *cobra.Command, args []string) {
 		log.Printf("Running Migrations...")
-		dbConfig, err := DBConfigFromViper()
+		config, err := ConfigFromViper()
 		if err != nil {
 			panic(err.Error())
 		}
-		db := data_store.New(dbConfig)
+		db := data_store.New(&config.Datastore)
 
-		migrations.Migrate(db, dbConfig.Migrations)
+		migrations.Migrate(db, config.Datastore.Migrations)
 		log.Printf("Migrations complete")
 	},
 }

diff --git a/components/originsrv/cmd/root.go b/components/originsrv/cmd/root.go
@@ -40,12 +40,12 @@ func init() {
 	// Cobra supports persistent flags, which, if defined here,
 	// will be global for your application.
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is /hab/svc/originsrv/config/config.toml)")
-	viper.SetDefault("host", "localhost")
-	viper.SetDefault("port", 26257)
-	viper.SetDefault("database", "originsrv")
-	viper.SetDefault("username", "root")
-	viper.SetDefault("password", "")
-	viper.SetDefault("ssl-mode", "disable")
+	viper.SetDefault("datastore.host", "localhost")
+	viper.SetDefault("datastore.port", 26257)
+	viper.SetDefault("datastore.database", "originsrv")
+	viper.SetDefault("datastore.username", "root")
+	viper.SetDefault("datastore.password", "")
+	viper.SetDefault("datastore.ssl-mode", "disable")
 }
 
 // initConfig reads in config file and ENV variables if set.
@@ -70,8 +70,8 @@ func initConfig() {
 }
 
 // DBConfigFromViper fetches database config from viper
-func DBConfigFromViper() (*config.DBConfig, error) {
-	cfg := &config.DBConfig{}
+func ConfigFromViper() (*config.Config, error) {
+	cfg := &config.Config{}
 	if err := viper.Unmarshal(cfg); err != nil {
 		panic(err.Error())
 	}

diff --git a/components/originsrv/cmd/start.go b/components/originsrv/cmd/start.go
@@ -50,11 +50,11 @@ func setup() {
 func runGRPC(lis net.Listener) {
 	server := grpc.NewServer()
 
-	dbConfig, err := DBConfigFromViper()
+	config, err := ConfigFromViper()
 	if err != nil {
 		panic(err.Error())
 	}
-	db := data_store.New(dbConfig)
+	db := data_store.New(&config.Datastore)
 	pbs.RegisterOriginsServer(server, srv.NewServer(db))
 	log.Printf("gRPC Listening on %s\n", lis.Addr().String())
 	server.Serve(lis)

diff --git a/components/originsrv/config/config.go b/components/originsrv/config/config.go
@@ -1,5 +1,5 @@
 package config
 
 type Config struct {
-	Database DBConfig
+	Datastore DBConfig
 }
diff --git a/components/originsrv/data_store/migrations/migrations.go b/components/originsrv/data_store/migrations/migrations.go
@@ -4,10 +4,10 @@ import (
 	"database/sql"
 	"fmt"
 
+	"github.com/golang-migrate/migrate"
+	"github.com/golang-migrate/migrate/database/postgres"
+	_ "github.com/golang-migrate/migrate/source/file"
 	_ "github.com/lib/pq"
-	"github.com/mattes/migrate"
-	"github.com/mattes/migrate/database/postgres"
-	_ "github.com/mattes/migrate/source/file"
 )
 
 func Migrate(db *sql.DB, migrations_dir string) {
@@ -18,5 +18,5 @@ func Migrate(db *sql.DB, migrations_dir string) {
 	m, err := migrate.NewWithDatabaseInstance(
 		fmt.Sprintf("file://%s", migrations_dir),
 		"postgres", driver)
-	m.Up()
+	m.Steps(0)
 }
diff --git a/components/originsrv/habitat/config/config.toml b/components/originsrv/habitat/config/config.toml
@@ -1,9 +1,9 @@
 [datastore]
 database = "{{cfg.datastore.database}}"
-{{#eachAlive bind.database.members as |member| ~}}
+{{#eachAlive bind.datastore.members as |member| ~}}
 {{#if @first ~}}
-host = "member.sys.ip"
-port = "member.cfg.port"
+host = "{{member.sys.ip}}"
+port = {{member.cfg.port}}
 {{/if ~}}
 {{/eachAlive ~}}
-migrations = {{pkg.path}}/migrations
+migrations = "{{pkg.path}}/migrations"
diff --git a/components/originsrv/habitat/hooks/init b/components/originsrv/habitat/hooks/init
@@ -2,4 +2,4 @@
 
 cockroach sql --execute="CREATE DATABASE IF NOT EXISTS {{cfg.datastore.database}};" --insecure
 
-originsrv migrate
+originsrv migrate --config "{{pkg.svc_config_path}}/config.toml"
diff --git a/components/originsrv/habitat/hooks/run b/components/originsrv/habitat/hooks/run
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+exec 2>&1
+
+exec {{pkg.name}} start --config "{{pkg.svc_config_path}}/config.toml"
diff --git a/components/originsrv/habitat/plan.sh b/components/originsrv/habitat/plan.sh
@@ -9,7 +9,6 @@ pkg_maintainer="The Habitat Maintainers <humans@habitat.sh>"
 pkg_bin_dirs=(bin)
 pkg_build_deps=(core/go core/git core/dep)
 pkg_deps=(core/cockroach)
-pkg_svc_run="${pkg_name} start"
 pkg_binds=(
   [datastore]="port"
 )
diff --git a/components/sessionsrv/cmd/migrate.go b/components/sessionsrv/cmd/migrate.go
@@ -13,13 +13,12 @@ var migrateCmd = &cobra.Command{
 	Short: "Run the database migrations for sessionsrv",
 	Run: func(cmd *cobra.Command, args []string) {
 		log.Printf("Running Migrations...")
-		dbConfig, err := DBConfigFromViper()
+		config, err := ConfigFromViper()
 		if err != nil {
 			panic(err.Error())
 		}
-		db := data_store.New(dbConfig)
-
-		migrations.Migrate(db, dbConfig.Migrations)
+		db := data_store.New(&config.Datastore)
+		migrations.Migrate(db, config.Datastore.Migrations)
 		log.Printf("Migrations complete")
 	},
 }

diff --git a/components/sessionsrv/cmd/root.go b/components/sessionsrv/cmd/root.go
@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"log"
 	"os"
 	"path/filepath"
 
@@ -35,12 +36,12 @@ func init() {
 	// Cobra supports persistent flags, which, if defined here,
 	// will be global for your application.
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is /hab/svc/sessionsrv/config/config.toml)")
-	viper.SetDefault("host", "localhost")
-	viper.SetDefault("port", 26257)
-	viper.SetDefault("database", "sessionsrv")
-	viper.SetDefault("username", "root")
-	viper.SetDefault("password", "")
-	viper.SetDefault("ssl-mode", "disable")
+	viper.SetDefault("datastore.host", "localhost")
+	viper.SetDefault("datastore.port", 26257)
+	viper.SetDefault("datastore.database", "sessionsrv")
+	viper.SetDefault("datastore.username", "root")
+	viper.SetDefault("datastore.password", "")
+	viper.SetDefault("datastore.ssl-mode", "disable")
 }
 
 // initConfig reads in config file and ENV variables if set.
@@ -51,7 +52,7 @@ func initConfig() {
 	} else {
 		home := filepath.Dir("/hab/svc/sessionsrv/config")
 
-		// Search config in home directory with name ".config" (without extension).
+		// Search config in home directory with name "config" (without extension).
 		viper.AddConfigPath(home)
 		viper.SetConfigName("config")
 	}
@@ -61,14 +62,18 @@ func initConfig() {
 	// If a config file is found, read it in.
 	if err := viper.ReadInConfig(); err == nil {
 		fmt.Println("Using config file:", viper.ConfigFileUsed())
+	} else {
+		fmt.Printf("Err: %v", err)
 	}
+
 }
 
-// DBConfigFromViper fetches database config from viper
-func DBConfigFromViper() (*config.DBConfig, error) {
-	cfg := &config.DBConfig{}
+// ConfigFromViper fetches database config from viper
+func ConfigFromViper() (*config.Config, error) {
+	cfg := &config.Config{}
 	if err := viper.Unmarshal(cfg); err != nil {
 		panic(err.Error())
 	}
+	log.Printf("CONFIG: %v", cfg)
 	return cfg, nil
 }
diff --git a/components/sessionsrv/cmd/start.go b/components/sessionsrv/cmd/start.go
@@ -45,11 +45,11 @@ func setup() {
 func runGRPC(lis net.Listener) {
 	server := grpc.NewServer()
 
-	dbConfig, err := DBConfigFromViper()
+	config, err := ConfigFromViper()
 	if err != nil {
 		panic(err.Error())
 	}
-	db := data_store.New(dbConfig)
+	db := data_store.New(&config.Datastore)
 	pb.RegisterAccountsServer(server, srv.NewServer(db))
 
 	log.Printf("gRPC Listening on %s\n", lis.Addr().String())

diff --git a/components/sessionsrv/config/config.go b/components/sessionsrv/config/config.go
@@ -1,5 +1,5 @@
 package config
 
 type Config struct {
-	Database DBConfig
+	Datastore DBConfig
 }
diff --git a/components/sessionsrv/data_store/migrations/migrations.go b/components/sessionsrv/data_store/migrations/migrations.go
@@ -3,20 +3,28 @@ package migrations
 import (
 	"database/sql"
 	"fmt"
+	"log"
 
+	"github.com/golang-migrate/migrate"
+	"github.com/golang-migrate/migrate/database/postgres"
+	_ "github.com/golang-migrate/migrate/source/file"
 	_ "github.com/lib/pq"
-	"github.com/mattes/migrate"
-	"github.com/mattes/migrate/database/postgres"
-	_ "github.com/mattes/migrate/source/file"
 )
 
 func Migrate(db *sql.DB, migrations_dir string) {
 	driver, err := postgres.WithInstance(db, &postgres.Config{})
 	if err != nil {
 		panic(err.Error())
 	}
+	log.Printf("file://%s", migrations_dir)
 	m, err := migrate.NewWithDatabaseInstance(
 		fmt.Sprintf("file://%s", migrations_dir),
 		"postgres", driver)
-	m.Up()
+	if err != nil {
+		panic(err.Error())
+	}
+	err = m.Up()
+	if err != nil {
+		panic(err.Error())
+	}
 }
diff --git a/components/sessionsrv/habitat/config/config.toml b/components/sessionsrv/habitat/config/config.toml
@@ -1,9 +1,9 @@
 [datastore]
 database = "{{cfg.datastore.database}}"
-{{#eachAlive bind.database.members as |member| ~}}
+{{#eachAlive bind.datastore.members as |member| ~}}
 {{#if @first ~}}
-host = "member.sys.ip"
-port = "member.cfg.port"
+host = "{{member.sys.ip}}"
+port = {{member.cfg.port}}
 {{/if ~}}
 {{/eachAlive ~}}
-migrations = {{pkg.path}}/migrations
+migrations = "{{pkg.path}}/migrations"
diff --git a/components/sessionsrv/habitat/hooks/init b/components/sessionsrv/habitat/hooks/init
@@ -2,4 +2,4 @@
 
 cockroach sql --execute="CREATE DATABASE IF NOT EXISTS {{cfg.datastore.database}};" --insecure
 
-sessionsrv migrate
+sessionsrv migrate --config  "{{pkg.svc_config_path}}/config.toml"
diff --git a/components/sessionsrv/habitat/hooks/run b/components/sessionsrv/habitat/hooks/run
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+exec 2>&1
+
+exec {{pkg.name}} start --config "{{pkg.svc_config_path}}/config.toml" 
diff --git a/components/sessionsrv/habitat/plan.sh b/components/sessionsrv/habitat/plan.sh
@@ -9,7 +9,6 @@ pkg_maintainer="The Habitat Maintainers <humans@habitat.sh>"
 pkg_bin_dirs=(bin)
 pkg_build_deps=(core/go core/git core/dep)
 pkg_deps=(core/cockroach)
-pkg_svc_run="${pkg_name} start"
 pkg_binds=(
   [datastore]="port"
 )
diff --git a/vendor/github.com/mattes/migrate/.gitignore → ...hub.com/golang-migrate/migrate/.gitignore b/vendor/github.com/mattes/migrate/.gitignore → ...hub.com/golang-migrate/migrate/.gitignore