From f5ed2f895024063af2217cecb29a7dcba2470200 Mon Sep 17 00:00:00 2001 From: Nathan Oyler Date: Thu, 7 Mar 2024 21:12:41 -0700 Subject: [PATCH 1/6] switch to env vars from conf --- docs/operators/config.md | 65 +++++++++++++---------- main.go | 110 +++++++++++++++++++-------------------- 2 files changed, 91 insertions(+), 84 deletions(-) diff --git a/docs/operators/config.md b/docs/operators/config.md index 9c2751df..4fccca41 100644 --- a/docs/operators/config.md +++ b/docs/operators/config.md @@ -1,45 +1,54 @@ # Configuration Guide -Hermes is configured using a TOML config file that is by default located in `etc/hermes/hermes.conf`. -An example configuration file is located in etc/ which can help you get started. +Hermes is configured using environment variables. You can set the following environment variables to configure Hermes: #### Main Hermes config -\[hermes\] -* PolicyFilePath - Location of [OpenStack policy file](https://docs.OpenStack.org/security-guide/identity/policies.html) - policy.json file for which roles are required to access audit events. -Example located in `etc/policy.json` +- `HERMES_DEBUG`: Enable debug logging. Set to `true` to enable debug mode. +- `HERMES_KEYSTONE_DRIVER`: Specifies the keystone driver to use. Default is `keystone`. +- `HERMES_STORAGE_DRIVER`: Specifies the storage driver to use. Default is `elasticsearch`. +- `HERMES_POLICY_FILE_PATH`: Location of [OpenStack policy file](https://docs.OpenStack.org/security-guide/identity/policies.html) - policy.json file for which roles are required to access audit events. Default is `etc/policy.json`. #### ElasticSearch configuration -Any data served by Hermes requires an underlying ElasticSearch installation to act as the Datastore. - -\[ElasticSearch\] -* url - Url for ElasticSearch - -#### Environment Variables - -To configure secure access to Elasticsearch, set the following environment variables: +Any data served by Hermes requires an underlying ElasticSearch installation to act as the Datastore. +- `HERMES_ES_URL`: URL for ElasticSearch. Default is `http://localhost:9200`. - `HERMES_ES_USERNAME`: The username for connecting to Elasticsearch. - `HERMES_ES_PASSWORD`: The password for connecting to Elasticsearch. +- `HERMES_ES_MAX_RESULT_WINDOW`: The maximum result window for Elasticsearch queries. Default is `20000`. -These environment variables can be set in the deployment environment, or you may include them in your Kubernetes configuration if you are deploying Hermes there. +#### Integration for OpenStack Keystone -#### Example usage: +- `HERMES_OS_AUTH_URL`: Location of v3 keystone identity - ex. `https://keystone.example.com/v3`. +- `HERMES_OS_USERNAME`: OpenStack *service* user to authenticate and authorize clients. +- `HERMES_OS_PASSWORD`: Password for the OpenStack service user. +- `HERMES_OS_USER_DOMAIN_NAME`: User domain name for the OpenStack service user. +- `HERMES_OS_PROJECT_NAME`: Project name for the OpenStack service user. +- `HERMES_OS_PROJECT_DOMAIN_NAME`: Project domain name for the OpenStack service user. +- `HERMES_OS_TOKEN_CACHE_TIME`: In order to improve responsiveness and protect Keystone from too much load, Hermes will re-check authorizations for users by default every 15 minutes (900 seconds). You can adjust this value as needed. +- `HERMES_OS_MEMCACHED_SERVERS`: Comma-separated list of memcached servers to use for token caching (optional). -```bash -export HERMES_ES_USERNAME="your_username_here" -export HERMES_ES_PASSWORD="your_password_here" -``` +#### API Configuration +- `HERMES_API_LISTEN_ADDRESS`: The address and port for the Hermes API server to listen on. Default is `0.0.0.0:8788`. -#### Integration for OpenStack Keystone -\[keystone\] -* auth_url - Location of v3 keystone identity - ex. https://keystone.example.com/v3 -* username - OpenStack *service* user to authenticate and authorize clients. -* password -* user_domain_name -* project_name -* token_cache_time - In order to improve responsiveness and protect Keystone from too much load, Hermes will -re-check authorizations for users by default every 15 minutes (900 seconds). +#### Example usage: +```bash +export HERMES_DEBUG=true +export HERMES_KEYSTONE_DRIVER=keystone +export HERMES_STORAGE_DRIVER=elasticsearch +export HERMES_POLICY_FILE_PATH=/path/to/policy.json +export HERMES_ES_URL=http://elasticsearch:9200 +export HERMES_ES_USERNAME=your_username_here +export HERMES_ES_PASSWORD=your_password_here +export HERMES_ES_MAX_RESULT_WINDOW=20000 +export HERMES_OS_AUTH_URL=https://keystone.example.com/v3 +export HERMES_OS_USERNAME=hermes_service_user +export HERMES_OS_PASSWORD=your_password_here +export HERMES_OS_USER_DOMAIN_NAME=Default +export HERMES_OS_PROJECT_NAME=service +export HERMES_OS_PROJECT_DOMAIN_NAME=Default +export HERMES_OS_TOKEN_CACHE_TIME=900 +export HERMES_API_LISTEN_ADDRESS=0.0.0.0:8788 \ No newline at end of file diff --git a/main.go b/main.go index 72770f3e..09484707 100644 --- a/main.go +++ b/main.go @@ -20,14 +20,9 @@ package main import ( - "flag" - "fmt" - "os" - policy "github.com/databus23/goslo.policy" "github.com/sapcc/go-bits/logg" "github.com/sapcc/go-bits/must" - "github.com/sapcc/go-bits/osext" "github.com/spf13/viper" "github.com/sapcc/hermes/internal/api" @@ -36,73 +31,74 @@ import ( "github.com/sapcc/hermes/internal/util" ) -var configPath *string - func main() { - logg.ShowDebug = osext.GetenvBool("HERMES_DEBUG") - parseCmdlineFlags() - setDefaultConfig() - readConfig(configPath) + bindEnvVariables() + + logg.ShowDebug = viper.GetBool("hermes.debug") + keystoneDriver := configuredKeystoneDriver() storageDriver := configuredStorageDriver() readPolicy() must.Succeed(api.Server(keystoneDriver, storageDriver)) } -func parseCmdlineFlags() { - // Get config file location - configPath = flag.String("f", "hermes.conf", "specifies the location of the TOML-format configuration file") - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) - flag.PrintDefaults() - } - flag.Parse() -} - func setDefaultConfig() { var nullEnforcer, err = policy.NewEnforcer(make(map[string]string)) if err != nil { panic(err) } + + viper.SetDefault("hermes.debug", false) + viper.SetDefault("hermes.keystone_driver", "keystone") viper.SetDefault("hermes.storage_driver", "elasticsearch") viper.SetDefault("hermes.PolicyEnforcer", &nullEnforcer) + viper.SetDefault("hermes.PolicyFilePath", "etc/policy.json") + + viper.SetDefault("elasticsearch.url", "http://localhost:9200") + + // Replace with your Keystone authentication URL + viper.SetDefault("keystone.auth_url", "https://identity-3.domain.com/v3/") + + // Replace with your Keystone authentication details + viper.SetDefault("keystone.username", "hermes") + viper.SetDefault("keystone.password", "PASSWORD") + viper.SetDefault("keystone.user_domain_name", "Default") + viper.SetDefault("keystone.project_name", "service") + viper.SetDefault("keystone.project_domain_name", "Default") + viper.SetDefault("keystone.token_cache_time", 900) + viper.SetDefault("keystone.memcached_servers", "") + viper.SetDefault("API.ListenAddress", "0.0.0.0:8788") - viper.SetDefault("elasticsearch.url", "localhost:9200") + // index.max_result_window defaults to 10000, as per // https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html // Increasing max_result_window to 20000, with corresponding changes to Elasticsearch to handle the increase. viper.SetDefault("elasticsearch.max_result_window", "20000") } -func readConfig(configPath *string) { - // Enable viper to read Environment Variables - viper.AutomaticEnv() - - // Bind the specific environment variable to a viper key - err := viper.BindEnv("elasticsearch.username", "HERMES_ES_USERNAME") - if err != nil { - logg.Fatal(err.Error()) - } - err = viper.BindEnv("elasticsearch.password", "HERMES_ES_PASSWORD") - if err != nil { - logg.Fatal(err.Error()) - } - - // Don't read config file if the default config file isn't there, - // as we will just fall back to config defaults in that case - var shouldReadConfig = true - if _, err := os.Stat(*configPath); os.IsNotExist(err) { - shouldReadConfig = *configPath != flag.Lookup("f").DefValue - } - // Now we sorted that out, read the config - logg.Debug("Should read config: %v, config file is %s", shouldReadConfig, *configPath) - if shouldReadConfig { - viper.SetConfigFile(*configPath) - viper.SetConfigType("toml") - must.Succeed(viper.ReadInConfig()) - } +// bindEnvVariables binds environment variables to viper keys +func bindEnvVariables() { + must.Succeed(viper.BindEnv("hermes.keystone_driver", "HERMES_KEYSTONE_DRIVER")) + must.Succeed(viper.BindEnv("hermes.storage_driver", "HERMES_STORAGE_DRIVER")) + must.Succeed(viper.BindEnv("hermes.PolicyFilePath", "HERMES_POLICY_FILE_PATH")) + + must.Succeed(viper.BindEnv("elasticsearch.url", "HERMES_ES_URL")) + + must.Succeed(viper.BindEnv("keystone.auth_url", "HERMES_OS_AUTH_URL")) + must.Succeed(viper.BindEnv("keystone.username", "HERMES_OS_USERNAME")) + must.Succeed(viper.BindEnv("keystone.password", "HERMES_OS_PASSWORD")) + must.Succeed(viper.BindEnv("keystone.user_domain_name", "HERMES_OS_USER_DOMAIN_NAME")) + must.Succeed(viper.BindEnv("keystone.project_name", "HERMES_OS_PROJECT_NAME")) + must.Succeed(viper.BindEnv("keystone.project_domain_name", "HERMES_OS_PROJECT_DOMAIN_NAME")) + must.Succeed(viper.BindEnv("keystone.token_cache_time", "HERMES_OS_TOKEN_CACHE_TIME")) + must.Succeed(viper.BindEnv("keystone.memcached_servers", "HERMES_OS_MEMCACHED_SERVERS")) + + must.Succeed(viper.BindEnv("API.ListenAddress", "HERMES_API_LISTEN_ADDRESS")) + must.Succeed(viper.BindEnv("elasticsearch.username", "HERMES_ES_USERNAME")) + must.Succeed(viper.BindEnv("elasticsearch.password", "HERMES_ES_PASSWORD")) + must.Succeed(viper.BindEnv("elasticsearch.max_result_window", "HERMES_ES_MAX_RESULT_WINDOW")) } var keystoneIdentity = identity.Keystone{} @@ -138,12 +134,14 @@ func configuredStorageDriver() storage.Storage { } func readPolicy() { - //load the policy file - policyEnforcer, err := util.LoadPolicyFile(viper.GetString("hermes.PolicyFilePath")) - if err != nil { - logg.Fatal(err.Error()) - } - if policyEnforcer != nil { - viper.Set("hermes.PolicyEnforcer", policyEnforcer) + policyFilePath := viper.GetString("hermes.PolicyFilePath") + if policyFilePath != "" { + policyEnforcer, err := util.LoadPolicyFile(policyFilePath) + if err != nil { + logg.Fatal(err.Error()) + } + if policyEnforcer != nil { + viper.Set("hermes.PolicyEnforcer", policyEnforcer) + } } } From 29ebfb7826ff33210fb1004fbcfdedb3ac819fd9 Mon Sep 17 00:00:00 2001 From: notque Date: Fri, 8 Mar 2024 07:58:43 -0700 Subject: [PATCH 2/6] Update docs/operators/config.md add closing block Co-authored-by: Sandro --- docs/operators/config.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/operators/config.md b/docs/operators/config.md index 4fccca41..cbad262c 100644 --- a/docs/operators/config.md +++ b/docs/operators/config.md @@ -51,4 +51,5 @@ export HERMES_OS_USER_DOMAIN_NAME=Default export HERMES_OS_PROJECT_NAME=service export HERMES_OS_PROJECT_DOMAIN_NAME=Default export HERMES_OS_TOKEN_CACHE_TIME=900 -export HERMES_API_LISTEN_ADDRESS=0.0.0.0:8788 \ No newline at end of file +export HERMES_API_LISTEN_ADDRESS=0.0.0.0:8788 +``` \ No newline at end of file From 322fec47c5d7eaf9f9e69fd9bdb97e5b570a57d3 Mon Sep 17 00:00:00 2001 From: notque Date: Fri, 8 Mar 2024 07:59:33 -0700 Subject: [PATCH 3/6] absolute path for PolicyFilePath Co-authored-by: Sandro --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 09484707..49775bae 100644 --- a/main.go +++ b/main.go @@ -54,7 +54,7 @@ func setDefaultConfig() { viper.SetDefault("hermes.keystone_driver", "keystone") viper.SetDefault("hermes.storage_driver", "elasticsearch") viper.SetDefault("hermes.PolicyEnforcer", &nullEnforcer) - viper.SetDefault("hermes.PolicyFilePath", "etc/policy.json") + viper.SetDefault("hermes.PolicyFilePath", "/etc/policy.json") viper.SetDefault("elasticsearch.url", "http://localhost:9200") From 5f363e9ca3490af6df98fbb533ed0c87ff12da30 Mon Sep 17 00:00:00 2001 From: Nathan Oyler Date: Fri, 8 Mar 2024 08:08:18 -0700 Subject: [PATCH 4/6] fatal error if required variables are not set --- main.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 49775bae..364bbe88 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,20 @@ func main() { setDefaultConfig() bindEnvVariables() + // Validate required Keystone authentication details + if viper.GetString("keystone.username") == "" { + logg.Fatal("Keystone username is not set") + } + if viper.GetString("keystone.password") == "" { + logg.Fatal("Keystone password is not set") + } + if viper.GetString("elasticsearch.url") == "" { + logg.Fatal("Elasticsearch URL is not set") + } + if viper.GetString("keystone.auth_url") == "" { + logg.Fatal("Keystone authentication URL is not set") + } + logg.ShowDebug = viper.GetBool("hermes.debug") keystoneDriver := configuredKeystoneDriver() @@ -56,14 +70,14 @@ func setDefaultConfig() { viper.SetDefault("hermes.PolicyEnforcer", &nullEnforcer) viper.SetDefault("hermes.PolicyFilePath", "/etc/policy.json") - viper.SetDefault("elasticsearch.url", "http://localhost:9200") + viper.SetDefault("elasticsearch.url", "") // Replace with your Keystone authentication URL - viper.SetDefault("keystone.auth_url", "https://identity-3.domain.com/v3/") + viper.SetDefault("keystone.auth_url", "") // Replace with your Keystone authentication details - viper.SetDefault("keystone.username", "hermes") - viper.SetDefault("keystone.password", "PASSWORD") + viper.SetDefault("keystone.username", "") + viper.SetDefault("keystone.password", "") viper.SetDefault("keystone.user_domain_name", "Default") viper.SetDefault("keystone.project_name", "service") viper.SetDefault("keystone.project_domain_name", "Default") From 0ddd7babb23034564a6e3866e3fd7fbe030caabe Mon Sep 17 00:00:00 2001 From: Nathan Oyler Date: Fri, 8 Mar 2024 08:09:00 -0700 Subject: [PATCH 5/6] gofmt main.go --- main.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index 364bbe88..4cc75dbe 100644 --- a/main.go +++ b/main.go @@ -36,12 +36,12 @@ func main() { bindEnvVariables() // Validate required Keystone authentication details - if viper.GetString("keystone.username") == "" { - logg.Fatal("Keystone username is not set") - } - if viper.GetString("keystone.password") == "" { - logg.Fatal("Keystone password is not set") - } + if viper.GetString("keystone.username") == "" { + logg.Fatal("Keystone username is not set") + } + if viper.GetString("keystone.password") == "" { + logg.Fatal("Keystone password is not set") + } if viper.GetString("elasticsearch.url") == "" { logg.Fatal("Elasticsearch URL is not set") } From c3a29c078dfd2e7c2ee4397a0e1021d48b97b642 Mon Sep 17 00:00:00 2001 From: Nathan Oyler Date: Mon, 11 Mar 2024 10:54:45 -0700 Subject: [PATCH 6/6] remove noop and defaults with no value --- main.go | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/main.go b/main.go index 4cc75dbe..b6bb4d98 100644 --- a/main.go +++ b/main.go @@ -20,7 +20,6 @@ package main import ( - policy "github.com/databus23/goslo.policy" "github.com/sapcc/go-bits/logg" "github.com/sapcc/go-bits/must" "github.com/spf13/viper" @@ -37,16 +36,16 @@ func main() { // Validate required Keystone authentication details if viper.GetString("keystone.username") == "" { - logg.Fatal("Keystone username is not set") + logg.Fatal("HERMES_OS_USERNAME is not set") } if viper.GetString("keystone.password") == "" { - logg.Fatal("Keystone password is not set") + logg.Fatal("HERMES_OS_PASSWORD is not set") } if viper.GetString("elasticsearch.url") == "" { - logg.Fatal("Elasticsearch URL is not set") + logg.Fatal("HERMES_ES_URL is not set") } if viper.GetString("keystone.auth_url") == "" { - logg.Fatal("Keystone authentication URL is not set") + logg.Fatal("HERMES_OS_AUTH_URL is not set") } logg.ShowDebug = viper.GetBool("hermes.debug") @@ -58,26 +57,12 @@ func main() { } func setDefaultConfig() { - var nullEnforcer, err = policy.NewEnforcer(make(map[string]string)) - if err != nil { - panic(err) - } - viper.SetDefault("hermes.debug", false) viper.SetDefault("hermes.keystone_driver", "keystone") viper.SetDefault("hermes.storage_driver", "elasticsearch") - viper.SetDefault("hermes.PolicyEnforcer", &nullEnforcer) viper.SetDefault("hermes.PolicyFilePath", "/etc/policy.json") - viper.SetDefault("elasticsearch.url", "") - - // Replace with your Keystone authentication URL - viper.SetDefault("keystone.auth_url", "") - - // Replace with your Keystone authentication details - viper.SetDefault("keystone.username", "") - viper.SetDefault("keystone.password", "") viper.SetDefault("keystone.user_domain_name", "Default") viper.SetDefault("keystone.project_name", "service") viper.SetDefault("keystone.project_domain_name", "Default")