Add named configurations (#255)

* Add support for named configurations

The new `topaz config` subcommand replaces and extends the functionality of `topaz configure`, enabling switching between configuration instances, each maintaining its own state.

topaz config 
```
topaz config new

topaz config use

topaz config list

topaz config rename
```

Co-authored-by: Ronen Hilewicz <ronen@aserto.com>
Co-authored-by: Gert Drapers <gert.drapers@live.com>

cmd/topaz/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/aserto-dev/topaz/pkg/cli/cc"
 	"github.com/aserto-dev/topaz/pkg/cli/cmd"
@@ -12,9 +13,34 @@ import (
 	"github.com/rs/zerolog"
 )
 
+const (
+	docLink = "https://www.topaz.sh/docs/command-line-interface/topaz-cli/configuration"
+)
+
 func main() {
 
 	cli := cmd.CLI{}
+
+	cliConfigFile := filepath.Join(cc.GetTopazDir(), cmd.CLIConfigurationFile)
+
+	oldDBPath := filepath.Join(cc.GetTopazDir(), "db")
+	warn, err := checkDBFiles(oldDBPath)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	ctx, err := cc.NewCommonContext(cli.NoCheck, cliConfigFile)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	if warn && len(os.Args) == 1 {
+		ctx.UI.Exclamation().Msgf("Detected directory db files in the old data location %q\nCheck the documentation on how to update your configuration:\n%s\n",
+			oldDBPath, docLink)
+	}
+
 	kongCtx := kong.Parse(&cli,
 		kong.Name(x.AppName),
 		kong.Description(x.AppDescription),
@@ -37,18 +63,11 @@ func main() {
 			"container_image":    cc.ContainerImage(),
 			"container_tag":      cc.ContainerTag(),
 			"container_platform": cc.ContainerPlatform(),
-			"container_name":     cc.ContainerName(),
+			"container_name":     cc.ContainerName(ctx.Config.Active.ConfigFile),
 		},
 	)
-
 	zerolog.SetGlobalLevel(logLevel(cli.LogLevel))
 
-	ctx, err := cc.NewCommonContext(cli.NoCheck)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err.Error())
-		os.Exit(1)
-	}
-
 	if err := kongCtx.Run(ctx); err != nil {
 		kongCtx.FatalIfErrorf(err)
 	}
@@ -72,3 +91,19 @@ func logLevel(level int) zerolog.Level {
 		return zerolog.Disabled
 	}
 }
+
+func checkDBFiles(topazDBDir string) (bool, error) {
+	if _, err := os.Stat(topazDBDir); os.IsNotExist(err) {
+		return false, nil
+	}
+	if topazDBDir == cc.GetTopazDataDir() {
+		return false, nil
+	}
+
+	files, err := os.ReadDir(topazDBDir)
+	if err != nil {
+		return false, err
+	}
+
+	return len(files) > 0, nil
+}

docs/config.md

@@ -1,6 +1,6 @@
 # Topaz configuration
 
-The main configuration for Topaz can be devided in 3 main sections:
+The main configuration for Topaz can be divided in 3 main sections:
 
 1. Common configuration
 2. Auth configuration - optional
@@ -9,9 +9,9 @@ The main configuration for Topaz can be devided in 3 main sections:
 
 ## Topaz configuration environment variables
 
----
-> The topaz service configuration is built using the [spf13/viper](https://github.com/spf13/viper) library so all configuration parameters can be passed to the topazd service as environment variable with the **TOPAZ_** prefix. 
----
+> 
+> The topaz service configuration uses the [spf13/viper](https://github.com/spf13/viper) library, which allows all configuration parameters to be passed to the `topazd` service as environment variables with the `TOPAZ_` prefix.  
+> 
 
 If you use topaz CLI to generate your configuration file by default it will add the TOPAZ_DIR environment variable to the path configurations. By default this is empty and considered an NOP addition, but it can easily allow you to specify the desired value to the run/start topaz CLI command with the `-e` flag. 
 
@@ -21,17 +21,24 @@ By default if you run/start the topaz container using the topaz CLI the followin
 - `TOPAZ_CFG_DIR` - default $HOME/.config/topaz/cfg - the directory from where topaz will load the configuration file
 - `TOPAZ_DB_DIR` - default $HOME/.config/topaz/db - the directory where topaz will store the edge directory DB
 
+- `TOPAZ_CERTS_DIR` - default $HOME/.config/topaz/certs - the directory where topaz will load/generate the certs
+- `TOPAZ_CFG_DIR` - default $HOME/.config/topaz/cfg - the directory from where topaz will load the configuration file
+- `TOPAZ_DB_DIR` - default $HOME/.config/topaz/db - the directory where topaz will store the edge directory DB
+
 Both run and start topaz CLI commands allow passing optional environment variables to your running container using the -e flag. This will allow you to use any desired environment variable in your configuration file as long as you pass it to the container. 
 
 ## 1. Common configuration
 
 ### a. Version
 
+
 The configuration version accepted by the version of topaz - current compatible version: 2
 
 ### b. Logging
 
+
 The [logging mechanism](https://github.com/aserto-dev/logger) for topaz is based on [zerolog](https://github.com/rs/zerolog) and has the following available settings:
+
  - *prod* - boolean - if set to false the entire log output will be written using a zerolog ConsoleWriter, setting this to true will write the errors to the stderr output and other logs to the stdout 
  - *log_level* - string - this value is parsed by zerolog to match desired logging level (default: info), available levels: trace, debug, info, warn, error, fatal and panic
  - *grpc_log_level* - string - same as above available values, however this is specific for the logged grpc messages, having the default value set to warn
@@ -44,35 +51,45 @@ logging:
 
 ### c. API
 
+
 The API section is defines the configuration for the health, metrics and services that topaz is able to spin up. 
 
 #### Health:
 
+
 The health configuration allows topaz to spin up a health server.
 
+
 - *listen_address* - string - allows the health service to spin up on the configured port (default: "0.0.0.0:9494") 
-- *certs* - certs.TLSCredsConfig - based on [aserto-dev/certs](https://github.com/aserto-dev/certs) package allows setting the paths of your certificate files. By default the certificates are not configured. 
+- *certs* - certs.TLSCredsConfig - based on [aserto-dev/certs](https://github.com/aserto-dev/certs) package allows setting the paths of your certificate files. By default the certificates are not configured.
 
 #### Metrics:
 
+
 The metrics configuration allows topaz to spin up a metric server.
 
+
 - *listen_address* - string - allows the metric service to spin up on the configured port (default: "0.0.0.0:9696") 
 - *certs* - certs.TLSCredsConfig - based on [aserto-dev/certs](https://github.com/aserto-dev/certs) package allows setting the paths of your certificate files. By default the certificates are not configured. 
 - *zpages* - bool - if enabled the metrics server will enable [zpages](https://opencensus.io/zpages/go/) on the "/debug" route
 
 #### Services APIs:
 
+
 #### 1. grpc
 
+
 The grpc section allows configuring the listen address, the connection timeout and the certificates. 
 
+
 - *listen_address* - string - allows the topaz GRPC server to spin up on the requested port (default: "0.0.0.0:8282")
 - *connection_timeout_seconds* - uint32 - sets the timeout for a [connection establishment](https://pkg.go.dev/google.golang.org/grpc#ConnectionTimeout) (default: 120)
 - *certs* - certs.TLSCredsConfig - based on [aserto-dev/certs](https://github.com/aserto-dev/certs) package allows setting the paths of your certificate files. If you do not have your certificates in the specified paths, topaz will generate self-signed certificates for you. By default topaz will generate the certificates in ` ~/.config/topaz/certs/` path
 
+
 Example:
 
+
 ```
 grpc:
   listen_address: "localhost:8282"
@@ -85,6 +102,7 @@ grpc:
 
 #### 2. gateway
 
+
 The gateway section allows configuring the [grpc gateway](https://github.com/grpc-ecosystem/grpc-gateway) for your topaz authorizer. 
 
 - *listen_address* - string - allows the topaz Gateway server to spin up on the requested port (default: "0.0.0.0:8383")
@@ -93,13 +111,15 @@ The gateway section allows configuring the [grpc gateway](https://github.com/grp
 - *allowed_origins* - []string - allows setting the paths for the [CORS handler](https://github.com/rs/cors)
 
 Detailed information about the gateway http server timeout configuration is available [here](https://pkg.go.dev/net/http#Server)
+
 - *read_timeout* - time.Duration - default value set to 2 * time.Second (default: 2000000000) 
 - *read_header_timeout* - time.Duration - default value set to 2 * time.Second (default: 2000000000) 
 - *write_timeout* - time.Duration - default value set to 2 * time.Second (default: 2000000000) 
 - *idle_timeout* - time.Duration - default is set to 30 * time.Second (default: 30000000000)
 
 Example:
 
+
 ```
 gateway:
   listen_address: "localhost:8383"
@@ -116,6 +136,7 @@ gateway:
 
 #### 3. needs
 
+
 The `needs` section allows adding a dependency between the services started. For example when using an edge directory with a reader service it is recommended to set the authorizer services to be dependent on the reader spin-up to be able to resolve the identity for the authorization calls.
 
 Example:
@@ -140,8 +161,10 @@ api:
 
 ### c. Directory
 
+
 The directory section allows setting the configuration for the topaz [local edge directory](https://github.com/aserto-dev/go-edge-ds).
 
+
  - *db_path* - string - path to the bbolt database file
  - *request_timeout* - time.Duration - request timeout setting for the bbolt database connection
  - *enable_v2* - boolean - enable directory version 2 services for backward compatibility (default: false)
@@ -152,12 +175,14 @@ Topaz is able to communicate with a directory service based on the [pb-directory
 
 The remote address can also be configured to a service that implements the proto definitions (for example, the Postgres-based Aserto directory service). In this case, Topaz will NOT spin-up a local edge directory service, and instead send all directory requests to this remote service.
 
+
 - *address* - string - address:port of the remote directory service
 - *api_key* - string - API key for the directory
 - *tenant_id* - string - the directory tenant ID 
 
 Example (using the hosted Aserto directory):
 
+
 ```
 remote_directory:
   address: "directory.prod.aserto.com:8443"
@@ -169,6 +194,7 @@ remote_directory:
 
 The OPA configuration section represent the [runtime configuration](https://github.com/aserto-dev/runtime/blob/main/config.go). The main elements of the runtime configuration are:
 
+
 - *local_bundles* - runtime.LocalBundlesConfig - allows the runtime to run with a local bundle (local path or local policy OCI image)
 - *instance_id* - string - represent the unique identifier of the runtime 
 - *plugins_error_limit* - int - represents the maximum number of errors that an OPA plugin can trigger before killing the runtime
@@ -192,6 +218,7 @@ The options section allows you to specify overrides for specific paths if you wa
 
 Example:
 
+
 ```
 auth:
   api_keys:
@@ -215,6 +242,7 @@ By default Topaz does not initiate a decision logger, however if you need to kee
 
 Example configuration:
 
+
 ```
 decision_logger:
   type: "file"
@@ -228,6 +256,7 @@ To use the decision logger the OPA configuration must contain the [configuration
 
 Example of the decision log plugin configuration:
 
+
 ```
 opa:
  instance_id: "-"
@@ -261,3 +290,4 @@ controller:
     client_cert_path: <path to client certificate>
     client_key_path: <path to client key>
 ```
+

docs/examples/config-local-image.yaml

@@ -1,4 +1,4 @@
-# If you follow the [topaz quickstart guide](https://github.com/aserto-dev/topaz#quickstart) and you wish to run an local image that you built with [policy CLI](https://github.com/opcr-io/policy) you can use the `topaz configure -d -s -l ghcr.io/default:latest` to generate this example configuration.
+# If you follow the [topaz quickstart guide](https://github.com/aserto-dev/topaz#quickstart) and you wish to run an local image that you built with [policy CLI](https://github.com/opcr-io/policy) you can use the `topaz config new -d -s -l ghcr.io/default:latest` to generate this example configuration.
 # In this case the `ghcr.io/default:latest` policy image is set as your *local_policy_image*. In this example this values is set with the assumption that you [build your policy image](https://openpolicycontainers.com/docs/cli/build) without setting a custom tag.
 # Using topaz with a local policy image gives you an easier method to ensure that your policies work as you desire before pushing an image to an upstream container registry. 
 # If you have started topaz with this configuration, if you rebuild the image, any changes will be reflected in your topaz runtime bundle. 

docs/examples/config-public-ghcr.yaml

@@ -1,4 +1,4 @@
-# If you follow the [topaz quickstart guide](https://github.com/aserto-dev/topaz#quickstart) this is the configuration generated by the `topaz configure -d -s -r ghcr.io/aserto-policies/policy-todo:latest -n todo` command. 
+# If you follow the [topaz quickstart guide](https://github.com/aserto-dev/topaz#quickstart) this is the configuration generated by the `topaz config new -d -s -r ghcr.io/aserto-policies/policy-todo:latest -n todo` command. 
 ---
 version: 2
 logging:

go.mod

@@ -12,6 +12,7 @@ go 1.21
 
 require (
 	github.com/Masterminds/semver/v3 v3.2.1
+	github.com/adrg/xdg v0.4.0
 	github.com/alecthomas/kong v0.9.0
 	github.com/aserto-dev/aserto-grpc v0.2.2
 	github.com/aserto-dev/aserto-management v0.9.4

go.sum

@@ -398,6 +398,8 @@ github.com/Microsoft/hcsshim v0.12.2/go.mod h1:RZV12pcHCXQ42XnlQ3pz6FZfmrC1C+R4g
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
+github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
+github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
 github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
 github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
 github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU=

makefile

@@ -11,26 +11,28 @@ GOARCH			:= $(shell go env GOARCH)
 GOPRIVATE		:= "github.com/aserto-dev"
 DOCKER_BUILDKIT	:= 1
 
+BIN_DIR			:= ./bin
+EXT_DIR			:= ./.ext
+EXT_BIN_DIR		:= ${EXT_DIR}/bin
+EXT_TMP_DIR		:= ${EXT_DIR}/tmp
+
 VAULT_VERSION	:= 1.8.12
 SVU_VERSION 	:= 1.12.0
 GOTESTSUM_VERSION := 1.11.0
 GOLANGCI-LINT_VERSION := 1.56.2
 GORELEASER_VERSION := 1.24.0
 WIRE_VERSION	:= 0.6.0
 
-BUF_USER		:= $(shell vault kv get -field ASERTO_BUF_USER kv/buf.build)
-BUF_TOKEN		:= $(shell vault kv get -field ASERTO_BUF_TOKEN kv/buf.build)
+BUF_USER		:= $(shell ${EXT_BIN_DIR}/vault kv get -field ASERTO_BUF_USER kv/buf.build)
+BUF_TOKEN		:= $(shell ${EXT_BIN_DIR}/vault kv get -field ASERTO_BUF_TOKEN kv/buf.build)
 BUF_REPO		:= "buf.build/aserto-dev/directory"
-BUF_LATEST		:= $(shell BUF_BETA_SUPPRESS_WARNINGS=1 buf beta registry tag list buf.build/aserto-dev/directory --format json --reverse | jq -r '.results[0].name')
+BUF_LATEST		:= $(shell BUF_BETA_SUPPRESS_WARNINGS=1 ${EXT_BIN_DIR}/buf beta registry tag list buf.build/aserto-dev/directory --format json --reverse | jq -r '.results[0].name')
 BUF_DEV_IMAGE	:= "../pb-directory/bin/directory.bin"
 BUF_VERSION 	:= 1.30.0
 
 RELEASE_TAG		:= $$(svu)
 
-BIN_DIR			:= ./bin
-EXT_DIR			:= ./.ext
-EXT_BIN_DIR		:= ${EXT_DIR}/bin
-EXT_TMP_DIR		:= ${EXT_DIR}/tmp
+.DEFAULT_GOAL 	:= build
 
 .PHONY: deps
 deps: info install-vault install-buf install-svu install-goreleaser install-golangci-lint install-gotestsum install-wire 
@@ -200,8 +202,8 @@ install-wire: ${EXT_TMP_DIR} ${EXT_BIN_DIR}
 .PHONY: clean
 clean:
 	@echo -e "$(ATTN_COLOR)==> $@ $(NO_COLOR)"
-	@rm -rf ./.ext
-	@rm -rf ./bin
+	@rm -rf ${EXT_DIR}
+	@rm -rf ${BIN_DIR}
 
 ${BIN_DIR}:
 	@echo -e "$(ATTN_COLOR)==> $@ $(NO_COLOR)"

pkg/cc/config/config.go

@@ -114,6 +114,9 @@ func NewConfig(configPath Path, log *zerolog.Logger, overrides Overrider, certsG
 		if err != nil {
 			return nil, err
 		}
+		if configLoader.HasTopazDir {
+			log.Warn().Msg("This configuration file still uses TOPAZ_DIR environment variable. Please change to using the new TOPAZ_DB_DIR and TOPAZ_CERTS_DIR environment variables.")
+		}
 
 		err = validateVersion(configLoader.Configuration.Version)
 		if err != nil {

pkg/cc/config/generator.go

@@ -41,7 +41,35 @@ func (g *Generator) WithEdgeDirectory(enabled bool) *Generator {
 }
 
 func (g *Generator) WithEnableDirectoryV2(enabled bool) *Generator {
-	g.EnableDirectoryV2 = enabled
+	g.EnableDirectoryV2 = false
+	return g
+}
+
+func (g *Generator) WithTenantID(tenantID string) *Generator {
+	g.TenantID = tenantID
+	return g
+}
+
+func (g *Generator) WithDiscovery(url, key string) *Generator {
+	g.DiscoveryURL = url
+	g.TenantKey = key
+	return g
+}
+
+func (g *Generator) WithController(url, clientCertPath, clientKeyPath string) *Generator {
+	g.ControlPlane.Enabled = true
+	g.ControlPlane.Address = url
+	g.ControlPlane.ClientCertPath = clientCertPath
+	g.ControlPlane.ClientKeyPath = clientKeyPath
+	return g
+}
+
+func (g *Generator) WithSelfDecisionLogger(emsURL, clientCertPath, clientKeyPath, storePath string) *Generator {
+	g.DecisionLogging = true
+	g.DecisionLogger.EMSAddress = emsURL
+	g.DecisionLogger.ClientCertPath = clientCertPath
+	g.DecisionLogger.ClientKeyPath = clientKeyPath
+	g.DecisionLogger.StorePath = storePath
 	return g
 }