-
Notifications
You must be signed in to change notification settings - Fork 74
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Introduce resolvers to allow for nested substitutions (#369)
- Loading branch information
Showing
30 changed files
with
1,087 additions
and
97 deletions.
There are no files selected for viewing
89 changes: 89 additions & 0 deletions
89
hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/AwsOps.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| package com.sksamuel.hoplite.aws | ||
|
|
||
| import com.amazonaws.AmazonClientException | ||
| import com.amazonaws.AmazonServiceException | ||
| import com.amazonaws.services.secretsmanager.AWSSecretsManager | ||
| import com.amazonaws.services.secretsmanager.model.DecryptionFailureException | ||
| import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest | ||
| import com.amazonaws.services.secretsmanager.model.GetSecretValueResult | ||
| import com.amazonaws.services.secretsmanager.model.InvalidParameterException | ||
| import com.amazonaws.services.secretsmanager.model.LimitExceededException | ||
| import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException | ||
| import com.sksamuel.hoplite.ConfigFailure | ||
| import com.sksamuel.hoplite.ConfigResult | ||
| import com.sksamuel.hoplite.DecoderContext | ||
| import com.sksamuel.hoplite.fp.invalid | ||
| import com.sksamuel.hoplite.fp.valid | ||
| import kotlinx.serialization.decodeFromString | ||
| import kotlinx.serialization.json.Json | ||
|
|
||
| class AwsOps(private val client: AWSSecretsManager) { | ||
|
|
||
| companion object { | ||
| const val ReportSection = "AWS Secrets Manager Lookups" | ||
|
|
||
| // check for index, so we can decode json stored in AWS | ||
| val keyRegex = "(.+)\\[(.+)]".toRegex() | ||
| } | ||
|
|
||
| fun report(context: DecoderContext, result: GetSecretValueResult) { | ||
| context.report( | ||
| ReportSection, | ||
| mapOf( | ||
| "Name" to result.name, | ||
| "Arn" to result.arn, | ||
| "Created Date" to result.createdDate.toString(), | ||
| "Version Id" to result.versionId, | ||
| ) | ||
| ) | ||
| } | ||
|
|
||
| fun extractIndex(path: String): Pair<String, String?> { | ||
| val keyMatch = keyRegex.matchEntire(path) | ||
| return if (keyMatch == null) | ||
| Pair(path, null) | ||
| else | ||
| Pair(keyMatch.groupValues[1], keyMatch.groupValues[2]) | ||
| } | ||
|
|
||
| fun parseSecret(result: GetSecretValueResult, index: String?): ConfigResult<String> { | ||
|
|
||
| val secret = result.secretString | ||
| return if (secret.isNullOrBlank()) | ||
| ConfigFailure.PreprocessorWarning("Empty secret '${result.name}' in AWS SecretsManager").invalid() | ||
| else { | ||
| if (index == null) { | ||
| secret.valid() | ||
| } else { | ||
| val map = runCatching { Json.Default.decodeFromString<Map<String, String>>(secret) }.getOrElse { emptyMap() } | ||
| val indexedValue = map[index] | ||
| if (indexedValue == null) | ||
| ConfigFailure.ResolverError( | ||
| "Index '$index' not present in AWS secret '${result.name}'. Present keys are ${map.keys.joinToString(",")}" | ||
| ).invalid() | ||
| else indexedValue.valid() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| fun fetchSecret(key: String): ConfigResult<GetSecretValueResult> { | ||
| return try { | ||
| val req = GetSecretValueRequest().withSecretId(key) | ||
| client.getSecretValue(req).valid() | ||
| } catch (e: ResourceNotFoundException) { | ||
| ConfigFailure.PreprocessorWarning("Could not locate resource '$key' in AWS SecretsManager").invalid() | ||
| } catch (e: DecryptionFailureException) { | ||
| ConfigFailure.PreprocessorWarning("Could not decrypt resource '$key' in AWS SecretsManager").invalid() | ||
| } catch (e: LimitExceededException) { | ||
| ConfigFailure.PreprocessorWarning("Could not load resource '$key' due to limits exceeded").invalid() | ||
| } catch (e: InvalidParameterException) { | ||
| ConfigFailure.PreprocessorWarning("Invalid parameter name '$key' in AWS SecretsManager").invalid() | ||
| } catch (e: AmazonServiceException) { | ||
| ConfigFailure.PreprocessorFailure("Failed loading secret '$key' from AWS SecretsManager", e).invalid() | ||
| } catch (e: AmazonClientException) { | ||
| ConfigFailure.PreprocessorFailure("Failed loading secret '$key' from AWS SecretsManager", e).invalid() | ||
| } catch (e: Exception) { | ||
| ConfigFailure.PreprocessorFailure("Failed loading secret '$key' from AWS SecretsManager", e).invalid() | ||
| } | ||
| } | ||
| } |
42 changes: 42 additions & 0 deletions
42
hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerPrefixResolver.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package com.sksamuel.hoplite.aws | ||
|
|
||
| import com.amazonaws.services.secretsmanager.AWSSecretsManager | ||
| import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder | ||
|
|
||
| /** | ||
| * Replaces strings of the form awssecretsmanager://path by looking up the path in AWS Secrets Manager. | ||
| */ | ||
| //class AwsSecretsManagerPrefixResolver( | ||
| // report: Boolean = false, | ||
| // createClient: () -> AWSSecretsManager = { AWSSecretsManagerClientBuilder.standard().build() }, | ||
| //) : AwsSecretsManagerRegexResolver(report, createClient) { | ||
| // override val regex: Regex = "awssecretsmanager://(.*)".toRegex() | ||
| //} | ||
| // | ||
| //class AwsSecretsManagerPrefixResolverHoplite1x( | ||
| // report: Boolean = false, | ||
| // createClient: () -> AWSSecretsManager = { AWSSecretsManagerClientBuilder.standard().build() }, | ||
| //) : AwsSecretsManagerRegexResolver(report, createClient) { | ||
| // override val regex: Regex = "awssm://(.*)".toRegex() | ||
| //} | ||
| // | ||
| //class AwsSecretsManagerPrefixResolverHoplite2x( | ||
| // report: Boolean = false, | ||
| // createClient: () -> AWSSecretsManager = { AWSSecretsManagerClientBuilder.standard().build() }, | ||
| //) : AwsSecretsManagerRegexResolver(report, createClient) { | ||
| // override val regex: Regex = "secretsmanager://(.*)".toRegex() | ||
| //} | ||
| // | ||
|
|
||
| /** | ||
| * Replaces strings of the form ${{ aws-secrets-manager:path }} by looking up the path in AWS Secrets Manager. | ||
| * The [AWSSecretsManager] client is created from the [createClient] function which by default | ||
| * uses the default builder. | ||
| */ | ||
| class AwsSecretsManagerContextResolver( | ||
| report: Boolean = false, | ||
| createClient: () -> AWSSecretsManager = { AWSSecretsManagerClientBuilder.standard().build() }, | ||
| ) : AwsSecretsManagerRegexResolver(report, createClient) { | ||
| override val contextKey: String = "aws-secrets-manager" | ||
| override val default: Boolean = false | ||
| } |
27 changes: 27 additions & 0 deletions
27
hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerRegexResolver.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| package com.sksamuel.hoplite.aws | ||
|
|
||
| import com.amazonaws.services.secretsmanager.AWSSecretsManager | ||
| import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder | ||
| import com.sksamuel.hoplite.ConfigResult | ||
| import com.sksamuel.hoplite.DecoderContext | ||
| import com.sksamuel.hoplite.Node | ||
| import com.sksamuel.hoplite.StringNode | ||
| import com.sksamuel.hoplite.fp.flatMap | ||
| import com.sksamuel.hoplite.resolver.ContextResolver | ||
|
|
||
| abstract class AwsSecretsManagerRegexResolver( | ||
| private val report: Boolean = false, | ||
| createClient: () -> AWSSecretsManager = { AWSSecretsManagerClientBuilder.standard().build() }, | ||
| ) : ContextResolver() { | ||
|
|
||
| // should stay lazy so still be added to config even when not used, eg locally | ||
| private val client by lazy { createClient() } | ||
| private val ops by lazy { AwsOps(client) } | ||
|
|
||
| override fun lookup(path: String, node: StringNode, root: Node, context: DecoderContext): ConfigResult<String?> { | ||
| val (key, index) = ops.extractIndex(path) | ||
| return ops.fetchSecret(key) | ||
| .onSuccess { if (report) ops.report(context, it) } | ||
| .flatMap { ops.parseSecret(it, index) } | ||
| } | ||
| } |
11 changes: 11 additions & 0 deletions
11
hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/CreateAwsSecretsManagerResolver.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| package com.sksamuel.hoplite.aws | ||
|
|
||
| import com.amazonaws.services.secretsmanager.AWSSecretsManager | ||
| import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder | ||
| import com.sksamuel.hoplite.resolver.CompositeResolver | ||
| import com.sksamuel.hoplite.resolver.Resolver | ||
|
|
||
| fun createAwsSecretsManagerResolver( | ||
| report: Boolean = false, | ||
| createClient: () -> AWSSecretsManager = { AWSSecretsManagerClientBuilder.standard().build() }, | ||
| ): Resolver = CompositeResolver(AwsSecretsManagerContextResolver(report, createClient)) |
113 changes: 113 additions & 0 deletions
113
hoplite-aws/src/test/kotlin/com/sksamuel/hoplite/aws/AwsSecretsManagerResolverTest.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| package com.sksamuel.hoplite.aws | ||
|
|
||
| import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder | ||
| import com.amazonaws.services.secretsmanager.model.CreateSecretRequest | ||
| import com.sksamuel.hoplite.ConfigException | ||
| import com.sksamuel.hoplite.ConfigFailure | ||
| import com.sksamuel.hoplite.ConfigLoaderBuilder | ||
| import com.sksamuel.hoplite.fp.Validated | ||
| import com.sksamuel.hoplite.parsers.PropsPropertySource | ||
| import io.kotest.assertions.throwables.shouldThrow | ||
| import io.kotest.core.extensions.install | ||
| import io.kotest.core.spec.style.FunSpec | ||
| import io.kotest.extensions.testcontainers.TestContainerExtension | ||
| import io.kotest.matchers.shouldBe | ||
| import io.kotest.matchers.string.shouldContain | ||
| import io.kotest.matchers.string.shouldNotContain | ||
| import io.kotest.matchers.types.shouldBeInstanceOf | ||
| import org.testcontainers.containers.localstack.LocalStackContainer | ||
| import org.testcontainers.utility.DockerImageName | ||
| import java.util.Properties | ||
|
|
||
| class AwsSecretsManagerResolverTest : FunSpec() { | ||
|
|
||
| private val localstack = LocalStackContainer(DockerImageName.parse("localstack/localstack:1.3.1")) | ||
| .withServices(LocalStackContainer.Service.SECRETSMANAGER) | ||
| .withEnv("SKIP_SSL_CERT_DOWNLOAD", "true") | ||
|
|
||
| init { | ||
|
|
||
| install(TestContainerExtension(localstack)) | ||
|
|
||
| val client = AWSSecretsManagerClientBuilder.standard() | ||
| .withEndpointConfiguration(localstack.getEndpointConfiguration(LocalStackContainer.Service.SECRETSMANAGER)) | ||
| .withCredentials(localstack.defaultCredentialsProvider) | ||
| .build() | ||
|
|
||
| client.createSecret(CreateSecretRequest().withName("foo").withSecretString("secret!")) | ||
| client.createSecret(CreateSecretRequest().withName("bubble").withSecretString("""{"f": "1", "g": "2"}""")) | ||
|
|
||
| test("placeholder should be detected and used") { | ||
| val props = Properties() | ||
| props["a"] = "\${{ aws-secrets-manager:foo }}" | ||
| ConfigLoaderBuilder.create() | ||
| .addResolver(createAwsSecretsManagerResolver { client }) | ||
| .addPropertySource(PropsPropertySource(props)) | ||
| .build() | ||
| .loadConfigOrThrow<ConfigHolder>() | ||
| .a.shouldBe("secret!") | ||
| } | ||
|
|
||
| test("unknown secret should return error and include key") { | ||
| val props = Properties() | ||
| props["a"] = "\${{ aws-secrets-manager:qwerty }}" | ||
| ConfigLoaderBuilder.create() | ||
| .addResolver(createAwsSecretsManagerResolver { client }) | ||
| .addPropertySource(PropsPropertySource(props)) | ||
| .build() | ||
| .loadConfig<ConfigHolder>() | ||
| .shouldBeInstanceOf<Validated.Invalid<ConfigFailure>>().error.description().shouldContain("qwerty") | ||
| } | ||
|
|
||
| test("empty secret should return error and include empty secret message") { | ||
| val props = Properties() | ||
| props["a"] = "\${{ aws-secrets-manager:bibblebobble }}" | ||
| client.createSecret(CreateSecretRequest().withName("bibblebobble").withSecretString("")) | ||
| ConfigLoaderBuilder.create() | ||
| .addResolver(createAwsSecretsManagerResolver { client }) | ||
| .addPropertySource(PropsPropertySource(props)) | ||
| .build() | ||
| .loadConfig<ConfigHolder>() | ||
| .shouldBeInstanceOf<Validated.Invalid<ConfigFailure>>().error.description().shouldContain("Empty secret") | ||
| } | ||
|
|
||
| test("unknown secret should return error and not include prefix") { | ||
| val props = Properties() | ||
| props["a"] = "\${{ aws-secrets-manager:unkunk }}" | ||
| ConfigLoaderBuilder.create() | ||
| .addResolver(createAwsSecretsManagerResolver { client }) | ||
| .addPropertySource(PropsPropertySource(props)) | ||
| .build() | ||
| .loadConfig<ConfigHolder>() | ||
| .shouldBeInstanceOf<Validated.Invalid<ConfigFailure>>().error.description() | ||
| .shouldNotContain("aws-secrets-manager://") | ||
| } | ||
|
|
||
| test("multiple errors should be returned at once") { | ||
| val props = Properties() | ||
| props["a"] = "\${{ aws-secrets-manager:foo.bar }}" | ||
| props["b"] = "\${{ aws-secrets-manager:bar.baz }}" | ||
| shouldThrow<ConfigException> { | ||
| ConfigLoaderBuilder.create() | ||
| .addResolver(createAwsSecretsManagerResolver { client }) | ||
| .addPropertySource(PropsPropertySource(props)) | ||
| .build() | ||
| .loadConfigOrThrow<ConfigHolder2>() | ||
| }.message.shouldContain("foo.bar").shouldContain("bar.baz") | ||
| } | ||
|
|
||
| test("should support index keys") { | ||
| val props = Properties() | ||
| props["a"] = "\${{ aws-secrets-manager:bubble[f] }}" | ||
| ConfigLoaderBuilder.create() | ||
| .addResolver(createAwsSecretsManagerResolver { client }) | ||
| .addPropertySource(PropsPropertySource(props)) | ||
| .build() | ||
| .loadConfigOrThrow<ConfigHolder>() | ||
| .a shouldBe "1" | ||
| } | ||
| } | ||
|
|
||
| data class ConfigHolder(val a: String) | ||
| data class ConfigHolder2(val a: String, val b: String) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.