Make sure zio-config doesn't support values as list without proper in…

…dexing (#1086) * Avoid scala-2.11 for all projects * Remove all fix checks * Add snapshot releases * Make sure fatal warnings exist for local development * Generate docs test * Avoid splitting values for Indexed configs * Make sure to fail if index is absent, and try to keep the order of list * Reformat all code * Allow keys to be present and allow empty list * Reformat examples
zio · Feb 26, 2023 · 17efd3c · 17efd3c
1 parent f37d40b
commit 17efd3c
Show file tree

Hide file tree

Showing 19 changed files with 127 additions and 98 deletions.
diff --git a/core/shared/src/main/scala/zio/config/syntax/ConfigSyntax.scala b/core/shared/src/main/scala/zio/config/syntax/ConfigSyntax.scala
@@ -235,11 +235,26 @@ trait ConfigSyntax {
               for {
                 keys <- indexedFlat
                           .enumerateChildrenIndexed(prefix)
-                          .map(set =>
-                            set.toList.flatMap { chunk =>
-                              chunk.headOption.toList
+                          .flatMap { set =>
+                            ZIO.foreach(set.toList) { configPath =>
+                              configPath.headOption match {
+                                case Some(value) =>
+                                  value match {
+                                    case index @ KeyComponent.Index(_) => ZIO.succeed(List(index))
+                                    case KeyComponent.KeyName(_)       =>
+                                      ZIO.fail(
+                                        Config.Error.InvalidData(
+                                          prefix.map(_.value),
+                                          "A list key should be always followed by an index which is an integer to consider the values as a list. Example: The key `employees` is a list only if there is exists `employees[0]`, `employees[1]` etc as keys"
+                                        )
+                                      )
+                                  }
+                                case None        =>
+                                  ZIO.succeed(Nil)
+                              }
                             }
-                          )
+                          }
+                          .map(_.flatten.sortBy(_.index))
 
                 values <-
                   ZIO
@@ -331,9 +346,8 @@ trait ConfigSyntax {
      * Constructs a ConfigProvider using a map and the specified delimiter string,
      * which determines how to split the keys in the map into path segments.
      */
-    def fromIndexedMap(map: Map[String, String], pathDelim: String = ".", seqDelim: String = ","): ConfigProvider =
+    def fromIndexedMap(map: Map[String, String], pathDelim: String = "."): ConfigProvider =
       fromIndexedFlat(new IndexedFlat {
-        val escapedSeqDelim  = java.util.regex.Pattern.quote(seqDelim)
         val escapedPathDelim = java.util.regex.Pattern.quote(pathDelim)
 
         def makePathString(path: Chunk[String]): String = path.mkString(pathDelim)
@@ -344,7 +358,6 @@ trait ConfigSyntax {
           trace: Trace
         ): IO[Config.Error, Chunk[A]] = {
           val pathString = makePathString(IndexedFlat.ConfigPath.toPath(path))
-          val name       = path.lastOption.getOrElse(IndexedFlat.KeyComponent.KeyName("<unnamed>"))
           val valueOpt   = map.get(pathString)
 
           for {
@@ -353,8 +366,11 @@ trait ConfigSyntax {
                          .mapError(_ =>
                            Config.Error.MissingData(path.map(_.value), s"Expected ${pathString} to be set in properties")
                          )
-            results <- ConfigProvider.Flat.util
-                         .parsePrimitive(value, path.map(_.value), name.value, primitive, escapedSeqDelim, split)
+            results <-
+              ZIO
+                .fromEither(primitive.parse(value))
+                .map(Chunk(_))
+                .mapError(_.prefixed(path.map(_.value)))
           } yield results
         }
 

diff --git a/docs/read-from-various-sources.md b/docs/read-from-various-sources.md
@@ -131,11 +131,39 @@ ConfigProvider.fromYamlString
 ## Xml String
 
 zio-config can read XML strings. Note that it's experimental with a dead simple native xml parser, 
-Currently it cannot XML comments, which will be fixed in the near future.
+Currently it cannot XML comments, and has not been tested with complex data types, which will be fixed in the near future.
 
 ```scala
-import zio.config.xml._
+import zio.config.xml.experimental._
+import zio.Config
 
-ConfigProvider.fromXmlString
+final case class Configuration(aws: Aws, database: Database)
+
+object Configuration {
+  val config: Config[Configuration] =
+    Aws.config.nested("aws").zip(Database.config.nested("database")).to[Configuration].nested("config")
+
+  final case class Aws(region: String, account: String)
+
+  object Aws {
+    val config: Config[Aws] = Config.string("region").zip(Config.string("account")).to[Aws]
+  }
+  final case class Database(port: Int, url: String)
+
+  object Database {
+    val config: Config[Database] = Config.int("port").zip(Config.string("url")).to[Database]
+  }
+}
+
+val config =
+  s"""
+     |<config>
+     |  <aws region="us-east" account="personal"></aws>
+     |  <database port="123" url="some url"></database>
+     |</config>
+     |
+     |""".stripMargin
+
+val parsed = ConfigProvider.fromYamlString(config).load(Configuration.config)
 
 ```
diff --git a/examples/shared/src/main/scala/zio/config/examples/typesafe/SealedTraitListExample.scala b/examples/shared/src/main/scala/zio/config/examples/typesafe/SealedTraitListExample.scala
@@ -0,0 +1,36 @@
+package zio.config.examples.typesafe
+
+import zio.ConfigProvider
+import zio.config.derivation.nameWithLabel
+import zio.config._, typesafe._, magnolia._
+
+object SealedTraitListExample extends App {
+
+  @nameWithLabel
+  sealed trait DataTransformation
+
+  case class CastColumns(dataTypeMapper: Map[String, String]) extends DataTransformation
+
+  case class TransformationRules(transformations: List[DataTransformation])
+
+  val transformations =
+    s"""
+       |transformations = [
+       |      {
+       |        type = "CastColumns"
+       |        dataTypeMapper = {
+       |          "col_A" = "string"
+       |          "col_B" = "double"
+       |          "col_C" = "decimal(19,2)"
+       |        }
+       |      }
+       |    ]
+       |""".stripMargin
+
+  val pgm = ConfigProvider.fromHoconString(transformations).load(deriveConfig[TransformationRules])
+
+  pgm equalM (TransformationRules(
+    List(CastColumns(Map("col_C" -> "decimal(19,2)", "col_B" -> "double", "col_A" -> "string")))
+  ))
+
+}
diff --git a/examples/shared/src/main/scala/zio/config/examples/typesafe/TypesafeConfigList.scala b/examples/shared/src/main/scala/zio/config/examples/typesafe/TypesafeConfigList.scala
@@ -1,10 +1,9 @@
 package zio.config.examples.typesafe
 
 import zio.config._
-import zio.config.examples.ZioOps
-import zio.{Config, ConfigProvider, IO}
-
+import zio.{Config, ConfigProvider, IO, Unsafe}
 import magnolia._
+import zio.Runtime.default
 
 object TypesafeConfigList extends App with EitherImpureOps {
   val configString: String =
@@ -146,58 +145,6 @@ object TypesafeConfigList extends App with EitherImpureOps {
   final case class B(c: List[C], table: String, columns: List[String])
   final case class A(b: List[B], x: X, w: W)
 
-  val expectedResult: A =
-    A(
-      List(
-        B(
-          List(
-            C(
-              List(
-                // NonEmptyList is simply scala.:: which is a List. However, if the list was empty you get a error
-                D(List(14, 14), List("e")),
-                D(List(12, 12), List("d")),
-                D(List(1, 1, 1), List("a", "b", "c")),
-                D(List(15, 15), List("f", "g"))
-              )
-            )
-          ),
-          "some_name",
-          List("aa")
-        ),
-        B(
-          List(
-            C(
-              List(
-                D(List(21, 21), List("af")),
-                D(List(24, 24, 24), List("l")),
-                D(List(22, 22), List("sa", "l")),
-                D(List(23, 23, 23), List("af", "l"))
-              )
-            )
-          ),
-          "some_name",
-          List("a", "b", "c", "d", "e")
-        ),
-        B(
-          List(
-            C(
-              List(
-                D(List(37), List("e", "f", "g", "h", "i")),
-                D(List(33, 33, 33), List("xx")),
-                D(List(32, 32), List("x")),
-                D(List(31), List("b")),
-                D(List(31, 31), List("bb"))
-              )
-            )
-          ),
-          "some_name",
-          List("a")
-        )
-      ),
-      X(Y("k")),
-      W(X(Y("k")))
-    )
-
   import zio.config.typesafe._
 
   // Since we already have a string with us, we don't need Config Service (or ZIO)
@@ -207,6 +154,6 @@ object TypesafeConfigList extends App with EitherImpureOps {
   val zioConfigResult: IO[Config.Error, A] =
     read(deriveConfig[A] from source)
 
-  assert(zioConfigResult equalM expectedResult)
+  println(Unsafe.unsafe(implicit u => default.unsafe.run(zioConfigResult).getOrThrowFiberFailure()))
 
 }
diff --git a/refined/shared/src/main/scala/zio/config/refined/package.scala b/refined/shared/src/main/scala/zio/config/refined/package.scala
@@ -55,7 +55,7 @@ package object refined {
    *     listOf("databases", deriveConfig[MyConfig])
    *
    *   val configDescriptor: Config[Refined[List[MyConfig], Size[Greater[W.`2`.T]]]] =
-   *     refined[Size[Greater[W.`2`.T]]](configs)
+   *     refine[Size[Greater[W.`2`.T]]](configs)
    * }}}
    *
    * If you don't care predicates specifically, and need to pass a fully formed Refined type

diff --git a/scalaz/shared/src/main/scala/zio/config/scalaz/package.scala b/scalaz/shared/src/main/scala/zio/config/scalaz/package.scala
@@ -2,7 +2,6 @@ package zio.config
 
 import _root_.scalaz.{==>>, IList, ISet, Maybe, Order}
 import zio.Config
-import zio.config._
 
 import Config._
 

diff --git a/...afe-magnolia-tests/shared/src/test/scala/zio/config/typesafe/TypesafeConfigListTest.scala b/...afe-magnolia-tests/shared/src/test/scala/zio/config/typesafe/TypesafeConfigListTest.scala
@@ -164,21 +164,21 @@ object TypesafeConfigListTest extends ZIOSpecDefault {
                   "di",
                   "ci",
                   List(
-                    Details3("ki", Left(3), List(1, 3, 5), Some(List(1, 2, 3))),
                     Details3("ki", Right(Right(Right(Right("bi")))), List(1, 1, 1), Some(Nil)),
-                    Details3("ki", Right(Right(Left(1.0882121))), List(1, 2, 1), Some(Nil)) //FIXME: It should be None
+                    Details3("ki", Right(Right(Left(1.0882121))), List(1, 2, 1), Some(Nil)), //FIXME: It should be None
+                    Details3("ki", Left(3), List(1, 3, 5), Some(List(1, 2, 3)))
                   )
                 ),
-                Details2("di", "ci", Nil),
                 Details2(
                   "di",
                   "ci",
                   List(
                     Details3("ki", Right(Right(Right(Right("bi")))), List(1, 1, 1), Some(Nil)),
-                    Details3("ki", Left(3), List(1, 3, 5), Some(List(1, 2, 3))),
-                    Details3("ki", Right(Right(Left(1.0882121))), List(1, 2, 1), Some(Nil)) //FIXME: It should be None
+                    Details3("ki", Right(Right(Left(1.0882121))), List(1, 2, 1), Some(Nil)), //FIXME: It should be None,
+                    Details3("ki", Left(3), List(1, 3, 5), Some(List(1, 2, 3)))
                   )
-                )
+                ),
+                Details2("di", "ci", Nil)
               )
             )
           ),

diff --git a/...e-magnolia-tests/shared/src/test/scala/zio/config/typesafe/TypesafeConfigSimpleSpec.scala b/...e-magnolia-tests/shared/src/test/scala/zio/config/typesafe/TypesafeConfigSimpleSpec.scala
@@ -69,9 +69,9 @@ object TypesafeConfigSimpleSpec extends ZIOSpecDefault {
       val expectedResult =
         AwsDetails(
           List(
-            Account(Some(Right("bb")), List("us-some", "ff", "gg"), None),
+            Account(Some(Right("jon")), List("us-east", "dd", "ee"), Some(Details("jaku", 10))),
             Account(Some(Left(123)), List("us-west", "ab", "cd"), Some(Details("zak", 11))),
-            Account(Some(Right("jon")), List("us-east", "dd", "ee"), Some(Details("jaku", 10)))
+            Account(Some(Right("bb")), List("us-some", "ff", "gg"), None)
           ),
           Database(Some(100), "postgres"),
           List(1, 2, 3)
@@ -88,9 +88,9 @@ object TypesafeConfigSimpleSpec extends ZIOSpecDefault {
       val expectedResult =
         AwsDetails(
           List(
-            Account(Some(Right("bb")), List("us-some", "ff", "gg"), None),
+            Account(Some(Right("jon")), List("us-east", "dd", "ee"), Some(Details("jaku", 10))),
             Account(Some(Left(123)), List("us-west", "ab", "cd"), Some(Details("zak", 11))),
-            Account(Some(Right("jon")), List("us-east", "dd", "ee"), Some(Details("jaku", 10)))
+            Account(Some(Right("bb")), List("us-some", "ff", "gg"), None)
           ),
           Database(Some(100), "postgres"),
           List(1, 2, 3)

diff --git a/typesafe/shared/src/main/scala/zio/config/typesafe/TypesafeConfigProvider.scala b/typesafe/shared/src/main/scala/zio/config/typesafe/TypesafeConfigProvider.scala
@@ -13,6 +13,7 @@ import scala.util.{Failure, Success, Try}
 
 @silent("Unused import")
 object TypesafeConfigProvider {
+
   import VersionSpecificSupport._
 
   /**
@@ -66,9 +67,11 @@ object TypesafeConfigProvider {
                   // Only possibility is a sequence of primitives
                   val result = config.getList(k).unwrapped().asScala.toList
 
-                  Map(
-                    kIterated ++ Chunk(KeyComponent.Index(0)) -> result.map(_.toString).mkString(",")
-                  )
+                  result.zipWithIndex
+                    .map({ case (result, index) =>
+                      (kIterated :+ KeyComponent.Index(index)) -> result.toString
+                    })
+                    .toMap
 
                 // Only possibility is a sequence of nested Configs
                 case Success(value) =>

diff --git a/...c/main/scala/zio/config/xml/Parsers.scala → ...zio/config/xml/experimental/Parsers.scala b/...c/main/scala/zio/config/xml/Parsers.scala → ...zio/config/xml/experimental/Parsers.scala
@@ -1,7 +1,7 @@
-package zio.config.xml
+package zio.config.xml.experimental
 
 import zio.Chunk
-import zio.config.xml.XmlObject.Text
+import zio.config.xml.experimental.XmlObject.Text
 import zio.parser.Parser
 
 object Parsers {

diff --git a/...la/zio/config/xml/XmlConfigProvider.scala → .../xml/experimental/XmlConfigProvider.scala b/...la/zio/config/xml/XmlConfigProvider.scala → .../xml/experimental/XmlConfigProvider.scala
@@ -1,4 +1,4 @@
-package zio.config.xml
+package zio.config.xml.experimental
 
 import com.github.ghik.silencer.silent
 import zio.ConfigProvider

diff --git a/...main/scala/zio/config/xml/XmlObject.scala → ...o/config/xml/experimental/XmlObject.scala b/...main/scala/zio/config/xml/XmlObject.scala → ...o/config/xml/experimental/XmlObject.scala
@@ -1,4 +1,4 @@
-package zio.config.xml
+package zio.config.xml.experimental
 
 import zio.Chunk
 import zio.config.IndexedFlat.KeyComponent

diff --git a/...main/scala/zio/config/xml/XmlParser.scala → ...o/config/xml/experimental/XmlParser.scala b/...main/scala/zio/config/xml/XmlParser.scala → ...o/config/xml/experimental/XmlParser.scala
@@ -1,8 +1,8 @@
-package zio.config.xml
+package zio.config.xml.experimental
 
 import zio.Chunk
-import zio.config.xml.Parsers._
-import zio.config.xml.XmlObject.TagElement
+import Parsers._
+import XmlObject.TagElement
 import zio.parser.Parser
 
 object XmlParser {

diff --git a/...c/main/scala/zio/config/xml/package.scala → ...zio/config/xml/experimental/package.scala b/...c/main/scala/zio/config/xml/package.scala → ...zio/config/xml/experimental/package.scala
@@ -1,8 +1,8 @@
-package zio.config
+package zio.config.xml
 
 import zio._
 
-package object xml {
+package object experimental {
   implicit class FromConfigSourceXml(c: ConfigProvider.type) {
     def fromYamlString(xml: String): ConfigProvider =
       XmlConfigProvider.fromXmlString(xml)

diff --git a/xml/shared/src/test/scala/zio/config/yaml/XmlParserSpec.scala b/xml/shared/src/test/scala/zio/config/yaml/XmlParserSpec.scala
@@ -1,7 +1,7 @@
 package zio.config.yaml
 
 import zio.Scope
-import zio.config.xml.XmlParser
+import zio.config.xml.experimental.XmlParser
 import zio.config.yaml.generators.{Space, WhiteSpacedXml}
 import zio.test.Assertion._
 import zio.test._

diff --git a/xml/shared/src/test/scala/zio/config/yaml/XmlProviderSpec.scala b/xml/shared/src/test/scala/zio/config/yaml/XmlProviderSpec.scala
@@ -1,7 +1,7 @@
 package zio.config.yaml
 
 import zio.config._
-import zio.config.xml._
+import zio.config.xml.experimental._
 import zio.config.yaml.XmlProviderSpec.Configuration.{Aws, Database}
 import zio.test.Assertion._
 import zio.test._

diff --git a/xml/shared/src/test/scala/zio/config/yaml/generators/WhiteSpacedAttributes.scala b/xml/shared/src/test/scala/zio/config/yaml/generators/WhiteSpacedAttributes.scala
@@ -1,6 +1,6 @@
 package zio.config.yaml.generators
 import zio.Chunk
-import zio.config.xml.XmlObject
+import zio.config.xml.experimental.XmlObject
 import zio.config.yaml.generators.WhiteSpacedAttributes.RandomAttribute
 import zio.test.Gen
 

diff --git a/xml/shared/src/test/scala/zio/config/yaml/generators/WhiteSpacedText.scala b/xml/shared/src/test/scala/zio/config/yaml/generators/WhiteSpacedText.scala
@@ -1,6 +1,6 @@
 package zio.config.yaml.generators
 
-import zio.config.xml.XmlObject
+import zio.config.xml.experimental.XmlObject
 import zio.test.Gen
 
 final case class WhiteSpacedText(preSpace: Space, value: String, postSpace: Space) {