Skip to content

Commit

Permalink
Adds logical and fixed types to code generation (#340)
Browse files Browse the repository at this point in the history 
* save wip

* tests pass

* fmt

* add toJson impl and keep precision and scale in mu model

* tag bigdecimal with precision and scale type literals

* get rid of nested tag

* remove shapeless dep

* move scala expected to it's own file

* fmt

* split logical into it's own file

* split logical into it's own file

* add nested example

* nested records kinda working

* add import example

* use avro hugger to parse idl files in test

* nested and imports working, fixed is wip

* fixed now represented

* drop uuid support for now

* fmt

* fix docs

* revert to deprecated converters

* work out cross build test failures

* revert some inadvertent change

* fix product comparison

* add invalid test and add Try to avro -> mu step

* re-add tests

* add invalid request case. clean up error handling

* fmt

* create error classes

* pr feedback

* remove old file

* add test coverage

* remove unused imports
  • Loading branch information
@dzanot
dzanot committed Oct 15, 2020
1 parent 24e7bf1 commit 05118bc
Show file tree
Hide file tree
Showing 30 changed files with 524 additions and 203 deletions.
40 changes: 20 additions & 20 deletions build.sbt
View file
Expand Up @@ -52,26 +52,26 @@ lazy val documentation = project
lazy val commonSettings = Seq(
scalacOptions ~= (_ filterNot Set("-Xfuture", "-Xfatal-warnings").contains),
libraryDependencies ++= Seq(
"org.typelevel" %% "cats-core" % "2.2.0",
"org.typelevel" %% "cats-effect" % "2.2.0",
"io.higherkindness" %% "droste-core" % "0.8.0",
"io.higherkindness" %% "droste-macros" % "0.8.0",
"org.apache.avro" % "avro" % "1.10.0",
"com.github.os72" % "protoc-jar" % "3.11.4",
"com.google.protobuf" % "protobuf-java" % "3.12.2",
"io.circe" %% "circe-core" % "0.13.0",
"io.circe" %% "circe-parser" % "0.13.0",
"io.circe" %% "circe-yaml" % "0.13.1",
"org.scalameta" %% "scalameta" % "4.3.24",
"org.scala-lang.modules" %% "scala-collection-compat" % "2.2.0",
"org.apache.avro" % "avro-compiler" % "1.10.0" % Test,
"org.typelevel" %% "cats-laws" % "2.2.0" % Test,
"io.circe" %% "circe-testing" % "0.13.0" % Test,
"org.typelevel" %% "discipline-specs2" % "1.1.0" % Test,
"org.specs2" %% "specs2-core" % "4.10.4" % Test,
"org.specs2" %% "specs2-scalacheck" % "4.10.4" % Test,
"org.scalacheck" %% "scalacheck" % "1.14.3" % Test,
"io.chrisdavenport" %% "cats-scalacheck" % "0.3.0" % Test
"org.typelevel" %% "cats-core" % "2.2.0",
"org.typelevel" %% "cats-effect" % "2.2.0",
"io.higherkindness" %% "droste-core" % "0.8.0",
"io.higherkindness" %% "droste-macros" % "0.8.0",
"org.apache.avro" % "avro" % "1.10.0",
"com.github.os72" % "protoc-jar" % "3.11.4",
"com.google.protobuf" % "protobuf-java" % "3.12.2",
"io.circe" %% "circe-core" % "0.13.0",
"io.circe" %% "circe-parser" % "0.13.0",
"io.circe" %% "circe-yaml" % "0.13.1",
"org.scalameta" %% "scalameta" % "4.3.24",
"com.julianpeeters" %% "avrohugger-core" % "1.0.0-RC22" % Test,
"org.typelevel" %% "cats-laws" % "2.2.0" % Test,
"io.circe" %% "circe-testing" % "0.13.0" % Test,
"org.typelevel" %% "discipline-specs2" % "1.1.0" % Test,
"org.specs2" %% "specs2-core" % "4.10.4" % Test,
"org.specs2" %% "specs2-scalacheck" % "4.10.4" % Test,
"org.scalacheck" %% "scalacheck" % "1.14.3" % Test,
"io.chrisdavenport" %% "cats-scalacheck" % "0.3.0" % Test,
"org.scalatra.scalate" %% "scalate-core" % "1.9.6" % Test
)
) ++ compilerPlugins ++ macroSettings

Expand Down
10 changes: 5 additions & 5 deletions microsite/docs/docs/optimizations.md
View file
Expand Up @@ -47,16 +47,16 @@ they're inside a product themselves. And we do this with the

```scala mdoc
def nestedNamedTypesTrans[T](implicit T: Basis[MuF, T]): Trans[MuF, MuF, T] = Trans {
case TProduct(name, fields, np, nc) =>
case TProduct(name, namespace, fields, np, nc) =>
def nameTypes(f: Field[T]): Field[T] = f.copy(tpe = namedTypes(T)(f.tpe))
TProduct[T](name, fields.map(nameTypes), np, nc)
TProduct[T](name, namespace, fields.map(nameTypes), np, nc)
case other => other
}

def namedTypesTrans[T]: Trans[MuF, MuF, T] = Trans {
case TProduct(name, _, _, _) => TNamedType[T](Nil, name)
case TSum(name, _) => TNamedType[T](Nil, name)
case other => other
case TProduct(name, ns, _, _, _) => TNamedType[T](ns.toList, name)
case TSum(name, _) => TNamedType[T](Nil, name)
case other => other
}

def namedTypes[T: Basis[MuF, ?]]: T => T = scheme.cata(namedTypesTrans.algebra)
Expand Down
10 changes: 10 additions & 0 deletions src/main/scala/higherkindness/skeuomorph/SkeuomorphError.scala
View file
Expand Up @@ -34,3 +34,13 @@ case class ProtobufNativeException(originalError: String) extends SkeuomorphErro
override val message = s"Failed to transform into Native descriptors with error: $originalError"
override def getMessage: String = message
}

case class UnsupportedResponseTypeException(originalError: String) extends SkeuomorphError {
override val message = s"Encountered an unsupported response type: ${originalError}"
override def getMessage: String = message
}

case class UnsupportedRequestTypeException(originalError: String) extends SkeuomorphError {
override val message = s"Encountered an unsupported request type: ${originalError}"
override def getMessage: String = message
}
65 changes: 39 additions & 26 deletions src/main/scala/higherkindness/skeuomorph/avro/Protocol.scala
View file
Expand Up @@ -18,12 +18,13 @@ package higherkindness.skeuomorph.avro

import cats.data.NonEmptyList
import cats.syntax.option._
import higherkindness.skeuomorph.mu
import higherkindness.skeuomorph.{mu, UnsupportedRequestTypeException, UnsupportedResponseTypeException}
import higherkindness.skeuomorph.mu.{MuF, SerializationType}
import io.circe.Json
import org.apache.avro.{Schema, Protocol => AvroProtocol}
import higherkindness.droste._
import higherkindness.droste.syntax.all._
import org.apache.avro.Schema.Type

import scala.collection.JavaConverters._

Expand Down Expand Up @@ -62,14 +63,20 @@ object Protocol {
if (req.getType == Schema.Type.NULL)
`null`[T]
else {
// Assume it's a record type.
// We don't support primitive types for RPC requests/responses.
val fields = req.getFields
if (fields.size == 0)
`null`[T]
else {
// Assume it's a record type.
// We don't support primitive types for RPC requests/responses.
val fieldSchema = fields.get(0).schema
namedType[T](fieldSchema.getNamespace, fieldSchema.getName)
fieldSchema.getType match {
case Type.RECORD => namedType[T](fieldSchema.getNamespace, fieldSchema.getName)
case nonRecord =>
throw UnsupportedRequestTypeException(
s"Skeuomorph only supports Record types for Avro requests. Encountered request schema with type $nonRecord"
)
}
}
}
}
Expand All @@ -80,7 +87,13 @@ object Protocol {
else
// Assume it's a record type.
// We don't support primitive types for RPC requests/responses.
namedType[T](resp.getNamespace, resp.getName)
resp.getType match {
case Type.RECORD => namedType[T](resp.getNamespace, resp.getName)
case nonRecord =>
throw UnsupportedResponseTypeException(
s"Skeuomorph only supports Record types for Avro responses. Encountered response schema with type $nonRecord"
)
}
}

def toMessage(kv: (String, AvroProtocol#Message)): Message[T] = {
Expand All @@ -90,7 +103,6 @@ object Protocol {
responseToAvroF(kv._2.getResponse).embed
)
}

Protocol(
proto.getName,
Option(proto.getNamespace),
Expand All @@ -113,28 +125,29 @@ object Protocol {

def fromMuSchema[T](implicit T: Basis[AvroF, T]): Trans[MuF, AvroF, T] =
Trans {
case MuF.TNull() => AvroF.`null`()
case MuF.TDouble() => AvroF.double()
case MuF.TFloat() => AvroF.float()
case MuF.TInt(MuF._32) => AvroF.int()
case MuF.TInt(MuF._64) => AvroF.long()
case MuF.TBoolean() => AvroF.boolean()
case MuF.TString() => AvroF.string()
case MuF.TByteArray() => AvroF.bytes()
case MuF.TNamedType(prefix, name) => AvroF.namedType(prefix.mkString("."), name)
case MuF.TOption(value) => AvroF.union(NonEmptyList(AvroF.`null`[T]().embed, List(value)))
case MuF.TEither(left, right) => AvroF.union(NonEmptyList(left, List(right)))
case MuF.TList(value) => AvroF.array(value)
case MuF.TMap(_, value) => AvroF.map(value)
case MuF.TGeneric(_, _) => ??? // WAT
case MuF.TContaining(_) => ??? // TBD
case MuF.TRequired(t) => T.coalgebra(t)
case MuF.TCoproduct(invariants) => AvroF.union(invariants)
case MuF.TSum(name, fields) => AvroF.enum(name, none[String], Nil, none[String], fields.map(_.name))
case MuF.TProduct(name, fields, _, _) =>
case MuF.TNull() => AvroF.`null`()
case MuF.TDouble() => AvroF.double()
case MuF.TFloat() => AvroF.float()
case MuF.TInt(MuF._32) => AvroF.int()
case MuF.TInt(MuF._64) => AvroF.long()
case MuF.TBoolean() => AvroF.boolean()
case MuF.TString() => AvroF.string()
case MuF.TByteArray(MuF.Length.Arbitrary) => AvroF.bytes()
case MuF.TByteArray(MuF.Length.Fixed(n, ns, l)) => AvroF.fixed(n, ns, Nil, l)
case MuF.TNamedType(prefix, name) => AvroF.namedType(prefix.mkString("."), name)
case MuF.TOption(value) => AvroF.union(NonEmptyList(AvroF.`null`[T]().embed, List(value)))
case MuF.TEither(left, right) => AvroF.union(NonEmptyList(left, List(right)))
case MuF.TList(value) => AvroF.array(value)
case MuF.TMap(_, value) => AvroF.map(value)
case MuF.TGeneric(_, _) => ??? // WAT
case MuF.TContaining(_) => ??? // TBD
case MuF.TRequired(t) => T.coalgebra(t)
case MuF.TCoproduct(invariants) => AvroF.union(invariants)
case MuF.TSum(name, fields) => AvroF.enum(name, none[String], Nil, none[String], fields.map(_.name))
case MuF.TProduct(name, namespace, fields, _, _) =>
TRecord(
name,
none[String],
namespace,
Nil,
none[String],
fields.map(f => Field(f.name, Nil, none[String], none[Order], f.tpe))
Expand Down
113 changes: 72 additions & 41 deletions src/main/scala/higherkindness/skeuomorph/avro/schema.scala
View file
Expand Up @@ -25,7 +25,7 @@ import cats.instances.int._
import cats.syntax.eq._
import higherkindness.droste.macros.deriveTraverse
import io.circe.Json
import org.apache.avro.Schema
import org.apache.avro.{LogicalType, LogicalTypes, Schema}
import org.apache.avro.Schema.Type
import higherkindness.droste.{Algebra, Coalgebra}

Expand Down Expand Up @@ -116,6 +116,9 @@ object AvroF {
) extends AvroF[A]
final case class TUnion[A](options: NonEmptyList[A]) extends AvroF[A]
final case class TFixed[A](name: String, namespace: Option[String], aliases: List[String], size: Int) extends AvroF[A]
final case class TDate[A]() extends AvroF[A]
final case class TTimestampMillis[A]() extends AvroF[A]
final case class TDecimal[A](precision: Int, scale: Int) extends AvroF[A]

implicit def eqAvroF[T: Eq]: Eq[AvroF[T]] =
Eq.instance {
Expand Down Expand Up @@ -177,49 +180,60 @@ object AvroF {
*/
def fromAvro: Coalgebra[AvroF, Schema] =
Coalgebra { sch =>
sch.getType match {
case Type.STRING => AvroF.TString()
case Type.BOOLEAN => AvroF.TBoolean()
case Type.BYTES => AvroF.TBytes()
case Type.DOUBLE => AvroF.TDouble()
case Type.FLOAT => AvroF.TFloat()
case Type.INT => AvroF.TInt()
case Type.LONG => AvroF.TLong()
case Type.NULL => AvroF.TNull()
case Type.MAP => AvroF.TMap(sch.getValueType)
case Type.ARRAY => AvroF.TArray(sch.getElementType)
case Type.RECORD =>
AvroF.TRecord(
sch.getName,
Option(sch.getNamespace),
sch.getAliases.asScala.toList,
Option(sch.getDoc),
sch.getFields.asScala.toList.map(field2Field)
)
case Type.ENUM =>
val symbols = sch.getEnumSymbols.asScala.toList
AvroF.TEnum(
sch.getName,
Option(sch.getNamespace),
sch.getAliases.asScala.toList,
Option(sch.getDoc),
symbols
)
case Type.UNION =>
val types = sch.getTypes.asScala.toList
AvroF.TUnion(
NonEmptyList.fromListUnsafe(types)
)
case Type.FIXED =>
AvroF.TFixed(
sch.getName,
Option(sch.getNamespace),
sch.getAliases.asScala.toList,
sch.getFixedSize
)
Option(sch.getLogicalType) match {
case Some(lt) => logicalType(lt)
case None => primitiveType(sch)
}
}

private def logicalType(logicalType: LogicalType): AvroF[Schema] = logicalType match {
case _: LogicalTypes.Date => AvroF.TDate()
case _: LogicalTypes.TimestampMillis => AvroF.TTimestampMillis()
case dec: LogicalTypes.Decimal => AvroF.TDecimal(dec.getPrecision, dec.getScale)
}

private def primitiveType(sch: Schema): AvroF[Schema] = sch.getType match {
case Type.STRING => AvroF.TString()
case Type.BOOLEAN => AvroF.TBoolean()
case Type.BYTES => AvroF.TBytes()
case Type.DOUBLE => AvroF.TDouble()
case Type.FLOAT => AvroF.TFloat()
case Type.INT => AvroF.TInt()
case Type.LONG => AvroF.TLong()
case Type.NULL => AvroF.TNull()
case Type.MAP => AvroF.TMap(sch.getValueType)
case Type.ARRAY => AvroF.TArray(sch.getElementType)
case Type.RECORD =>
AvroF.TRecord(
sch.getName,
Option(sch.getNamespace),
sch.getAliases.asScala.toList,
Option(sch.getDoc),
sch.getFields.asScala.toList.map(field2Field)
)
case Type.ENUM =>
val symbols = sch.getEnumSymbols.asScala.toList
AvroF.TEnum(
sch.getName,
Option(sch.getNamespace),
sch.getAliases.asScala.toList,
Option(sch.getDoc),
symbols
)
case Type.UNION =>
val types = sch.getTypes.asScala.toList
AvroF.TUnion(
NonEmptyList.fromListUnsafe(types)
)
case Type.FIXED =>
AvroF.TFixed(
sch.getName,
Option(sch.getNamespace),
sch.getAliases.asScala.toList,
sch.getFixedSize
)
}

def toJson: Algebra[AvroF, Json] =
Algebra {
case TNull() => Json.fromString("Null")
Expand Down Expand Up @@ -263,5 +277,22 @@ object AvroF {
"name" -> Json.fromString(name),
"size" -> Json.fromInt(size)
)
case TDate() =>
Json.obj(
"type" -> Json.fromString("int"),
"logicalType" -> Json.fromString("date")
)
case TTimestampMillis() =>
Json.obj(
"type" -> Json.fromString("long"),
"logicalType" -> Json.fromString("timestamp-millis")
)
case TDecimal(precision, scale) =>
Json.obj(
"type" -> Json.fromString("bytes"),
"logicalType" -> Json.fromString("decimal"),
"precision" -> Json.fromInt(precision),
"scale" -> Json.fromInt(scale)
)
}
}
13 changes: 8 additions & 5 deletions src/main/scala/higherkindness/skeuomorph/mu/Optimize.scala
View file
Expand Up @@ -44,15 +44,17 @@ object Optimize {
*/
def nestedNamedTypesTrans[T](implicit T: Basis[MuF, T]): Trans[MuF, MuF, T] =
Trans {
case TProduct(name, fields, nestedProducts, nestedCoproducts) =>
case TProduct(name, namespace, fields, nestedProducts, nestedCoproducts) =>
def nameTypes(f: Field[T]): Field[T] = f.copy(tpe = namedTypes(T)(f.tpe))
TProduct[T](
name,
namespace,
fields.map(nameTypes),
nestedProducts,
nestedCoproducts
)
case other => other
case other =>
other
}

def nestedOptionInCoproductsTrans[T](implicit T: Basis[MuF, T]): Trans[MuF, MuF, T] =
Expand All @@ -69,9 +71,10 @@ object Optimize {

def namedTypesTrans[T]: Trans[MuF, MuF, T] =
Trans {
case TProduct(name, _, _, _) => TNamedType[T](Nil, name)
case TSum(name, _) => TNamedType[T](Nil, name)
case other => other
case TProduct(name, ns, _, _, _) => TNamedType[T](ns.map(n => n.split('.').toList).getOrElse(Nil), name)
case TSum(name, _) => TNamedType[T](Nil, name)
case TByteArray(Length.Fixed(n, ns, _)) => TNamedType(ns.map(_.split('.').toList).getOrElse(Nil) :+ n, n)
case other => other
}

def namedTypes[T: Basis[MuF, ?]]: T => T = scheme.cata(namedTypesTrans.algebra)
Expand Down

0 comments on commit 05118bc

Please sign in to comment.