refactor(triplestore): ZIO-fying triplestore service (DSP-904)

build.sbt

@@ -105,7 +105,8 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
   .settings(
     name := "webapi",
     resolvers ++= Seq(
-      Resolver.bintrayRepo("hseeberger", "maven")
+      Resolver.bintrayRepo("hseeberger", "maven"),
+      "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots"
     ),
     libraryDependencies ++= Dependencies.webapiLibraryDependencies
   )
@@ -143,9 +144,9 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
     logLevel := Level.Info,
     run / javaOptions := webapiJavaRunOptions,
     javaAgents += Dependencies.aspectjweaver,
-    fork := true,                       // run tests in a forked JVM
-    Test / testForkedParallel := false, // run forked tests in parallel
-    Test / parallelExecution := false,  // run non-forked tests in parallel
+    Test / fork := true,                // run tests in a forked JVM
+    Test / testForkedParallel := false, // not run forked tests in parallel
+    Test / parallelExecution := false,  // not run non-forked tests in parallel
     // Global / concurrentRestrictions += Tags.limit(Tags.Test, 1), // restrict the number of concurrently executing tests in all projects
     Test / javaOptions ++= Seq("-Dconfig.resource=fuseki.conf") ++ webapiJavaTestOptions,
     // Test / javaOptions ++= Seq("-Dakka.log-config-on-start=on"), // prints out akka config
@@ -236,6 +237,7 @@ lazy val apiMain = project
   .settings(
     name := "dsp-api-main",
     libraryDependencies ++= Dependencies.dspApiMainLibraryDependencies,
+    resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
     testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
   )
   .dependsOn(schemaCore, schemaRepo, schemaApi)
@@ -257,6 +259,7 @@ lazy val schemaApi = project
   .settings(
     name := "schemaApi",
     libraryDependencies ++= Dependencies.schemaApiLibraryDependencies,
+    resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
     testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
   )
   .dependsOn(schemaCore)
@@ -266,6 +269,7 @@ lazy val schemaCore = project
   .settings(
     name := "schemaCore",
     libraryDependencies ++= Dependencies.schemaCoreLibraryDependencies,
+    resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
     testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
   )
 
@@ -274,6 +278,7 @@ lazy val schemaRepo = project
   .settings(
     name := "schemaRepo",
     libraryDependencies ++= Dependencies.schemaRepoLibraryDependencies,
+    resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
     testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
   )
   .dependsOn(schemaCore)
@@ -283,6 +288,7 @@ lazy val schemaRepoEventStoreService = project
   .settings(
     name := "schemaRepoEventstoreService",
     libraryDependencies ++= Dependencies.schemaRepoEventStoreServiceLibraryDependencies,
+    resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
     testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
   )
   .dependsOn(schemaRepo)
@@ -292,6 +298,7 @@ lazy val schemaRepoSearchService = project
   .settings(
     name := "dsp-schema-repo-search-service",
     libraryDependencies ++= Dependencies.schemaRepoSearchServiceLibraryDependencies,
+    resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
     testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
   )
   .dependsOn(schemaRepo)

docs/architecture/decisions/0004-change-triplestore-service-manager-and-fuseki-implementation-to-zlayer.md

@@ -0,0 +1,20 @@
+# 4. Change Triplestore Service Manager and Fuseki implementation to ZLayer
+
+Date: 2022-05-23
+
+## Status
+
+Accepted
+
+## Context
+
+Both `org.knora.webapi.store.triplestore.TriplestoreServiceManager` and `org.knora.webapi.store.triplestore.impl.TriplestoreServiceHttpConnectorImpl`
+where implemented as Akka-Actors.
+
+## Decision
+
+As part of the move from `Akka` to `ZIO`, it was decided that the `TriplestoreServiceManager` and the `TriplestoreServiceHttpConnectorImpl` is refactored using ZIO.
+
+## Consequences
+
+The usage from other actors stays the same. The actor messages and responses don't change.

dsp-api-main/src/main/scala/dsp/api/main/MainApp.scala

@@ -6,13 +6,13 @@ import zio.Console.printLine
 import zio._
 
 object MainApp extends ZIOAppDefault {
-  val effect: ZIO[Console with SchemaRepo, Nothing, Unit] =
+  val effect: ZIO[SchemaRepo, Nothing, Unit] =
     for {
       profile <- SchemaRepo.lookup("user1").orDie
-      _       <- printLine(profile).orDie
-      _       <- printLine(42).orDie
+      // _       <- printLine(profile).orDie
+      // _       <- printLine(42).orDie
     } yield ()
 
-  val mainApp: ZIO[Any, Nothing, Unit] = effect.provide(Console.live ++ SchemaRepoLive.layer)
+  val mainApp: ZIO[Any, Nothing, Unit] = effect.provide(SchemaRepoLive.layer)
   def run: ZIO[Any, Nothing, Unit]     = mainApp
 }

dsp-shared/src/main/scala/dsp/errors/Errors.scala

@@ -292,12 +292,15 @@ object TriplestoreConnectionException {
  * @param message a description of the error.
  * @param cause   the original exception representing the cause of the error, if any.
  */
-case class TriplestoreTimeoutException(message: String, cause: Option[Throwable] = None)
+final case class TriplestoreTimeoutException(message: String, cause: Option[Throwable] = None)
     extends TriplestoreException(message, cause)
 
 object TriplestoreTimeoutException {
   def apply(message: String, e: Throwable, log: Logger): TriplestoreTimeoutException =
     TriplestoreTimeoutException(message, Some(ExceptionUtil.logAndWrapIfNotSerializable(e, log)))
+
+  def apply(message: String, cause: Throwable): TriplestoreTimeoutException =
+    TriplestoreTimeoutException(message, Some(cause))
 }
 
 /**
@@ -334,12 +337,15 @@ object TriplestoreInternalException {
  * @param message a description of the error.
  * @param cause   the original exception representing the cause of the error, if any.
  */
-case class TriplestoreResponseException(message: String, cause: Option[Throwable] = None)
+final case class TriplestoreResponseException(message: String, cause: Option[Throwable] = None)
     extends TriplestoreException(message, cause)
 
 object TriplestoreResponseException {
   def apply(message: String, e: Throwable, log: Logger): TriplestoreResponseException =
     TriplestoreResponseException(message, Some(ExceptionUtil.logAndWrapIfNotSerializable(e, log)))
+
+  def apply(message: String): TriplestoreResponseException =
+    TriplestoreResponseException(message)
 }
 
 /**
@@ -373,7 +379,7 @@ object InvalidApiJsonException {
 }
 
 /**
- * Indicates that during caching with [[org.knora.webapi.store.cacheservice.api.CacheService]] something went wrong.
+ * Indicates that during caching with [[org.knora.webapi.store.cache.api.CacheService]] something went wrong.
  *
  * @param message a description of the error.
  */

project/Dependencies.scala

@@ -20,12 +20,12 @@ object Dependencies {
   val jenaVersion       = "4.4.0"
   val metricsVersion    = "4.0.1"
   val scalaVersion      = "2.13.8"
-  val ZioVersion        = "2.0.0-RC6"
+  val ZioVersion        = "2.0.0"
   val ZioHttpVersion    = "2.0.0-RC4"
   val ZioJsonVersion    = "0.3.0-RC3"
-  val ZioConfigVersion  = "3.0.0-RC9"
-  val ZioSchemaVersion  = "0.2.0-RC5"
-  val ZioLoggingVersion = "2.0.0-RC9"
+  val ZioConfigVersion  = "3.0.1"
+  val ZioSchemaVersion  = "0.2.0"
+  val ZioLoggingVersion = "2.0.0"
   val ZioZmxVersion     = "2.0.0-RC4"
   val ZioPreludeVersion = "1.0.0-RC13"
 
@@ -36,11 +36,12 @@ object Dependencies {
   val zioJson           = "dev.zio" %% "zio-json"            % ZioJsonVersion
   val zioPrelude        = "dev.zio" %% "zio-prelude"         % ZioPreludeVersion
   val zioLogging        = "dev.zio" %% "zio-logging"         % ZioLoggingVersion
+  val zioLoggingSlf4j   = "dev.zio" %% "zio-logging-slf4j"   % ZioLoggingVersion
   val zioConfig         = "dev.zio" %% "zio-config"          % ZioConfigVersion
   val zioConfigMagnolia = "dev.zio" %% "zio-config-magnolia" % ZioConfigVersion
   val zioConfigTypesafe = "dev.zio" %% "zio-config-typesafe" % ZioConfigVersion
-  val zioTest           = "dev.zio" %% "zio-test"            % ZioVersion
-  val zioTestSbt        = "dev.zio" %% "zio-test-sbt"        % ZioVersion
+  val zioTest           = "dev.zio" %% "zio-test"            % "2.0.0"
+  val zioTestSbt        = "dev.zio" %% "zio-test-sbt"        % "2.0.0"
 
   // akka
   val akkaActor         = "com.typesafe.akka" %% "akka-actor"           % akkaVersion     // Scala 3 compatible
@@ -100,8 +101,8 @@ object Dependencies {
   val akkaTestkit          = "com.typesafe.akka"    %% "akka-testkit"              % akkaVersion     // Scala 3 compatible
   val gatlingHighcharts    = "io.gatling.highcharts" % "gatling-charts-highcharts" % "3.7.6"
   val gatlingTestFramework = "io.gatling"            % "gatling-test-framework"    % "3.7.6"
-  val scalaTest            = "org.scalatest"        %% "scalatest"                 % "3.2.2"         // Scala 3 compatible
-  val testcontainers       = "org.testcontainers"    % "testcontainers"            % "1.16.3"
+  val scalaTest            = "org.scalatest"        %% "scalatest"                 % "3.2.12"        // Scala 3 compatible
+  val testcontainers       = "org.testcontainers"    % "testcontainers"            % "1.17.2"
 
   // found/added by the plugin but deleted anyway
   val commonsLang3 = "org.apache.commons" % "commons-lang3" % "3.12.0"
@@ -152,6 +153,7 @@ object Dependencies {
     zioHttp,
     zioJson,
     zioLogging,
+    zioLoggingSlf4j,
     zioMacros,
     zioPrelude,
     zioTest    % Test,

webapi/src/main/resources/application.conf

@@ -264,36 +264,7 @@ akka-http-cors {
 }
 
 app {
-    feature-toggles {
-        jena-rdf-library {
-            description = "Use the Jena API for RDF processing. If turned off, use the RDF4J API."
-
-            available-versions = [ 1 ]
-            default-version = 1
-            enabled-by-default = yes
-            override-allowed = yes
-
-            developer-emails = [
-                "Benjamin Geer <benjamin.geer@dasch.swiss>"
-            ]
-        }
-
-        gravsearch-dependency-optimisation {
-            description = "Optimise Gravsearch queries by reordering query patterns according to their dependencies."
-
-            available-versions = [ 1 ]
-            default-version = 1
-            enabled-by-default = yes
-            override-allowed = yes
-            expiration-date = "2021-12-01T00:00:00Z"
-
-            developer-emails = [
-                "Sepideh Alassi <sepideh.alassi@dasch.swiss>"
-                "Benjamin Geer <benjamin.geer@dasch.swiss>"
-            ]
-        }
-    }
-
+
     shacl {
         # The directory that SHACL shapes are loaded from.
         shapes-dir = "../test_data/shacl"
@@ -541,131 +512,11 @@ app {
 
         reload-on-start = false // ignored if "memory" as it will always reload
 
-        # This data is automatically loaded during resetting of the triple store content initiated
-        # through the `ResetTriplestoreContent` message. The main usage is in unit testing, where
-        # we want a known state of the triple store data. If additional triples need to be loaded,
-        # then a list of `RdfDataObject` instances containing the path and the name of the named graph
-        # can be supplied to the `ResetTriplestoreContent` message.
-        default-rdf-data = [
-            {
-                path = "knora-ontologies/knora-admin.ttl"
-                name = "http://www.knora.org/ontology/knora-admin"
-            }
-            {
-                path = "knora-ontologies/knora-base.ttl"
-                name = "http://www.knora.org/ontology/knora-base"
-            }
-            {
-                path = "knora-ontologies/standoff-onto.ttl"
-                name = "http://www.knora.org/ontology/standoff"
-            }
-            {
-                path = "knora-ontologies/standoff-data.ttl"
-                name = "http://www.knora.org/data/standoff"
-            }
-            {
-                path = "knora-ontologies/salsah-gui.ttl"
-                name = "http://www.knora.org/ontology/salsah-gui"
-            }
-            {
-                path = "test_data/all_data/admin-data.ttl"
-                name = "http://www.knora.org/data/admin"
-            }
-            {
-                path = "test_data/all_data/permissions-data.ttl"
-                name = "http://www.knora.org/data/permissions"
-            }
-            {
-                path = "test_data/all_data/system-data.ttl"
-                name = "http://www.knora.org/data/0000/SystemProject"
-            }
-            {
-                path = "test_data/ontologies/anything-onto.ttl"
-                name = "http://www.knora.org/ontology/0001/anything"
-            }
-            {
-                path = "test_data/ontologies/something-onto.ttl"
-                name = "http://www.knora.org/ontology/0001/something"
-            }
-            {
-                path = "test_data/ontologies/images-onto.ttl"
-                name = "http://www.knora.org/ontology/00FF/images"
-            }
-            {
-                path = "test_data/ontologies/beol-onto.ttl"
-                name = "http://www.knora.org/ontology/0801/beol"
-            }
-            {
-                path = "test_data/ontologies/biblio-onto.ttl"
-                name = "http://www.knora.org/ontology/0801/biblio"
-            }
-            {
-                path = "test_data/ontologies/incunabula-onto.ttl"
-                name = "http://www.knora.org/ontology/0803/incunabula"
-            }
-            {
-                path = "test_data/ontologies/dokubib-onto.ttl"
-                name = "http://www.knora.org/ontology/0804/dokubib"
-            }
-            {
-                path = "test_data/ontologies/webern-onto.ttl"
-                name = "http://www.knora.org/ontology/0806/webern"
-            }
-        ]
-
-        # This data is loaded if the `webapi` server is started with the `loadDemoData` flag, which initiates
-        # sending of `ResetTriplestoreContent` with a list of `RdfDataObject` instances containing this data.
-        # The result will be a triplestore containing `default-rdf-data` and `rdf-data`
-        rdf-data = [
-            {
-                path = "test_data/ontologies/incunabula-onto.ttl"
-                name = "http://www.knora.org/ontology/0803/incunabula"
-            }
-            {
-                path = "test_data/demo_data/incunabula-demo-data.ttl"
-                name = "http://www.knora.org/data/incunabula"
-            }
-            {
-                path = "test_data/ontologies/images-onto.ttl"
-                name = "http://www.knora.org/ontology/00FF/images"
-            }
-            {
-                path = "test_data/demo_data/images-demo-data.ttl"
-                name = "http://www.knora.org/data/00FF/images"
-            }
-
-        ]
-
         // If true, the time taken by each SPARQL query is logged at DEBUG level. To see these messages,
         // set loglevel = "DEBUG" above, and
         // <logger name="org.knora.webapi.store.triplestore.http.HttpTriplestoreConnector" level="DEBUG"/>
         // in logback.xml.
         profile-queries = false
-
-        ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
-        // Fake triplestore settings
-        //
-        // The application can generate a fake triplestore, consisting of SPARQL queries and responses saved in
-        // text files (in fake-triplestore-query-dir). This is useful for:
-        //
-        // * measuring the response time of the application minus the response time of the triplestore.
-        //
-        // * debugging SPARQL queries, because you can open the generated queries in a text editor and copy and paste
-        //   them into something like GraphDB Workbench to experiment with them.
-        //
-        // * benchmarking triplestores, because you can feed the whole fake triplestore file structure to RDFBench.
-        //
-        // To generate a fake triplestore, set fake-triplestore to "prepare", start the application, and run one or more
-        // API operations. The fake triplestore will contain all the SPARQL queries and responses involved in those
-        // operations.
-        //
-        // To have the application use the fake triplestore, set fake-triplestore to "use", and restart the application.
-        // The entire contents of the fake triplestore will be loaded when the application starts, and all SPARQL queries
-        // will simply be hashtable lookups in this in-memory data.
-        //
-        // To just use a real triplestore, set fake-triplestore to "off".
-        fake-triplestore = "off"
-        fake-triplestore-data-dir = "src/main/resources/query-log"
     }
 
     cache-service {

webapi/src/main/resources/logback.xml

@@ -33,6 +33,9 @@
     <logger name="org.apache.http" level="INFO"/>
     <logger name="org.apache.http.wire" level="INFO"/>
 
+    <!-- Logging inside ZIO -->
+    <logger name="zio-slf4j-logger" level="INFO"/>
+
     <!-- V1 -->
 
     <logger name="org.knora" level="INFO"/>