Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
refactor(triplestore): ZIO-fying triplestore service (DSP-904) (#2059)
  • Loading branch information
subotic committed Jul 1, 2022
1 parent 3d660a3 commit 9e038ec
Show file tree
Hide file tree
Showing 251 changed files with 5,964 additions and 10,847 deletions.
15 changes: 14 additions & 1 deletion Makefile
Expand Up @@ -203,7 +203,20 @@ test-repository-upgrade: build init-db-test-minimal ## runs DB upgrade integrati

.PHONY: test
test: build ## runs all tests
sbt -v +test
sbt -v "schemaApi/test"
sbt -v "schemaCore/test"
sbt -v "schemaRepo/test"
sbt -v "schemaRepoEventStoreService/test"
sbt -v "schemaRepoSearchService/test"
sbt -v "shared/test"
sbt -v "sipi/test"
sbt -v "userCore/test"
sbt -v "userHandler/test"
sbt -v "userInterface/test"
sbt -v "userRepo/test"
sbt -v "valueObjects/test"
sbt -v "webapi/test"


#################################
## Database Management
Expand Down
15 changes: 11 additions & 4 deletions build.sbt
Expand Up @@ -105,7 +105,8 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
.settings(
name := "webapi",
resolvers ++= Seq(
Resolver.bintrayRepo("hseeberger", "maven")
Resolver.bintrayRepo("hseeberger", "maven"),
"Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots"
),
libraryDependencies ++= Dependencies.webapiLibraryDependencies
)
Expand Down Expand Up @@ -143,9 +144,9 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
logLevel := Level.Info,
run / javaOptions := webapiJavaRunOptions,
javaAgents += Dependencies.aspectjweaver,
fork := true, // run tests in a forked JVM
Test / testForkedParallel := false, // run forked tests in parallel
Test / parallelExecution := false, // run non-forked tests in parallel
Test / fork := true, // run tests in a forked JVM
Test / testForkedParallel := false, // not run forked tests in parallel
Test / parallelExecution := false, // not run non-forked tests in parallel
// Global / concurrentRestrictions += Tags.limit(Tags.Test, 1), // restrict the number of concurrently executing tests in all projects
Test / javaOptions ++= Seq("-Dconfig.resource=fuseki.conf") ++ webapiJavaTestOptions,
// Test / javaOptions ++= Seq("-Dakka.log-config-on-start=on"), // prints out akka config
Expand Down Expand Up @@ -236,6 +237,7 @@ lazy val apiMain = project
.settings(
name := "dsp-api-main",
libraryDependencies ++= Dependencies.dspApiMainLibraryDependencies,
resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
)
.dependsOn(schemaCore, schemaRepo, schemaApi)
Expand All @@ -257,6 +259,7 @@ lazy val schemaApi = project
.settings(
name := "schemaApi",
libraryDependencies ++= Dependencies.schemaApiLibraryDependencies,
resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
)
.dependsOn(schemaCore)
Expand All @@ -266,6 +269,7 @@ lazy val schemaCore = project
.settings(
name := "schemaCore",
libraryDependencies ++= Dependencies.schemaCoreLibraryDependencies,
resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
)

Expand All @@ -274,6 +278,7 @@ lazy val schemaRepo = project
.settings(
name := "schemaRepo",
libraryDependencies ++= Dependencies.schemaRepoLibraryDependencies,
resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
)
.dependsOn(schemaCore)
Expand All @@ -283,6 +288,7 @@ lazy val schemaRepoEventStoreService = project
.settings(
name := "schemaRepoEventstoreService",
libraryDependencies ++= Dependencies.schemaRepoEventStoreServiceLibraryDependencies,
resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
)
.dependsOn(schemaRepo)
Expand All @@ -292,6 +298,7 @@ lazy val schemaRepoSearchService = project
.settings(
name := "dsp-schema-repo-search-service",
libraryDependencies ++= Dependencies.schemaRepoSearchServiceLibraryDependencies,
resolvers += "Sonatype" at "https://oss.sonatype.org/content/repositories/snapshots",
testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
)
.dependsOn(schemaRepo)
Expand Down
@@ -0,0 +1,20 @@
# 4. Change Triplestore Service Manager and Fuseki implementation to ZLayer

Date: 2022-05-23

## Status

Accepted

## Context

Both `org.knora.webapi.store.triplestore.TriplestoreServiceManager` and `org.knora.webapi.store.triplestore.impl.TriplestoreServiceHttpConnectorImpl`
where implemented as Akka-Actors.

## Decision

As part of the move from `Akka` to `ZIO`, it was decided that the `TriplestoreServiceManager` and the `TriplestoreServiceHttpConnectorImpl` is refactored using ZIO.

## Consequences

The usage from other actors stays the same. The actor messages and responses don't change.
8 changes: 4 additions & 4 deletions dsp-api-main/src/main/scala/dsp/api/main/MainApp.scala
Expand Up @@ -6,13 +6,13 @@ import zio.Console.printLine
import zio._

object MainApp extends ZIOAppDefault {
val effect: ZIO[Console with SchemaRepo, Nothing, Unit] =
val effect: ZIO[SchemaRepo, Nothing, Unit] =
for {
profile <- SchemaRepo.lookup("user1").orDie
_ <- printLine(profile).orDie
_ <- printLine(42).orDie
// _ <- printLine(profile).orDie
// _ <- printLine(42).orDie
} yield ()

val mainApp: ZIO[Any, Nothing, Unit] = effect.provide(Console.live ++ SchemaRepoLive.layer)
val mainApp: ZIO[Any, Nothing, Unit] = effect.provide(SchemaRepoLive.layer)
def run: ZIO[Any, Nothing, Unit] = mainApp
}
12 changes: 9 additions & 3 deletions dsp-shared/src/main/scala/dsp/errors/Errors.scala
Expand Up @@ -292,12 +292,15 @@ object TriplestoreConnectionException {
* @param message a description of the error.
* @param cause the original exception representing the cause of the error, if any.
*/
case class TriplestoreTimeoutException(message: String, cause: Option[Throwable] = None)
final case class TriplestoreTimeoutException(message: String, cause: Option[Throwable] = None)
extends TriplestoreException(message, cause)

object TriplestoreTimeoutException {
def apply(message: String, e: Throwable, log: Logger): TriplestoreTimeoutException =
TriplestoreTimeoutException(message, Some(ExceptionUtil.logAndWrapIfNotSerializable(e, log)))

def apply(message: String, cause: Throwable): TriplestoreTimeoutException =
TriplestoreTimeoutException(message, Some(cause))
}

/**
Expand Down Expand Up @@ -334,12 +337,15 @@ object TriplestoreInternalException {
* @param message a description of the error.
* @param cause the original exception representing the cause of the error, if any.
*/
case class TriplestoreResponseException(message: String, cause: Option[Throwable] = None)
final case class TriplestoreResponseException(message: String, cause: Option[Throwable] = None)
extends TriplestoreException(message, cause)

object TriplestoreResponseException {
def apply(message: String, e: Throwable, log: Logger): TriplestoreResponseException =
TriplestoreResponseException(message, Some(ExceptionUtil.logAndWrapIfNotSerializable(e, log)))

def apply(message: String): TriplestoreResponseException =
TriplestoreResponseException(message)
}

/**
Expand Down Expand Up @@ -373,7 +379,7 @@ object InvalidApiJsonException {
}

/**
* Indicates that during caching with [[org.knora.webapi.store.cacheservice.api.CacheService]] something went wrong.
* Indicates that during caching with [[org.knora.webapi.store.cache.api.CacheService]] something went wrong.
*
* @param message a description of the error.
*/
Expand Down
18 changes: 10 additions & 8 deletions project/Dependencies.scala
Expand Up @@ -20,12 +20,12 @@ object Dependencies {
val jenaVersion = "4.4.0"
val metricsVersion = "4.0.1"
val scalaVersion = "2.13.8"
val ZioVersion = "2.0.0-RC6"
val ZioVersion = "2.0.0"
val ZioHttpVersion = "2.0.0-RC4"
val ZioJsonVersion = "0.3.0-RC3"
val ZioConfigVersion = "3.0.0-RC9"
val ZioSchemaVersion = "0.2.0-RC5"
val ZioLoggingVersion = "2.0.0-RC9"
val ZioConfigVersion = "3.0.1"
val ZioSchemaVersion = "0.2.0"
val ZioLoggingVersion = "2.0.0"
val ZioZmxVersion = "2.0.0-RC4"
val ZioPreludeVersion = "1.0.0-RC13"

Expand All @@ -36,11 +36,12 @@ object Dependencies {
val zioJson = "dev.zio" %% "zio-json" % ZioJsonVersion
val zioPrelude = "dev.zio" %% "zio-prelude" % ZioPreludeVersion
val zioLogging = "dev.zio" %% "zio-logging" % ZioLoggingVersion
val zioLoggingSlf4j = "dev.zio" %% "zio-logging-slf4j" % ZioLoggingVersion
val zioConfig = "dev.zio" %% "zio-config" % ZioConfigVersion
val zioConfigMagnolia = "dev.zio" %% "zio-config-magnolia" % ZioConfigVersion
val zioConfigTypesafe = "dev.zio" %% "zio-config-typesafe" % ZioConfigVersion
val zioTest = "dev.zio" %% "zio-test" % ZioVersion
val zioTestSbt = "dev.zio" %% "zio-test-sbt" % ZioVersion
val zioTest = "dev.zio" %% "zio-test" % "2.0.0"
val zioTestSbt = "dev.zio" %% "zio-test-sbt" % "2.0.0"

// akka
val akkaActor = "com.typesafe.akka" %% "akka-actor" % akkaVersion // Scala 3 compatible
Expand Down Expand Up @@ -100,8 +101,8 @@ object Dependencies {
val akkaTestkit = "com.typesafe.akka" %% "akka-testkit" % akkaVersion // Scala 3 compatible
val gatlingHighcharts = "io.gatling.highcharts" % "gatling-charts-highcharts" % "3.7.6"
val gatlingTestFramework = "io.gatling" % "gatling-test-framework" % "3.7.6"
val scalaTest = "org.scalatest" %% "scalatest" % "3.2.2" // Scala 3 compatible
val testcontainers = "org.testcontainers" % "testcontainers" % "1.16.3"
val scalaTest = "org.scalatest" %% "scalatest" % "3.2.12" // Scala 3 compatible
val testcontainers = "org.testcontainers" % "testcontainers" % "1.17.2"

// found/added by the plugin but deleted anyway
val commonsLang3 = "org.apache.commons" % "commons-lang3" % "3.12.0"
Expand Down Expand Up @@ -152,6 +153,7 @@ object Dependencies {
zioHttp,
zioJson,
zioLogging,
zioLoggingSlf4j,
zioMacros,
zioPrelude,
zioTest % Test,
Expand Down
151 changes: 1 addition & 150 deletions webapi/src/main/resources/application.conf
Expand Up @@ -264,36 +264,7 @@ akka-http-cors {
}

app {
feature-toggles {
jena-rdf-library {
description = "Use the Jena API for RDF processing. If turned off, use the RDF4J API."

available-versions = [ 1 ]
default-version = 1
enabled-by-default = yes
override-allowed = yes

developer-emails = [
"Benjamin Geer <benjamin.geer@dasch.swiss>"
]
}

gravsearch-dependency-optimisation {
description = "Optimise Gravsearch queries by reordering query patterns according to their dependencies."

available-versions = [ 1 ]
default-version = 1
enabled-by-default = yes
override-allowed = yes
expiration-date = "2021-12-01T00:00:00Z"

developer-emails = [
"Sepideh Alassi <sepideh.alassi@dasch.swiss>"
"Benjamin Geer <benjamin.geer@dasch.swiss>"
]
}
}


shacl {
# The directory that SHACL shapes are loaded from.
shapes-dir = "../test_data/shacl"
Expand Down Expand Up @@ -541,131 +512,11 @@ app {

reload-on-start = false // ignored if "memory" as it will always reload

# This data is automatically loaded during resetting of the triple store content initiated
# through the `ResetTriplestoreContent` message. The main usage is in unit testing, where
# we want a known state of the triple store data. If additional triples need to be loaded,
# then a list of `RdfDataObject` instances containing the path and the name of the named graph
# can be supplied to the `ResetTriplestoreContent` message.
default-rdf-data = [
{
path = "knora-ontologies/knora-admin.ttl"
name = "http://www.knora.org/ontology/knora-admin"
}
{
path = "knora-ontologies/knora-base.ttl"
name = "http://www.knora.org/ontology/knora-base"
}
{
path = "knora-ontologies/standoff-onto.ttl"
name = "http://www.knora.org/ontology/standoff"
}
{
path = "knora-ontologies/standoff-data.ttl"
name = "http://www.knora.org/data/standoff"
}
{
path = "knora-ontologies/salsah-gui.ttl"
name = "http://www.knora.org/ontology/salsah-gui"
}
{
path = "test_data/all_data/admin-data.ttl"
name = "http://www.knora.org/data/admin"
}
{
path = "test_data/all_data/permissions-data.ttl"
name = "http://www.knora.org/data/permissions"
}
{
path = "test_data/all_data/system-data.ttl"
name = "http://www.knora.org/data/0000/SystemProject"
}
{
path = "test_data/ontologies/anything-onto.ttl"
name = "http://www.knora.org/ontology/0001/anything"
}
{
path = "test_data/ontologies/something-onto.ttl"
name = "http://www.knora.org/ontology/0001/something"
}
{
path = "test_data/ontologies/images-onto.ttl"
name = "http://www.knora.org/ontology/00FF/images"
}
{
path = "test_data/ontologies/beol-onto.ttl"
name = "http://www.knora.org/ontology/0801/beol"
}
{
path = "test_data/ontologies/biblio-onto.ttl"
name = "http://www.knora.org/ontology/0801/biblio"
}
{
path = "test_data/ontologies/incunabula-onto.ttl"
name = "http://www.knora.org/ontology/0803/incunabula"
}
{
path = "test_data/ontologies/dokubib-onto.ttl"
name = "http://www.knora.org/ontology/0804/dokubib"
}
{
path = "test_data/ontologies/webern-onto.ttl"
name = "http://www.knora.org/ontology/0806/webern"
}
]

# This data is loaded if the `webapi` server is started with the `loadDemoData` flag, which initiates
# sending of `ResetTriplestoreContent` with a list of `RdfDataObject` instances containing this data.
# The result will be a triplestore containing `default-rdf-data` and `rdf-data`
rdf-data = [
{
path = "test_data/ontologies/incunabula-onto.ttl"
name = "http://www.knora.org/ontology/0803/incunabula"
}
{
path = "test_data/demo_data/incunabula-demo-data.ttl"
name = "http://www.knora.org/data/incunabula"
}
{
path = "test_data/ontologies/images-onto.ttl"
name = "http://www.knora.org/ontology/00FF/images"
}
{
path = "test_data/demo_data/images-demo-data.ttl"
name = "http://www.knora.org/data/00FF/images"
}

]

// If true, the time taken by each SPARQL query is logged at DEBUG level. To see these messages,
// set loglevel = "DEBUG" above, and
// <logger name="org.knora.webapi.store.triplestore.http.HttpTriplestoreConnector" level="DEBUG"/>
// in logback.xml.
profile-queries = false

///////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Fake triplestore settings
//
// The application can generate a fake triplestore, consisting of SPARQL queries and responses saved in
// text files (in fake-triplestore-query-dir). This is useful for:
//
// * measuring the response time of the application minus the response time of the triplestore.
//
// * debugging SPARQL queries, because you can open the generated queries in a text editor and copy and paste
// them into something like GraphDB Workbench to experiment with them.
//
// * benchmarking triplestores, because you can feed the whole fake triplestore file structure to RDFBench.
//
// To generate a fake triplestore, set fake-triplestore to "prepare", start the application, and run one or more
// API operations. The fake triplestore will contain all the SPARQL queries and responses involved in those
// operations.
//
// To have the application use the fake triplestore, set fake-triplestore to "use", and restart the application.
// The entire contents of the fake triplestore will be loaded when the application starts, and all SPARQL queries
// will simply be hashtable lookups in this in-memory data.
//
// To just use a real triplestore, set fake-triplestore to "off".
fake-triplestore = "off"
fake-triplestore-data-dir = "src/main/resources/query-log"
}

cache-service {
Expand Down

0 comments on commit 9e038ec

Please sign in to comment.