Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
refactor(cache-service): add in-memory implementation (#1870)
  • Loading branch information
subotic committed Jun 15, 2021
1 parent ff65323 commit 61531ab
Show file tree
Hide file tree
Showing 52 changed files with 1,488 additions and 463 deletions.
3 changes: 2 additions & 1 deletion README.md
Expand Up @@ -188,7 +188,8 @@ Lukas Rosenthaler `<lukas.rosenthaler@unibas.ch>`

## Commit Message Schema

When writing commit messages, we follow the [Conventional Commit messages](https://www.conventionalcommits.org/) rules. Get more information in our official [DSP Contribution Documentation](https://docs.dasch.swiss/developers/dsp/contribution/#git-commit-guidelines)
When writing commit messages, we follow the [Conventional Commit messages](https://www.conventionalcommits.org/) rules.
Get more information in our official [DSP Contribution Documentation](https://docs.dasch.swiss/developers/dsp/contribution/#git-commit-guidelines)

## Release Versioning Convention

Expand Down
2 changes: 0 additions & 2 deletions WORKSPACE
Expand Up @@ -289,8 +289,6 @@ http_archive(
url = "https://github.com/bazelbuild/buildtools/archive/master.zip",
)



#####################################
# rules_pkg - basic packaging rules #
#####################################
Expand Down
2 changes: 2 additions & 0 deletions third_party/dependencies.bzl
Expand Up @@ -164,6 +164,8 @@ ALL_WEBAPI_MAIN_DEPENDENCIES = [
"//webapi/src/main/scala/org/knora/webapi/routing",
"//webapi/src/main/scala/org/knora/webapi/settings",
"//webapi/src/main/scala/org/knora/webapi/store",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice/redis",
"//webapi/src/main/scala/org/knora/webapi/util",
"//webapi/src/main/scala/org/knora/webapi/util/cache",
]
Expand Down
5 changes: 5 additions & 0 deletions webapi/BUILD.bazel
Expand Up @@ -30,6 +30,7 @@ scala_library(
"//webapi/scripts:fuseki_repository_config_ttl_template",
"//webapi/src/main/resources",
],
scalacopts = ["-deprecation"],
unused_dependency_checker_mode = "warn",
runtime_deps = [
"@maven//:ch_qos_logback_logback_classic",
Expand Down Expand Up @@ -140,6 +141,7 @@ scala_library(
"//sipi/config",
"//webapi/src/test/resources",
],
scalacopts = ["-deprecation"],
unused_dependency_checker_mode = "warn",
runtime_deps = [
"@maven//:ch_qos_logback_logback_classic",
Expand All @@ -161,6 +163,9 @@ scala_library(
"//webapi/src/main/scala/org/knora/webapi/routing",
"//webapi/src/main/scala/org/knora/webapi/settings",
"//webapi/src/main/scala/org/knora/webapi/store",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice/inmem",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice/settings",
"//webapi/src/main/scala/org/knora/webapi/util",
# Logging
"@maven//:com_typesafe_scala_logging_scala_logging_2_13",
Expand Down
23 changes: 19 additions & 4 deletions webapi/src/main/scala/org/knora/webapi/app/ApplicationActor.scala
Expand Up @@ -50,7 +50,7 @@ import org.knora.webapi.messages.store.cacheservicemessages.{
}
import org.knora.webapi.messages.store.sipimessages.{IIIFServiceGetStatus, IIIFServiceStatusNOK, IIIFServiceStatusOK}
import org.knora.webapi.messages.store.triplestoremessages._
import org.knora.webapi.messages.util.KnoraSystemInstances
import org.knora.webapi.messages.util.{KnoraSystemInstances, ResponderData}
import org.knora.webapi.messages.v1.responder.KnoraRequestV1
import org.knora.webapi.messages.v2.responder.ontologymessages.LoadOntologiesRequestV2
import org.knora.webapi.messages.v2.responder.{KnoraRequestV2, SuccessResponseV2}
Expand All @@ -61,6 +61,8 @@ import org.knora.webapi.routing.v1._
import org.knora.webapi.routing.v2._
import org.knora.webapi.settings.{KnoraDispatchers, KnoraSettings, KnoraSettingsImpl, _}
import org.knora.webapi.store.StoreManager
import org.knora.webapi.store.cacheservice.inmem.CacheServiceInMemImpl
import org.knora.webapi.store.cacheservice.settings.CacheServiceSettings
import org.knora.webapi.util.cache.CacheUtil
import redis.clients.jedis.exceptions.JedisConnectionException

Expand All @@ -69,6 +71,7 @@ import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success}

trait Managers {
implicit val system: ActorSystem
val responderManager: ActorRef
val storeManager: ActorRef
}
Expand All @@ -80,7 +83,7 @@ trait LiveManagers extends Managers {
* The actor that forwards messages to actors that deal with persistent storage.
*/
lazy val storeManager: ActorRef = context.actorOf(
Props(new StoreManager(self) with LiveActorMaker)
Props(new StoreManager(appActor = self, cs = CacheServiceInMemImpl) with LiveActorMaker)
.withDispatcher(KnoraDispatchers.KnoraActorDispatcher),
name = StoreManagerActorName
)
Expand All @@ -89,7 +92,14 @@ trait LiveManagers extends Managers {
* The actor that forwards messages to responder actors to handle API requests.
*/
lazy val responderManager: ActorRef = context.actorOf(
Props(new ResponderManager(self) with LiveActorMaker)
Props(
new ResponderManager(
appActor = self,
responderData = ResponderData(system = context.system,
appActor = self,
knoraSettings = KnoraSettings(system),
cacheServiceSettings = new CacheServiceSettings(system.settings.config))
) with LiveActorMaker)
.withDispatcher(KnoraDispatchers.KnoraActorDispatcher),
name = RESPONDER_MANAGER_ACTOR_NAME
)
Expand All @@ -115,6 +125,11 @@ class ApplicationActor extends Actor with Stash with LazyLogging with AroundDire
*/
implicit val knoraSettings: KnoraSettingsImpl = KnoraSettings(system)

/**
* The Cache Service's configuration.
*/
implicit val cacheServiceSettings: CacheServiceSettings = new CacheServiceSettings(system.settings.config)

/**
* The default feature factory configuration, which is used during startup.
*/
Expand Down Expand Up @@ -169,7 +184,7 @@ class ApplicationActor extends Actor with Stash with LazyLogging with AroundDire
private var printConfigState = false
private var ignoreRepository = true
private var withIIIFService = true
private val withCacheService = knoraSettings.cacheServiceEnabled
private val withCacheService = cacheServiceSettings.cacheServiceEnabled

/**
* Startup of the ApplicationActor is a two step process:
Expand Down
4 changes: 4 additions & 0 deletions webapi/src/main/scala/org/knora/webapi/app/BUILD.bazel
Expand Up @@ -19,6 +19,10 @@ scala_library(
"//webapi/src/main/scala/org/knora/webapi/routing",
"//webapi/src/main/scala/org/knora/webapi/settings",
"//webapi/src/main/scala/org/knora/webapi/store",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice/inmem",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice/redis",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice/settings",
"//webapi/src/main/scala/org/knora/webapi/util/cache",
"@maven//:ch_megard_akka_http_cors_2_13",
"@maven//:com_github_swagger_akka_http_swagger_akka_http_2_13",
Expand Down
Expand Up @@ -383,11 +383,11 @@ object InvalidApiJsonException {
}

/**
* Indicates that the during caching with Redis something went wrong.
* Indicates that the during caching with the [[org.knora.webapi.store.cacheservice.CacheService]] something went wrong.
*
* @param message a description of the error.
*/
abstract class RedisException(message: String) extends InternalServerException(message)
abstract class CacheServiceException(message: String) extends InternalServerException(message)

/**
* Indicates that an application lock could not be acquired.
Expand Down
2 changes: 2 additions & 0 deletions webapi/src/main/scala/org/knora/webapi/messages/BUILD.bazel
Expand Up @@ -5,13 +5,15 @@ load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
scala_library(
name = "messages",
srcs = glob(["**/*.scala"]) + ["//webapi/src/main/twirl:twirl_sources"],
scalacopts = ["-deprecation"],
unused_dependency_checker_mode = "warn",
deps = [
"//webapi/src/main/scala/org/knora/webapi",
"//webapi/src/main/scala/org/knora/webapi/annotation",
"//webapi/src/main/scala/org/knora/webapi/exceptions",
"//webapi/src/main/scala/org/knora/webapi/feature",
"//webapi/src/main/scala/org/knora/webapi/settings",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice/settings",
"//webapi/src/main/scala/org/knora/webapi/util",
"//webapi/src/main/scala/org/knora/webapi/util/cache",
"@maven//:com_apicatalog_titanium_json_ld",
Expand Down
Expand Up @@ -20,11 +20,16 @@
package org.knora.webapi.messages.util

import akka.actor.{ActorRef, ActorSystem}
import org.knora.webapi.settings.KnoraSettingsImpl
import org.knora.webapi.store.cacheservice.settings.CacheServiceSettings

/**
* Data needed to be passed to each responder.
*
* @param system the actor system.
* @param appActor the main application actor.
*/
case class ResponderData(system: ActorSystem, appActor: ActorRef)
case class ResponderData(system: ActorSystem,
appActor: ActorRef,
knoraSettings: KnoraSettingsImpl,
cacheServiceSettings: CacheServiceSettings)
Expand Up @@ -172,7 +172,7 @@ object GravsearchParser {
wherePatterns.toSeq
}

private def unsupported(node: algebra.QueryModelNode) {
private def unsupported(node: algebra.QueryModelNode): Unit = {
throw GravsearchException(s"SPARQL feature not supported in Gravsearch query: $node")
}

Expand Down
2 changes: 2 additions & 0 deletions webapi/src/main/scala/org/knora/webapi/responders/BUILD.bazel
Expand Up @@ -5,6 +5,7 @@ load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
scala_library(
name = "responders",
srcs = glob(["**/*.scala"]),
scalacopts = ["-deprecation"],
unused_dependency_checker_mode = "warn",
deps = [
"//webapi/src/main/scala/org/knora/webapi",
Expand All @@ -15,6 +16,7 @@ scala_library(
"//webapi/src/main/scala/org/knora/webapi/instrumentation",
"//webapi/src/main/scala/org/knora/webapi/messages",
"//webapi/src/main/scala/org/knora/webapi/settings",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice/settings",
"//webapi/src/main/scala/org/knora/webapi/util",
"//webapi/src/main/scala/org/knora/webapi/util/cache",
"@maven//:com_typesafe_akka_akka_actor_2_13",
Expand Down
Expand Up @@ -33,6 +33,7 @@ import akka.http.scaladsl.util.FastFuture
import akka.pattern._
import akka.util.Timeout
import com.typesafe.scalalogging.{LazyLogging, Logger}
import org.knora.webapi.store.cacheservice.settings.CacheServiceSettings

import scala.concurrent.{ExecutionContext, Future}
import scala.language.postfixOps
Expand Down Expand Up @@ -75,7 +76,12 @@ abstract class Responder(responderData: ResponderData) extends LazyLogging {
/**
* The application settings.
*/
protected val settings: KnoraSettingsImpl = KnoraSettings(system)
protected val settings: KnoraSettingsImpl = responderData.knoraSettings

/**
* The Cache Service settings.
*/
protected val cacheServiceSettings: CacheServiceSettings = responderData.cacheServiceSettings

/**
* The main application actor.
Expand Down
Expand Up @@ -60,7 +60,7 @@ import scala.concurrent.ExecutionContext
*
* @param appActor the main application actor.
*/
class ResponderManager(appActor: ActorRef) extends Actor with ActorLogging {
class ResponderManager(appActor: ActorRef, responderData: ResponderData) extends Actor with ActorLogging {
this: ActorMaker =>

/**
Expand All @@ -74,14 +74,6 @@ class ResponderManager(appActor: ActorRef) extends Actor with ActorLogging {
protected implicit val executionContext: ExecutionContext =
system.dispatchers.lookup(KnoraDispatchers.KnoraActorDispatcher)

/**
* The responder data.
*/
private val responderData = ResponderData(
system = system,
appActor = appActor
)

// A subclass can replace the standard responders with custom responders, e.g. for testing. To do this, it must
// override one or more of the protected val members below representing responder classes. To construct a default
// responder, a subclass can call one of the protected methods below.
Expand Down
Expand Up @@ -1153,7 +1153,7 @@ class ProjectsResponderADM(responderData: ResponderData) extends Responder(respo
private def getProjectFromCacheOrTriplestore(
identifier: ProjectIdentifierADM,
featureFactoryConfig: FeatureFactoryConfig): Future[Option[ProjectADM]] = {
if (settings.cacheServiceEnabled) {
if (cacheServiceSettings.cacheServiceEnabled) {
// caching enabled
getProjectFromCache(identifier)
.flatMap {
Expand Down Expand Up @@ -1405,7 +1405,7 @@ class ProjectsResponderADM(responderData: ResponderData) extends Responder(respo
* Removes the project from cache.
*/
private def invalidateCachedProjectADM(maybeProject: Option[ProjectADM]): Future[Boolean] = {
if (settings.cacheServiceEnabled) {
if (cacheServiceSettings.cacheServiceEnabled) {
val keys: Set[String] =
Seq(maybeProject.map(_.id), maybeProject.map(_.shortname), maybeProject.map(_.shortcode)).flatten.toSet
// only send to Redis if keys are not empty
Expand Down
Expand Up @@ -1626,7 +1626,7 @@ class UsersResponderADM(responderData: ResponderData) extends Responder(responde
*/
private def getUserFromCacheOrTriplestore(identifier: UserIdentifierADM,
featureFactoryConfig: FeatureFactoryConfig): Future[Option[UserADM]] = {
if (settings.cacheServiceEnabled) {
if (cacheServiceSettings.cacheServiceEnabled) {
// caching enabled
getUserFromCache(identifier)
.flatMap {
Expand Down Expand Up @@ -1981,7 +1981,7 @@ class UsersResponderADM(responderData: ResponderData) extends Responder(responde
* Removes the user from cache.
*/
private def invalidateCachedUserADM(maybeUser: Option[UserADM]): Future[Boolean] = {
if (settings.cacheServiceEnabled) {
if (cacheServiceSettings.cacheServiceEnabled) {
val keys: Set[String] = Seq(maybeUser.map(_.id), maybeUser.map(_.email), maybeUser.map(_.username)).flatten.toSet
// only send to Redis if keys are not empty
if (keys.nonEmpty) {
Expand Down
Expand Up @@ -5,6 +5,7 @@ load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
scala_library(
name = "settings",
srcs = glob(["*.scala"]),
scalacopts = ["-deprecation"],
unused_dependency_checker_mode = "warn",
deps = [
"//webapi/src/main/scala/org/knora/webapi/exceptions",
Expand Down
Expand Up @@ -270,11 +270,6 @@ class KnoraSettingsImpl(config: Config, log: LoggingAdapter) extends Extension {

val bcryptPasswordStrength: Int = config.getInt("app.bcrypt-password-strength")

// Cache Service
val cacheServiceEnabled: Boolean = config.getBoolean("app.cache-service.enabled")
val cacheServiceRedisHost: String = config.getString("app.cache-service.redis.host")
val cacheServiceRedisPort: Int = config.getInt("app.cache-service.redis.port")

// Client test data service

val collectClientTestData: Boolean = if (config.hasPath("app.client-test-data-service.collect-client-test-data")) {
Expand All @@ -297,8 +292,8 @@ class KnoraSettingsImpl(config: Config, log: LoggingAdapter) extends Extension {

private def getFiniteDuration(path: String, underlying: Config): FiniteDuration =
Duration(underlying.getString(path)) match {
case x: FiniteDuration x
case _ throw new ConfigurationException(s"Config setting '$path' must be a finite duration")
case x: FiniteDuration => x
case _ => throw new ConfigurationException(s"Config setting '$path' must be a finite duration")
}

val prometheusEndpoint: Boolean = config.getBoolean("app.monitoring.prometheus-endpoint")
Expand Down
1 change: 1 addition & 0 deletions webapi/src/main/scala/org/knora/webapi/store/BUILD.bazel
Expand Up @@ -15,6 +15,7 @@ scala_library(
"//webapi/src/main/scala/org/knora/webapi/messages",
"//webapi/src/main/scala/org/knora/webapi/routing",
"//webapi/src/main/scala/org/knora/webapi/settings",
"//webapi/src/main/scala/org/knora/webapi/store/cacheservice",
"//webapi/src/main/scala/org/knora/webapi/util",
"@maven//:com_twitter_chill_2_13",
"@maven//:com_typesafe_akka_akka_actor_2_13",
Expand Down
14 changes: 7 additions & 7 deletions webapi/src/main/scala/org/knora/webapi/store/StoreManager.scala
Expand Up @@ -28,7 +28,7 @@ import org.knora.webapi.messages.store.cacheservicemessages.CacheServiceRequest
import org.knora.webapi.messages.store.sipimessages.IIIFRequest
import org.knora.webapi.messages.store.triplestoremessages.TriplestoreRequest
import org.knora.webapi.settings.{KnoraDispatchers, KnoraSettings, KnoraSettingsImpl, _}
import org.knora.webapi.store.cacheservice.CacheServiceManager
import org.knora.webapi.store.cacheservice.{CacheService, CacheServiceManager}
import org.knora.webapi.store.iiif.IIIFManager
import org.knora.webapi.store.triplestore.TriplestoreManager

Expand All @@ -42,7 +42,7 @@ import scala.concurrent.ExecutionContext
*
* @param appActor a reference to the main application actor.
*/
class StoreManager(appActor: ActorRef) extends Actor with ActorLogging {
class StoreManager(appActor: ActorRef, cs: CacheService) extends Actor with ActorLogging {
this: ActorMaker =>

/**
Expand Down Expand Up @@ -89,14 +89,14 @@ class StoreManager(appActor: ActorRef) extends Actor with ActorLogging {
/**
* Instantiates the Redis Manager
*/
protected lazy val redisManager: ActorRef = makeActor(
Props(new CacheServiceManager).withDispatcher(KnoraDispatchers.KnoraActorDispatcher),
protected lazy val cacheServiceManager: ActorRef = makeActor(
Props(new CacheServiceManager(cs)).withDispatcher(KnoraDispatchers.KnoraActorDispatcher),
RedisManagerActorName)

def receive: Receive = LoggingReceive {
case tripleStoreMessage: TriplestoreRequest => triplestoreManager forward tripleStoreMessage
case iiifMessages: IIIFRequest => iiifManager forward iiifMessages
case redisMessages: CacheServiceRequest => redisManager forward redisMessages
case tripleStoreMessage: TriplestoreRequest => triplestoreManager forward tripleStoreMessage
case iiifMessages: IIIFRequest => iiifManager forward iiifMessages
case cacheServiceMessages: CacheServiceRequest => cacheServiceManager forward cacheServiceMessages
case other =>
sender ! Status.Failure(UnexpectedMessageException(s"StoreManager received an unexpected message: $other"))
}
Expand Down

0 comments on commit 61531ab

Please sign in to comment.