refactor: add metrics endpoint (DEV-1555)

build.sbt

@@ -232,7 +232,7 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
     dockerUpdateLatest        := true,
     dockerBaseImage           := "eclipse-temurin:17-jre-focal",
     Docker / maintainer       := "support@dasch.swiss",
-    Docker / dockerExposedPorts ++= Seq(3333),
+    Docker / dockerExposedPorts ++= Seq(3333, 3339),
     Docker / defaultLinuxInstallLocation := "/opt/docker",
     // use filterNot to return all items that do NOT meet the criteria
     dockerCommands := dockerCommands.value.filterNot {

docker-compose.yml

@@ -53,6 +53,7 @@ services:
     image: daschswiss/knora-api:latest
     ports:
       - "3333:3333"
+      - "3339:3339"
     volumes:
       - /tmp:/tmp
     networks:

webapi/src/it/scala/org/knora/webapi/CoreSpec.scala

@@ -9,21 +9,21 @@ import akka.actor
 import akka.testkit.ImplicitSender
 import akka.testkit.TestKitBase
 import com.typesafe.scalalogging.Logger
-import org.scalatest.BeforeAndAfterAll
-import org.scalatest.matchers.should.Matchers
-import org.scalatest.wordspec.AnyWordSpec
-import zio._
-import zio.logging.backend.SLF4J
-
-import scala.concurrent.ExecutionContext
-
 import org.knora.webapi.config.AppConfig
+import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
 import org.knora.webapi.core.TestStartupUtils
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
 import org.knora.webapi.messages.util.ResponderData
 import org.knora.webapi.store.cache.settings.CacheServiceSettings
 import org.knora.webapi.util.LogAspect
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+import zio._
+import zio.logging.backend.SLF4J
+
+import scala.concurrent.ExecutionContext
 
 abstract class CoreSpec
     extends AnyWordSpec
@@ -71,7 +71,7 @@ abstract class CoreSpec
   /**
    * Create router and config by unsafe running them.
    */
-  private val (router, config) =
+  private val (router: AppRouter, config: AppConfig) =
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
@@ -97,8 +97,9 @@ abstract class CoreSpec
       runtime.unsafe
         .run(
           (for {
+            _ <- AppServer.testWithoutSipi
             _ <- prepareRepository(rdfDataObjects) @@ LogAspect.logSpan("prepare-repo")
-          } yield ()).provideSomeLayer(AppServer.testWithoutSipi)
+          } yield ())
         )
         .getOrThrow()
 

webapi/src/it/scala/org/knora/webapi/E2ESpec.scala

@@ -28,9 +28,9 @@ import scala.concurrent.Await
 import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
 import scala.concurrent.duration.FiniteDuration
-
 import dsp.errors.FileWriteException
 import org.knora.webapi.config.AppConfig
+import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
 import org.knora.webapi.core.TestStartupUtils
 import org.knora.webapi.messages.store.sipimessages.SipiUploadResponse
@@ -95,7 +95,7 @@ abstract class E2ESpec
   /**
    * Create router and config by unsafe running them.
    */
-  val (router, config) =
+  val (router: AppRouter, config: AppConfig) =
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
@@ -120,9 +120,10 @@ abstract class E2ESpec
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
-          (for {
+          for {
+            _ <- AppServer.testWithoutSipi
             _ <- prepareRepository(rdfDataObjects) @@ LogAspect.logSpan("prepare-repo")
-          } yield ()).provideSomeLayer(AppServer.testWithoutSipi)
+          } yield ()
         )
         .getOrThrow()
     }

webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala

@@ -22,8 +22,8 @@ import scala.concurrent.Await
 import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
 import scala.concurrent.duration.FiniteDuration
-
 import org.knora.webapi.config.AppConfig
+import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
 import org.knora.webapi.core.TestStartupUtils
 import org.knora.webapi.messages.store.sipimessages._
@@ -87,7 +87,7 @@ abstract class ITKnoraLiveSpec
   /**
    * Create router and config by unsafe running them.
    */
-  val (router, config) =
+  val (router: AppRouter, config: AppConfig) =
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
@@ -112,9 +112,10 @@ abstract class ITKnoraLiveSpec
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
-          (for {
+          for {
+            _ <- AppServer.testWithSipi
             _ <- prepareRepository(rdfDataObjects) @@ LogAspect.logSpan("prepare-repo")
-          } yield ()).provideSomeLayer(AppServer.testWithSipi)
+          } yield ()
         )
         .getOrThrow()
     }

webapi/src/it/scala/org/knora/webapi/R2RSpec.scala

@@ -20,8 +20,8 @@ import java.nio.file.Paths
 import java.util.concurrent.TimeUnit
 import scala.concurrent.Await
 import scala.concurrent.Future
-
 import org.knora.webapi.config.AppConfig
+import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
 import org.knora.webapi.core.TestStartupUtils
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
@@ -78,7 +78,7 @@ abstract class R2RSpec
   /**
    * Create router and config by unsafe running them.
    */
-  private val (router, config) =
+  private val (router: AppRouter, config: AppConfig) =
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
@@ -101,9 +101,10 @@ abstract class R2RSpec
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
-          (for {
+          for {
+            _ <- AppServer.testWithoutSipi
             _ <- prepareRepository(rdfDataObjects) @@ LogAspect.logSpan("prepare-repo")
-          } yield ()).provideSomeLayer(AppServer.testWithoutSipi)
+          } yield ()
         )
         .getOrThrow()
     }

webapi/src/it/scala/org/knora/webapi/core/LayersTest.scala

@@ -1,7 +1,6 @@
 package org.knora.webapi.core
 
 import zio.ZLayer
-
 import org.knora.webapi.auth.JWTService
 import org.knora.webapi.config.AppConfigForTestContainers
 import org.knora.webapi.routing.ApiRoutes

webapi/src/main/resources/application.conf

@@ -501,6 +501,11 @@ app {
         collect-client-test-data = false
     }
 
+    prometheus-server-config {
+        port = 3339
+        interval = 5 seconds
+    }
+
     monitoring {
         prometheus-endpoint: false
         prometheus-endpoint: ${?KNORA_WEBAPI_PROMETHEUS_ENDPOINT}

webapi/src/main/scala/org/knora/webapi/Main.scala

@@ -9,6 +9,7 @@ import zio._
 import zio.logging.backend.SLF4J
 
 import org.knora.webapi.core._
+import org.knora.webapi.instrumentation.metrics.PrometheusServer
 
 object Main extends ZIOApp {
 
@@ -30,6 +31,10 @@ object Main extends ZIOApp {
   ] = ZLayer.empty ++ Runtime.removeDefaultLoggers ++ SLF4J.slf4j ++ LayersLive.dspLayersLive
 
   /* Here we start our Application */
-  override def run = AppServer.live.launch
-
+  override def run = for {
+    f1 <- PrometheusServer.make.forkDaemon
+    f2 <- (AppServer.live *> ZIO.never).forkDaemon
+    _  <- f1.join
+    _  <- f2.join
+  } yield ()
 }

webapi/src/main/scala/org/knora/webapi/config/AppConfig.scala

@@ -12,6 +12,7 @@ import zio.config._
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
+import java.time.Duration
 import scala.concurrent.duration
 import scala.util.Failure
 import scala.util.Success
@@ -55,7 +56,8 @@ final case class AppConfig(
   triplestore: Triplestore,
   shacl: Shacl,
   cacheService: CacheService,
-  clientTestDataService: ClientTestDataService
+  clientTestDataService: ClientTestDataService,
+  prometheusServerConfig: PrometheusServerConfig
 ) {
   val jwtLongevityAsDuration = scala.concurrent.duration.Duration(jwtLongevity)
   val defaultTimeoutAsDuration =
@@ -237,6 +239,11 @@ final case class ClientTestDataService(
   collectClientTestData: Boolean
 )
 
+final case class PrometheusServerConfig(
+  port: Int,
+  interval: Duration
+)
+
 /**
  * Loads the applicaton configuration using ZIO-Config. ZIO-Config is capable of loading
  * the Typesafe-Config format.

webapi/src/main/scala/org/knora/webapi/core/AppServer.scala

@@ -179,37 +179,31 @@ object AppServer {
     } yield appServer
 
   /**
-   * The AppServer live layer
+   * The live AppServer
    */
-  val live: ZLayer[AppServerEnvironment, Nothing, Unit] =
-    ZLayer {
-      for {
-        appServer <- AppServer.init()
-        _         <- appServer.start(requiresAdditionalRepositoryChecks = true, requiresIIIFService = true)
-      } yield ()
-    }
+  val live: ZIO[AppServerEnvironment, Nothing, Unit] =
+    for {
+      appServer <- AppServer.init()
+      _         <- appServer.start(requiresAdditionalRepositoryChecks = true, requiresIIIFService = true)
+    } yield ()
 
   /**
-   * The AppServer test layer with Sipi, which initiates the startup checks. Before this layer does what it does,
+   * The test AppServer with Sipi, which initiates the startup checks. Before this effect does what it does,
    * the complete server should have already been started.
    */
-  val testWithSipi: ZLayer[AppServerEnvironment, Nothing, Unit] =
-    ZLayer {
-      for {
-        appServer <- AppServer.init()
-        _         <- appServer.start(requiresAdditionalRepositoryChecks = false, requiresIIIFService = true)
-      } yield ()
-    }
+  val testWithSipi: ZIO[AppServerEnvironment, Nothing, Unit] =
+    for {
+      appServer <- AppServer.init()
+      _         <- appServer.start(requiresAdditionalRepositoryChecks = false, requiresIIIFService = true)
+    } yield ()
 
   /**
-   * The AppServer test layer without Sipi, which initiates the startup checks. Before this layer does what it does,
+   * The test AppServer without Sipi, which initiates the startup checks. Before this effect does what it does,
    * the complete server should have already been started.
    */
-  val testWithoutSipi: ZLayer[AppServerEnvironment, Nothing, Unit] =
-    ZLayer {
-      for {
-        appServer <- AppServer.init()
-        _         <- appServer.start(requiresAdditionalRepositoryChecks = false, requiresIIIFService = false)
-      } yield ()
-    }
+  val testWithoutSipi: ZIO[AppServerEnvironment, Nothing, Unit] =
+    for {
+      appServer <- AppServer.init()
+      _         <- appServer.start(requiresAdditionalRepositoryChecks = false, requiresIIIFService = false)
+    } yield ()
 }

webapi/src/main/scala/org/knora/webapi/core/HttpServerWithZIOHttp.scala

@@ -10,7 +10,6 @@ import zio.ZLayer
 import zio._
 
 import org.knora.webapi.config.AppConfig
-import org.knora.webapi.core._
 import org.knora.webapi.routing.ApiRoutesWithZIOHttp
 
 object HttpServerWithZIOHttp {

webapi/src/main/scala/org/knora/webapi/instrumentation/metrics/PrometheusServer.scala

@@ -0,0 +1,70 @@
+package org.knora.webapi.instrumentation.metrics
+
+import zhttp.html._
+import zhttp.http._
+import zhttp.service.EventLoopGroup
+import zhttp.service.Server
+import zhttp.service.server.ServerChannelFactory
+import zio.Runtime
+import zio.ZIO
+import zio.ZLayer
+import zio.metrics.connectors.MetricsConfig
+import zio.metrics.connectors.prometheus
+import zio.metrics.connectors.prometheus.PrometheusPublisher
+import zio.metrics.jvm.DefaultJvmMetrics
+
+import org.knora.webapi.config.AppConfig
+
+object PrometheusServer {
+
+  private val nThreads = 5
+
+  private lazy val indexPage =
+    """<html>
+      |<title>Simple Server</title>
+      |<body>
+      |<p><a href="/metrics">Prometheus Metrics</a></p>
+      |</body
+      |</html>""".stripMargin
+
+  private lazy val static =
+    Http.collect[Request] { case Method.GET -> !! => Response.html(Html.fromString(indexPage)) }
+
+  private lazy val prometheusRouter =
+    Http
+      .collectZIO[Request] { case Method.GET -> !! / "metrics" =>
+        ZIO.serviceWithZIO[PrometheusPublisher](_.get.map(Response.text))
+      }
+
+  private def server(config: AppConfig): Server[PrometheusPublisher, Throwable] =
+    Server.port(config.prometheusServerConfig.port) ++ Server.app(static ++ prometheusRouter)
+
+  private def runHttpServer(config: AppConfig) =
+    ZIO.logInfo("starting prometheus server in a separate root fiber.") *>
+      server(config).start
+
+  val make: ZIO[AppConfig, Throwable, Nothing] =
+    ZIO
+      .service[AppConfig]
+      .flatMap(config =>
+        runHttpServer(config)
+          .provide(
+            ServerChannelFactory.auto,
+            EventLoopGroup.auto(nThreads),
+
+            // Metrics config
+            ZLayer.succeed(MetricsConfig(config.prometheusServerConfig.interval)),
+
+            // The prometheus reporting layer
+            prometheus.publisherLayer,
+            prometheus.prometheusLayer,
+
+            // Enable the ZIO internal metrics and the default JVM metricsConfig
+            // Do NOT forget the .unit for the JVM metrics layer
+            Runtime.enableRuntimeMetrics,
+            Runtime.enableFiberRoots,
+            DefaultJvmMetrics.live.unit.orDie
+          )
+      )
+
+}

webapi/src/test/scala/org/knora/webapi/config/AppConfigZSpec.scala

@@ -24,7 +24,10 @@ object AppConfigZSpec extends ZIOSpecDefault {
         assertTrue(appConfig.printExtendedConfig == false) &&
         assertTrue(appConfig.jwtLongevityAsDuration == FiniteDuration(30L, TimeUnit.DAYS)) &&
         assertTrue(appConfig.sipi.timeoutInSeconds == FiniteDuration(120L, TimeUnit.SECONDS)) &&
-        assertTrue(appConfig.bcryptPasswordStrength == User.PasswordStrength(12))
+        assertTrue(appConfig.bcryptPasswordStrength == User.PasswordStrength(12)) &&
+        assertTrue(
+          appConfig.prometheusServerConfig.interval == java.time.Duration.ofSeconds(5)
+        )
       }
     }.provideLayer(AppConfig.live)
   )