refactor: add metrics endpoint (DEV-1555) (#2331)

build.sbt

@@ -232,7 +232,7 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
     dockerUpdateLatest        := true,
     dockerBaseImage           := "eclipse-temurin:17-jre-focal",
     Docker / maintainer       := "support@dasch.swiss",
-    Docker / dockerExposedPorts ++= Seq(3333),
+    Docker / dockerExposedPorts ++= Seq(3333, 3339),
     Docker / defaultLinuxInstallLocation := "/opt/docker",
     // use filterNot to return all items that do NOT meet the criteria
     dockerCommands := dockerCommands.value.filterNot {

docker-compose.yml

@@ -53,6 +53,7 @@ services:
     image: daschswiss/knora-api:latest
     ports:
       - "3333:3333"
+      - "3339:3339"
     volumes:
       - /tmp:/tmp
     networks:

docs/03-endpoints/instrumentation/introduction.md

@@ -0,0 +1,14 @@
+<!---
+ * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
+ * SPDX-License-Identifier: Apache-2.0
+-->
+
+# Instrumentation
+
+The instrumentation endpoints are running on a separate port (default `3339`)
+defined in `application.conf` under the key: `app.instrumentaion-server-config.port`
+and can also be set through the environment variable: `KNORA_INSTRUMENTATION_SERVER_PORT`.
+
+The exposed endpoints are:
+ - `/metrics` - a metrics endpoint, backed by the ZIO metrics backend exposing metrics in the prometheus format
+ - `/health` - provides information about the health state, see [Health Endpoint](./health.md)

docs/03-endpoints/instrumentation/metrics.md

@@ -0,0 +1,39 @@
+<!---
+ * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
+ * SPDX-License-Identifier: Apache-2.0
+-->
+
+# Metrics Endpoint
+
+The metrics endpoint exposes metrics gathered through the ZIO metrics frontend in the Prometheus
+format. Additionally, ZIO runtime and JVM metrics are also exposed.
+
+## Configuration
+
+The refresh interval is configured in `application.conf` under the key: `app.instrumentaion-server-config.interval`
+which es per default set to `5 seconds`.
+
+
+## Example request
+
+`GET /metrics`
+
+
+## Example response
+
+```text
+# TYPE jvm_memory_pool_allocated_bytes_total counter
+# HELP jvm_memory_pool_allocated_bytes_total Some help
+jvm_memory_pool_allocated_bytes_total{pool="G1 Survivor Space"}  4828024.0 1671021037947
+# TYPE jvm_memory_pool_allocated_bytes_total counter
+# HELP jvm_memory_pool_allocated_bytes_total Some help
+jvm_memory_pool_allocated_bytes_total{pool="G1 Eden Space"}  3.3554432E7 1671021037947
+# TYPE zio_fiber_successes counter
+# HELP zio_fiber_successes Some help
+zio_fiber_successes 17.0 1671021037947
+# TYPE zio_fiber_lifetimes histogram
+# HELP zio_fiber_lifetimes Some help
+zio_fiber_lifetimes_bucket{le="1.0"}  17.0 1671021037947
+zio_fiber_lifetimes_bucket{le="2.0"}  17.0 1671021037947
+...
+```

mkdocs.yml

@@ -51,8 +51,13 @@ nav:
                           - Permissions Endpoint: 03-endpoints/api-admin/permissions.md
                           - Stores Endpoint: 03-endpoints/api-admin/stores.md
                   - Util API:
-                          - Health: 03-endpoints/api-util/health.md
+
                           - Version: 03-endpoints/api-util/version.md
+                  - Instrumentation API:
+                          - Introduction: 03-endpoints/instrumentation/introduction.md
+                          - Health Endpoint: 03-endpoints/instrumentation/health.md
+                          - Metrics Endpoint: 03-endpoints/instrumentation/metrics.md
+
         - Publishing and Deployment:
                   - Publishing: 04-publishing-deployment/publishing.md
                   - Configuration: 04-publishing-deployment/configuration.md
@@ -98,7 +103,6 @@ nav:
                             - Setup Visual Studio Code for development of DSP-API: 05-internals/development/vscode-config.md
                             - Testing: 05-internals/development/testing.md
                             - Docker Cheat Sheet: 05-internals/development/docker-cheat-sheet.md
-                            - Monitoring DSP-API: 05-internals/development/monitoring.md
                             - Starting the DSP-Stack inside Docker Container: 05-internals/development/docker-compose.md
                             - Updating Repositories: 05-internals/development/updating-repositories.md
                             - Generating Client Test Data: 05-internals/development/generating-client-test-data.md

webapi/src/it/scala/org/knora/webapi/CoreSpec.scala

@@ -9,21 +9,21 @@ import akka.actor
 import akka.testkit.ImplicitSender
 import akka.testkit.TestKitBase
 import com.typesafe.scalalogging.Logger
-import org.scalatest.BeforeAndAfterAll
-import org.scalatest.matchers.should.Matchers
-import org.scalatest.wordspec.AnyWordSpec
-import zio._
-import zio.logging.backend.SLF4J
-
-import scala.concurrent.ExecutionContext
-
 import org.knora.webapi.config.AppConfig
+import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
 import org.knora.webapi.core.TestStartupUtils
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
 import org.knora.webapi.messages.util.ResponderData
 import org.knora.webapi.store.cache.settings.CacheServiceSettings
 import org.knora.webapi.util.LogAspect
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+import zio._
+import zio.logging.backend.SLF4J
+
+import scala.concurrent.ExecutionContext
 
 abstract class CoreSpec
     extends AnyWordSpec
@@ -71,7 +71,7 @@ abstract class CoreSpec
   /**
    * Create router and config by unsafe running them.
    */
-  private val (router, config) =
+  private val (router: AppRouter, config: AppConfig) =
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
@@ -97,8 +97,9 @@ abstract class CoreSpec
       runtime.unsafe
         .run(
           (for {
+            _ <- AppServer.testWithoutSipi
             _ <- prepareRepository(rdfDataObjects) @@ LogAspect.logSpan("prepare-repo")
-          } yield ()).provideSomeLayer(AppServer.testWithoutSipi)
+          } yield ())
         )
         .getOrThrow()
 

webapi/src/it/scala/org/knora/webapi/E2ESpec.scala

@@ -28,9 +28,9 @@ import scala.concurrent.Await
 import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
 import scala.concurrent.duration.FiniteDuration
-
 import dsp.errors.FileWriteException
 import org.knora.webapi.config.AppConfig
+import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
 import org.knora.webapi.core.TestStartupUtils
 import org.knora.webapi.messages.store.sipimessages.SipiUploadResponse
@@ -95,7 +95,7 @@ abstract class E2ESpec
   /**
    * Create router and config by unsafe running them.
    */
-  val (router, config) =
+  val (router: AppRouter, config: AppConfig) =
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
@@ -120,9 +120,10 @@ abstract class E2ESpec
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
-          (for {
+          for {
+            _ <- AppServer.testWithoutSipi
             _ <- prepareRepository(rdfDataObjects) @@ LogAspect.logSpan("prepare-repo")
-          } yield ()).provideSomeLayer(AppServer.testWithoutSipi)
+          } yield ()
         )
         .getOrThrow()
     }

webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala

@@ -22,8 +22,8 @@ import scala.concurrent.Await
 import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
 import scala.concurrent.duration.FiniteDuration
-
 import org.knora.webapi.config.AppConfig
+import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
 import org.knora.webapi.core.TestStartupUtils
 import org.knora.webapi.messages.store.sipimessages._
@@ -87,7 +87,7 @@ abstract class ITKnoraLiveSpec
   /**
    * Create router and config by unsafe running them.
    */
-  val (router, config) =
+  val (router: AppRouter, config: AppConfig) =
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
@@ -112,9 +112,10 @@ abstract class ITKnoraLiveSpec
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
-          (for {
+          for {
+            _ <- AppServer.testWithSipi
             _ <- prepareRepository(rdfDataObjects) @@ LogAspect.logSpan("prepare-repo")
-          } yield ()).provideSomeLayer(AppServer.testWithSipi)
+          } yield ()
         )
         .getOrThrow()
     }

webapi/src/it/scala/org/knora/webapi/R2RSpec.scala

@@ -20,8 +20,8 @@ import java.nio.file.Paths
 import java.util.concurrent.TimeUnit
 import scala.concurrent.Await
 import scala.concurrent.Future
-
 import org.knora.webapi.config.AppConfig
+import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
 import org.knora.webapi.core.TestStartupUtils
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
@@ -78,7 +78,7 @@ abstract class R2RSpec
   /**
    * Create router and config by unsafe running them.
    */
-  private val (router, config) =
+  private val (router: AppRouter, config: AppConfig) =
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
@@ -101,9 +101,10 @@ abstract class R2RSpec
     Unsafe.unsafe { implicit u =>
       runtime.unsafe
         .run(
-          (for {
+          for {
+            _ <- AppServer.testWithoutSipi
             _ <- prepareRepository(rdfDataObjects) @@ LogAspect.logSpan("prepare-repo")
-          } yield ()).provideSomeLayer(AppServer.testWithoutSipi)
+          } yield ()
         )
         .getOrThrow()
     }

webapi/src/it/scala/org/knora/webapi/core/LayersTest.scala

@@ -1,7 +1,6 @@
 package org.knora.webapi.core
 
 import zio.ZLayer
-
 import org.knora.webapi.auth.JWTService
 import org.knora.webapi.config.AppConfigForTestContainers
 import org.knora.webapi.routing.ApiRoutes

webapi/src/main/resources/application.conf

@@ -501,18 +501,9 @@ app {
         collect-client-test-data = false
     }
 
-    monitoring {
-        prometheus-endpoint: false
-        prometheus-endpoint: ${?KNORA_WEBAPI_PROMETHEUS_ENDPOINT}
+    instrumentation-server-config {
+        port = 3339
+        port = ${?KNORA_INSTRUMENTATION_SERVER_PORT}
+        interval = 5 seconds
     }
 }
-
-kamon.prometheus.embedded-server {
-
-    # Hostname and port used by the embedded web server to publish the
-    # prometheus scraping enpoint.
-    hostname = 0.0.0.0
-    hostname = ${?KNORA_WEBAPI_PROMETHEUS_HOST}
-    port = 9095
-    port = ${?KNORA_WEBAPI_PROMETHEUS_PORT}
-}

webapi/src/main/scala/org/knora/webapi/Main.scala

@@ -30,6 +30,10 @@ object Main extends ZIOApp {
   ] = ZLayer.empty ++ Runtime.removeDefaultLoggers ++ SLF4J.slf4j ++ LayersLive.dspLayersLive
 
   /* Here we start our Application */
-  override def run = AppServer.live.launch
-
+  override def run = for {
+    f <- ZIO.never.forkDaemon
+    _ <- InstrumentationHttpServer.make
+    _ <- AppServer.live
+    _ <- f.join
+  } yield ()
 }

webapi/src/main/scala/org/knora/webapi/config/AppConfig.scala

@@ -12,6 +12,7 @@ import zio.config._
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
+import java.time.Duration
 import scala.concurrent.duration
 import scala.util.Failure
 import scala.util.Success
@@ -55,7 +56,8 @@ final case class AppConfig(
   triplestore: Triplestore,
   shacl: Shacl,
   cacheService: CacheService,
-  clientTestDataService: ClientTestDataService
+  clientTestDataService: ClientTestDataService,
+  instrumentationServerConfig: InstrumentationServerConfig
 ) {
   val jwtLongevityAsDuration = scala.concurrent.duration.Duration(jwtLongevity)
   val defaultTimeoutAsDuration =
@@ -237,6 +239,11 @@ final case class ClientTestDataService(
   collectClientTestData: Boolean
 )
 
+final case class InstrumentationServerConfig(
+  port: Int,
+  interval: Duration
+)
+
 /**
  * Loads the applicaton configuration using ZIO-Config. ZIO-Config is capable of loading
  * the Typesafe-Config format.