Merge branch 'main' into DEV-545-remove-graph-db-support

dasch-swiss · Apr 13, 2022 · a9e1155 · a9e1155
2 parents d1f996e + f72e7a0
commit a9e1155
Show file tree

Hide file tree

Showing 56 changed files with 2,121 additions and 1,844 deletions.
diff --git a/.adr-dir b/.adr-dir
@@ -0,0 +1 @@
+docs/architecture/decisions
diff --git a/.gitignore b/.gitignore
@@ -18,6 +18,8 @@ webapi-it
 webapi-test
 upgrade/project
 sipi/test
+/docs/architecture/.structurizr
+/docs/architecture/workspace.json
 
 **/target/
 *.aux
@@ -31,6 +33,7 @@ sipi/test
 *.tdo
 *.toc
 *.bak
+*.rdb
 .sbtrc
 
 dependencies.txt

diff --git a/Makefile b/Makefile
@@ -33,6 +33,11 @@ docs-install-requirements: ## install requirements
 docs-clean: ## cleans the project directory
 	@rm -rf site/
 
+.PHONY: structurizer
+structurizer: ## starts the structurizer and serves c4 architecture docs
+	@docker pull structurizr/lite
+	@docker run -it --rm -p 8080:8080 -v $(CURRENT_DIR)/docs/architecture:/usr/local/structurizr structurizr/lite
+
 #################################
 # Docker targets
 #################################

diff --git a/build.sbt b/build.sbt
@@ -100,7 +100,7 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
   .settings(buildSettings)
   .enablePlugins(SbtTwirl, JavaAppPackaging, DockerPlugin, GatlingPlugin, JavaAgent, RevolverPlugin, BuildInfoPlugin)
   .settings(
-    webApiCommonSettings,
+    name := "webapi",
     resolvers ++= Seq(
       Resolver.bintrayRepo("hseeberger", "maven")
     ),
@@ -112,19 +112,19 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
   .settings(
     // add needed files to production jar
     Compile / packageBin / mappings ++= Seq(
-      (rootBaseDir.value / "knora-ontologies" / "knora-admin.ttl") -> "knora-ontologies/knora-admin.ttl",
-      (rootBaseDir.value / "knora-ontologies" / "knora-base.ttl") -> "knora-ontologies/knora-base.ttl",
-      (rootBaseDir.value / "knora-ontologies" / "salsah-gui.ttl") -> "knora-ontologies/salsah-gui.ttl",
-      (rootBaseDir.value / "knora-ontologies" / "standoff-data.ttl") -> "knora-ontologies/standoff-data.ttl",
-      (rootBaseDir.value / "knora-ontologies" / "standoff-onto.ttl") -> "knora-ontologies/standoff-onto.ttl",
+      (rootBaseDir.value / "knora-ontologies" / "knora-admin.ttl")                         -> "knora-ontologies/knora-admin.ttl",
+      (rootBaseDir.value / "knora-ontologies" / "knora-base.ttl")                          -> "knora-ontologies/knora-base.ttl",
+      (rootBaseDir.value / "knora-ontologies" / "salsah-gui.ttl")                          -> "knora-ontologies/salsah-gui.ttl",
+      (rootBaseDir.value / "knora-ontologies" / "standoff-data.ttl")                       -> "knora-ontologies/standoff-data.ttl",
+      (rootBaseDir.value / "knora-ontologies" / "standoff-onto.ttl")                       -> "knora-ontologies/standoff-onto.ttl",
       (rootBaseDir.value / "webapi" / "scripts" / "fuseki-repository-config.ttl.template") -> "webapi/scripts/fuseki-repository-config.ttl.template" // needed for initialization of triplestore
     ),
     // use packaged jars (through packageBin) on classpaths instead of class directories for production
     Compile / exportJars := true,
     // add needed files to test jar
     Test / packageBin / mappings ++= Seq(
       (rootBaseDir.value / "webapi" / "scripts" / "fuseki-repository-config.ttl.template") -> "webapi/scripts/fuseki-repository-config.ttl.template", // needed for initialization of triplestore
-      (rootBaseDir.value / "sipi" / "config" / "sipi.docker-config.lua") -> "sipi/config/sipi.docker-config.lua"
+      (rootBaseDir.value / "sipi" / "config" / "sipi.docker-config.lua")                   -> "sipi/config/sipi.docker-config.lua"
     ),
     // use packaged jars (through packageBin) on classpaths instead of class directories for test
     Test / exportJars := true
@@ -140,12 +140,14 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
     javaAgents += Dependencies.aspectjweaver,
     fork := true, // run tests in a forked JVM
     Test / testForkedParallel := false, // run forked tests in parallel
-    Test / parallelExecution := false, // run non-forked tests in parallel
+    Test / parallelExecution := false,  // run non-forked tests in parallel
     // Global / concurrentRestrictions += Tags.limit(Tags.Test, 1), // restrict the number of concurrently executing tests in all projects
     Test / javaOptions ++= Seq("-Dconfig.resource=fuseki.conf") ++ webapiJavaTestOptions,
     // Test / javaOptions ++= Seq("-Dakka.log-config-on-start=on"), // prints out akka config
     // Test / javaOptions ++= Seq("-Dconfig.trace=loads"), // prints out config locations
-    Test / testOptions += Tests.Argument("-oDF") // show full stack traces and test case durations
+    Test / testOptions += Tests.Argument("-oDF"), // show full stack traces and test case durations
+    // add test framework for running zio-tests
+    Test / testFrameworks ++= Seq(new TestFramework("zio.test.sbt.ZTestFramework"))
   )
   .settings(
     // prepare for publishing
@@ -189,8 +191,8 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
       name,
       version,
       "akkaHttp" -> Dependencies.akkaHttpVersion,
-      "sipi" -> Dependencies.sipiImage,
-      "fuseki" -> Dependencies.fusekiImage
+      "sipi"     -> Dependencies.sipiImage,
+      "fuseki"   -> Dependencies.fusekiImage
     ),
     buildInfoPackage := "org.knora.webapi.http.version"
   )
@@ -219,6 +221,10 @@ lazy val webapiJavaTestOptions = Seq(
   //"-XX:MaxMetaspaceSize=4096m"
 )
 
+//////////////////////////////////////
+// DSP's new codebase
+//////////////////////////////////////
+
 lazy val apiMain = project
   .in(file("dsp-api-main"))
   .settings(

diff --git a/docs/01-introduction/data-formats.md b/docs/01-introduction/data-formats.md
diff --git a/docs/01-introduction/file-formats.md b/docs/01-introduction/file-formats.md
@@ -0,0 +1,24 @@
+<!---
+ * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
+ * SPDX-License-Identifier: Apache-2.0
+-->
+
+# File Formats in DSP-API
+
+Currently, only a limited number of file formats is accepted to be uploaded onto DSP. Some metadata is extracted from the files during the ingest but the file formats are not validated. Only image file formats are currently migrated into another format. Both, the migrated version of the file and the original are kept.
+
+The following table shows the accepted file formats:
+
+| Category              | Accepted format           | Converted during ingest?                                                   |
+| --------------------- | ------------------------- | -------------------------------------------------------------------------- |
+| Text, XML<sup>1</sup> | TXT, XML, XSL, XSD        | No                                                                         |
+| Tables                | CSV, XLS, XLSX            | No                                                                         |
+| 2D Images             | JPEG, PNG, TIFF, JP2      | Yes, converted to JPEG 2000 by [Sipi](https://github.com/dasch-swiss/sipi) |
+| Audio                 | MPEG (MP3), MP4, WAV      | No                                                                         |
+| Video                 | MP4                       | No                                                                         |
+| Office                | PDF, DOC, DOCX, PPT, PPTX | No                                                                         |
+| Archives              | ZIP, TAR, ISO, GZIP, 7Z   | No                                                                         |
+
+
+1: If your XML files represent text with markup (e.g. [TEI/XML](http://www.tei-c.org/)),
+the recommended approach is to allow Knora to store it as [Standoff/RDF](standoff-rdf.md).
diff --git a/docs/01-introduction/index.md b/docs/01-introduction/index.md
@@ -6,6 +6,6 @@
 # Introduction
 
 * [What Is DSP and DSP-API (previous Knora)?](what-is-knora.md)
-* [Data Formats in DSP-API](data-formats.md)
+* [File Formats in DSP-API](file-formats.md)
 * [Standoff/RDF Text Markup](standoff-rdf.md)
 * [An Example Project](example-project.md)
diff --git a/docs/01-introduction/what-is-knora.md b/docs/01-introduction/what-is-knora.md
@@ -23,15 +23,11 @@ DSP solves this problem by keeping the data alive. You can query all the data
 in a DSP repository, not just the metadata. You can import thousands of databases into
 DSP, and run queries that search through all of them at once.
 
-Another problem is that researchers use a multitude of different data formats, many of
+Another problem is that researchers use a multitude of different file formats, many of
 which are proprietary and quickly become obsolete. It is not practical to maintain
-all the programs that were used to create and read old data files, or even
-all the operating systems that these programs ran on.
-
-Instead of preserving all these data formats, DSP supports
-the conversion of all sorts of data to a [small number of formats](data-formats.md)
-that are suitable for long-term preservation, and that maintain the data's meaning and
-structure:
+all the programs that were used to create and read old files, or even
+all the operating systems that these programs ran on. Therefore, DSP only accepts a
+certain number of [file formats](file-formats.md).
 
 - Non-binary data is stored as
   [RDF](http://www.w3.org/TR/2014/NOTE-rdf11-primer-20140624/), in a dedicated

diff --git a/docs/architecture/README.md b/docs/architecture/README.md
@@ -0,0 +1,15 @@
+# C4 Model and ADRs
+
+## Installation
+
+```bash
+$ brew install adr-tools
+```
+
+## Usage
+
+Run the following command from the root directory to start the C4 model browser:
+
+```bash
+$ make structurizer
+```
diff --git a/docs/architecture/decisions/0001-record-architecture-decisions.md b/docs/architecture/decisions/0001-record-architecture-decisions.md
@@ -0,0 +1,19 @@
+# 1. Record architecture decisions
+
+Date: 2022-03-14
+
+## Status
+
+Accepted
+
+## Context
+
+We need to record the architectural decisions made on this project.
+
+## Decision
+
+We will use Architecture Decision Records, as [described by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions).
+
+## Consequences
+
+See Michael Nygard's article, linked above. For a lightweight ADR toolset, see Nat Pryce's [adr-tools](https://github.com/npryce/adr-tools).
diff --git a/...ecture/decisions/0002-change-cache-service-manager-from-akka-actor-to-zlayer.md b/...ecture/decisions/0002-change-cache-service-manager-from-akka-actor-to-zlayer.md
@@ -0,0 +1,19 @@
+# 2. Change Cache Service Manager from Akka-Actor to ZLayer
+
+Date: 2022-04-06
+
+## Status
+
+Accepted
+
+## Context
+
+The `org.knora.webapi.store.cacheservice.CacheServiceManager` was implemented as an `Akka-Actor`.
+
+## Decision
+
+As part of the move from `Akka` to `ZIO`, it was decided that the `CacheServiceManager` and the whole implementation of the in-memory and Redis backed cache is refactored using ZIO.
+
+## Consequences
+
+The usage from other actors stays the same. The actor messages and responses don't change.
diff --git a/docs/architecture/workspace.dsl b/docs/architecture/workspace.dsl
@@ -0,0 +1,21 @@
+workspace {
+
+    model {
+        user = person "User"
+        softwareSystem = softwareSystem "Software System"
+
+        user -> softwareSystem "Uses"
+    }
+
+    views {
+        systemContext softwareSystem "Diagram1" {
+            include *
+            autoLayout
+        }
+
+        theme default
+    }
+
+    !adrs decisions
+
+}
diff --git a/docs/faq/index.md b/docs/faq/index.md
@@ -5,11 +5,11 @@
 
 # Frequently Asked Questions
 
-## Data Formats
+## File Formats
 
-### What data formats does Knora store?
+### What file formats does Knora store?
 
-See [Data Formats in Knora](../01-introduction/data-formats.md).
+See [File Formats in Knora](../01-introduction/file-formats.md).
 
 ### Does Knora store XML files?
 

diff --git a/dsp-schema/repo/src/main/scala/dsp/schema/repo/SchemaRepoLive.scala b/dsp-schema/repo/src/main/scala/dsp/schema/repo/SchemaRepoLive.scala
@@ -14,6 +14,6 @@ case class SchemaRepoLive() extends SchemaRepo {
 
 }
 
-object SchemaRepoLive extends (() => SchemaRepo) {
-  val layer: URLayer[Any, SchemaRepo] = (SchemaRepoLive.apply _).toLayer
+object SchemaRepoLive {
+  val layer: URLayer[Any, SchemaRepo] = ZLayer.succeed(SchemaRepoLive())
 }
diff --git a/dsp-schema/repo/src/main/scala/dsp/schema/repo/SchemaRepoTest.scala b/dsp-schema/repo/src/main/scala/dsp/schema/repo/SchemaRepoTest.scala
@@ -7,18 +7,18 @@ case class SchemaRepoTest() extends SchemaRepo {
   private var map: Map[UserID, UserProfile] = Map()
 
   def setTestData(map0: Map[UserID, UserProfile]): Task[Unit] =
-    Task { map = map0 }
+    Task.succeed { map = map0 }
 
   def getTestData: Task[Map[UserID, UserProfile]] =
-    Task(map)
+    Task.succeed(map)
 
   def lookup(id: UserID): Task[UserProfile] =
-    Task(map(id))
+    Task.succeed(map(id))
 
   def update(id: UserID, profile: UserProfile): Task[Unit] =
     Task.attempt { map = map + (id -> profile) }
 }
 
 object SchemaRepoTest extends (() => SchemaRepo) {
-  val layer: URLayer[Any, SchemaRepo] = (SchemaRepoTest.apply _).toLayer
+  val layer: URLayer[Any, SchemaRepo] = ZLayer.succeed(SchemaRepoTest())
 }
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -10,7 +10,7 @@ nav:
         - Introduction:
                   - Index: 01-introduction/index.md
                   - What is DSP?: 01-introduction/what-is-knora.md
-                  - Data Formats in DSP-API: 01-introduction/data-formats.md
+                  - File Formats in DSP-API: 01-introduction/file-formats.md
                   - Standoff/RDF Text Markup: 01-introduction/standoff-rdf.md
                   - An Example Project: 01-introduction/example-project.md
         - DSP Ontologies: