From 747d13de9a0eee479decd48b7d3a186b49f81a2b Mon Sep 17 00:00:00 2001 From: irinaschubert Date: Mon, 14 Mar 2022 19:42:39 +0100 Subject: [PATCH] build(test): get tests to run in vs code (DEV-601) (#2020) Co-authored-by: Ivan Subotic <400790+subotic@users.noreply.github.com> --- build.sbt | 18 +- .../http/HttpTriplestoreConnector.scala | 4 +- .../fuseki-repository-config.ttl.template | 53 +++++ .../resources/sipi.knora-docker-config.lua | 219 ++++++++++++++++++ .../org/knora/webapi/TestContainersAll.scala | 5 +- 5 files changed, 287 insertions(+), 12 deletions(-) create mode 100644 webapi/src/test/resources/fuseki-repository-config.ttl.template create mode 100644 webapi/src/test/resources/sipi.knora-docker-config.lua diff --git a/build.sbt b/build.sbt index c34ffeac55..223396c787 100644 --- a/build.sbt +++ b/build.sbt @@ -111,9 +111,7 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi")) inConfig(Test)(Defaults.testTasks ++ baseAssemblySettings) ) .settings( - exportJars := true, - Compile / unmanagedResourceDirectories += (rootBaseDir.value / "knora-ontologies"), - // add needed files to jar + // add needed files to production jar Compile / packageBin / mappings ++= Seq( (rootBaseDir.value / "knora-ontologies" / "knora-admin.ttl") -> "knora-ontologies/knora-admin.ttl", (rootBaseDir.value / "knora-ontologies" / "knora-base.ttl") -> "knora-ontologies/knora-base.ttl", @@ -122,10 +120,15 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi")) (rootBaseDir.value / "knora-ontologies" / "standoff-onto.ttl") -> "knora-ontologies/standoff-onto.ttl", (rootBaseDir.value / "webapi" / "scripts" / "fuseki-repository-config.ttl.template") -> "webapi/scripts/fuseki-repository-config.ttl.template" // needed for initialization of triplestore ), - // put additional files into the jar when running tests which are needed by testcontainers + // use packaged jars (through packageBin) on classpaths instead of class directories for production + Compile / exportJars := true, + // add needed files to test jar Test / packageBin / mappings ++= Seq( + (rootBaseDir.value / "webapi" / "scripts" / "fuseki-repository-config.ttl.template") -> "webapi/scripts/fuseki-repository-config.ttl.template", // needed for initialization of triplestore (rootBaseDir.value / "sipi" / "config" / "sipi.knora-docker-config.lua") -> "sipi/config/sipi.knora-docker-config.lua" - ) + ), + // use packaged jars (through packageBin) on classpaths instead of class directories for test + Test / exportJars := true ) .settings( scalacOptions ++= Seq("-feature", "-unchecked", "-deprecation", "-Yresolve-term-conflict:package"), @@ -144,11 +147,6 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi")) // Test / javaOptions ++= Seq("-Dakka.log-config-on-start=on"), // prints out akka config // Test / javaOptions ++= Seq("-Dconfig.trace=loads"), // prints out config locations Test / testOptions += Tests.Argument("-oDF") // show full stack traces and test case durations - - // enable publishing the jars for test and it - // Test / packageBin / publishArtifact := true, - // IntegrationTest / packageBin / publishArtifact := true, - // addArtifact(artifact in (IntegrationTest, packageBin), packageBin in IntegrationTest) ) .settings( // prepare for publishing diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala index 21a5a4c147..f1dcff1463 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala @@ -693,7 +693,9 @@ class HttpTriplestoreConnector extends Actor with ActorLogging with Instrumentat */ private def initJenaFusekiTriplestore(): Try[CheckTriplestoreResponse] = { - val configFileName = s"webapi/scripts/fuseki-repository-config.ttl.template" + // TODO: Needs https://github.com/scalameta/metals/issues/3623 to be resolved + // val configFileName = s"webapi/scripts/fuseki-repository-config.ttl.template" + val configFileName = s"fuseki-repository-config.ttl.template" val triplestoreConfig: String = try { diff --git a/webapi/src/test/resources/fuseki-repository-config.ttl.template b/webapi/src/test/resources/fuseki-repository-config.ttl.template new file mode 100644 index 0000000000..7d3abbe3be --- /dev/null +++ b/webapi/src/test/resources/fuseki-repository-config.ttl.template @@ -0,0 +1,53 @@ +@prefix : . +@prefix fuseki: . +@prefix ja: . +@prefix tdb2: . +@prefix rdf: . +@prefix rdfs: . +@prefix text: . +@prefix knora-base: . + +[] rdf:type fuseki:Server ; + fuseki:services :service_tdb_all ; + ja:loadClass "org.apache.jena.query.text.TextQuery" . + + +:service_tdb_all a fuseki:Service ; + rdfs:label "TDB2 @REPOSITORY@" ; + fuseki:dataset :text_dataset ; + fuseki:name "@REPOSITORY@" ; + fuseki:serviceQuery "query" , "sparql" ; + fuseki:serviceReadGraphStore "get" ; + fuseki:serviceReadWriteGraphStore "data" ; + fuseki:serviceUpdate "update" ; + fuseki:serviceUpload "upload" . + +## --------------------------------------------------------------- +## This URI must be fixed - it's used to assemble the text dataset. + +:text_dataset rdf:type text:TextDataset ; + text:dataset :tdb_dataset_readwrite ; + text:index :indexLucene . + +# A TDB2 dataset used for RDF storage +:tdb_dataset_readwrite a tdb2:DatasetTDB2 ; + tdb2:unionDefaultGraph true ; + tdb2:location "/fuseki/databases/@REPOSITORY@" . + +# Text index description +:indexLucene a text:TextIndexLucene ; + text:directory "/fuseki/lucene/@REPOSITORY@" ; + text:entityMap :entMap . + +# Mapping in the index +# URI stored in field "uri" +# knora-base:valueHasString is mapped to field "text" +:entMap a text:EntityMap ; + text:entityField "uri" ; + text:defaultField "text" ; + text:uidField "uid" ; + text:map ( + [ text:field "text" ; text:predicate rdfs:label ] + [ text:field "text" ; text:predicate knora-base:valueHasString ] + [ text:field "text" ; text:predicate knora-base:valueHasComment ] + ) . diff --git a/webapi/src/test/resources/sipi.knora-docker-config.lua b/webapi/src/test/resources/sipi.knora-docker-config.lua new file mode 100644 index 0000000000..c16b30ec7e --- /dev/null +++ b/webapi/src/test/resources/sipi.knora-docker-config.lua @@ -0,0 +1,219 @@ +-- Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. +-- SPDX-License-Identifier: Apache-2.0 + +-- +-- ATTENTION: This configuration file should only be used for integration testing. It has additional routes defined!!! +-- +sipi = { + -- + -- The user under which the Sipi server should run. Use this only if Sipi should setuid to a particular user after + -- starting. Otherwise, leave this commented out. If this setting is used, Sipi must be started as root. + -- + -- userid = '_www', + + -- + -- Sipi's hostname as returned in the thumbnail response, default is "localhost". + -- If sipi is run behind a proxy, then this external FQDN needs to be set here. + -- + hostname = '0.0.0.0', + + -- + -- port number the server is listening to + -- + port = 1024, + + -- + -- Number of threads to use + -- + nthreads = 8, + + -- + -- SIPI is using libjpeg to generate the JPEG images. libjpeg requires a quality value which + -- corresponds to the compression rate. 100 is (almost) no compression and best quality, 0 + -- would be full compression and no quality. Reasonable values are between 30 and 95... + -- + jpeg_quality = 60, + + -- + -- For scaling images, SIPI offers two methods. The value "high" offers best quality using expensive + -- algorithms: bilinear interpolation, if downscaling the image is first scaled up to an integer + -- multiple of the requires size, and then downscaled using averaging. This results in the best + -- image quality. "medium" uses bilinear interpolation but does not do upscaling before + -- downscaling. If scaling quality is set to "low", then just a lookup table and nearest integer + -- interpolation is being used to scale the images. + -- Recognized values are: "high", "medium", "low". + -- + scaling_quality = { + jpeg = "medium", + tiff = "high", + png = "high", + j2k = "high" + }, + + -- + -- Number of seconds a connection (socket) remains open + -- + keep_alive = 5, + + -- + -- Maximal size of a post request + -- + max_post_size = '250M', + + -- + -- indicates the path to the root of the image directory. Depending on the settings of the variable + -- "prefix_as_path" the images are search at // (prefix_as_path = TRUE) + -- or / (prefix_as_path = FALSE). Please note that "prefix" and "imageid" are + -- expected to be urlencoded. Both will be decoded. That is, "/" will be recoignized and expanded + -- in the final path the image file! + -- + imgroot = '/sipi/images', -- make sure that this directory exists + + -- + -- If FALSE, the prefix is not used to build the path to the image files + -- + prefix_as_path = true, + + -- + -- In order not to accumulate to many files into one diretory (which slows down file + -- access considerabely), the images are stored in recursive subdirectories 'A'-'Z'. + -- If subdir_levels is equal 0, no subdirectories are used. The maximum is 6. + -- The recommandeation is that on average there should not me more than a few + -- thousand files in a unix directory (your mileage may vay depending on the + -- file system used). + -- + subdir_levels = 0, + + -- + -- if subdir_levels is > 0 and if prefix_as_path is true, all prefixes will be + -- regarded as directories under imgroot. Thus, the subdirs 'A'-'Z' will be + -- created in these directories for the prefixes. However, it may make sense + -- for certain prefixes *not* to use subdirs. A list of these prefix-directories + -- can be given with this configuration parameter. + -- + subdir_excludes = { "knora", "thumbs" }, + + -- + -- Lua script which is executed on initialization of the Lua interpreter + -- + initscript = '/sipi/scripts/sipi.init-knora.lua', + + -- + -- path to the caching directory + -- + cachedir = '/sipi/cache', + + -- + -- maximal size of the cache + -- + cachesize = '100M', + + -- + -- if the cache becomes full, the given percentage of file space is marked for reuase + -- + cache_hysteresis = 0.15, + + -- + -- Path to the directory where the scripts for the routes defined below are to be found + -- + scriptdir = '/sipi/scripts', + + --- + --- Size of the thumbnails (to be used within Lua) + --- + thumb_size = '!128,128', + + -- + -- Path to the temporary directory + -- + tmpdir = '/tmp', + + -- + -- Maximum age of temporary files, in seconds (requires Knora's upload.lua). + -- Defaults to 86400 seconds (1 day). + -- + max_temp_file_age = 86400, + + -- + -- Path to Knora Application + -- + knora_path = 'api', + + -- + -- Port of Knora Application + -- + knora_port = '3333', + + -- + -- The secret for generating JWT's (JSON Web Tokens) (42 characters) + -- + jwt_secret = 'UP 4888, nice 4-8-4 steam engine', + -- 12345678901234567890123456789012 + + -- + -- Name of the logfile (a ".txt" is added...) + -- + -- logfile = "sipi.log", + + + -- + -- loglevel, one of "DEBUG", "INFO", "NOTICE", "WARNING", "ERR", + -- "CRIT", "ALERT", "EMERG" + -- + loglevel = "DEBUG" + +} + + +fileserver = { + -- + -- directory where the documents for the normal webserver are located + -- + docroot = '/sipi/server', + + -- + -- route under which the normal webserver shouöd respond to requests + -- + wwwroute = '/server' +} + +-- +-- Custom routes. Each route is an URL path associated with a Lua script. +-- +routes = { + { + method = 'POST', + route = '/upload', + script = 'upload.lua' + }, + { + method = 'POST', + route = '/store', + script = 'store.lua' + }, + { + method = 'DELETE', + route = '/delete_temp_file', + script = 'delete_temp_file.lua' + }, + -- + -- additional routes used for testing. should not be defined in production. + -- + { + method = 'GET', + route = '/test_functions', + script = 'test_functions.lua' + }, + { + method = 'GET', + route = '/test_file_info', + script = 'test_file_info.lua' + }, + { + method = 'GET', + route = '/test_knora_session_cookie', + script = 'test_knora_session_cookie.lua' + } + +} + diff --git a/webapi/src/test/scala/org/knora/webapi/TestContainersAll.scala b/webapi/src/test/scala/org/knora/webapi/TestContainersAll.scala index 8bff7ac0b1..0d5d3fc905 100644 --- a/webapi/src/test/scala/org/knora/webapi/TestContainersAll.scala +++ b/webapi/src/test/scala/org/knora/webapi/TestContainersAll.scala @@ -42,8 +42,11 @@ object TestContainersAll { SipiContainer.withEnv("SIPI_WEBAPI_HOSTNAME", localIpAddress) SipiContainer.withEnv("SIPI_WEBAPI_PORT", "3333") SipiContainer.withCommand("--config=/sipi/config/sipi.knora-docker-config.lua") + + // TODO: Needs https://github.com/scalameta/metals/issues/3623 to be resolved SipiContainer.withClasspathResourceMapping( - "/sipi/config/sipi.knora-docker-config.lua", + // "/sipi/config/sipi.knora-docker-config.lua" + "/sipi.knora-docker-config.lua", "/sipi/config/sipi.knora-docker-config.lua", BindMode.READ_ONLY )