From 7d420a4e88f355b1465aa1403508c433deb0ae8d Mon Sep 17 00:00:00 2001 From: Ivan Subotic <400790+subotic@users.noreply.github.com> Date: Thu, 14 Jul 2022 10:37:38 +0200 Subject: [PATCH] fix(authentication): make cookie name unique between environments (#2095) --- build.sbt | 2 +- docker-compose.yml | 1 + sipi/config/sipi.docker-test-config.lua | 87 ++++++---- sipi/scripts/basexx.lua | 20 ++- sipi/scripts/get_knora_session.lua | 12 +- sipi/scripts/sipi.init.lua | 13 +- .../knora/webapi/app/ApplicationActor.scala | 2 +- ...SipiRouteADM.scala => FilesRouteADM.scala} | 2 +- webapi/src/test/resources/logback-test.xml | 10 +- .../e2e/v2/AuthenticationV2E2ESpec.scala | 19 +++ .../it/v2/KnoraSipiAuthenticationITSpec.scala | 151 ++++++++++++++++++ .../testcontainers/SipiTestContainer.scala | 9 ++ 12 files changed, 272 insertions(+), 56 deletions(-) rename webapi/src/main/scala/org/knora/webapi/routing/admin/{SipiRouteADM.scala => FilesRouteADM.scala} (95%) create mode 100644 webapi/src/test/scala/org/knora/webapi/it/v2/KnoraSipiAuthenticationITSpec.scala diff --git a/build.sbt b/build.sbt index 94c6c35ec2..62ffcbb6ff 100644 --- a/build.sbt +++ b/build.sbt @@ -70,7 +70,7 @@ lazy val sipi: Project = Project(id = "sipi", base = file("sipi")) Docker / defaultLinuxInstallLocation := "/sipi", Universal / mappings ++= { // copy the sipi/scripts folder - directory("sipi/scripts") + directory("sipi/scripts"), }, // use filterNot to return all items that do NOT meet the criteria dockerCommands := dockerCommands.value.filterNot { diff --git a/docker-compose.yml b/docker-compose.yml index 23e8afbbf3..8800c8fd1c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,6 +38,7 @@ services: - KNORA_WEBAPI_KNORA_API_EXTERNAL_HOST=0.0.0.0 - KNORA_WEBAPI_KNORA_API_EXTERNAL_PORT=3333 # entrypoint: [ "valgrind", "--leak-check=yes", "/sipi/sipi" ] ## uncomment to run SIPI under valgrind + # command: --config=/sipi/config/sipi.docker-test-config.lua ## command variant to start the sipi container with test routes enabled command: --config=/sipi/config/sipi.docker-config.lua api: diff --git a/sipi/config/sipi.docker-test-config.lua b/sipi/config/sipi.docker-test-config.lua index a89c455b0d..26d5683b53 100644 --- a/sipi/config/sipi.docker-test-config.lua +++ b/sipi/config/sipi.docker-test-config.lua @@ -1,8 +1,8 @@ --- * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. --- * SPDX-License-Identifier: Apache-2.0 +-- Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. +-- SPDX-License-Identifier: Apache-2.0 -- --- configuration file for use with Knora +-- ATTENTION: This configuration file should only be used for integration testing. It has additional routes defined!!! -- sipi = { -- @@ -22,6 +22,34 @@ sipi = { -- port = 1024, + -- + -- Number of threads to use + -- + nthreads = 8, + + -- + -- SIPI is using libjpeg to generate the JPEG images. libjpeg requires a quality value which + -- corresponds to the compression rate. 100 is (almost) no compression and best quality, 0 + -- would be full compression and no quality. Reasonable values are between 30 and 95... + -- + jpeg_quality = 60, + + -- + -- For scaling images, SIPI offers two methods. The value "high" offers best quality using expensive + -- algorithms: bilinear interpolation, if downscaling the image is first scaled up to an integer + -- multiple of the requires size, and then downscaled using averaging. This results in the best + -- image quality. "medium" uses bilinear interpolation but does not do upscaling before + -- downscaling. If scaling quality is set to "low", then just a lookup table and nearest integer + -- interpolation is being used to scale the images. + -- Recognized values are: "high", "medium", "low". + -- + scaling_quality = { + jpeg = "medium", + tiff = "high", + png = "high", + j2k = "high" + }, + -- -- Number of seconds a connection (socket) remains open -- @@ -30,16 +58,16 @@ sipi = { -- -- Maximal size of a post request -- - max_post_size = '30M', + max_post_size = '250M', - -- + -- -- indicates the path to the root of the image directory. Depending on the settings of the variable -- "prefix_as_path" the images are search at // (prefix_as_path = TRUE) -- or / (prefix_as_path = FALSE). Please note that "prefix" and "imageid" are -- expected to be urlencoded. Both will be decoded. That is, "/" will be recoignized and expanded -- in the final path the image file! -- - imgroot = './test/_test_data/images', -- directory for Knora Sipi integration testing + imgroot = '/sipi/images', -- make sure that this directory exists -- -- If FALSE, the prefix is not used to build the path to the image files @@ -68,38 +96,44 @@ sipi = { -- -- Lua script which is executed on initialization of the Lua interpreter -- - initscript = './scripts/sipi.init-test.lua', + initscript = '/sipi/scripts/sipi.init.lua', -- -- path to the caching directory -- - cachedir = './cache', + cachedir = '/sipi/cache', -- - -- maxcimal size of the cache + -- maximal size of the cache -- cachesize = '100M', -- -- if the cache becomes full, the given percentage of file space is marked for reuase -- - cache_hysteresis = 0.1, + cache_hysteresis = 0.15, -- -- Path to the directory where the scripts for the routes defined below are to be found -- - scriptdir = './scripts', + scriptdir = '/sipi/scripts', --- - --- Size of the thumbnails + --- Size of the thumbnails (to be used within Lua) --- - thumb_size = 'pct:4', + thumb_size = '!128,128', -- -- Path to the temporary directory -- tmpdir = '/tmp', + -- + -- Maximum age of temporary files, in seconds (requires Knora's upload.lua). + -- Defaults to 86400 seconds (1 day). + -- + max_temp_file_age = 86400, + -- -- Path to Knora Application -- @@ -110,26 +144,6 @@ sipi = { -- knora_port = '3333', - -- - -- If compiled with SSL support, the port the server is listening for secure connections - -- - -- ssl_port = 1025, - - -- - -- If compiled with SSL support, the path to the certificate (must be .pem file) - -- The follow commands can be used to generate a self-signed certificate - -- # openssl genrsa -out key.pem 2048 - -- # openssl req -new -key key.pem -out csr.pem - -- #openssl req -x509 -days 365 -key key.pem -in csr.pem -out certificate.pem - -- - -- ssl_certificate = './certificate/certificate.pem', - - -- - -- If compiled with SSL support, the path to the key file (see above to create) - -- - -- ssl_key = './certificate/key.pem', - - -- -- The secret for generating JWT's (JSON Web Tokens) (42 characters) -- @@ -139,20 +153,23 @@ sipi = { -- -- Name of the logfile (a ".txt" is added...) -- - logfile = "sipi.log", + -- logfile = "sipi.log", + -- -- loglevel, one of "DEBUG", "INFO", "NOTICE", "WARNING", "ERR", -- "CRIT", "ALERT", "EMERG" -- loglevel = "DEBUG" + } + fileserver = { -- -- directory where the documents for the normal webserver are located -- - docroot = './server', + docroot = '/sipi/server', -- -- route under which the normal webserver shouöd respond to requests diff --git a/sipi/scripts/basexx.lua b/sipi/scripts/basexx.lua index e6d617a82c..1d28418233 100644 --- a/sipi/scripts/basexx.lua +++ b/sipi/scripts/basexx.lua @@ -109,7 +109,7 @@ end -- generic function to decode and encode base32/base64 -------------------------------------------------------------------------------- -local function from_basexx( str, alphabet, bits ) +function from_basexx( str, alphabet, bits ) local result = {} for i = 1, #str do local c = string.sub( str, i, i ) @@ -127,7 +127,7 @@ local function from_basexx( str, alphabet, bits ) return pure_from_bit( string.sub( value, 1, #value - pad ) ) end -local function to_basexx( str, alphabet, bits, pad ) +function to_basexx( str, alphabet, bits, pad ) local bitString = basexx.to_bit( str ) local chunks = divide_string( bitString, bits ) @@ -160,6 +160,22 @@ function basexx.to_base32( str ) return to_basexx( str, base32Alphabet, 5, base32PadMap[ #str % 5 + 1 ] ) end +-------------------------------------------------------------------------------- +-- dsp-api custom variant +-------------------------------------------------------------------------------- + +local base32CustomAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" +local base32CustomPadMap = { "", "999999", "9999", "999", "9" } + +function basexx.from_base32Custom( str, ignore ) + str = ignore_set( str, ignore ) + return from_basexx( string.upper( str ), base32CustomAlphabet, 5 ) +end + +function basexx.to_base32Custom( str ) + return to_basexx( str, base32CustomAlphabet, 5, base32CustomPadMap[ #str % 5 + 1 ] ) +end + -------------------------------------------------------------------------------- -- crockford: http://www.crockford.com/wrmg/base32.html -------------------------------------------------------------------------------- diff --git a/sipi/scripts/get_knora_session.lua b/sipi/scripts/get_knora_session.lua index 10b14c5674..c31f77dd24 100644 --- a/sipi/scripts/get_knora_session.lua +++ b/sipi/scripts/get_knora_session.lua @@ -36,12 +36,14 @@ function get_session_id(cookie) host_port = webapi_hostname .. ':' .. webapi_port server.log("host_port: " .. host_port, server.loglevel.LOG_DEBUG) - local customPadMap = { "", "999999", "9999", "999", "9" } - host_port_base32 = basexx.to_basexx(host_port, base32Alphabet, 5, customPadMap) + host_port_base32 = basexx.to_base32Custom(host_port) server.log("host_port_base32: " .. host_port_base32, server.loglevel.LOG_DEBUG) + + + -- tries to extract the Knora session id from the cookie: -- gets the digits between "sid=" and the closing ";" (only given in case of several key value pairs) -- ";" is expected to separate different key value pairs (https://tools.ietf.org/html/rfc6265#section-4.2.1) @@ -51,6 +53,10 @@ function get_session_id(cookie) local session_id = string.match(cookie, "KnoraAuthentication" .. host_port_base32 .. "=([^%s;]+)") server.log("extracted session_id: " .. session_id, server.loglevel.LOG_DEBUG) - return session_id + local session = {} + session["id"] = session_id + session["name"] = "KnoraAuthentication" .. host_port_base32 + + return session end diff --git a/sipi/scripts/sipi.init.lua b/sipi/scripts/sipi.init.lua index 8334197ff2..5aada21dd0 100644 --- a/sipi/scripts/sipi.init.lua +++ b/sipi/scripts/sipi.init.lua @@ -42,17 +42,17 @@ function pre_flight(prefix, identifier, cookie) if cookie ~='' then - -- tries to extract the Knora session id from the cookie: + -- tries to extract the Knora session name and id from the cookie: -- gets the digits between "sid=" and the closing ";" (only given in case of several key value pairs) -- returns nil if it cannot find it - session_id = get_session_id(cookie) + session = get_session_id(cookie) - if session_id == nil then - -- no session_id could be extracted - print("cookie key is invalid: " .. cookie) + if session == nil then + -- no session could be extracted server.log("cookie key is invalid: " .. cookie, server.loglevel.LOG_ERR) else - knora_cookie_header = { Cookie = "KnoraAuthentication=" .. session_id } + knora_cookie_header = { Cookie = session["name"] .. "=" .. session["id"] } + server.log("pre_flight - knora_cookie_header: " .. knora_cookie_header["Cookie"], server.loglevel.LOG_DEBUG) end end @@ -78,7 +78,6 @@ function pre_flight(prefix, identifier, cookie) -- print("knora_url: " .. knora_url) server.log("pre_flight - knora_url: " .. knora_url, server.loglevel.LOG_DEBUG) - server.log("pre_flight - knora_cookie_header: " .. tostring(knora_cookie_header), server.loglevel.LOG_DEBUG) success, result = server.http("GET", knora_url, knora_cookie_header, 5000) diff --git a/webapi/src/main/scala/org/knora/webapi/app/ApplicationActor.scala b/webapi/src/main/scala/org/knora/webapi/app/ApplicationActor.scala index 857db02030..f715d8cbbf 100644 --- a/webapi/src/main/scala/org/knora/webapi/app/ApplicationActor.scala +++ b/webapi/src/main/scala/org/knora/webapi/app/ApplicationActor.scala @@ -433,7 +433,7 @@ class ApplicationActor( new ProjectsRouteADM(routeData).knoraApiPath ~ new StoreRouteADM(routeData).knoraApiPath ~ new UsersRouteADM(routeData).knoraApiPath ~ - new SipiRouteADM(routeData).knoraApiPath ~ + new FilesRouteADM(routeData).knoraApiPath ~ new SwaggerApiDocsRoute(routeData).knoraApiPath } } diff --git a/webapi/src/main/scala/org/knora/webapi/routing/admin/SipiRouteADM.scala b/webapi/src/main/scala/org/knora/webapi/routing/admin/FilesRouteADM.scala similarity index 95% rename from webapi/src/main/scala/org/knora/webapi/routing/admin/SipiRouteADM.scala rename to webapi/src/main/scala/org/knora/webapi/routing/admin/FilesRouteADM.scala index d0991f85f4..a732dac8ac 100644 --- a/webapi/src/main/scala/org/knora/webapi/routing/admin/SipiRouteADM.scala +++ b/webapi/src/main/scala/org/knora/webapi/routing/admin/FilesRouteADM.scala @@ -17,7 +17,7 @@ import org.knora.webapi.routing.RouteUtilADM /** * Provides a routing function for the API that Sipi connects to. */ -class SipiRouteADM(routeData: KnoraRouteData) extends KnoraRoute(routeData) with Authenticator { +class FilesRouteADM(routeData: KnoraRouteData) extends KnoraRoute(routeData) with Authenticator { /** * A routing function for the API that Sipi connects to. diff --git a/webapi/src/test/resources/logback-test.xml b/webapi/src/test/resources/logback-test.xml index 13f83bb972..f9f996aed3 100644 --- a/webapi/src/test/resources/logback-test.xml +++ b/webapi/src/test/resources/logback-test.xml @@ -37,12 +37,10 @@ - - - - - - + + + + diff --git a/webapi/src/test/scala/org/knora/webapi/e2e/v2/AuthenticationV2E2ESpec.scala b/webapi/src/test/scala/org/knora/webapi/e2e/v2/AuthenticationV2E2ESpec.scala index c4b38089a7..3df4b4958a 100644 --- a/webapi/src/test/scala/org/knora/webapi/e2e/v2/AuthenticationV2E2ESpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/e2e/v2/AuthenticationV2E2ESpec.scala @@ -21,6 +21,7 @@ import org.knora.webapi.util.MutableTestString import scala.concurrent.Await import scala.concurrent.duration._ +import org.knora.webapi.routing.Authenticator object AuthenticationV2E2ESpec { val config: Config = ConfigFactory.parseString(""" @@ -212,6 +213,24 @@ class AuthenticationV2E2ESpec assert(response.status === StatusCodes.OK) } + "authenticate with token in cookie" in { + val KnoraAuthenticationCookieName = Authenticator.calculateCookieName(settings) + val cookieHeader = headers.Cookie(KnoraAuthenticationCookieName, token.get) + + val request = Get(baseApiUrl + "/v2/authentication") ~> addHeader(cookieHeader) + val response = singleAwaitingRequest(request) + assert(response.status === StatusCodes.OK) + } + + "fail authentication with invalid token in cookie" in { + val KnoraAuthenticationCookieName = Authenticator.calculateCookieName(settings) + val cookieHeader = headers.Cookie(KnoraAuthenticationCookieName, "not_a_valid_token") + + val request = Get(baseApiUrl + "/v2/authentication") ~> addHeader(cookieHeader) + val response = singleAwaitingRequest(request) + assert(response.status === StatusCodes.Unauthorized) + } + "logout when providing token in header" in { // do logout with stored token val request = diff --git a/webapi/src/test/scala/org/knora/webapi/it/v2/KnoraSipiAuthenticationITSpec.scala b/webapi/src/test/scala/org/knora/webapi/it/v2/KnoraSipiAuthenticationITSpec.scala new file mode 100644 index 0000000000..15c89237d0 --- /dev/null +++ b/webapi/src/test/scala/org/knora/webapi/it/v2/KnoraSipiAuthenticationITSpec.scala @@ -0,0 +1,151 @@ +/* + * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.it.v2 + +import akka.http.scaladsl.model._ +import akka.http.scaladsl.model.headers.BasicHttpCredentials +import akka.http.scaladsl.unmarshalling.Unmarshal +import com.typesafe.config.Config +import com.typesafe.config.ConfigFactory +import org.knora.webapi._ +import dsp.errors.AssertionException +import dsp.errors.BadRequestException +import org.knora.webapi.messages.IriConversions._ +import org.knora.webapi.messages.OntologyConstants +import org.knora.webapi.messages.SmartIri +import org.knora.webapi.messages.StringFormatter +import org.knora.webapi.messages.store.sipimessages._ +import org.knora.webapi.messages.store.triplestoremessages.TriplestoreJsonProtocol +import org.knora.webapi.messages.util.rdf._ +import org.knora.webapi.messages.v2.routing.authenticationmessages._ +import org.knora.webapi.models.filemodels._ +import org.knora.webapi.sharedtestdata.SharedTestDataADM +import org.knora.webapi.testservices.FileToUpload +import org.knora.webapi.util.MutableTestIri + +import java.net.URLEncoder +import java.nio.file.Files +import java.nio.file.Paths +import scala.concurrent.Await +import scala.concurrent.duration._ +import org.knora.webapi.routing.Authenticator +import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject + +object KnoraSipiAuthenticationITSpec { + val config: Config = ConfigFactory.parseString(""" + |akka.loglevel = "DEBUG" + |akka.stdout-loglevel = "DEBUG" + """.stripMargin) +} + +/** + * Tests interaction between Knora and Sipi using Knora API v2. + */ +class KnoraSipiAuthenticationITSpec + extends ITKnoraLiveSpec(KnoraSipiIntegrationV2ITSpec.config) + with AuthenticationV2JsonProtocol + with TriplestoreJsonProtocol { + private implicit val stringFormatter: StringFormatter = StringFormatter.getGeneralInstance + + private val anythingUserEmail = SharedTestDataADM.anythingAdminUser.email + private val incunabulaUserEmail = SharedTestDataADM.incunabulaMemberUser.email + private val password = SharedTestDataADM.testPass + + private val marblesOriginalFilename = "marbles.tif" + private val pathToMarbles = Paths.get("..", s"test_data/test_route/images/$marblesOriginalFilename") + private val incunabulaImageDirPath = Paths.get("..", "sipi/images/0803") + + override lazy val rdfDataObjects: List[RdfDataObject] = List( + RdfDataObject(path = "test_data/all_data/incunabula-data.ttl", name = "http://www.knora.org/data/0803/incunabula"), + RdfDataObject(path = "test_data/demo_data/images-demo-data.ttl", name = "http://www.knora.org/data/00FF/images") + ) + + "The Knora/Sipi authentication" should { + var loginToken: String = "" + + "log in as a Knora user" in { + /* Correct username and correct password */ + + val params = + s""" + |{ + | "email": "$anythingUserEmail", + | "password": "$password" + |} + """.stripMargin + + val request = Post(baseApiUrl + s"/v2/authentication", HttpEntity(ContentTypes.`application/json`, params)) + val response: HttpResponse = singleAwaitingRequest(request) + assert(response.status == StatusCodes.OK) + + val lr: LoginResponse = Await.result(Unmarshal(response.entity).to[LoginResponse], 1.seconds) + loginToken = lr.token + + loginToken.nonEmpty should be(true) + } + + "successfuly get an image with provided credentials inside cookie" in { + + // using cookie to authenticate when accessing sipi (test for cookie parsing in sipi) + val KnoraAuthenticationCookieName = Authenticator.calculateCookieName(settings) + val cookieHeader = headers.Cookie(KnoraAuthenticationCookieName, loginToken) + + // Request the permanently stored image from Sipi. + val sipiGetImageRequest = + Get( + "http://0.0.0.0:1024/0803/incunabula_0000000002.jp2/full/max/0/default.jpg".replace( + "http://0.0.0.0:1024", + baseInternalSipiUrl + ) + ) ~> addHeader(cookieHeader) + + val response = singleAwaitingRequest(sipiGetImageRequest) + assert(response.status === StatusCodes.OK) + } + + "accept a token in Sipi that has been signed by Knora" in { + val invalidToken = "a_invalid_token" + + // The image to be uploaded. + assert(Files.exists(pathToMarbles), s"File $pathToMarbles does not exist") + + // A multipart/form-data request containing the image. + val sipiFormData = Multipart.FormData( + Multipart.FormData.BodyPart( + "file", + HttpEntity.fromPath(MediaTypes.`image/tiff`, pathToMarbles), + Map("filename" -> pathToMarbles.getFileName.toString) + ) + ) + + // Send a POST request to Sipi, asking it to convert the image to JPEG 2000 and store it in a temporary file. + val sipiRequest = Post(s"$baseInternalSipiUrl/upload?token=$loginToken", sipiFormData) + val sipiResponse = singleAwaitingRequest(sipiRequest) + assert(sipiResponse.status == StatusCodes.OK) + } + + "not accept a token in Sipi that hasn't been signed by Knora" in { + val invalidToken = "a_invalid_token" + + // The image to be uploaded. + assert(Files.exists(pathToMarbles), s"File $pathToMarbles does not exist") + + // A multipart/form-data request containing the image. + val sipiFormData = Multipart.FormData( + Multipart.FormData.BodyPart( + "file", + HttpEntity.fromPath(MediaTypes.`image/tiff`, pathToMarbles), + Map("filename" -> pathToMarbles.getFileName.toString) + ) + ) + + // Send a POST request to Sipi, asking it to convert the image to JPEG 2000 and store it in a temporary file. + val sipiRequest = Post(s"$baseInternalSipiUrl/upload?token=$invalidToken", sipiFormData) + val sipiResponse = singleAwaitingRequest(sipiRequest) + assert(sipiResponse.status == StatusCodes.Unauthorized) + } + } +} diff --git a/webapi/src/test/scala/org/knora/webapi/testcontainers/SipiTestContainer.scala b/webapi/src/test/scala/org/knora/webapi/testcontainers/SipiTestContainer.scala index 4dbf8d2d80..653ad45dc3 100644 --- a/webapi/src/test/scala/org/knora/webapi/testcontainers/SipiTestContainer.scala +++ b/webapi/src/test/scala/org/knora/webapi/testcontainers/SipiTestContainer.scala @@ -9,6 +9,7 @@ import zio._ import java.net.NetworkInterface import java.net.UnknownHostException import scala.jdk.CollectionConverters._ +import java.nio.file.Paths final case class SipiTestContainer(container: GenericContainer[Nothing]) @@ -45,6 +46,14 @@ object SipiTestContainer { "/sipi/config/sipi.docker-config.lua", BindMode.READ_ONLY ) + + val incunabulaImageDirPath = Paths.get("..", "sipi/images/0803/incunabula_0000000002.jp2") + sipiContainer.withFileSystemBind( + incunabulaImageDirPath.toString(), + "/sipi/images/0803/incunabula_0000000002.jp2", + BindMode.READ_ONLY + ) + sipiContainer.start() sipiContainer }.tap(_ => ZIO.debug(">>> Acquire Sipi TestContainer <<<"))