fix: failing repository upgrade at startup (DSP-654) (#1712)

dasch-swiss · Sep 18, 2020 · 0d6b4ee · 0d6b4ee
1 parent 7f95dcc
commit 0d6b4ee
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 10 deletions.
diff --git a/Makefile b/Makefile
@@ -282,6 +282,11 @@ init-db-test-minimal: stack-db-remove stack-db-only ## initializes the knora-tes
 	@echo $@
 	@$(MAKE) -C webapi/scripts fuseki-init-knora-test-minimal
 
+.PHONY: init-db-test-empty
+init-db-test-empty: stack-db-remove stack-db-only ## initializes the knora-test repository with minimal data
+	@echo $@
+	@$(MAKE) -C webapi/scripts fuseki-init-knora-test-empty
+
 .PHONY: init-db-test-unit
 init-db-test-unit: stack-db-remove stack-db-only ## initializes the knora-test-unit repository
 	@echo $@

diff --git a/webapi/scripts/Makefile b/webapi/scripts/Makefile
@@ -6,6 +6,10 @@ fuseki-init-knora-test: ## initializes Fuseki with the knora-test repository and
 fuseki-init-knora-test-minimal: ## initializes Fuseki with the knora-test repository and minimal data
 	./fuseki-init-knora-test-minimal.sh
 
+.PHONY: fuseki-init-knora-test-empty
+fuseki-init-knora-test-empty: ## initializes Fuseki with the knora-test repository and no data
+	./fuseki-init-knora-test-empty.sh
+
 .PHONY: fuseki-init-knora-test-unit
 fuseki-init-knora-test-unit: ## initializes Fuseki with the knora-test-unit repository and no data
 	./fuseki-init-knora-test-unit.sh

diff --git a/webapi/scripts/fuseki-init-knora-test-empty.sh b/webapi/scripts/fuseki-init-knora-test-empty.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# Including fuseki-funcions.sh implementing delete, create, and upload.
+source fuseki-functions.sh
+
+# Name of the repository
+REPOSITORY="knora-test"
+
+# delete-repository // delete dos not work correctly. need to delete database manually.
+create-repository
diff --git a/webapi/src/main/scala/org/knora/webapi/app/BUILD.bazel b/webapi/src/main/scala/org/knora/webapi/app/BUILD.bazel
@@ -45,6 +45,8 @@ scala_binary(
     ],
     main_class = "org.knora.webapi.app.Main",
     resources = [
+        "//knora-ontologies",
+        "//webapi/scripts:fuseki_repository_config_ttl_template",
         "//webapi/src/main/resources",
     ],
     runtime_deps = [

diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala
@@ -86,7 +86,7 @@ class HttpTriplestoreConnector extends Actor with ActorLogging with Instrumentat
     private val credsProvider: BasicCredentialsProvider = new BasicCredentialsProvider
     credsProvider.setCredentials(new AuthScope(targetHost.getHostName, targetHost.getPort), new UsernamePasswordCredentials(settings.triplestoreUsername, settings.triplestorePassword))
 
-    // Reading data should be quick.
+    // Reading data should be quick, except when it is not ;-)
     private val queryTimeoutMillis = settings.triplestoreQueryTimeout.toMillis.toInt
 
     private val queryRequestConfig = RequestConfig.custom()
@@ -775,7 +775,7 @@ class HttpTriplestoreConnector extends Actor with ActorLogging with Instrumentat
                 }
 
                 val took = System.currentTimeMillis() - start
-                log.info(s"[$statusCode] GraphDB Query took: ${took}ms")
+                log.info(s"[$statusCode] DB Query took: ${took}ms")
 
                 Option(maybeResponse.get.getEntity) match {
                     case Some(responseEntity: HttpEntity) =>
@@ -921,6 +921,20 @@ class HttpTriplestoreConnector extends Actor with ActorLogging with Instrumentat
             var maybeResponse: Option[CloseableHttpResponse] = None
 
             try {
+
+                val queryTimeoutMillis = settings.triplestoreQueryTimeout.toMillis.toInt * 10
+
+                val queryRequestConfig = RequestConfig.custom()
+                  .setConnectTimeout(queryTimeoutMillis)
+                  .setConnectionRequestTimeout(queryTimeoutMillis)
+                  .setSocketTimeout(queryTimeoutMillis)
+                  .build
+
+                val queryHttpClient: CloseableHttpClient = HttpClients.custom
+                  .setDefaultCredentialsProvider(credsProvider)
+                  .setDefaultRequestConfig(queryRequestConfig)
+                  .build
+
                 maybeResponse = Some(queryHttpClient.execute(targetHost, httpGet, httpContext))
 
                 val statusCode: Int = maybeResponse.get.getStatusLine.getStatusCode
@@ -932,7 +946,7 @@ class HttpTriplestoreConnector extends Actor with ActorLogging with Instrumentat
                 }
 
                 val took = System.currentTimeMillis() - start
-                log.info(s"[$statusCode] GraphDB Query took: ${took}ms")
+                log.info(s"[$statusCode] DB Query took: ${took}ms")
 
                 Option(maybeResponse.get.getEntity) match {
                     case Some(responseEntity: HttpEntity) =>

diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdater.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdater.scala
@@ -79,19 +79,22 @@ class RepositoryUpdater(system: ActorSystem,
      */
     def maybeUpdateRepository: Future[RepositoryUpdatedResponse] = {
         for {
-            maybeRepositoryVersionString <- getRepositoryVersion
+            foundRepositoryVersion: Option[String] <- getRepositoryVersion
+            requiredRepositoryVersion = org.knora.webapi.KnoraBaseVersion
 
             // Is the repository up to date?
-            repositoryUpdatedResponse: RepositoryUpdatedResponse <- if (maybeRepositoryVersionString.contains(org.knora.webapi.KnoraBaseVersion)) {
+            repositoryUpToData = foundRepositoryVersion.contains(requiredRepositoryVersion)
+            repositoryUpdatedResponse: RepositoryUpdatedResponse <- if (repositoryUpToData) {
                 // Yes. Nothing more to do.
-                FastFuture.successful(RepositoryUpdatedResponse(s"Repository is up to date at ${org.knora.webapi.KnoraBaseVersion}"))
+                FastFuture.successful(RepositoryUpdatedResponse(s"Repository is up to date at $requiredRepositoryVersion"))
             } else {
                 // No. Construct the list of updates that it needs.
-                val pluginsForNeededUpdates: Seq[PluginForKnoraBaseVersion] = selectPluginsForNeededUpdates(maybeRepositoryVersionString)
-                log.info(s"Updating repository with transformations: ${pluginsForNeededUpdates.map(_.versionString).mkString(", ")}")
+                log.info(s"Repository not up-to-date. Found: $foundRepositoryVersion, Required: $requiredRepositoryVersion")
+                val selectedPlugins: Seq[PluginForKnoraBaseVersion] = selectPluginsForNeededUpdates(foundRepositoryVersion)
+                log.info(s"Updating repository with transformations: ${selectedPlugins.map(_.versionString).mkString(", ")}")
 
                 // Update it with those plugins.
-                updateRepository(pluginsForNeededUpdates)
+                updateRepositoryWithSelectedPlugins(selectedPlugins)
             }
         } yield repositoryUpdatedResponse
     }
@@ -144,7 +147,7 @@ class RepositoryUpdater(system: ActorSystem,
      * @param pluginsForNeededUpdates the plugins needed to update the repository.
      * @return a [[RepositoryUpdatedResponse]] indicating what was done.
      */
-    private def updateRepository(pluginsForNeededUpdates: Seq[PluginForKnoraBaseVersion]): Future[RepositoryUpdatedResponse] = {
+    private def updateRepositoryWithSelectedPlugins(pluginsForNeededUpdates: Seq[PluginForKnoraBaseVersion]): Future[RepositoryUpdatedResponse] = {
         // Was a download directory specified in the application settings?
         val downloadDir: File = settings.upgradeDownloadDir match {
             case Some(configuredDir) =>