fix!: add upgrade plugin that fixes invalid date serialisations

.gitignore

@@ -36,7 +36,7 @@ sipi/test
 *.bak
 *.rdb
 .sbtrc
-*.trig
+/*.trig
 
 dependencies.txt
 /client-test-data.zip

Makefile

@@ -273,7 +273,10 @@ init-db-test-from-prod: db_prod_dump.trig init-db-test-empty ## init local datab
 	@curl -X POST -H "Content-Type: application/trig" --data-binary "@${CURRENT_DIR}/db_prod_dump.trig" -u "admin:test" "http://localhost:3030/knora-test"
 
 .PHONY: init-db-test-from-ls-test-server
-init-db-test-from-ls-test-server: db_ls_test_server_dump.trig init-db-test-empty ## init local database with data from ls-test-server
+init-db-test-from-ls-test-server: db_ls_test_server_dump.trig init-db-test-from-ls-test-server-trig-file ## init local database with data from ls-test-server
+
+.PHONY: init-db-test-from-ls-test-server-trig-file
+init-db-test-from-ls-test-server-trig-file: init-db-test-empty ## init local database with data from a local ls-test-server dump
 	@echo $@
 	@curl -X POST -H "Content-Type: application/sparql-update" -d "DROP ALL" -u "admin:test" "http://localhost:3030/knora-test"
 	@curl -X POST -H "Content-Type: application/trig" --data-binary "@${CURRENT_DIR}/db_ls_test_server_dump.trig" -u "admin:test" "http://localhost:3030/knora-test"

knora-ontologies/knora-base.ttl

@@ -19,7 +19,7 @@
     rdf:type           owl:Ontology ;
     rdfs:label         "The Knora base ontology"@en ;
     :attachedToProject knora-admin:SystemProject ;
-    :ontologyVersion   "knora-base v21" .
+    :ontologyVersion   "knora-base v22" .
 
 
 #################################################################

test_data/upgrade/pr2081.trig

@@ -0,0 +1,36 @@
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix knora-base: <http://www.knora.org/ontology/knora-base#> .
+
+# resources with old date serialization
+
+<http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9ma>
+    a                            knora-base:Resource ;
+    knora-base:attachedToUser    <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
+    knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
+    knora-base:hasPermissions    "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ;
+    knora-base:creationDate      "2019-11-29T10:00:00.673298+00:00"^^xsd:dateTime ;
+    rdfs:label                   "a thing" ;
+    knora-base:isDeleted         false .
+
+<http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mb>
+    a                               knora-base:Resource ;
+    knora-base:attachedToUser       <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
+    knora-base:attachedToProject    <http://rdfh.ch/projects/0001> ;
+    knora-base:hasPermissions       "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ;
+    knora-base:creationDate         "2018-08-07T13:27:18.518+00:00"^^xsd:dateTime ;
+    knora-base:lastModificationDate "2019-11-29T10:00:01.673298+00:00"^^xsd:dateTime ;
+    rdfs:label                      "a thing" ;
+    knora-base:isDeleted            false .
+
+<http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mc>
+    a                               knora-base:Resource ;
+    knora-base:attachedToUser       <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
+    knora-base:attachedToProject    <http://rdfh.ch/projects/0001> ;
+    knora-base:hasPermissions       "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ;
+    knora-base:creationDate         "2018-08-07T13:27:18.518123Z"^^xsd:dateTime ;
+    knora-base:lastModificationDate "2019-11-29T10:00:01.673298+02:00"^^xsd:dateTime ;
+    knora-base:deleteDate           "2020-04-07T14:59:28.960124+00:00"^^xsd:dateTime ;
+    rdfs:label                      "a thing" ;
+    knora-base:isDeleted            false .

webapi/src/main/scala/org/knora/webapi/package.scala

@@ -11,7 +11,7 @@ package object webapi {
    * The version of `knora-base` and of the other built-in ontologies that this version of Knora requires.
    * Must be the same as the object of `knora-base:ontologyVersion` in the `knora-base` ontology being used.
    */
-  val KnoraBaseVersion: String = "knora-base v21"
+  val KnoraBaseVersion: String = "knora-base v22"
 
   /**
    * `IRI` is a synonym for `String`, used to improve code readability.

webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala

@@ -51,7 +51,8 @@ object RepositoryUpdatePlan {
       PluginForKnoraBaseVersion(versionNumber = 13, plugin = new UpgradePluginPR1921(featureFactoryConfig, log)),
       PluginForKnoraBaseVersion(versionNumber = 14, plugin = new NoopPlugin), // PR 1992
       PluginForKnoraBaseVersion(versionNumber = 20, plugin = new UpgradePluginPR2018(featureFactoryConfig, log)),
-      PluginForKnoraBaseVersion(versionNumber = 21, plugin = new UpgradePluginPR2079(featureFactoryConfig, log))
+      PluginForKnoraBaseVersion(versionNumber = 21, plugin = new UpgradePluginPR2079(featureFactoryConfig, log)),
+      PluginForKnoraBaseVersion(versionNumber = 22, plugin = new UpgradePluginPR2081(featureFactoryConfig, log))
       // KEEP IT ON THE BOTTOM
       // From "versionNumber = 6" don't use prBasedVersionString!
     )

webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR2081.scala

@@ -0,0 +1,52 @@
+/*
+ * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.knora.webapi.store.triplestore.upgrade.plugins
+
+import com.typesafe.scalalogging.Logger
+import org.knora.webapi.feature.FeatureFactoryConfig
+import org.knora.webapi.messages.OntologyConstants
+import org.knora.webapi.messages.util.rdf._
+import org.knora.webapi.store.triplestore.upgrade.UpgradePlugin
+import java.time.Instant
+
+/**
+ * Transforms a repository for Knora PR 2081.
+ * Fixes wrong date serialisations (all `xsd:dateTime` in the database should end on `Z` rather than specifying a time zone).
+ */
+class UpgradePluginPR2081(featureFactoryConfig: FeatureFactoryConfig, log: Logger) extends UpgradePlugin {
+  private val nodeFactory: RdfNodeFactory = RdfFeatureFactory.getRdfNodeFactory(featureFactoryConfig)
+
+  override def transform(model: RdfModel): Unit = {
+    val statementsToRemove: collection.mutable.Set[Statement] = collection.mutable.Set.empty
+    val statementsToAdd: collection.mutable.Set[Statement]    = collection.mutable.Set.empty
+
+    val newObjectValue: String => DatatypeLiteral = (in: String) =>
+      nodeFactory.makeDatatypeLiteral(Instant.parse(in).toString, OntologyConstants.Xsd.DateTime)
+    val shouldTransform: DatatypeLiteral => Boolean = (literal: DatatypeLiteral) =>
+      (literal.datatype == OntologyConstants.Xsd.DateTime &&
+        literal.value != newObjectValue(literal.value).value)
+
+    for (statement: Statement <- model) {
+      statement.obj match {
+        case literal: DatatypeLiteral if shouldTransform(literal) =>
+          val newValue = newObjectValue(literal.value)
+          log.debug(s"Transformed ${literal.value} => ${newValue.value}")
+          statementsToRemove += statement
+          statementsToAdd += nodeFactory.makeStatement(
+            subj = statement.subj,
+            pred = statement.pred,
+            obj = newValue,
+            context = statement.context
+          )
+        case other => ()
+      }
+    }
+
+    model.removeStatements(statementsToRemove.toSet)
+    model.addStatements(statementsToAdd.toSet)
+  }
+
+}

webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/changeResourceMetadata.scala.txt

@@ -36,7 +36,7 @@ DELETE {
     GRAPH ?dataNamedGraph {
         @maybeLastModificationDate match {
             case Some(lastModificationDate) => {
-                ?resource knora-base:lastModificationDate "@lastModificationDate"^^xsd:dateTime .
+                ?resource knora-base:lastModificationDate ?lastModificationDate .
             }
 
             case None => {}
@@ -90,7 +90,7 @@ WHERE {
 
         @maybeLastModificationDate match {
             case Some(lastModificationDate) => {
-                ?resource knora-base:lastModificationDate "@lastModificationDate"^^xsd:dateTime .
+                ?resource knora-base:lastModificationDate ?lastModificationDate .
             }
 
             case None => {

webapi/src/test/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR2081Spec.scala

@@ -0,0 +1,76 @@
+/*
+ * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.knora.webapi.store.triplestore.upgrade.plugins
+
+import com.typesafe.scalalogging.LazyLogging
+import dsp.errors.AssertionException
+import org.knora.webapi.messages.OntologyConstants
+import org.knora.webapi.messages.util.rdf._
+
+class UpgradePluginPR2081Spec extends UpgradePluginSpec with LazyLogging {
+  private val nodeFactory: RdfNodeFactory = RdfFeatureFactory.getRdfNodeFactory(defaultFeatureFactoryConfig)
+
+  private def getDateValue(model: RdfModel, subj: IriNode, pred: IriNode): String = {
+    val statement = model.find(subj = Some(subj), pred = Some(pred), obj = None).toSet.head
+    statement.obj match {
+      case literal: DatatypeLiteral if literal.datatype == OntologyConstants.Xsd.DateTime =>
+        literal.value
+      case other => throw AssertionException(s"Unexpected object for $pred: $other")
+    }
+  }
+
+  "Upgrade plugin PR2081" should {
+
+    "fix invalid date serializations" in {
+      val resource1            = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9ma")
+      val resource2            = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mb")
+      val resource3            = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mc")
+      val creationDate         = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.CreationDate)
+      val lastModificationDate = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.LastModificationDate)
+      val deletionDate         = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.DeleteDate)
+
+      // Parse the input file.
+      val model: RdfModel = trigFileToModel("../test_data/upgrade/pr2081.trig")
+
+      // Store previous values
+      val resource1CreationDate         = getDateValue(model, resource1, creationDate)
+      val resource2CreationDate         = getDateValue(model, resource2, creationDate)
+      val resource2LastModificationDate = getDateValue(model, resource2, lastModificationDate)
+      val resource3CreationDate         = getDateValue(model, resource3, creationDate) // only this one should stay the same
+      val resource3LastModificationDate = getDateValue(model, resource3, lastModificationDate)
+      val resource3DeletionDate         = getDateValue(model, resource3, deletionDate)
+
+      // Use the plugin to transform the input.
+      val plugin = new UpgradePluginPR2081(defaultFeatureFactoryConfig, log)
+      plugin.transform(model)
+
+      // get the new values after transformation
+      val newResource1CreationDate         = getDateValue(model, resource1, creationDate)
+      val newResource2CreationDate         = getDateValue(model, resource2, creationDate)
+      val newResource2LastModificationDate = getDateValue(model, resource2, lastModificationDate)
+      val newResource3CreationDate =
+        getDateValue(model, resource3, creationDate) // only this one should have stayed the same
+      val newResource3LastModificationDate = getDateValue(model, resource3, lastModificationDate)
+      val newResource3DeletionDate         = getDateValue(model, resource3, deletionDate)
+
+      // Check that the dates were fixed.
+      newResource1CreationDate should not equal (resource1CreationDate)
+      newResource1CreationDate should endWith("Z")
+
+      newResource2CreationDate should not equal (resource2CreationDate)
+      newResource2CreationDate should endWith("Z")
+      newResource2LastModificationDate should not equal (resource2CreationDate)
+      newResource2LastModificationDate should endWith("Z")
+
+      newResource3CreationDate should equal(resource3CreationDate) // is equal!
+      newResource3CreationDate should endWith("Z")
+      newResource3LastModificationDate should not equal (resource3CreationDate)
+      newResource3LastModificationDate should endWith("Z")
+      newResource3DeletionDate should not equal (resource3CreationDate)
+      newResource3DeletionDate should endWith("Z")
+    }
+  }
+}