diff --git a/.gitignore b/.gitignore index 7369dd2801..009d44d69d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,7 @@ sipi/test *.bak *.rdb .sbtrc -*.trig +/*.trig dependencies.txt /client-test-data.zip diff --git a/Makefile b/Makefile index cee1b8e9ee..00d763a32f 100644 --- a/Makefile +++ b/Makefile @@ -273,7 +273,10 @@ init-db-test-from-prod: db_prod_dump.trig init-db-test-empty ## init local datab @curl -X POST -H "Content-Type: application/trig" --data-binary "@${CURRENT_DIR}/db_prod_dump.trig" -u "admin:test" "http://localhost:3030/knora-test" .PHONY: init-db-test-from-ls-test-server -init-db-test-from-ls-test-server: db_ls_test_server_dump.trig init-db-test-empty ## init local database with data from ls-test-server +init-db-test-from-ls-test-server: db_ls_test_server_dump.trig init-db-test-from-ls-test-server-trig-file ## init local database with data from ls-test-server + +.PHONY: init-db-test-from-ls-test-server-trig-file +init-db-test-from-ls-test-server-trig-file: init-db-test-empty ## init local database with data from a local ls-test-server dump @echo $@ @curl -X POST -H "Content-Type: application/sparql-update" -d "DROP ALL" -u "admin:test" "http://localhost:3030/knora-test" @curl -X POST -H "Content-Type: application/trig" --data-binary "@${CURRENT_DIR}/db_ls_test_server_dump.trig" -u "admin:test" "http://localhost:3030/knora-test" diff --git a/knora-ontologies/knora-base.ttl b/knora-ontologies/knora-base.ttl index 9bfa15ad61..94fe5047f2 100644 --- a/knora-ontologies/knora-base.ttl +++ b/knora-ontologies/knora-base.ttl @@ -19,7 +19,7 @@ rdf:type owl:Ontology ; rdfs:label "The Knora base ontology"@en ; :attachedToProject knora-admin:SystemProject ; - :ontologyVersion "knora-base v21" . + :ontologyVersion "knora-base v22" . ################################################################# diff --git a/test_data/upgrade/pr2081.trig b/test_data/upgrade/pr2081.trig new file mode 100644 index 0000000000..8a859e8fe8 --- /dev/null +++ b/test_data/upgrade/pr2081.trig @@ -0,0 +1,36 @@ +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +@prefix knora-base: . + +# resources with old date serialization + + + a knora-base:Resource ; + knora-base:attachedToUser ; + knora-base:attachedToProject ; + knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ; + knora-base:creationDate "2019-11-29T10:00:00.673298+00:00"^^xsd:dateTime ; + rdfs:label "a thing" ; + knora-base:isDeleted false . + + + a knora-base:Resource ; + knora-base:attachedToUser ; + knora-base:attachedToProject ; + knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ; + knora-base:creationDate "2018-08-07T13:27:18.518+00:00"^^xsd:dateTime ; + knora-base:lastModificationDate "2019-11-29T10:00:01.673298+00:00"^^xsd:dateTime ; + rdfs:label "a thing" ; + knora-base:isDeleted false . + + + a knora-base:Resource ; + knora-base:attachedToUser ; + knora-base:attachedToProject ; + knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ; + knora-base:creationDate "2018-08-07T13:27:18.518123Z"^^xsd:dateTime ; + knora-base:lastModificationDate "2019-11-29T10:00:01.673298+02:00"^^xsd:dateTime ; + knora-base:deleteDate "2020-04-07T14:59:28.960124+00:00"^^xsd:dateTime ; + rdfs:label "a thing" ; + knora-base:isDeleted false . diff --git a/webapi/src/main/scala/org/knora/webapi/package.scala b/webapi/src/main/scala/org/knora/webapi/package.scala index 3f9c2139e5..51ef10bd45 100644 --- a/webapi/src/main/scala/org/knora/webapi/package.scala +++ b/webapi/src/main/scala/org/knora/webapi/package.scala @@ -11,7 +11,7 @@ package object webapi { * The version of `knora-base` and of the other built-in ontologies that this version of Knora requires. * Must be the same as the object of `knora-base:ontologyVersion` in the `knora-base` ontology being used. */ - val KnoraBaseVersion: String = "knora-base v21" + val KnoraBaseVersion: String = "knora-base v22" /** * `IRI` is a synonym for `String`, used to improve code readability. diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala index 97188e90d1..205b1e7a69 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala @@ -51,7 +51,8 @@ object RepositoryUpdatePlan { PluginForKnoraBaseVersion(versionNumber = 13, plugin = new UpgradePluginPR1921(featureFactoryConfig, log)), PluginForKnoraBaseVersion(versionNumber = 14, plugin = new NoopPlugin), // PR 1992 PluginForKnoraBaseVersion(versionNumber = 20, plugin = new UpgradePluginPR2018(featureFactoryConfig, log)), - PluginForKnoraBaseVersion(versionNumber = 21, plugin = new UpgradePluginPR2079(featureFactoryConfig, log)) + PluginForKnoraBaseVersion(versionNumber = 21, plugin = new UpgradePluginPR2079(featureFactoryConfig, log)), + PluginForKnoraBaseVersion(versionNumber = 22, plugin = new UpgradePluginPR2081(featureFactoryConfig, log)) // KEEP IT ON THE BOTTOM // From "versionNumber = 6" don't use prBasedVersionString! ) diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR2081.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR2081.scala new file mode 100644 index 0000000000..aba224d5ea --- /dev/null +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR2081.scala @@ -0,0 +1,52 @@ +/* + * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.store.triplestore.upgrade.plugins + +import com.typesafe.scalalogging.Logger +import org.knora.webapi.feature.FeatureFactoryConfig +import org.knora.webapi.messages.OntologyConstants +import org.knora.webapi.messages.util.rdf._ +import org.knora.webapi.store.triplestore.upgrade.UpgradePlugin +import java.time.Instant + +/** + * Transforms a repository for Knora PR 2081. + * Fixes wrong date serialisations (all `xsd:dateTime` in the database should end on `Z` rather than specifying a time zone). + */ +class UpgradePluginPR2081(featureFactoryConfig: FeatureFactoryConfig, log: Logger) extends UpgradePlugin { + private val nodeFactory: RdfNodeFactory = RdfFeatureFactory.getRdfNodeFactory(featureFactoryConfig) + + override def transform(model: RdfModel): Unit = { + val statementsToRemove: collection.mutable.Set[Statement] = collection.mutable.Set.empty + val statementsToAdd: collection.mutable.Set[Statement] = collection.mutable.Set.empty + + val newObjectValue: String => DatatypeLiteral = (in: String) => + nodeFactory.makeDatatypeLiteral(Instant.parse(in).toString, OntologyConstants.Xsd.DateTime) + val shouldTransform: DatatypeLiteral => Boolean = (literal: DatatypeLiteral) => + (literal.datatype == OntologyConstants.Xsd.DateTime && + literal.value != newObjectValue(literal.value).value) + + for (statement: Statement <- model) { + statement.obj match { + case literal: DatatypeLiteral if shouldTransform(literal) => + val newValue = newObjectValue(literal.value) + log.debug(s"Transformed ${literal.value} => ${newValue.value}") + statementsToRemove += statement + statementsToAdd += nodeFactory.makeStatement( + subj = statement.subj, + pred = statement.pred, + obj = newValue, + context = statement.context + ) + case other => () + } + } + + model.removeStatements(statementsToRemove.toSet) + model.addStatements(statementsToAdd.toSet) + } + +} diff --git a/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/changeResourceMetadata.scala.txt b/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/changeResourceMetadata.scala.txt index f54d22fe68..8832b1de03 100644 --- a/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/changeResourceMetadata.scala.txt +++ b/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/changeResourceMetadata.scala.txt @@ -36,7 +36,7 @@ DELETE { GRAPH ?dataNamedGraph { @maybeLastModificationDate match { case Some(lastModificationDate) => { - ?resource knora-base:lastModificationDate "@lastModificationDate"^^xsd:dateTime . + ?resource knora-base:lastModificationDate ?lastModificationDate . } case None => {} @@ -90,7 +90,7 @@ WHERE { @maybeLastModificationDate match { case Some(lastModificationDate) => { - ?resource knora-base:lastModificationDate "@lastModificationDate"^^xsd:dateTime . + ?resource knora-base:lastModificationDate ?lastModificationDate . } case None => { diff --git a/webapi/src/test/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR2081Spec.scala b/webapi/src/test/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR2081Spec.scala new file mode 100644 index 0000000000..fc1c869cba --- /dev/null +++ b/webapi/src/test/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR2081Spec.scala @@ -0,0 +1,76 @@ +/* + * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.store.triplestore.upgrade.plugins + +import com.typesafe.scalalogging.LazyLogging +import dsp.errors.AssertionException +import org.knora.webapi.messages.OntologyConstants +import org.knora.webapi.messages.util.rdf._ + +class UpgradePluginPR2081Spec extends UpgradePluginSpec with LazyLogging { + private val nodeFactory: RdfNodeFactory = RdfFeatureFactory.getRdfNodeFactory(defaultFeatureFactoryConfig) + + private def getDateValue(model: RdfModel, subj: IriNode, pred: IriNode): String = { + val statement = model.find(subj = Some(subj), pred = Some(pred), obj = None).toSet.head + statement.obj match { + case literal: DatatypeLiteral if literal.datatype == OntologyConstants.Xsd.DateTime => + literal.value + case other => throw AssertionException(s"Unexpected object for $pred: $other") + } + } + + "Upgrade plugin PR2081" should { + + "fix invalid date serializations" in { + val resource1 = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9ma") + val resource2 = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mb") + val resource3 = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mc") + val creationDate = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.CreationDate) + val lastModificationDate = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.LastModificationDate) + val deletionDate = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.DeleteDate) + + // Parse the input file. + val model: RdfModel = trigFileToModel("../test_data/upgrade/pr2081.trig") + + // Store previous values + val resource1CreationDate = getDateValue(model, resource1, creationDate) + val resource2CreationDate = getDateValue(model, resource2, creationDate) + val resource2LastModificationDate = getDateValue(model, resource2, lastModificationDate) + val resource3CreationDate = getDateValue(model, resource3, creationDate) // only this one should stay the same + val resource3LastModificationDate = getDateValue(model, resource3, lastModificationDate) + val resource3DeletionDate = getDateValue(model, resource3, deletionDate) + + // Use the plugin to transform the input. + val plugin = new UpgradePluginPR2081(defaultFeatureFactoryConfig, log) + plugin.transform(model) + + // get the new values after transformation + val newResource1CreationDate = getDateValue(model, resource1, creationDate) + val newResource2CreationDate = getDateValue(model, resource2, creationDate) + val newResource2LastModificationDate = getDateValue(model, resource2, lastModificationDate) + val newResource3CreationDate = + getDateValue(model, resource3, creationDate) // only this one should have stayed the same + val newResource3LastModificationDate = getDateValue(model, resource3, lastModificationDate) + val newResource3DeletionDate = getDateValue(model, resource3, deletionDate) + + // Check that the dates were fixed. + newResource1CreationDate should not equal (resource1CreationDate) + newResource1CreationDate should endWith("Z") + + newResource2CreationDate should not equal (resource2CreationDate) + newResource2CreationDate should endWith("Z") + newResource2LastModificationDate should not equal (resource2CreationDate) + newResource2LastModificationDate should endWith("Z") + + newResource3CreationDate should equal(resource3CreationDate) // is equal! + newResource3CreationDate should endWith("Z") + newResource3LastModificationDate should not equal (resource3CreationDate) + newResource3LastModificationDate should endWith("Z") + newResource3DeletionDate should not equal (resource3CreationDate) + newResource3DeletionDate should endWith("Z") + } + } +}