Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
fix!: add upgrade plugin that fixes invalid date serialisations (#2081)
* feat: add plugin to upgrade plan

* write upgrade script

* change gitignore so that not all .trig files are excluded

* bump upgrade plugin version

* test data for upgrade plugin

* finish tests

* add make target for uploading the ls trig file without loading it first

* make label updating twirl template more robust

* fix comment

* chore: streamline ls-test make targets
  • Loading branch information
BalduinLandolt committed Jun 27, 2022
1 parent 09688f5 commit 3a0902e
Show file tree
Hide file tree
Showing 9 changed files with 175 additions and 7 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Expand Up @@ -36,7 +36,7 @@ sipi/test
*.bak
*.rdb
.sbtrc
*.trig
/*.trig

dependencies.txt
/client-test-data.zip
Expand Down
5 changes: 4 additions & 1 deletion Makefile
Expand Up @@ -273,7 +273,10 @@ init-db-test-from-prod: db_prod_dump.trig init-db-test-empty ## init local datab
@curl -X POST -H "Content-Type: application/trig" --data-binary "@${CURRENT_DIR}/db_prod_dump.trig" -u "admin:test" "http://localhost:3030/knora-test"

.PHONY: init-db-test-from-ls-test-server
init-db-test-from-ls-test-server: db_ls_test_server_dump.trig init-db-test-empty ## init local database with data from ls-test-server
init-db-test-from-ls-test-server: db_ls_test_server_dump.trig init-db-test-from-ls-test-server-trig-file ## init local database with data from ls-test-server

.PHONY: init-db-test-from-ls-test-server-trig-file
init-db-test-from-ls-test-server-trig-file: init-db-test-empty ## init local database with data from a local ls-test-server dump
@echo $@
@curl -X POST -H "Content-Type: application/sparql-update" -d "DROP ALL" -u "admin:test" "http://localhost:3030/knora-test"
@curl -X POST -H "Content-Type: application/trig" --data-binary "@${CURRENT_DIR}/db_ls_test_server_dump.trig" -u "admin:test" "http://localhost:3030/knora-test"
Expand Down
2 changes: 1 addition & 1 deletion knora-ontologies/knora-base.ttl
Expand Up @@ -19,7 +19,7 @@
rdf:type owl:Ontology ;
rdfs:label "The Knora base ontology"@en ;
:attachedToProject knora-admin:SystemProject ;
:ontologyVersion "knora-base v21" .
:ontologyVersion "knora-base v22" .


#################################################################
Expand Down
36 changes: 36 additions & 0 deletions test_data/upgrade/pr2081.trig
@@ -0,0 +1,36 @@
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
@prefix knora-base: <http://www.knora.org/ontology/knora-base#> .

# resources with old date serialization

<http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9ma>
a knora-base:Resource ;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ;
knora-base:creationDate "2019-11-29T10:00:00.673298+00:00"^^xsd:dateTime ;
rdfs:label "a thing" ;
knora-base:isDeleted false .

<http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mb>
a knora-base:Resource ;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ;
knora-base:creationDate "2018-08-07T13:27:18.518+00:00"^^xsd:dateTime ;
knora-base:lastModificationDate "2019-11-29T10:00:01.673298+00:00"^^xsd:dateTime ;
rdfs:label "a thing" ;
knora-base:isDeleted false .

<http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mc>
a knora-base:Resource ;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ;
knora-base:creationDate "2018-08-07T13:27:18.518123Z"^^xsd:dateTime ;
knora-base:lastModificationDate "2019-11-29T10:00:01.673298+02:00"^^xsd:dateTime ;
knora-base:deleteDate "2020-04-07T14:59:28.960124+00:00"^^xsd:dateTime ;
rdfs:label "a thing" ;
knora-base:isDeleted false .
2 changes: 1 addition & 1 deletion webapi/src/main/scala/org/knora/webapi/package.scala
Expand Up @@ -11,7 +11,7 @@ package object webapi {
* The version of `knora-base` and of the other built-in ontologies that this version of Knora requires.
* Must be the same as the object of `knora-base:ontologyVersion` in the `knora-base` ontology being used.
*/
val KnoraBaseVersion: String = "knora-base v21"
val KnoraBaseVersion: String = "knora-base v22"

/**
* `IRI` is a synonym for `String`, used to improve code readability.
Expand Down
Expand Up @@ -51,7 +51,8 @@ object RepositoryUpdatePlan {
PluginForKnoraBaseVersion(versionNumber = 13, plugin = new UpgradePluginPR1921(featureFactoryConfig, log)),
PluginForKnoraBaseVersion(versionNumber = 14, plugin = new NoopPlugin), // PR 1992
PluginForKnoraBaseVersion(versionNumber = 20, plugin = new UpgradePluginPR2018(featureFactoryConfig, log)),
PluginForKnoraBaseVersion(versionNumber = 21, plugin = new UpgradePluginPR2079(featureFactoryConfig, log))
PluginForKnoraBaseVersion(versionNumber = 21, plugin = new UpgradePluginPR2079(featureFactoryConfig, log)),
PluginForKnoraBaseVersion(versionNumber = 22, plugin = new UpgradePluginPR2081(featureFactoryConfig, log))
// KEEP IT ON THE BOTTOM
// From "versionNumber = 6" don't use prBasedVersionString!
)
Expand Down
@@ -0,0 +1,52 @@
/*
* Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
* SPDX-License-Identifier: Apache-2.0
*/

package org.knora.webapi.store.triplestore.upgrade.plugins

import com.typesafe.scalalogging.Logger
import org.knora.webapi.feature.FeatureFactoryConfig
import org.knora.webapi.messages.OntologyConstants
import org.knora.webapi.messages.util.rdf._
import org.knora.webapi.store.triplestore.upgrade.UpgradePlugin
import java.time.Instant

/**
* Transforms a repository for Knora PR 2081.
* Fixes wrong date serialisations (all `xsd:dateTime` in the database should end on `Z` rather than specifying a time zone).
*/
class UpgradePluginPR2081(featureFactoryConfig: FeatureFactoryConfig, log: Logger) extends UpgradePlugin {
private val nodeFactory: RdfNodeFactory = RdfFeatureFactory.getRdfNodeFactory(featureFactoryConfig)

override def transform(model: RdfModel): Unit = {
val statementsToRemove: collection.mutable.Set[Statement] = collection.mutable.Set.empty
val statementsToAdd: collection.mutable.Set[Statement] = collection.mutable.Set.empty

val newObjectValue: String => DatatypeLiteral = (in: String) =>
nodeFactory.makeDatatypeLiteral(Instant.parse(in).toString, OntologyConstants.Xsd.DateTime)
val shouldTransform: DatatypeLiteral => Boolean = (literal: DatatypeLiteral) =>
(literal.datatype == OntologyConstants.Xsd.DateTime &&
literal.value != newObjectValue(literal.value).value)

for (statement: Statement <- model) {
statement.obj match {
case literal: DatatypeLiteral if shouldTransform(literal) =>
val newValue = newObjectValue(literal.value)
log.debug(s"Transformed ${literal.value} => ${newValue.value}")
statementsToRemove += statement
statementsToAdd += nodeFactory.makeStatement(
subj = statement.subj,
pred = statement.pred,
obj = newValue,
context = statement.context
)
case other => ()
}
}

model.removeStatements(statementsToRemove.toSet)
model.addStatements(statementsToAdd.toSet)
}

}
Expand Up @@ -36,7 +36,7 @@ DELETE {
GRAPH ?dataNamedGraph {
@maybeLastModificationDate match {
case Some(lastModificationDate) => {
?resource knora-base:lastModificationDate "@lastModificationDate"^^xsd:dateTime .
?resource knora-base:lastModificationDate ?lastModificationDate .
}

case None => {}
Expand Down Expand Up @@ -90,7 +90,7 @@ WHERE {

@maybeLastModificationDate match {
case Some(lastModificationDate) => {
?resource knora-base:lastModificationDate "@lastModificationDate"^^xsd:dateTime .
?resource knora-base:lastModificationDate ?lastModificationDate .
}

case None => {
Expand Down
@@ -0,0 +1,76 @@
/*
* Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
* SPDX-License-Identifier: Apache-2.0
*/

package org.knora.webapi.store.triplestore.upgrade.plugins

import com.typesafe.scalalogging.LazyLogging
import dsp.errors.AssertionException
import org.knora.webapi.messages.OntologyConstants
import org.knora.webapi.messages.util.rdf._

class UpgradePluginPR2081Spec extends UpgradePluginSpec with LazyLogging {
private val nodeFactory: RdfNodeFactory = RdfFeatureFactory.getRdfNodeFactory(defaultFeatureFactoryConfig)

private def getDateValue(model: RdfModel, subj: IriNode, pred: IriNode): String = {
val statement = model.find(subj = Some(subj), pred = Some(pred), obj = None).toSet.head
statement.obj match {
case literal: DatatypeLiteral if literal.datatype == OntologyConstants.Xsd.DateTime =>
literal.value
case other => throw AssertionException(s"Unexpected object for $pred: $other")
}
}

"Upgrade plugin PR2081" should {

"fix invalid date serializations" in {
val resource1 = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9ma")
val resource2 = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mb")
val resource3 = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mc")
val creationDate = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.CreationDate)
val lastModificationDate = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.LastModificationDate)
val deletionDate = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.DeleteDate)

// Parse the input file.
val model: RdfModel = trigFileToModel("../test_data/upgrade/pr2081.trig")

// Store previous values
val resource1CreationDate = getDateValue(model, resource1, creationDate)
val resource2CreationDate = getDateValue(model, resource2, creationDate)
val resource2LastModificationDate = getDateValue(model, resource2, lastModificationDate)
val resource3CreationDate = getDateValue(model, resource3, creationDate) // only this one should stay the same
val resource3LastModificationDate = getDateValue(model, resource3, lastModificationDate)
val resource3DeletionDate = getDateValue(model, resource3, deletionDate)

// Use the plugin to transform the input.
val plugin = new UpgradePluginPR2081(defaultFeatureFactoryConfig, log)
plugin.transform(model)

// get the new values after transformation
val newResource1CreationDate = getDateValue(model, resource1, creationDate)
val newResource2CreationDate = getDateValue(model, resource2, creationDate)
val newResource2LastModificationDate = getDateValue(model, resource2, lastModificationDate)
val newResource3CreationDate =
getDateValue(model, resource3, creationDate) // only this one should have stayed the same
val newResource3LastModificationDate = getDateValue(model, resource3, lastModificationDate)
val newResource3DeletionDate = getDateValue(model, resource3, deletionDate)

// Check that the dates were fixed.
newResource1CreationDate should not equal (resource1CreationDate)
newResource1CreationDate should endWith("Z")

newResource2CreationDate should not equal (resource2CreationDate)
newResource2CreationDate should endWith("Z")
newResource2LastModificationDate should not equal (resource2CreationDate)
newResource2LastModificationDate should endWith("Z")

newResource3CreationDate should equal(resource3CreationDate) // is equal!
newResource3CreationDate should endWith("Z")
newResource3LastModificationDate should not equal (resource3CreationDate)
newResource3LastModificationDate should endWith("Z")
newResource3DeletionDate should not equal (resource3CreationDate)
newResource3DeletionDate should endWith("Z")
}
}
}

0 comments on commit 3a0902e

Please sign in to comment.