From 8541519c9a4120b4cfa96b3fea4e071956a2e0c4 Mon Sep 17 00:00:00 2001 From: Balduin Landolt <33053745+BalduinLandolt@users.noreply.github.com> Date: Wed, 25 May 2022 17:26:47 +0200 Subject: [PATCH] fix(cache): cache does not update correctly when an ontology is modified (DEV-939) (#2068) * tests: write test to reproduce the bug at hand (should fail for now) * feat: use new cacheUpdatedOntology method * fix: update subclasses in case of editing an ontology * refactor: make direct cache access private to enforce using a clean API * refactor: rename method * ensure to avoid race conditions in cache * docs: add docstrings * refactor: remove some code duplication * remove unused code * use correct cache method for class deletion * restore original makeOntologyCache method * fix typo Co-authored-by: Marcin Procyk Co-authored-by: irinaschubert Co-authored-by: Marcin Procyk --- .../responders/v2/OntologyResponderV2.scala | 198 ++-------- .../webapi/responders/v2/ontology/Cache.scala | 359 +++++++++++++++--- .../v2/ontology/Cardinalities.scala | 9 +- .../org/knora/webapi/AsyncCoreSpec.scala | 2 +- .../test/scala/org/knora/webapi/E2ESpec.scala | 6 + .../webapi/e2e/v2/OntologyV2R2RSpec.scala | 1 - .../e2e/v2/ResourcesRouteV2E2ESpec.scala | 129 +++++++ .../responders/v2/ontology/CacheSpec.scala | 9 +- 8 files changed, 498 insertions(+), 215 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v2/OntologyResponderV2.scala b/webapi/src/main/scala/org/knora/webapi/responders/v2/OntologyResponderV2.scala index 4823347d9c..7905e5d359 100644 --- a/webapi/src/main/scala/org/knora/webapi/responders/v2/OntologyResponderV2.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v2/OntologyResponderV2.scala @@ -20,6 +20,7 @@ import org.knora.webapi.messages.store.triplestoremessages.SmartIriLiteralV2 import org.knora.webapi.messages.store.triplestoremessages.SparqlUpdateRequest import org.knora.webapi.messages.store.triplestoremessages.SparqlUpdateResponse import org.knora.webapi.messages.store.triplestoremessages.StringLiteralV2 +import scala.concurrent.duration._ import org.knora.webapi.messages.util.ErrorHandlingMap import org.knora.webapi.messages.util.ResponderData import org.knora.webapi.messages.v2.responder.CanDoResponseV2 @@ -37,6 +38,10 @@ import org.knora.webapi.util._ import java.time.Instant import scala.concurrent.Future +import scala.concurrent.Await +import org.knora.webapi.messages.util.KnoraSystemInstances +import org.knora.webapi.feature.TestFeatureFactoryConfig +import org.knora.webapi.feature.KnoraSettingsFeatureFactoryConfig /** * Responds to requests dealing with ontologies. @@ -585,13 +590,10 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon // Update the ontology cache with the unescaped metadata. - _ = - Cache.storeCacheData( - cacheData.copy( - ontologies = - cacheData.ontologies + (internalOntologyIri -> ReadOntologyV2(ontologyMetadata = unescapedNewMetadata)) - ) - ) + _ <- Cache.cacheUpdatedOntologyWithoutUpdatingMaps( + internalOntologyIri, + ReadOntologyV2(ontologyMetadata = unescapedNewMetadata) + ) } yield ReadOntologyMetadataV2(ontologies = Set(unescapedNewMetadata)) @@ -741,14 +743,13 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon } // Update the ontology cache with the unescaped metadata. - - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> cacheData - .ontologies(internalOntologyIri) - .copy(ontologyMetadata = unescapedNewMetadata)) - ) - ) + updatedOntology = cacheData + .ontologies(internalOntologyIri) + .copy(ontologyMetadata = unescapedNewMetadata) + _ <- Cache.cacheUpdatedOntologyWithoutUpdatingMaps( + internalOntologyIri, + updatedOntology + ) } yield ReadOntologyMetadataV2(ontologies = Set(unescapedNewMetadata)) @@ -840,13 +841,10 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon // Update the ontology cache with the unescaped metadata. - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> cacheData - .ontologies(internalOntologyIri) - .copy(ontologyMetadata = unescapedNewMetadata)) - ) - ) + updatedOntology = cacheData + .ontologies(internalOntologyIri) + .copy(ontologyMetadata = unescapedNewMetadata) + _ <- Cache.cacheUpdatedOntologyWithoutUpdatingMaps(internalOntologyIri, updatedOntology) } yield ReadOntologyMetadataV2(ontologies = Set(unescapedNewMetadata)) @@ -1036,9 +1034,6 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon // Update the cache. - updatedSubClassOfRelations = cacheData.subClassOfRelations + (internalClassIri -> allBaseClassIris) - updatedSuperClassOfRelations = OntologyHelpers.calculateSuperClassOfRelations(updatedSubClassOfRelations) - updatedOntology = ontology.copy( ontologyMetadata = ontology.ontologyMetadata.copy( lastModificationDate = Some(currentTime) @@ -1046,13 +1041,7 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon classes = ontology.classes + (internalClassIri -> readClassInfo) ) - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology), - subClassOfRelations = updatedSubClassOfRelations, - superClassOfRelations = updatedSuperClassOfRelations - ) - ) + _ <- Cache.cacheUpdatedOntologyWithClass(internalOntologyIri, updatedOntology, internalClassIri) // Read the data back from the cache. @@ -1234,14 +1223,7 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon // Update subclasses and write the cache. - _ = Cache.storeCacheData( - Cache.updateSubClasses( - baseClassIri = internalClassIri, - cacheData = cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology) - ) - ) - ) + _ <- Cache.cacheUpdatedOntologyWithClass(internalOntologyIri, updatedOntology, internalClassIri) // Read the data back from the cache. @@ -1481,14 +1463,7 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon classes = ontology.classes + (internalClassIri -> readClassInfo) ) - _ = Cache.storeCacheData( - Cache.updateSubClasses( - baseClassIri = internalClassIri, - cacheData = cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology) - ) - ) - ) + _ <- Cache.cacheUpdatedOntologyWithClass(internalOntologyIri, updatedOntology, internalClassIri) // Read the data back from the cache. @@ -1739,11 +1714,7 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon classes = ontology.classes + (internalClassIri -> readClassInfo) ) - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology) - ) - ) + _ <- Cache.cacheUpdatedOntologyWithClass(internalOntologyIri, updatedOntology, internalClassIri) // Read the data back from the cache. @@ -1964,20 +1935,8 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon classes = ontology.classes - internalClassIri ) - updatedSubClassOfRelations = - (cacheData.subClassOfRelations - internalClassIri).map { case (subClass, baseClasses) => - subClass -> (baseClasses.toSet - internalClassIri).toSeq - } - - updatedSuperClassOfRelations = OntologyHelpers.calculateSuperClassOfRelations(updatedSubClassOfRelations) + _ <- Cache.cacheUpdatedOntology(internalOntologyIri, updatedOntology) - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology), - subClassOfRelations = updatedSubClassOfRelations, - superClassOfRelations = updatedSuperClassOfRelations - ) - ) } yield ReadOntologyMetadataV2(Set(updatedOntology.ontologyMetadata)) for { @@ -2141,24 +2100,16 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon propertiesToRemoveFromCache = Set(internalPropertyIri) ++ maybeInternalLinkValuePropertyIri - updatedOntology = ontology.copy( - ontologyMetadata = ontology.ontologyMetadata.copy( - lastModificationDate = Some(currentTime) - ), - properties = ontology.properties -- propertiesToRemoveFromCache - ) + updatedOntology = + ontology.copy( + ontologyMetadata = ontology.ontologyMetadata.copy( + lastModificationDate = Some(currentTime) + ), + properties = ontology.properties -- propertiesToRemoveFromCache + ) - updatedSubPropertyOfRelations = - (cacheData.subPropertyOfRelations -- propertiesToRemoveFromCache).map { case (subProperty, baseProperties) => - subProperty -> (baseProperties -- propertiesToRemoveFromCache) - } + _ <- Cache.cacheUpdatedOntology(internalOntologyIri, updatedOntology) - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology), - subPropertyOfRelations = updatedSubPropertyOfRelations - ) - ) } yield ReadOntologyMetadataV2(Set(updatedOntology.ontologyMetadata)) for { @@ -2274,35 +2225,7 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon } // Remove the ontology from the cache. - - updatedSubClassOfRelations = cacheData.subClassOfRelations.filterNot { case (subClass, _) => - subClass.getOntologyFromEntity == internalOntologyIri - }.map { case (subClass, baseClasses) => - subClass -> baseClasses.filterNot(_.getOntologyFromEntity == internalOntologyIri) - } - - updatedSuperClassOfRelations = OntologyHelpers.calculateSuperClassOfRelations(updatedSubClassOfRelations) - - updatedSubPropertyOfRelations = cacheData.subPropertyOfRelations.filterNot { case (subProperty, _) => - subProperty.getOntologyFromEntity == internalOntologyIri - }.map { case (subProperty, baseProperties) => - subProperty -> baseProperties.filterNot( - _.getOntologyFromEntity == internalOntologyIri - ) - } - - updatedStandoffProperties = - cacheData.standoffProperties.filterNot(_.getOntologyFromEntity == internalOntologyIri) - - updatedCacheData = cacheData.copy( - ontologies = cacheData.ontologies - internalOntologyIri, - subClassOfRelations = updatedSubClassOfRelations, - superClassOfRelations = updatedSuperClassOfRelations, - subPropertyOfRelations = updatedSubPropertyOfRelations, - standoffProperties = updatedStandoffProperties - ) - - _ = Cache.storeCacheData(updatedCacheData) + _ <- Cache.deleteOntology(internalOntologyIri) } yield SuccessResponseV2(s"Ontology ${internalOntologyIri.toOntologySchema(ApiV2Complex)} has been deleted") for { @@ -2644,28 +2567,7 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon ontology.properties ++ maybeLinkValuePropertyCacheEntry + (internalPropertyIri -> readPropertyInfo) ) - // if a link value property was created, add its subproperty relation to the ontology's subPropertyOfRelations map - // note: this is only needed for the special case of link properties that are subproperties of custom link properties - - maybeSubPropertyOfRelationsForLinkValueProperty: Option[(SmartIri, Set[SmartIri])] = - maybeLinkValuePropertyCacheEntry.map { case (smartIri: SmartIri, readPropertyInfoV2: ReadPropertyInfoV2) => - smartIri -> readPropertyInfoV2.entityInfoContent.subPropertyOf - } - - newSubPropertyOfRelations: Map[SmartIri, Set[SmartIri]] = - maybeSubPropertyOfRelationsForLinkValueProperty match { - case Some(smartIriSetOfSmartIris: (SmartIri, Set[SmartIri])) => - Map(internalPropertyIri -> allKnoraSuperPropertyIris, smartIriSetOfSmartIris) - case None => - Map(internalPropertyIri -> allKnoraSuperPropertyIris) - } - - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology), - subPropertyOfRelations = cacheData.subPropertyOfRelations ++ newSubPropertyOfRelations - ) - ) + _ <- Cache.cacheUpdatedOntology(internalOntologyIri, updatedOntology) // Read the data back from the cache. response <- getPropertyDefinitionsFromOntologyV2( @@ -2888,11 +2790,7 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon ontology.properties ++ maybeLinkValuePropertyCacheEntry + (internalPropertyIri -> newReadPropertyInfo) ) - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology) - ) - ) + _ <- Cache.cacheUpdatedOntology(internalOntologyIri, updatedOntology) // Read the data back from the cache. @@ -3096,11 +2994,7 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon ontology.properties ++ maybeLinkValuePropertyCacheEntry + (internalPropertyIri -> newReadPropertyInfo) ) - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology) - ) - ) + _ <- Cache.cacheUpdatedOntologyWithoutUpdatingMaps(internalOntologyIri, updatedOntology) // Read the data back from the cache. @@ -3238,11 +3132,7 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon classes = ontology.classes + (internalClassIri -> newReadClassInfo) ) - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology) - ) - ) + _ <- Cache.cacheUpdatedOntologyWithoutUpdatingMaps(internalOntologyIri, updatedOntology) // Read the data back from the cache. @@ -3436,11 +3326,7 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon ontology.properties ++ maybeLinkValuePropertyCacheEntry + (internalPropertyIri -> newReadPropertyInfo) ) - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology) - ) - ) + _ <- Cache.cacheUpdatedOntologyWithoutUpdatingMaps(internalOntologyIri, updatedOntology) // Read the data back from the cache. @@ -3595,11 +3481,7 @@ class OntologyResponderV2(responderData: ResponderData) extends Responder(respon classes = ontology.classes + (internalClassIri -> newReadClassInfo) ) - _ = Cache.storeCacheData( - cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology) - ) - ) + _ <- Cache.cacheUpdatedOntologyWithoutUpdatingMaps(internalOntologyIri, updatedOntology) // Read the data back from the cache. diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v2/ontology/Cache.scala b/webapi/src/main/scala/org/knora/webapi/responders/v2/ontology/Cache.scala index 69904d922b..c4722df626 100644 --- a/webapi/src/main/scala/org/knora/webapi/responders/v2/ontology/Cache.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v2/ontology/Cache.scala @@ -152,49 +152,47 @@ object Cache extends LazyLogging { } // Get the contents of each named graph containing an ontology. - ontologyGraphResponseFutures: Iterable[Future[OntologyGraph]] = allOntologyMetadata.keys.map { ontologyIri => - val ontology: OntologyMetadataV2 = - allOntologyMetadata.get(ontologyIri).get - val lastModificationDate: Option[Instant] = - ontology.lastModificationDate - val attachedToProject: Option[SmartIri] = - ontology.projectIri - - // throw an expception if ontology doesn't have lastModificationDate property and isn't attached to system project - lastModificationDate match { - case None => - attachedToProject match { - case Some(iri: SmartIri) => - if ( - iri != OntologyConstants.KnoraAdmin.SystemProject.toSmartIri - ) { - throw MissingLastModificationDateOntologyException( - s"Required property knora-base:lastModificationDate is missing in `$ontologyIri`" - ) - } - case _ => () - } - case _ => () - } - - val ontologyGraphConstructQuery = - org.knora.webapi.messages.twirl.queries.sparql.v2.txt - .getOntologyGraph( - ontologyGraph = ontologyIri - ) - .toString - - (storeManager ? SparqlExtendedConstructRequest( - sparql = ontologyGraphConstructQuery, - featureFactoryConfig = featureFactoryConfig - )).mapTo[SparqlExtendedConstructResponse].map { - response => - OntologyGraph( - ontologyIri = ontologyIri, - constructResponse = response - ) - } - } + ontologyGraphResponseFutures: Iterable[Future[OntologyGraph]] = + allOntologyMetadata.keys.map { ontologyIri => + val ontology: OntologyMetadataV2 = + allOntologyMetadata.get(ontologyIri).get + val lastModificationDate: Option[Instant] = + ontology.lastModificationDate + val attachedToProject: Option[SmartIri] = + ontology.projectIri + + // throw an expception if ontology doesn't have lastModificationDate property and isn't attached to system project + lastModificationDate match { + case None => + attachedToProject match { + case Some(iri: SmartIri) => + if (iri != OntologyConstants.KnoraAdmin.SystemProject.toSmartIri) { + throw MissingLastModificationDateOntologyException( + s"Required property knora-base:lastModificationDate is missing in `$ontologyIri`" + ) + } + case _ => () + } + case _ => () + } + + val ontologyGraphConstructQuery = + org.knora.webapi.messages.twirl.queries.sparql.v2.txt + .getOntologyGraph( + ontologyGraph = ontologyIri + ) + .toString + + (storeManager ? SparqlExtendedConstructRequest( + sparql = ontologyGraphConstructQuery, + featureFactoryConfig = featureFactoryConfig + )).mapTo[SparqlExtendedConstructResponse].map { response => + OntologyGraph( + ontologyIri = ontologyIri, + constructResponse = response + ) + } + } ontologyGraphs: Iterable[OntologyGraph] <- Future.sequence(ontologyGraphResponseFutures) @@ -214,6 +212,196 @@ object Cache extends LazyLogging { } } + /** + * Creates an [[OntologyCacheData]] object on the basis of a map of ontology IRIs to the corresponding [[ReadOntologyV2]]. + * + * @param ontologies a map of ontology IRIs to the corresponding [[ReadOntologyV2]] + * @return An [[OntologyCacheData]] object + */ + def make(ontologies: Map[SmartIri, ReadOntologyV2]): OntologyCacheData = { + implicit val sf: StringFormatter = StringFormatter.getGeneralInstance + + // A map of ontology IRIs to class IRIs in each ontology. + val classIrisPerOntology: Map[SmartIri, Set[SmartIri]] = ontologies.map { + case (iri, ontology) => { + val classIris = ontology.classes.values.map { case classInfo: ReadClassInfoV2 => + classInfo.entityInfoContent.classIri + }.toSet + (iri -> classIris) + } + } + + // A map of ontology IRIs to property IRIs in each ontology. + val propertyIrisPerOntology: Map[SmartIri, Set[SmartIri]] = ontologies.map { + case (iri, ontology) => { + val propertyIris = ontology.properties.values.map { case propertyInfo: ReadPropertyInfoV2 => + propertyInfo.entityInfoContent.propertyIri + }.toSet + (iri -> propertyIris) + } + } + + // A map of OWL named individual IRIs to named individuals. + val allIndividuals = ontologies.flatMap { + case (_, ontology) => { + ontology.individuals.map { + case (individualIri, readIndividual) => { + individualIri -> readIndividual.entityInfoContent + } + } + } + } + // A map of salsah-gui:GuiElement individuals to their GUI attribute definitions. + val allGuiAttributeDefinitions: Map[SmartIri, Set[SalsahGuiAttributeDefinition]] = + OntologyHelpers.makeGuiAttributeDefinitions(allIndividuals) + + // Construct entity definitions. + + val readClassInfos: Map[SmartIri, ReadClassInfoV2] = ontologies.flatMap { case (_, ontology) => + ontology.classes + } + val readPropertyInfos: Map[SmartIri, ReadPropertyInfoV2] = ontologies.flatMap { case (_, ontology) => + ontology.properties + } + // A map of class IRIs to class definitions. + val allClassDefs: Map[SmartIri, ClassInfoContentV2] = readClassInfos.map { case (iri, classInfo) => + iri -> classInfo.entityInfoContent + } + // A map of property IRIs to property definitions. + val allPropertyDefs: Map[SmartIri, PropertyInfoContentV2] = readPropertyInfos.map { case (iri, propertyInfo) => + iri -> propertyInfo.entityInfoContent + } + + // Determine relations between entities. + + // A map of class IRIs to their immediate super classes. + val directSubClassOfRelations: Map[SmartIri, Set[SmartIri]] = allClassDefs.map { case (classIri, classDef) => + classIri -> classDef.subClassOf + } + + // A map of property IRIs to their immediate super properties. + val directSubPropertyOfRelations: Map[SmartIri, Set[SmartIri]] = allPropertyDefs.map { + case (propertyIri, propertyDef) => propertyIri -> propertyDef.subPropertyOf + } + + val allClassIris = readClassInfos.keySet + val allPropertyIris = readPropertyInfos.keySet + + // A map in which each class IRI points to the full sequence of its super classes. + val allSubClassOfRelations: Map[SmartIri, Seq[SmartIri]] = allClassIris.toSeq.map { classIri => + // get the hierarchically ordered super classes. + val superClasses: Seq[SmartIri] = OntologyUtil.getAllBaseDefs(classIri, directSubClassOfRelations) + // prepend the classIri to the sequence of super classes because a class is also a subclass of itself. + (classIri, classIri +: superClasses) + }.toMap + + // Make a map in which each property IRI points to the full set of its base properties. A property is also a subproperty of itself. + val allSubPropertyOfRelations: Map[SmartIri, Set[SmartIri]] = allPropertyIris.map { propertyIri => + (propertyIri, OntologyUtil.getAllBaseDefs(propertyIri, directSubPropertyOfRelations).toSet + propertyIri) + }.toMap + + // A map in which each class IRI points to the full set of its subclasses. A class is also a subclass of itself. + val allSuperClassOfRelations: Map[SmartIri, Set[SmartIri]] = + OntologyHelpers.calculateSuperClassOfRelations(allSubClassOfRelations) + + // Make a map in which each property IRI points to the full set of its subproperties. A property is also a superproperty of itself. + val allSuperPropertyOfRelations: Map[SmartIri, Set[SmartIri]] = + OntologyHelpers.calculateSuperPropertiesOfRelations(allSubPropertyOfRelations) + + // A set of the IRIs of all properties used in cardinalities in standoff classes. + val propertiesUsedInStandoffCardinalities: Set[SmartIri] = readClassInfos.flatMap { case (_, readClassInfo) => + if (readClassInfo.isStandoffClass) { + readClassInfo.allCardinalities.keySet + } else { + Set.empty[SmartIri] + } + }.toSet + + // A set of the IRIs of all properties whose subject class constraint is a standoff class. + val propertiesWithStandoffTagSubjects: Set[SmartIri] = readPropertyInfos.flatMap { + case (propertyIri, readPropertyInfo) => + readPropertyInfo.entityInfoContent.getPredicateIriObject( + OntologyConstants.KnoraBase.SubjectClassConstraint.toSmartIri + ) match { + case Some(subjectClassConstraint: SmartIri) => + readClassInfos.get(subjectClassConstraint) match { + case Some(subjectReadClassInfo: ReadClassInfoV2) => + if (subjectReadClassInfo.isStandoffClass) { + Some(propertyIri) + } else { + None + } + + case None => None + } + + case None => None + } + }.toSet + + val classDefinedInOntology = classIrisPerOntology.flatMap { case (ontoIri, classIris) => + classIris.map(_ -> ontoIri) + } + val propertyDefinedInOntology = propertyIrisPerOntology.flatMap { case (ontoIri, propertyIris) => + propertyIris.map(_ -> ontoIri) + } + + val ontologiesErrorMap = + new ErrorHandlingMap[SmartIri, ReadOntologyV2](ontologies, key => s"Ontology not found in ontologies: $key") + val subClassOfRelationsErrorMap = + new ErrorHandlingMap[SmartIri, Seq[SmartIri]]( + allSubClassOfRelations, + key => s"Class not found in subClassOfRelations: $key" + ) + val superClassOfRelationsErrorMap = + new ErrorHandlingMap[SmartIri, Set[SmartIri]]( + allSuperClassOfRelations, + key => s"Class not found in superClassOfRelations: $key" + ) + val subPropertyOfRelationsErrorMap = + new ErrorHandlingMap[SmartIri, Set[SmartIri]]( + allSubPropertyOfRelations, + key => s"Property not found in allSubPropertyOfRelations: $key" + ) + val superPropertyOfRelationsErrorMap = + new ErrorHandlingMap[SmartIri, Set[SmartIri]]( + allSuperPropertyOfRelations, + key => s"Property not found in allSuperPropertyOfRelations: $key" + ) + val classDefinedInOntologyErrorMap = + new ErrorHandlingMap[SmartIri, SmartIri]( + classDefinedInOntology, + key => s"Class not found classDefinedInOntology: $key" + ) + val propertyDefinedInOntologyErrorMap = + new ErrorHandlingMap[SmartIri, SmartIri]( + propertyDefinedInOntology, + key => s"Property not found in propertyDefinedInOntology: $key" + ) + val entityDefinedInOntologyErrorMap = new ErrorHandlingMap[SmartIri, SmartIri]( + propertyDefinedInOntology ++ classDefinedInOntology, + key => s"Property not found in propertyDefinedInOntology: $key" + ) + val guiAttributeDefinitions = new ErrorHandlingMap[SmartIri, Set[SalsahGuiAttributeDefinition]]( + allGuiAttributeDefinitions, + key => s"salsah-gui:Guielement not found in allGuiAttributeDefinitions: $key" + ) + val standoffProperties = propertiesUsedInStandoffCardinalities ++ propertiesWithStandoffTagSubjects + + OntologyCacheData( + ontologies = ontologiesErrorMap, + subClassOfRelations = subClassOfRelationsErrorMap, + superClassOfRelations = superClassOfRelationsErrorMap, + subPropertyOfRelations = subPropertyOfRelationsErrorMap, + superPropertyOfRelations = superPropertyOfRelationsErrorMap, + classDefinedInOntology = classDefinedInOntologyErrorMap, + propertyDefinedInOntology = propertyDefinedInOntologyErrorMap, + entityDefinedInOntology = entityDefinedInOntologyErrorMap, + guiAttributeDefinitions = guiAttributeDefinitions, + standoffProperties = standoffProperties + ) + } + /** * Given ontology metdata and ontology graphs read from the triplestore, constructs the ontology cache. * @@ -779,7 +967,7 @@ object Cache extends LazyLogging { * * @param cacheData the updated data to be cached. */ - def storeCacheData(cacheData: OntologyCacheData): Unit = + private def storeCacheData(cacheData: OntologyCacheData): Unit = CacheUtil.put(cacheName = OntologyCacheName, key = OntologyCacheKey, value = cacheData) /** @@ -870,4 +1058,87 @@ object Cache extends LazyLogging { } } + /** + * Updates an existing ontology in the cache and ensures that the sub- and superclasses of a (presumably changed) class get updated correctly. + * + * @param updatedOntologyIri the IRI of the updated ontology + * @param updatedOntologyData the [[ReadOntologyV2]] representation of the updated ontology + * @param updatedClassIri the IRI of the changed class + * @return the updated cache data + */ + def cacheUpdatedOntologyWithClass( + updatedOntologyIri: SmartIri, + updatedOntologyData: ReadOntologyV2, + updatedClassIri: SmartIri + )(implicit + ec: ExecutionContext + ): Future[OntologyCacheData] = + for { + ontologyCache <- getCacheData + newOntologies = ontologyCache.ontologies + (updatedOntologyIri -> updatedOntologyData) + newOntologyCacheData = make(newOntologies) + updatedCacheData = updateSubClasses(updatedClassIri, newOntologyCacheData) + _ = storeCacheData(updatedCacheData) + updatedOntologyCache <- getCacheData + } yield updatedOntologyCache + + /** + * Updates an existing ontology in the cache. If a class has changed, use `cacheUpdatedOntologyWithClass()`. + * + * @param updatedOntologyIri the IRI of the updated ontology + * @param updatedOntologyData the [[ReadOntologyV2]] representation of the updated ontology + * @return the updated cache data + */ + def cacheUpdatedOntology( + updatedOntologyIri: SmartIri, + updatedOntologyData: ReadOntologyV2 + )(implicit + ec: ExecutionContext + ): Future[OntologyCacheData] = + for { + ontologyCache <- getCacheData + newOntologies = ontologyCache.ontologies + (updatedOntologyIri -> updatedOntologyData) + newOntologyCacheData = make(newOntologies) + _ = storeCacheData(newOntologyCacheData) + updatedOntologyCache <- getCacheData + } yield updatedOntologyCache + + /** + * Updates an existing ontology in the cache without updating the cache lookup maps. This should only be used if only the ontology metadata has changed. + * + * @param updatedOntologyIri the IRI of the updated ontology + * @param updatedOntologyData the [[ReadOntologyV2]] representation of the updated ontology + * @return the updated cache data + */ + def cacheUpdatedOntologyWithoutUpdatingMaps( + updatedOntologyIri: SmartIri, + updatedOntologyData: ReadOntologyV2 + )(implicit + ec: ExecutionContext + ): Future[OntologyCacheData] = + for { + ontologyCache <- getCacheData + newOntologies = ontologyCache.ontologies + (updatedOntologyIri -> updatedOntologyData) + updatedCacheData = ontologyCache.copy(ontologies = newOntologies) + _ = storeCacheData(updatedCacheData) + updatedOntologyCache <- getCacheData + } yield updatedOntologyCache + + /** + * Deletes an ontology from the cache. + * + * @param updatedOntologyIri the IRI of the ontology to delete + * @return the updated cache data + */ + def deleteOntology(ontologyIri: SmartIri)(implicit + ec: ExecutionContext + ): Future[OntologyCacheData] = + for { + ontologyCache <- getCacheData + newOntologies = ontologyCache.ontologies - ontologyIri + newOntologyCacheData = make(newOntologies) + _ = storeCacheData(newOntologyCacheData) + updatedOntologyCache <- getCacheData + } yield updatedOntologyCache + } diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v2/ontology/Cardinalities.scala b/webapi/src/main/scala/org/knora/webapi/responders/v2/ontology/Cardinalities.scala index 96958e6f6c..b8e5877924 100644 --- a/webapi/src/main/scala/org/knora/webapi/responders/v2/ontology/Cardinalities.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v2/ontology/Cardinalities.scala @@ -434,14 +434,7 @@ object Cardinalities { classes = ontology.classes + (internalClassIri -> readClassInfo) ) - _ = Cache.storeCacheData( - Cache.updateSubClasses( - baseClassIri = internalClassIri, - cacheData = cacheData.copy( - ontologies = cacheData.ontologies + (internalOntologyIri -> updatedOntology) - ) - ) - ) + _ <- Cache.cacheUpdatedOntologyWithClass(internalOntologyIri, updatedOntology, internalClassIri) // Read the data back from the cache. diff --git a/webapi/src/test/scala/org/knora/webapi/AsyncCoreSpec.scala b/webapi/src/test/scala/org/knora/webapi/AsyncCoreSpec.scala index 08027f4051..f8417237d1 100644 --- a/webapi/src/test/scala/org/knora/webapi/AsyncCoreSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/AsyncCoreSpec.scala @@ -202,7 +202,7 @@ abstract class AsyncCoreSpec(_system: ActorSystem) case Failure(e) => logger.error(s"Loading test data failed: ${e.getMessage}") } - logger.info("Loading load ontologies into cache started ...") + logger.info("Loading ontologies into cache started ...") Try( Await.result( appActor ? LoadOntologiesRequestV2( diff --git a/webapi/src/test/scala/org/knora/webapi/E2ESpec.scala b/webapi/src/test/scala/org/knora/webapi/E2ESpec.scala index e23feb2d10..c0637666d4 100644 --- a/webapi/src/test/scala/org/knora/webapi/E2ESpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/E2ESpec.scala @@ -32,6 +32,7 @@ import org.knora.webapi.messages.store.sipimessages.SipiUploadResponse import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject import org.knora.webapi.messages.store.triplestoremessages.TriplestoreJsonProtocol import org.knora.webapi.messages.util.rdf._ +import org.knora.webapi.routing.KnoraRouteData import org.knora.webapi.settings._ import org.knora.webapi.store.cacheservice.CacheServiceManager import org.knora.webapi.store.cacheservice.impl.CacheServiceInMemImpl @@ -313,4 +314,9 @@ class E2ESpec(_system: ActorSystem) } } } + + val routeData: KnoraRouteData = KnoraRouteData( + system = system, + appActor = appActor + ) } diff --git a/webapi/src/test/scala/org/knora/webapi/e2e/v2/OntologyV2R2RSpec.scala b/webapi/src/test/scala/org/knora/webapi/e2e/v2/OntologyV2R2RSpec.scala index 96d3de9396..e5f98c3286 100644 --- a/webapi/src/test/scala/org/knora/webapi/e2e/v2/OntologyV2R2RSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/e2e/v2/OntologyV2R2RSpec.scala @@ -1122,7 +1122,6 @@ class OntologyV2R2RSpec extends R2RSpec { val responseFromJsonLD: InputOntologyV2 = InputOntologyV2.fromJsonLD(responseJsonDoc, parsingMode = TestResponseParsingModeV2).unescape - println(responseFromJsonLD) responseFromJsonLD.classes.head._2.predicates.toSet should ===( expectedResponseToCompare.classes.head._2.predicates.toSet ) diff --git a/webapi/src/test/scala/org/knora/webapi/e2e/v2/ResourcesRouteV2E2ESpec.scala b/webapi/src/test/scala/org/knora/webapi/e2e/v2/ResourcesRouteV2E2ESpec.scala index 053d4de988..abde33c856 100644 --- a/webapi/src/test/scala/org/knora/webapi/e2e/v2/ResourcesRouteV2E2ESpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/e2e/v2/ResourcesRouteV2E2ESpec.scala @@ -23,13 +23,17 @@ import org.knora.webapi.e2e.TestDataFileContent import org.knora.webapi.e2e.TestDataFilePath import org.knora.webapi.e2e.v2.ResponseCheckerV2._ import org.knora.webapi.exceptions.AssertionException +import org.knora.webapi.http.directives.DSPApiDirectives import org.knora.webapi.messages.IriConversions._ import org.knora.webapi.messages.OntologyConstants import org.knora.webapi.messages.StringFormatter import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject import org.knora.webapi.messages.util._ import org.knora.webapi.messages.util.rdf._ +import org.knora.webapi.messages.v2.responder.ontologymessages.InputOntologyV2 import org.knora.webapi.routing.RouteUtilV2 +import org.knora.webapi.routing.v2.OntologiesRouteV2 +import org.knora.webapi.sharedtestdata.SharedOntologyTestDataADM import org.knora.webapi.sharedtestdata.SharedTestDataADM import org.knora.webapi.util._ import org.xmlunit.builder.DiffBuilder @@ -2272,6 +2276,131 @@ class ResourcesRouteV2E2ESpec extends E2ESpec(ResourcesRouteV2E2ESpec.config) { assert(responseJson == expectedJson) } + + "correctly update the ontology cache when adding a resource, so that the resource can afterwards be found by gravsearch" in { + var freetestLastModDate: Instant = Instant.parse("2012-12-12T12:12:12.12Z") + val ontologiesPath = DSPApiDirectives.handleErrors(system)(new OntologiesRouteV2(routeData).knoraApiPath) + val auth = BasicHttpCredentials(SharedTestDataADM.anythingAdminUser.email, SharedTestDataADM.testPass) + + // create a new resource class and add a property with cardinality to it + val createResourceClass = + s"""{ + | "@id" : "${SharedOntologyTestDataADM.FREETEST_ONTOLOGY_IRI_LocalHost}", + | "@type" : "owl:Ontology", + | "knora-api:lastModificationDate" : { + | "@type" : "xsd:dateTimeStamp", + | "@value" : "$freetestLastModDate" + | }, + | "@graph" : [ { + | "@id" : "freetest:NewClass", + | "@type" : "owl:Class", + | "rdfs:label" : { + | "@language" : "en", + | "@value" : "New resource class" + | }, + | "rdfs:subClassOf" : [ + | { + | "@id": "knora-api:Resource" + | }, + | { + | "@type": "http://www.w3.org/2002/07/owl#Restriction", + | "owl:maxCardinality": 1, + | "owl:onProperty": { + | "@id": "freetest:hasName" + | }, + | "salsah-gui:guiOrder": 1 + | } + | ] + | } ], + | "@context" : { + | "rdf" : "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + | "knora-api" : "http://api.knora.org/ontology/knora-api/v2#", + | "salsah-gui" : "http://api.knora.org/ontology/salsah-gui/v2#", + | "owl" : "http://www.w3.org/2002/07/owl#", + | "rdfs" : "http://www.w3.org/2000/01/rdf-schema#", + | "xsd" : "http://www.w3.org/2001/XMLSchema#", + | "freetest" : "http://0.0.0.0:3333/ontology/0001/freetest/v2#" + | } + |}""".stripMargin + + val createResourceClassRequest = Post( + s"$baseApiUrl/v2/ontologies/classes", + HttpEntity(RdfMediaTypes.`application/ld+json`, createResourceClass) + ) ~> addCredentials(auth) + val createResourceClassResponse: HttpResponse = singleAwaitingRequest(createResourceClassRequest) + + assert(createResourceClassResponse.status == StatusCodes.OK, createResourceClassResponse.toString) + val responseJsonDoc: JsonLDDocument = responseToJsonLDDocument(createResourceClassResponse) + + // create an instance of the class + val createResourceWithValues: String = + """{ + | "@type" : "freetest:NewClass", + | "freetest:hasName" : { + | "@type" : "knora-api:TextValue", + | "knora-api:valueAsString" : "The new text value" + | }, + | "knora-api:attachedToProject" : { + | "@id" : "http://rdfh.ch/projects/0001" + | }, + | "rdfs:label" : "test resource instance", + | "@context" : { + | "rdf" : "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + | "knora-api" : "http://api.knora.org/ontology/knora-api/v2#", + | "rdfs" : "http://www.w3.org/2000/01/rdf-schema#", + | "xsd" : "http://www.w3.org/2001/XMLSchema#", + | "freetest" : "http://0.0.0.0:3333/ontology/0001/freetest/v2#" + | } + |}""".stripMargin + + val resourceRequest = Post( + s"$baseApiUrl/v2/resources", + HttpEntity(RdfMediaTypes.`application/ld+json`, createResourceWithValues) + ) ~> addCredentials(auth) + val resourceResponse: HttpResponse = singleAwaitingRequest(resourceRequest) + + assert(resourceResponse.status == StatusCodes.OK, resourceResponse.toString) + val resourceIri: IRI = + responseToJsonLDDocument(resourceResponse).body + .requireStringWithValidation(JsonLDKeywords.ID, stringFormatter.validateAndEscapeIri) + + // get resource back + val resourceComplexGetRequest = Get( + s"$baseApiUrl/v2/resources/${URLEncoder.encode(resourceIri, "UTF-8")}" + ) ~> addCredentials(auth) + val resourceComplexGetResponse: HttpResponse = singleAwaitingRequest(resourceComplexGetRequest) + + val valueObject = responseToJsonLDDocument(resourceComplexGetResponse).body + .requireObject("http://0.0.0.0:3333/ontology/0001/freetest/v2#hasName") + val valueIri: IRI = valueObject.requireString("@id") + val valueAsString: IRI = valueObject.requireString("http://api.knora.org/ontology/knora-api/v2#valueAsString") + + assert(valueAsString == "The new text value") + + // try to edit the value which requires the class to be properly cached + val editValue = + s"""{ + | "@id" : "$resourceIri", + | "@type" : "freetest:NewClass", + | "freetest:hasName" : { + | "@id" : "$valueIri", + | "@type" : "knora-api:TextValue", + | "knora-api:valueAsString" : "changed value" + | }, + | "@context" : { + | "knora-api" : "http://api.knora.org/ontology/knora-api/v2#", + | "freetest" : "http://0.0.0.0:3333/ontology/0001/freetest/v2#" + | } + |}""".stripMargin + + val editValueRequest = + Put(baseApiUrl + "/v2/values", HttpEntity(RdfMediaTypes.`application/ld+json`, editValue)) ~> addCredentials( + auth + ) + val editValueResponse: HttpResponse = singleAwaitingRequest(editValueRequest) + val editValueResponseDoc = responseToJsonLDDocument(editValueResponse) + assert(editValueResponse.status == StatusCodes.OK, responseToString(editValueResponse)) + } } } diff --git a/webapi/src/test/scala/org/knora/webapi/responders/v2/ontology/CacheSpec.scala b/webapi/src/test/scala/org/knora/webapi/responders/v2/ontology/CacheSpec.scala index 0430a87d9e..dd0630d1db 100644 --- a/webapi/src/test/scala/org/knora/webapi/responders/v2/ontology/CacheSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/responders/v2/ontology/CacheSpec.scala @@ -115,7 +115,8 @@ class CacheSpec extends IntegrationSpec(TestContainerFuseki.PortConfig) { val newCacheData = previousCacheData.copy( ontologies = previousCacheData.ontologies + (iri -> newBooks) ) - Cache.storeCacheData(newCacheData) + val updatedCacheFuture = Cache.cacheUpdatedOntologyWithoutUpdatingMaps(iri, newBooks) + Await.ready(updatedCacheFuture, 1.second) // read back the cache val newCachedCacheDataFuture = for { @@ -210,7 +211,8 @@ class CacheSpec extends IntegrationSpec(TestContainerFuseki.PortConfig) { val newCacheData = previousCacheData.copy( ontologies = previousCacheData.ontologies + (iri -> newBooks) ) - Cache.storeCacheData(newCacheData) + val updatedCacheFuture = Cache.cacheUpdatedOntologyWithoutUpdatingMaps(iri, newBooks) + Await.ready(updatedCacheFuture, 1.second) // read back the cache val newCachedCacheDataFuture = for { @@ -306,7 +308,8 @@ class CacheSpec extends IntegrationSpec(TestContainerFuseki.PortConfig) { val newCacheData = previousCacheData.copy( ontologies = previousCacheData.ontologies + (ontologyIri -> newBooks) ) - Cache.storeCacheData(newCacheData) + val updatedCacheFuture = Cache.cacheUpdatedOntologyWithoutUpdatingMaps(ontologyIri, newBooks) + Await.ready(updatedCacheFuture, 1.second) // read back the cache val newCachedCacheData = Await.result(Cache.getCacheData, 2 seconds)