From cc3b06c8638838ead6ea5753d8898a31e4fb1c40 Mon Sep 17 00:00:00 2001 From: irinaschubert Date: Thu, 21 Oct 2021 10:46:33 +0200 Subject: [PATCH] fix(list): find list labels in full-text search (#1922) * add e2e test * fix full-text search for list values * reactivate tests that were commented out * update query to find list values * add rosetta files generated by sipi to gitignore --- .gitignore | 2 + test_data/all_data/books-data.ttl | 95 +++++++++++++++---- test_data/ontologies/books-onto.ttl | 23 ++--- .../v2/searchFulltextStandard.scala.txt | 29 +++++- .../scala/org/knora/webapi/CoreSpec.scala | 2 +- .../responders/v2/SearchResponderV2Spec.scala | 44 +++++++-- .../v2/SearchResponderV2SpecFullData.scala | 38 ++++++++ 7 files changed, 189 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index e75f10cb9f..2a3849bc95 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,5 @@ dependencies.txt /.vscode /cleandeps.sh /.metals +/sipi/images/082E/* +/sipi/images/originals/082E/* diff --git a/test_data/all_data/books-data.ttl b/test_data/all_data/books-data.ttl index 4c044db029..0e643fab05 100644 --- a/test_data/all_data/books-data.ttl +++ b/test_data/all_data/books-data.ttl @@ -12,46 +12,105 @@ a knora-base:TextValue ; - knora-base:valueHasUUID "OTAzYmIzOTktYzA4Ni00NmM0LWFlZGItYzIyNzcyN2FkNjFk"^^xsd:string ; + knora-base:valueHasUUID "SZyeLLmOTcCCuS3B0VksHQ"^^xsd:string ; knora-base:isDeleted false ; knora-base:valueCreationDate "2018-05-28T15:52:03.897Z"^^xsd:dateTime ; knora-base:valueHasOrder 0 ; knora-base:valueHasString "Handbuch Projektmanagement" ; - knora-base:hasPermissions - "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ; + knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ; knora-base:attachedToUser . - - - a books:Book ; - knora-base:attachedToUser ; - knora-base:attachedToProject ; - knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ; - knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ; - books:hasTitle ; - rdfs:label "instance of a book" ; - knora-base:isDeleted false . - + + a knora-base:TextValue ; + knora-base:valueHasUUID "IN4R19yYR0ygi3K2VEHpUQ"^^xsd:string ; + knora-base:isDeleted false ; + knora-base:valueCreationDate "2018-05-29T16:42:04.381Z"^^xsd:dateTime ; + knora-base:valueHasOrder 0 ; + knora-base:valueHasString "A book with a list value" ; + knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ; + knora-base:attachedToUser . a knora-base:IntValue ; - knora-base:valueHasUUID "MmI4YTg1YTctZjM4Yy00MzE2LTkwNTgtYzdjODk1NjcwZmFm"^^xsd:string ; + knora-base:valueHasUUID "SZyeLLmOTcCCuS3B0VksHQ"^^xsd:string ; knora-base:isDeleted false ; knora-base:valueCreationDate "2018-05-28T15:52:03.897Z"^^xsd:dateTime ; knora-base:valueHasInteger 1 ; knora-base:valueHasOrder 0 ; knora-base:valueHasString "1" ; - knora-base:hasPermissions - "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ; + knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ; knora-base:attachedToUser . + a knora-base:ListValue; + knora-base:valueHasUUID "list_value"^^xsd:string; + knora-base:isDeleted false; + knora-base:attachedToUser ; + knora-base:valueHasString "http://rdfh.ch/lists/0001/ynm02-03"; + knora-base:valueHasOrder 0; + knora-base:valueHasListNode ; + knora-base:valueCreationDate "2018-05-29T16:42:04.381Z"^^xsd:dateTime; + knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" . + + + a books:Book ; + knora-base:attachedToUser ; + knora-base:attachedToProject ; + knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ; + knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ; + books:hasTitle ; + rdfs:label "instance of a book" ; + knora-base:isDeleted false . + + + a books:Book ; + knora-base:attachedToUser ; + knora-base:attachedToProject ; + knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ; + knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ; + books:hasTitle ; + books:hasTextType ; + rdfs:label "instance of a book with a list value" ; + knora-base:isDeleted false . a books:Page ; knora-base:attachedToUser ; knora-base:attachedToProject ; - knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ; + knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ; knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ; books:hasPageNumber ; rdfs:label "instance of a page" ; knora-base:isDeleted false . + + a knora-base:ListNode ; + knora-base:isRootNode true ; + rdfs:label "Text category"@en ; + rdfs:label "Textsorte"@de ; + rdfs:comment "This list is used to categorize books."@en ; + rdfs:comment "Diese Liste dient dazu, Bücher zu kategorisieren."@de ; + knora-base:attachedToProject knora-admin:SystemProject ; + knora-base:hasSubListNode , , . + + a knora-base:ListNode ; + knora-base:listNodeName "novel" ; + knora-base:hasRootNode ; + knora-base:listNodePosition 0 ; + rdfs:label "Novel"@en ; + rdfs:label "Roman"@de . + + a knora-base:ListNode ; + knora-base:listNodeName "short-story" ; + knora-base:hasRootNode ; + knora-base:listNodePosition 1 ; + rdfs:label "Short story"@en ; + rdfs:label "Kurzgeschichte"@de . + + a knora-base:ListNode ; + knora-base:listNodeName "non-fiction" ; + knora-base:hasRootNode ; + knora-base:listNodePosition 2 ; + rdfs:label "Non fiction"@en ; + rdfs:label "Sachbuch"@de . + + + diff --git a/test_data/ontologies/books-onto.ttl b/test_data/ontologies/books-onto.ttl index 808bddaa61..1761cf0b03 100644 --- a/test_data/ontologies/books-onto.ttl +++ b/test_data/ontologies/books-onto.ttl @@ -29,21 +29,14 @@ salsah-gui:guiElement salsah-gui:SimpleText ; salsah-gui:guiAttribute "size=80", "maxlength=255" . - -#:hasPage -# rdf:type owl:ObjectProperty ; -# rdfs:subPropertyOf knora-base:hasLinkTo ; -# rdfs:label "Seite im Buch"@de, "Page in the book"@en ; -# knora-base:subjectClassConstraint :Book ; -# knora-base:objectClassConstraint :Page ; -# salsah-gui:guiElement salsah-gui:Searchbox . -# -# -#:hasPageValue -# rdf:type owl:ObjectProperty ; -# rdfs:subPropertyOf knora-base:hasLinkToValue ; -# rdfs:label "Seite im Buch"@de, "Page in the book"@en ; -# knora-base:objectClassConstraint knora-base:LinkValue . +:hasTextType + rdf:type owl:ObjectProperty ; + rdfs:subPropertyOf knora-base:hasValue ; + rdfs:label "Hat Textsorte"@de, "Has text type"@en ; + knora-base:subjectClassConstraint :Book ; + knora-base:objectClassConstraint knora-base:ListValue ; + salsah-gui:guiElement salsah-gui:List ; + salsah-gui:guiAttribute "hlist=" . :Book diff --git a/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/searchFulltextStandard.scala.txt b/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/searchFulltextStandard.scala.txt index ba8900323a..6b58ca4aeb 100644 --- a/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/searchFulltextStandard.scala.txt +++ b/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/searchFulltextStandard.scala.txt @@ -109,16 +109,39 @@ WHERE { ?property rdfs:subPropertyOf* knora-base:hasValue . FILTER NOT EXISTS { - ?matchingSubject knora-base:isDeleted true . + ?matchingSubject knora-base:isDeleted true } # this variable will only be bound if the search matched a value object BIND(?matchingSubject AS ?valueObject) } - # If the previous OPTIONAL clause was executed, ?matchingSubject is a value object, and ?containingResource will be set. + OPTIONAL { + ?matchingSubject a ?valueObjectType . + + ?valueObjectType rdfs:subClassOf *knora-base:ListNode . + + ?listValue knora-base:valueHasListNode ?matchingSubject . + + ?subjectWithListValue ?predicate ?listValue . + + FILTER NOT EXISTS { + ?matchingSubject knora-base:isDeleted true + } + + # this variable will only be bound if the search matched a list node + BIND(?listValue AS ?valueObject) + } + + # If the first OPTIONAL clause was executed, ?matchingSubject is a value object, and ?containingResource will be set as ?valueObject. + # If the second OPTIONAL clause was executed, ?matchingSubject is a list node, and ?listValue will be set as ?valueObject. # Otherwise, ?matchingSubject is a resource (its rdfs:label matched the search pattern). - BIND(COALESCE(?containingResource, ?matchingSubject) AS ?resource) + BIND( + COALESCE( + ?containingResource, + ?subjectWithListValue, + ?matchingSubject) + AS ?resource) ?resource a ?resourceClass . diff --git a/webapi/src/test/scala/org/knora/webapi/CoreSpec.scala b/webapi/src/test/scala/org/knora/webapi/CoreSpec.scala index d90bd3a426..cc13b620d6 100644 --- a/webapi/src/test/scala/org/knora/webapi/CoreSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/CoreSpec.scala @@ -181,7 +181,7 @@ abstract class CoreSpec(_system: ActorSystem) } logger.info("Flush Redis cache started ...") - Try(Await.result(appActor ? CacheServiceFlushDB(KnoraSystemInstances.Users.SystemUser), 5 seconds)) match { + Try(Await.result(appActor ? CacheServiceFlushDB(KnoraSystemInstances.Users.SystemUser), 15 seconds)) match { case Success(res) => logger.info("... flushing Redis cache done.") case Failure(e) => logger.error(s"Flushing Redis cache failed: ${e.getMessage}") } diff --git a/webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2Spec.scala b/webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2Spec.scala index 545e7cd48d..b51f7d33bb 100644 --- a/webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2Spec.scala +++ b/webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2Spec.scala @@ -21,15 +21,20 @@ package org.knora.webapi.responders.v2 import akka.testkit.ImplicitSender import org.knora.webapi.messages.IriConversions._ -import org.knora.webapi.messages.StringFormatter import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject import org.knora.webapi.messages.v2.responder.resourcemessages._ import org.knora.webapi.messages.v2.responder.searchmessages._ -import org.knora.webapi.messages.v2.responder.valuemessages.{ReadValueV2, StillImageFileValueContentV2} +import org.knora.webapi.messages.v2.responder.valuemessages.{ + HierarchicalListValueContentV2, + ReadValueV2, + StillImageFileValueContentV2 +} +import org.knora.webapi.messages.{SmartIri, StringFormatter} import org.knora.webapi.responders.v2.ResourcesResponseCheckerV2.compareReadResourcesSequenceV2Response import org.knora.webapi.sharedtestdata.SharedTestDataADM -import org.knora.webapi.{ApiV2Complex, CoreSpec, SchemaOptions} +import org.knora.webapi.{ApiV2Complex, CoreSpec, IRI, SchemaOptions} +import java.util.UUID import scala.concurrent.duration._ /** @@ -38,14 +43,16 @@ import scala.concurrent.duration._ class SearchResponderV2Spec extends CoreSpec() with ImplicitSender { private implicit val stringFormatter: StringFormatter = StringFormatter.getGeneralInstance - private val searchResponderV2SpecFullData = new SearchResponderV2SpecFullData - override lazy val rdfDataObjects = List( RdfDataObject(path = "test_data/all_data/incunabula-data.ttl", name = "http://www.knora.org/data/0803/incunabula"), RdfDataObject(path = "test_data/demo_data/images-demo-data.ttl", name = "http://www.knora.org/data/00FF/images"), - RdfDataObject(path = "test_data/all_data/anything-data.ttl", name = "http://www.knora.org/data/0001/anything") + RdfDataObject(path = "test_data/all_data/anything-data.ttl", name = "http://www.knora.org/data/0001/anything"), + RdfDataObject(path = "test_data/all_data/incunabula-data.ttl", name = "http://www.knora.org/data/0803/incunabula"), + RdfDataObject(path = "test_data/ontologies/books-onto.ttl", name = "http://www.knora.org/ontology/0001/anything"), + RdfDataObject(path = "test_data/all_data/books-data.ttl", name = "http://www.knora.org/data/0001/anything") ) - + private val searchResponderV2SpecFullData = new SearchResponderV2SpecFullData + private val anythingUserProfile = SharedTestDataADM.anythingUser2 // The default timeout for receiving reply messages from actors. private val timeout = 10.seconds @@ -250,6 +257,29 @@ class SearchResponderV2Spec extends CoreSpec() with ImplicitSender { } } + "search for list label" in { + + responderManager ! FulltextSearchRequestV2( + searchValue = "non fiction", + offset = 0, + limitToProject = None, + limitToResourceClass = None, + limitToStandoffClass = None, + returnFiles = false, + targetSchema = ApiV2Complex, + schemaOptions = SchemaOptions.ForStandoffWithTextValues, + featureFactoryConfig = defaultFeatureFactoryConfig, + requestingUser = SharedTestDataADM.anythingUser1 + ) + + expectMsgPF(timeout) { case response: ReadResourcesSequenceV2 => + compareReadResourcesSequenceV2Response( + expected = searchResponderV2SpecFullData.fulltextSearchForListNodeLabel, + received = response + ) + } + } + } } diff --git a/webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2SpecFullData.scala b/webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2SpecFullData.scala index a41643e42d..290eb04597 100644 --- a/webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2SpecFullData.scala +++ b/webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2SpecFullData.scala @@ -2039,4 +2039,42 @@ class SearchResponderV2SpecFullData(implicit stringFormatter: StringFormatter) { ), querySchema = Some(ApiV2Simple) ) + + val fulltextSearchForListNodeLabel: ReadResourcesSequenceV2 = ReadResourcesSequenceV2( + resources = Vector( + ReadResourceV2( + label = "instance of a book with a list value", + resourceIri = "http://rdfh.ch/0001/book-instance-02", + permissions = + "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser", + userPermission = ChangeRightsPermission, + attachedToUser = "http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q", + resourceClassIri = "http://www.knora.org/ontology/0001/books#Book".toSmartIri, + projectADM = SharedTestDataADM.anythingProject, + creationDate = Instant.parse("2019-11-29T10:00:00.673298Z"), + values = Map( + "http://www.knora.org/ontology/0001/books#hasTextType".toSmartIri -> Vector( + ReadOtherValueV2( + valueContent = HierarchicalListValueContentV2( + ontologySchema = InternalSchema, + valueHasListNode = "http://rdfh.ch/lists/0001/ynm02-03" + ), + valueIri = "http://rdfh.ch/0001/book-instance-02/values/has-list-value-01", + valueHasUUID = stringFormatter.decodeUuid("d34d34d3-4d34-d34d-3496-2b2dfef6a5b9"), + permissions = + "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser", + userPermission = ModifyPermission, + previousValueIri = None, + valueCreationDate = Instant.parse("2018-05-29T16:42:04.381Z"), + attachedToUser = "http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ", + deletionInfo = None + ) + ) + ), + lastModificationDate = None, + versionDate = None, + deletionInfo = None + ) + ) + ) }