Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
fix(list): find list labels in full-text search (#1922)
* add e2e test

* fix full-text search for list values

* reactivate tests that were commented out

* update query to find list values

* add rosetta files generated by sipi to gitignore
  • Loading branch information
irinaschubert committed Oct 21, 2021
1 parent 83212af commit cc3b06c
Show file tree
Hide file tree
Showing 7 changed files with 189 additions and 44 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Expand Up @@ -60,3 +60,5 @@ dependencies.txt
/.vscode
/cleandeps.sh
/.metals
/sipi/images/082E/*
/sipi/images/originals/082E/*
95 changes: 77 additions & 18 deletions test_data/all_data/books-data.ttl
Expand Up @@ -12,46 +12,105 @@

<http://rdfh.ch/0001/book-instance-01/values/has-title-value-01>
a knora-base:TextValue ;
knora-base:valueHasUUID "OTAzYmIzOTktYzA4Ni00NmM0LWFlZGItYzIyNzcyN2FkNjFk"^^xsd:string ;
knora-base:valueHasUUID "SZyeLLmOTcCCuS3B0VksHQ"^^xsd:string ;
knora-base:isDeleted false ;
knora-base:valueCreationDate "2018-05-28T15:52:03.897Z"^^xsd:dateTime ;
knora-base:valueHasOrder 0 ;
knora-base:valueHasString "Handbuch Projektmanagement" ;
knora-base:hasPermissions
"CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:attachedToUser <http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ> .


<http://rdfh.ch/0001/book-instance-01>
a books:Book ;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ;
knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ;
books:hasTitle <http://rdfh.ch/0001/book-instance-01/values/has-title-value-01> ;
rdfs:label "instance of a book" ;
knora-base:isDeleted false .

<http://rdfh.ch/0001/book-instance-01/values/has-title-value-02>
a knora-base:TextValue ;
knora-base:valueHasUUID "IN4R19yYR0ygi3K2VEHpUQ"^^xsd:string ;
knora-base:isDeleted false ;
knora-base:valueCreationDate "2018-05-29T16:42:04.381Z"^^xsd:dateTime ;
knora-base:valueHasOrder 0 ;
knora-base:valueHasString "A book with a list value" ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:attachedToUser <http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ> .

<http://rdfh.ch/0001/page-instance-01/values/has-page-number-value-01>
a knora-base:IntValue ;
knora-base:valueHasUUID "MmI4YTg1YTctZjM4Yy00MzE2LTkwNTgtYzdjODk1NjcwZmFm"^^xsd:string ;
knora-base:valueHasUUID "SZyeLLmOTcCCuS3B0VksHQ"^^xsd:string ;
knora-base:isDeleted false ;
knora-base:valueCreationDate "2018-05-28T15:52:03.897Z"^^xsd:dateTime ;
knora-base:valueHasInteger 1 ;
knora-base:valueHasOrder 0 ;
knora-base:valueHasString "1" ;
knora-base:hasPermissions
"CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:attachedToUser <http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ> .

<http://rdfh.ch/0001/book-instance-02/values/has-list-value-01> a knora-base:ListValue;
knora-base:valueHasUUID "list_value"^^xsd:string;
knora-base:isDeleted false;
knora-base:attachedToUser <http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ>;
knora-base:valueHasString "http://rdfh.ch/lists/0001/ynm02-03";
knora-base:valueHasOrder 0;
knora-base:valueHasListNode <http://rdfh.ch/lists/0001/ynm02-03>;
knora-base:valueCreationDate "2018-05-29T16:42:04.381Z"^^xsd:dateTime;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" .

<http://rdfh.ch/0001/book-instance-01>
a books:Book ;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ;
books:hasTitle <http://rdfh.ch/0001/book-instance-01/values/has-title-value-01> ;
rdfs:label "instance of a book" ;
knora-base:isDeleted false .

<http://rdfh.ch/0001/book-instance-02>
a books:Book ;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ;
books:hasTitle <http://rdfh.ch/0001/book-instance-01/values/has-title-value-02> ;
books:hasTextType <http://rdfh.ch/0001/book-instance-02/values/has-list-value-01> ;
rdfs:label "instance of a book with a list value" ;
knora-base:isDeleted false .

<http://rdfh.ch/0001/page-instance-01>
a books:Page ;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ;
books:hasPageNumber <http://rdfh.ch/0001/page-instance-01/values/has-page-number-value-01> ;
rdfs:label "instance of a page" ;
knora-base:isDeleted false .

<http://rdfh.ch/lists/0001/ynm02> a knora-base:ListNode ;
knora-base:isRootNode true ;
rdfs:label "Text category"@en ;
rdfs:label "Textsorte"@de ;
rdfs:comment "This list is used to categorize books."@en ;
rdfs:comment "Diese Liste dient dazu, Bücher zu kategorisieren."@de ;
knora-base:attachedToProject knora-admin:SystemProject ;
knora-base:hasSubListNode <http://rdfh.ch/lists/0001/ynm02-01> , <http://rdfh.ch/lists/0001/ynm02-02> , <http://rdfh.ch/lists/0001/ynm02-03> .

<http://rdfh.ch/lists/0001/ynm02-01> a knora-base:ListNode ;
knora-base:listNodeName "novel" ;
knora-base:hasRootNode <http://rdfh.ch/lists/0001/ynm02> ;
knora-base:listNodePosition 0 ;
rdfs:label "Novel"@en ;
rdfs:label "Roman"@de .

<http://rdfh.ch/lists/0001/ynm02-02> a knora-base:ListNode ;
knora-base:listNodeName "short-story" ;
knora-base:hasRootNode <http://rdfh.ch/lists/0001/ynm02> ;
knora-base:listNodePosition 1 ;
rdfs:label "Short story"@en ;
rdfs:label "Kurzgeschichte"@de .

<http://rdfh.ch/lists/0001/ynm02-03> a knora-base:ListNode ;
knora-base:listNodeName "non-fiction" ;
knora-base:hasRootNode <http://rdfh.ch/lists/0001/ynm02> ;
knora-base:listNodePosition 2 ;
rdfs:label "Non fiction"@en ;
rdfs:label "Sachbuch"@de .



23 changes: 8 additions & 15 deletions test_data/ontologies/books-onto.ttl
Expand Up @@ -29,21 +29,14 @@
salsah-gui:guiElement salsah-gui:SimpleText ;
salsah-gui:guiAttribute "size=80", "maxlength=255" .


#:hasPage
# rdf:type owl:ObjectProperty ;
# rdfs:subPropertyOf knora-base:hasLinkTo ;
# rdfs:label "Seite im Buch"@de, "Page in the book"@en ;
# knora-base:subjectClassConstraint :Book ;
# knora-base:objectClassConstraint :Page ;
# salsah-gui:guiElement salsah-gui:Searchbox .
#
#
#:hasPageValue
# rdf:type owl:ObjectProperty ;
# rdfs:subPropertyOf knora-base:hasLinkToValue ;
# rdfs:label "Seite im Buch"@de, "Page in the book"@en ;
# knora-base:objectClassConstraint knora-base:LinkValue .
:hasTextType
rdf:type owl:ObjectProperty ;
rdfs:subPropertyOf knora-base:hasValue ;
rdfs:label "Hat Textsorte"@de, "Has text type"@en ;
knora-base:subjectClassConstraint :Book ;
knora-base:objectClassConstraint knora-base:ListValue ;
salsah-gui:guiElement salsah-gui:List ;
salsah-gui:guiAttribute "hlist=<http://rdfh.ch/lists/0001/ynm02>" .


:Book
Expand Down
Expand Up @@ -109,16 +109,39 @@ WHERE {
?property rdfs:subPropertyOf* knora-base:hasValue .

FILTER NOT EXISTS {
?matchingSubject knora-base:isDeleted true .
?matchingSubject knora-base:isDeleted true
}

# this variable will only be bound if the search matched a value object
BIND(?matchingSubject AS ?valueObject)
}

# If the previous OPTIONAL clause was executed, ?matchingSubject is a value object, and ?containingResource will be set.
OPTIONAL {
?matchingSubject a ?valueObjectType .

?valueObjectType rdfs:subClassOf *knora-base:ListNode .

?listValue knora-base:valueHasListNode ?matchingSubject .

?subjectWithListValue ?predicate ?listValue .

FILTER NOT EXISTS {
?matchingSubject knora-base:isDeleted true
}

# this variable will only be bound if the search matched a list node
BIND(?listValue AS ?valueObject)
}

# If the first OPTIONAL clause was executed, ?matchingSubject is a value object, and ?containingResource will be set as ?valueObject.
# If the second OPTIONAL clause was executed, ?matchingSubject is a list node, and ?listValue will be set as ?valueObject.
# Otherwise, ?matchingSubject is a resource (its rdfs:label matched the search pattern).
BIND(COALESCE(?containingResource, ?matchingSubject) AS ?resource)
BIND(
COALESCE(
?containingResource,
?subjectWithListValue,
?matchingSubject)
AS ?resource)

?resource a ?resourceClass .

Expand Down
2 changes: 1 addition & 1 deletion webapi/src/test/scala/org/knora/webapi/CoreSpec.scala
Expand Up @@ -181,7 +181,7 @@ abstract class CoreSpec(_system: ActorSystem)
}

logger.info("Flush Redis cache started ...")
Try(Await.result(appActor ? CacheServiceFlushDB(KnoraSystemInstances.Users.SystemUser), 5 seconds)) match {
Try(Await.result(appActor ? CacheServiceFlushDB(KnoraSystemInstances.Users.SystemUser), 15 seconds)) match {
case Success(res) => logger.info("... flushing Redis cache done.")
case Failure(e) => logger.error(s"Flushing Redis cache failed: ${e.getMessage}")
}
Expand Down
Expand Up @@ -21,15 +21,20 @@ package org.knora.webapi.responders.v2

import akka.testkit.ImplicitSender
import org.knora.webapi.messages.IriConversions._
import org.knora.webapi.messages.StringFormatter
import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
import org.knora.webapi.messages.v2.responder.resourcemessages._
import org.knora.webapi.messages.v2.responder.searchmessages._
import org.knora.webapi.messages.v2.responder.valuemessages.{ReadValueV2, StillImageFileValueContentV2}
import org.knora.webapi.messages.v2.responder.valuemessages.{
HierarchicalListValueContentV2,
ReadValueV2,
StillImageFileValueContentV2
}
import org.knora.webapi.messages.{SmartIri, StringFormatter}
import org.knora.webapi.responders.v2.ResourcesResponseCheckerV2.compareReadResourcesSequenceV2Response
import org.knora.webapi.sharedtestdata.SharedTestDataADM
import org.knora.webapi.{ApiV2Complex, CoreSpec, SchemaOptions}
import org.knora.webapi.{ApiV2Complex, CoreSpec, IRI, SchemaOptions}

import java.util.UUID
import scala.concurrent.duration._

/**
Expand All @@ -38,14 +43,16 @@ import scala.concurrent.duration._
class SearchResponderV2Spec extends CoreSpec() with ImplicitSender {

private implicit val stringFormatter: StringFormatter = StringFormatter.getGeneralInstance
private val searchResponderV2SpecFullData = new SearchResponderV2SpecFullData

override lazy val rdfDataObjects = List(
RdfDataObject(path = "test_data/all_data/incunabula-data.ttl", name = "http://www.knora.org/data/0803/incunabula"),
RdfDataObject(path = "test_data/demo_data/images-demo-data.ttl", name = "http://www.knora.org/data/00FF/images"),
RdfDataObject(path = "test_data/all_data/anything-data.ttl", name = "http://www.knora.org/data/0001/anything")
RdfDataObject(path = "test_data/all_data/anything-data.ttl", name = "http://www.knora.org/data/0001/anything"),
RdfDataObject(path = "test_data/all_data/incunabula-data.ttl", name = "http://www.knora.org/data/0803/incunabula"),
RdfDataObject(path = "test_data/ontologies/books-onto.ttl", name = "http://www.knora.org/ontology/0001/anything"),
RdfDataObject(path = "test_data/all_data/books-data.ttl", name = "http://www.knora.org/data/0001/anything")
)

private val searchResponderV2SpecFullData = new SearchResponderV2SpecFullData
private val anythingUserProfile = SharedTestDataADM.anythingUser2
// The default timeout for receiving reply messages from actors.
private val timeout = 10.seconds

Expand Down Expand Up @@ -250,6 +257,29 @@ class SearchResponderV2Spec extends CoreSpec() with ImplicitSender {
}
}

"search for list label" in {

responderManager ! FulltextSearchRequestV2(
searchValue = "non fiction",
offset = 0,
limitToProject = None,
limitToResourceClass = None,
limitToStandoffClass = None,
returnFiles = false,
targetSchema = ApiV2Complex,
schemaOptions = SchemaOptions.ForStandoffWithTextValues,
featureFactoryConfig = defaultFeatureFactoryConfig,
requestingUser = SharedTestDataADM.anythingUser1
)

expectMsgPF(timeout) { case response: ReadResourcesSequenceV2 =>
compareReadResourcesSequenceV2Response(
expected = searchResponderV2SpecFullData.fulltextSearchForListNodeLabel,
received = response
)
}
}

}

}
Expand Up @@ -2039,4 +2039,42 @@ class SearchResponderV2SpecFullData(implicit stringFormatter: StringFormatter) {
),
querySchema = Some(ApiV2Simple)
)

val fulltextSearchForListNodeLabel: ReadResourcesSequenceV2 = ReadResourcesSequenceV2(
resources = Vector(
ReadResourceV2(
label = "instance of a book with a list value",
resourceIri = "http://rdfh.ch/0001/book-instance-02",
permissions =
"CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser",
userPermission = ChangeRightsPermission,
attachedToUser = "http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q",
resourceClassIri = "http://www.knora.org/ontology/0001/books#Book".toSmartIri,
projectADM = SharedTestDataADM.anythingProject,
creationDate = Instant.parse("2019-11-29T10:00:00.673298Z"),
values = Map(
"http://www.knora.org/ontology/0001/books#hasTextType".toSmartIri -> Vector(
ReadOtherValueV2(
valueContent = HierarchicalListValueContentV2(
ontologySchema = InternalSchema,
valueHasListNode = "http://rdfh.ch/lists/0001/ynm02-03"
),
valueIri = "http://rdfh.ch/0001/book-instance-02/values/has-list-value-01",
valueHasUUID = stringFormatter.decodeUuid("d34d34d3-4d34-d34d-3496-2b2dfef6a5b9"),
permissions =
"CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser",
userPermission = ModifyPermission,
previousValueIri = None,
valueCreationDate = Instant.parse("2018-05-29T16:42:04.381Z"),
attachedToUser = "http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ",
deletionInfo = None
)
)
),
lastModificationDate = None,
versionDate = None,
deletionInfo = None
)
)
)
}

0 comments on commit cc3b06c

Please sign in to comment.