Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(list): find list labels in full-text search #1922

Merged
merged 5 commits into from Oct 21, 2021
Merged

fix(list): find list labels in full-text search #1922

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3829214
add e2e test
Oct 18, 2021
fbc4186
fix full-text search for list values
Oct 19, 2021
6620cd2
reactivate tests that were commented out
Oct 19, 2021
0020342
update query to find list values
Oct 20, 2021
903c22f
add rosetta files generated by sipi to gitignore
Oct 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Expand Up @@ -60,3 +60,5 @@ dependencies.txt
/.vscode
/cleandeps.sh
/.metals
/sipi/images/082E/*
/sipi/images/originals/082E/*
95 changes: 77 additions & 18 deletions test_data/all_data/books-data.ttl
View file
Open in desktop
Expand Up @@ -12,46 +12,105 @@

<http://rdfh.ch/0001/book-instance-01/values/has-title-value-01>
a knora-base:TextValue ;
knora-base:valueHasUUID "OTAzYmIzOTktYzA4Ni00NmM0LWFlZGItYzIyNzcyN2FkNjFk"^^xsd:string ;
knora-base:valueHasUUID "SZyeLLmOTcCCuS3B0VksHQ"^^xsd:string ;
knora-base:isDeleted false ;
knora-base:valueCreationDate "2018-05-28T15:52:03.897Z"^^xsd:dateTime ;
knora-base:valueHasOrder 0 ;
knora-base:valueHasString "Handbuch Projektmanagement" ;
knora-base:hasPermissions
"CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:attachedToUser <http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ> .


<http://rdfh.ch/0001/book-instance-01>
a books:Book ;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ;
knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ;
books:hasTitle <http://rdfh.ch/0001/book-instance-01/values/has-title-value-01> ;
rdfs:label "instance of a book" ;
knora-base:isDeleted false .

<http://rdfh.ch/0001/book-instance-01/values/has-title-value-02>
a knora-base:TextValue ;
knora-base:valueHasUUID "IN4R19yYR0ygi3K2VEHpUQ"^^xsd:string ;
knora-base:isDeleted false ;
knora-base:valueCreationDate "2018-05-29T16:42:04.381Z"^^xsd:dateTime ;
knora-base:valueHasOrder 0 ;
knora-base:valueHasString "A book with a list value" ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:attachedToUser <http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ> .

<http://rdfh.ch/0001/page-instance-01/values/has-page-number-value-01>
a knora-base:IntValue ;
knora-base:valueHasUUID "MmI4YTg1YTctZjM4Yy00MzE2LTkwNTgtYzdjODk1NjcwZmFm"^^xsd:string ;
knora-base:valueHasUUID "SZyeLLmOTcCCuS3B0VksHQ"^^xsd:string ;
knora-base:isDeleted false ;
knora-base:valueCreationDate "2018-05-28T15:52:03.897Z"^^xsd:dateTime ;
knora-base:valueHasInteger 1 ;
knora-base:valueHasOrder 0 ;
knora-base:valueHasString "1" ;
knora-base:hasPermissions
"CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:attachedToUser <http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ> .

<http://rdfh.ch/0001/book-instance-02/values/has-list-value-01> a knora-base:ListValue;
knora-base:valueHasUUID "list_value"^^xsd:string;
knora-base:isDeleted false;
knora-base:attachedToUser <http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ>;
knora-base:valueHasString "http://rdfh.ch/lists/0001/ynm02-03";
knora-base:valueHasOrder 0;
knora-base:valueHasListNode <http://rdfh.ch/lists/0001/ynm02-03>;
knora-base:valueCreationDate "2018-05-29T16:42:04.381Z"^^xsd:dateTime;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" .

<http://rdfh.ch/0001/book-instance-01>
a books:Book ;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ;
books:hasTitle <http://rdfh.ch/0001/book-instance-01/values/has-title-value-01> ;
rdfs:label "instance of a book" ;
knora-base:isDeleted false .

<http://rdfh.ch/0001/book-instance-02>
a books:Book ;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ;
books:hasTitle <http://rdfh.ch/0001/book-instance-01/values/has-title-value-02> ;
books:hasTextType <http://rdfh.ch/0001/book-instance-02/values/has-list-value-01> ;
rdfs:label "instance of a book with a list value" ;
knora-base:isDeleted false .

<http://rdfh.ch/0001/page-instance-01>
a books:Page ;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ;
knora-base:attachedToProject <http://rdfh.ch/projects/0001> ;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ;
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser" ;
knora-base:creationDate "2019-11-29T10:00:00.673298Z"^^xsd:dateTime ;
books:hasPageNumber <http://rdfh.ch/0001/page-instance-01/values/has-page-number-value-01> ;
rdfs:label "instance of a page" ;
knora-base:isDeleted false .

<http://rdfh.ch/lists/0001/ynm02> a knora-base:ListNode ;
knora-base:isRootNode true ;
rdfs:label "Text category"@en ;
rdfs:label "Textsorte"@de ;
rdfs:comment "This list is used to categorize books."@en ;
rdfs:comment "Diese Liste dient dazu, Bücher zu kategorisieren."@de ;
knora-base:attachedToProject knora-admin:SystemProject ;
knora-base:hasSubListNode <http://rdfh.ch/lists/0001/ynm02-01> , <http://rdfh.ch/lists/0001/ynm02-02> , <http://rdfh.ch/lists/0001/ynm02-03> .

<http://rdfh.ch/lists/0001/ynm02-01> a knora-base:ListNode ;
knora-base:listNodeName "novel" ;
knora-base:hasRootNode <http://rdfh.ch/lists/0001/ynm02> ;
knora-base:listNodePosition 0 ;
rdfs:label "Novel"@en ;
rdfs:label "Roman"@de .

<http://rdfh.ch/lists/0001/ynm02-02> a knora-base:ListNode ;
knora-base:listNodeName "short-story" ;
knora-base:hasRootNode <http://rdfh.ch/lists/0001/ynm02> ;
knora-base:listNodePosition 1 ;
rdfs:label "Short story"@en ;
rdfs:label "Kurzgeschichte"@de .

<http://rdfh.ch/lists/0001/ynm02-03> a knora-base:ListNode ;
knora-base:listNodeName "non-fiction" ;
knora-base:hasRootNode <http://rdfh.ch/lists/0001/ynm02> ;
knora-base:listNodePosition 2 ;
rdfs:label "Non fiction"@en ;
rdfs:label "Sachbuch"@de .



23 changes: 8 additions & 15 deletions test_data/ontologies/books-onto.ttl
View file
Open in desktop
Expand Up @@ -29,21 +29,14 @@
salsah-gui:guiElement salsah-gui:SimpleText ;
salsah-gui:guiAttribute "size=80", "maxlength=255" .


#:hasPage
# rdf:type owl:ObjectProperty ;
# rdfs:subPropertyOf knora-base:hasLinkTo ;
# rdfs:label "Seite im Buch"@de, "Page in the book"@en ;
# knora-base:subjectClassConstraint :Book ;
# knora-base:objectClassConstraint :Page ;
# salsah-gui:guiElement salsah-gui:Searchbox .
#
#
#:hasPageValue
# rdf:type owl:ObjectProperty ;
# rdfs:subPropertyOf knora-base:hasLinkToValue ;
# rdfs:label "Seite im Buch"@de, "Page in the book"@en ;
# knora-base:objectClassConstraint knora-base:LinkValue .
:hasTextType
rdf:type owl:ObjectProperty ;
rdfs:subPropertyOf knora-base:hasValue ;
rdfs:label "Hat Textsorte"@de, "Has text type"@en ;
knora-base:subjectClassConstraint :Book ;
knora-base:objectClassConstraint knora-base:ListValue ;
salsah-gui:guiElement salsah-gui:List ;
salsah-gui:guiAttribute "hlist=<http://rdfh.ch/lists/0001/ynm02>" .


:Book
Expand Down
29 changes: 26 additions & 3 deletions .../twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/searchFulltextStandard.scala.txt
View file
Open in desktop
Expand Up @@ -109,16 +109,39 @@ WHERE {
?property rdfs:subPropertyOf* knora-base:hasValue .

FILTER NOT EXISTS {
?matchingSubject knora-base:isDeleted true .
?matchingSubject knora-base:isDeleted true
}

# this variable will only be bound if the search matched a value object
BIND(?matchingSubject AS ?valueObject)
}

# If the previous OPTIONAL clause was executed, ?matchingSubject is a value object, and ?containingResource will be set.
OPTIONAL {
?matchingSubject a ?valueObjectType .

?valueObjectType rdfs:subClassOf *knora-base:ListNode .

?listValue knora-base:valueHasListNode ?matchingSubject .

?subjectWithListValue ?predicate ?listValue .

FILTER NOT EXISTS {
?matchingSubject knora-base:isDeleted true
}

# this variable will only be bound if the search matched a list node
BIND(?listValue AS ?valueObject)
}

# If the first OPTIONAL clause was executed, ?matchingSubject is a value object, and ?containingResource will be set as ?valueObject.
# If the second OPTIONAL clause was executed, ?matchingSubject is a list node, and ?listValue will be set as ?valueObject.
# Otherwise, ?matchingSubject is a resource (its rdfs:label matched the search pattern).
BIND(COALESCE(?containingResource, ?matchingSubject) AS ?resource)
BIND(
COALESCE(
?containingResource,
?subjectWithListValue,
?matchingSubject)
AS ?resource)

?resource a ?resourceClass .

Expand Down
2 changes: 1 addition & 1 deletion webapi/src/test/scala/org/knora/webapi/CoreSpec.scala
View file
Open in desktop
Expand Up @@ -181,7 +181,7 @@ abstract class CoreSpec(_system: ActorSystem)
}

logger.info("Flush Redis cache started ...")
Try(Await.result(appActor ? CacheServiceFlushDB(KnoraSystemInstances.Users.SystemUser), 5 seconds)) match {
Try(Await.result(appActor ? CacheServiceFlushDB(KnoraSystemInstances.Users.SystemUser), 15 seconds)) match {
case Success(res) => logger.info("... flushing Redis cache done.")
case Failure(e) => logger.error(s"Flushing Redis cache failed: ${e.getMessage}")
}
Expand Down
44 changes: 37 additions & 7 deletions webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2Spec.scala
View file
Open in desktop
Expand Up @@ -21,15 +21,20 @@ package org.knora.webapi.responders.v2

import akka.testkit.ImplicitSender
import org.knora.webapi.messages.IriConversions._
import org.knora.webapi.messages.StringFormatter
import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
import org.knora.webapi.messages.v2.responder.resourcemessages._
import org.knora.webapi.messages.v2.responder.searchmessages._
import org.knora.webapi.messages.v2.responder.valuemessages.{ReadValueV2, StillImageFileValueContentV2}
import org.knora.webapi.messages.v2.responder.valuemessages.{
HierarchicalListValueContentV2,
ReadValueV2,
StillImageFileValueContentV2
}
import org.knora.webapi.messages.{SmartIri, StringFormatter}
import org.knora.webapi.responders.v2.ResourcesResponseCheckerV2.compareReadResourcesSequenceV2Response
import org.knora.webapi.sharedtestdata.SharedTestDataADM
import org.knora.webapi.{ApiV2Complex, CoreSpec, SchemaOptions}
import org.knora.webapi.{ApiV2Complex, CoreSpec, IRI, SchemaOptions}

import java.util.UUID
import scala.concurrent.duration._

/**
Expand All @@ -38,14 +43,16 @@ import scala.concurrent.duration._
class SearchResponderV2Spec extends CoreSpec() with ImplicitSender {

private implicit val stringFormatter: StringFormatter = StringFormatter.getGeneralInstance
private val searchResponderV2SpecFullData = new SearchResponderV2SpecFullData

override lazy val rdfDataObjects = List(
RdfDataObject(path = "test_data/all_data/incunabula-data.ttl", name = "http://www.knora.org/data/0803/incunabula"),
RdfDataObject(path = "test_data/demo_data/images-demo-data.ttl", name = "http://www.knora.org/data/00FF/images"),
RdfDataObject(path = "test_data/all_data/anything-data.ttl", name = "http://www.knora.org/data/0001/anything")
RdfDataObject(path = "test_data/all_data/anything-data.ttl", name = "http://www.knora.org/data/0001/anything"),
RdfDataObject(path = "test_data/all_data/incunabula-data.ttl", name = "http://www.knora.org/data/0803/incunabula"),
RdfDataObject(path = "test_data/ontologies/books-onto.ttl", name = "http://www.knora.org/ontology/0001/anything"),
RdfDataObject(path = "test_data/all_data/books-data.ttl", name = "http://www.knora.org/data/0001/anything")
)

private val searchResponderV2SpecFullData = new SearchResponderV2SpecFullData
private val anythingUserProfile = SharedTestDataADM.anythingUser2
// The default timeout for receiving reply messages from actors.
private val timeout = 10.seconds

Expand Down Expand Up @@ -250,6 +257,29 @@ class SearchResponderV2Spec extends CoreSpec() with ImplicitSender {
}
}

"search for list label" in {

responderManager ! FulltextSearchRequestV2(
searchValue = "non fiction",
offset = 0,
limitToProject = None,
limitToResourceClass = None,
limitToStandoffClass = None,
returnFiles = false,
targetSchema = ApiV2Complex,
schemaOptions = SchemaOptions.ForStandoffWithTextValues,
featureFactoryConfig = defaultFeatureFactoryConfig,
requestingUser = SharedTestDataADM.anythingUser1
)

expectMsgPF(timeout) { case response: ReadResourcesSequenceV2 =>
compareReadResourcesSequenceV2Response(
expected = searchResponderV2SpecFullData.fulltextSearchForListNodeLabel,
received = response
)
}
}

}

}
38 changes: 38 additions & 0 deletions webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2SpecFullData.scala
View file
Open in desktop
Expand Up @@ -2039,4 +2039,42 @@ class SearchResponderV2SpecFullData(implicit stringFormatter: StringFormatter) {
),
querySchema = Some(ApiV2Simple)
)

val fulltextSearchForListNodeLabel: ReadResourcesSequenceV2 = ReadResourcesSequenceV2(
resources = Vector(
ReadResourceV2(
label = "instance of a book with a list value",
resourceIri = "http://rdfh.ch/0001/book-instance-02",
permissions =
"CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser",
userPermission = ChangeRightsPermission,
attachedToUser = "http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q",
resourceClassIri = "http://www.knora.org/ontology/0001/books#Book".toSmartIri,
projectADM = SharedTestDataADM.anythingProject,
creationDate = Instant.parse("2019-11-29T10:00:00.673298Z"),
values = Map(
"http://www.knora.org/ontology/0001/books#hasTextType".toSmartIri -> Vector(
ReadOtherValueV2(
valueContent = HierarchicalListValueContentV2(
ontologySchema = InternalSchema,
valueHasListNode = "http://rdfh.ch/lists/0001/ynm02-03"
),
valueIri = "http://rdfh.ch/0001/book-instance-02/values/has-list-value-01",
valueHasUUID = stringFormatter.decodeUuid("d34d34d3-4d34-d34d-3496-2b2dfef6a5b9"),
permissions =
"CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|RV knora-admin:UnknownUser",
userPermission = ModifyPermission,
previousValueIri = None,
valueCreationDate = Instant.parse("2018-05-29T16:42:04.381Z"),
attachedToUser = "http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ",
deletionInfo = None
)
)
),
lastModificationDate = None,
versionDate = None,
deletionInfo = None
)
)
)
}