Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat(api-v2): Optionally return file values in full-text search resul…
…ts (DSP-1191) (#1776)
  • Loading branch information
Benjamin Geer committed Jan 11, 2021
1 parent 9a5fb77 commit 01f59bd
Show file tree
Hide file tree
Showing 12 changed files with 695 additions and 18 deletions.
3 changes: 3 additions & 0 deletions docs/03-apis/api-v2/reading-and-searching-resources.md
Expand Up @@ -509,6 +509,9 @@ in `app/v2` in `application.conf`.
If the parameter `limitToStandoffClass` is provided, Knora will look for search terms
that are marked up with the indicated standoff class.

If the parameter `returnFiles=true` is provided, Knora will return any
file value attached to each matching resource.

To request the number of results rather than the results themselves, you can
do a count query:

Expand Down
579 changes: 579 additions & 0 deletions test_data/searchR2RV2/FulltextSearchWithImage.jsonld

Large diffs are not rendered by default.

Expand Up @@ -60,6 +60,7 @@ case class FullTextSearchCountRequestV2(searchValue: String,
* @param offset the offset to be used for paging.
* @param limitToProject limit search to given project.
* @param limitToResourceClass limit search to given resource class.
* @param returnFiles if true, return any file value value attached to each matching resource.
* @param targetSchema the target API schema.
* @param schemaOptions the schema options submitted with the request.
* @param featureFactoryConfig the feature factory configuration.
Expand All @@ -70,6 +71,7 @@ case class FulltextSearchRequestV2(searchValue: String,
limitToProject: Option[IRI],
limitToResourceClass: Option[SmartIri],
limitToStandoffClass: Option[SmartIri],
returnFiles: Boolean,
targetSchema: ApiV2Schema,
schemaOptions: Set[SchemaOption],
featureFactoryConfig: FeatureFactoryConfig,
Expand Down
Expand Up @@ -80,34 +80,42 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
limitToStandoffClass,
featureFactoryConfig,
requestingUser)

case FulltextSearchRequestV2(searchValue,
offset,
limitToProject,
limitToResourceClass,
limitToStandoffClass,
returnFiles,
targetSchema,
schemaOptions,
featureFactoryConfig,
requestingUser) =>
fulltextSearchV2(searchValue,
offset,
limitToProject,
limitToResourceClass,
limitToStandoffClass,
targetSchema,
schemaOptions,
featureFactoryConfig,
requestingUser)
fulltextSearchV2(
searchValue,
offset,
limitToProject,
limitToResourceClass,
limitToStandoffClass,
returnFiles,
targetSchema,
schemaOptions,
featureFactoryConfig,
requestingUser
)

case GravsearchCountRequestV2(query, featureFactoryConfig, requestingUser) =>
gravsearchCountV2(inputQuery = query,
featureFactoryConfig = featureFactoryConfig,
requestingUser = requestingUser)

case GravsearchRequestV2(query, targetSchema, schemaOptions, featureFactoryConfig, requestingUser) =>
gravsearchV2(inputQuery = query,
targetSchema = targetSchema,
schemaOptions = schemaOptions,
featureFactoryConfig = featureFactoryConfig,
requestingUser = requestingUser)

case SearchResourceByLabelCountRequestV2(searchValue,
limitToProject,
limitToResourceClass,
Expand All @@ -118,6 +126,7 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
limitToResourceClass,
featureFactoryConfig,
requestingUser)

case SearchResourceByLabelRequestV2(searchValue,
offset,
limitToProject,
Expand All @@ -132,8 +141,10 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
targetSchema,
featureFactoryConfig,
requestingUser)

case resourcesInProjectGetRequestV2: SearchResourcesByProjectAndClassRequestV2 =>
searchResourcesByProjectAndClassV2(resourcesInProjectGetRequestV2)

case other => handleUnexpectedMessage(other, log, this.getClass.getName)
}

Expand Down Expand Up @@ -167,7 +178,8 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
searchTerms = searchTerms,
limitToProject = limitToProject,
limitToResourceClass = limitToResourceClass.map(_.toString),
limitToStandoffClass.map(_.toString),
limitToStandoffClass = limitToStandoffClass.map(_.toString),
returnFiles = false, // not relevant for a count query
separator = None, // no separator needed for count query
limit = 1,
offset = 0,
Expand Down Expand Up @@ -197,6 +209,7 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
* @param offset the offset to be used for paging.
* @param limitToProject limit search to given project.
* @param limitToResourceClass limit search to given resource class.
* @param returnFiles if true, return any file value attached to each matching resource.
* @param targetSchema the target API schema.
* @param schemaOptions the schema options submitted with the request.
* @param featureFactoryConfig the feature factory configuration.
Expand All @@ -208,6 +221,7 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
limitToProject: Option[IRI],
limitToResourceClass: Option[SmartIri],
limitToStandoffClass: Option[SmartIri],
returnFiles: Boolean,
targetSchema: ApiV2Schema,
schemaOptions: Set[SchemaOption],
featureFactoryConfig: FeatureFactoryConfig,
Expand All @@ -227,6 +241,7 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
limitToProject = limitToProject,
limitToResourceClass = limitToResourceClass.map(_.toString),
limitToStandoffClass = limitToStandoffClass.map(_.toString),
returnFiles = returnFiles,
separator = Some(groupConcatSeparator),
limit = settings.v2ResultsPerPage,
offset = offset * settings.v2ResultsPerPage, // determine the actual offset
Expand Down Expand Up @@ -364,7 +379,6 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
* @return a [[ResourceCountV2]] representing the number of resources that have been found.
*/
private def gravsearchCountV2(inputQuery: ConstructQuery,
apiSchema: ApiV2Schema = ApiV2Simple,
featureFactoryConfig: FeatureFactoryConfig,
requestingUser: UserADM): Future[ResourceCountV2] = {

Expand Down
Expand Up @@ -41,6 +41,7 @@ class SearchRouteV2(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit
private val LIMIT_TO_RESOURCE_CLASS = "limitToResourceClass"
private val OFFSET = "offset"
private val LIMIT_TO_STANDOFF_CLASS = "limitToStandoffClass"
private val RETURN_FILES = "returnFiles"

/**
* Returns the route.
Expand Down Expand Up @@ -236,6 +237,11 @@ class SearchRouteV2(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit

val limitToStandoffClass: Option[SmartIri] = getStandoffClass(params)

val returnFiles: Boolean = stringFormatter.optionStringToBoolean(
params.get(RETURN_FILES),
throw BadRequestException(s"Invalid boolean value for '$RETURN_FILES'")
)

val targetSchema: ApiV2Schema = RouteUtilV2.getOntologySchema(requestContext)
val schemaOptions: Set[SchemaOption] = RouteUtilV2.getSchemaOptions(requestContext)

Expand All @@ -250,7 +256,8 @@ class SearchRouteV2(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit
offset = offset,
limitToProject = limitToProject,
limitToResourceClass = limitToResourceClass,
limitToStandoffClass,
limitToStandoffClass = limitToStandoffClass,
returnFiles = returnFiles,
featureFactoryConfig = featureFactoryConfig,
requestingUser = requestingUser,
targetSchema = targetSchema,
Expand Down
Expand Up @@ -147,7 +147,6 @@ WHERE {
OPTIONAL {
?resource knora-base:hasStillImageFileValue ?fileValue .
?fileValue a knora-base:StillImageFileValue .
?fileValue knora-base:isPreview true .
?fileValue knora-base:internalFilename ?previewPath .

OPTIONAL {
Expand Down
Expand Up @@ -32,6 +32,7 @@
* @param limitToProject limit search to the given project.
* @param limitToResourceClass limit search to given resource class.
* @param limitToStandoffClass limit the search to given standoff class.
* @param returnFiles if true, return any file value attached to each matching resource.
* @param separator the separator to be used in aggregation functions.
* @param limit maximal amount of rows to be returned
* @param offset offset for paging (starts with 0)
Expand All @@ -42,6 +43,7 @@
limitToProject: Option[IRI],
limitToResourceClass: Option[IRI],
limitToStandoffClass: Option[IRI],
returnFiles: Boolean,
separator: Option[Char],
limit: Int,
offset: Int,
Expand All @@ -53,6 +55,7 @@
limitToProject = limitToProject,
limitToResourceClass = limitToResourceClass,
limitToStandoffClass = limitToStandoffClass,
returnFiles = returnFiles,
separator = separator,
limit = limit,
offset = offset,
Expand All @@ -65,6 +68,7 @@
limitToProject = limitToProject,
limitToResourceClass = limitToResourceClass,
limitToStandoffClass = limitToStandoffClass,
returnFiles = returnFiles,
separator = separator,
limit = limit,
offset = offset,
Expand Down
Expand Up @@ -36,6 +36,7 @@
* @param limitToProject limit search to the given project.
* @param limitToResourceClass limit search to given resource class.
* @param limitToStandoffClass limit the search to given standoff class.
* @param returnFiles if true, return any file value attached to each matching resource.
* @param limit maximal amount of rows to be returned
* @param offset offset for paging (starts with 0)
* @param countQuery indicates whether it is a count query or the actual resources should be returned.
Expand All @@ -44,6 +45,7 @@
limitToProject: Option[IRI],
limitToResourceClass: Option[IRI],
limitToStandoffClass: Option[IRI],
returnFiles: Boolean,
separator: Option[Char],
limit: Int,
offset: Int,
Expand All @@ -53,7 +55,10 @@ PREFIX knora-base: <http://www.knora.org/ontology/knora-base#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>

@if(!countQuery) {
SELECT DISTINCT ?resource (GROUP_CONCAT(IF(BOUND(?valueObject), STR(?valueObject), ""); separator="@separator.getOrElse(throw SparqlGenerationException("Separator expected for non count query, but none given"))") AS ?valueObjectConcat)
SELECT DISTINCT ?resource
(GROUP_CONCAT(IF(BOUND(?valueObject), STR(?valueObject), "");
separator="@separator.getOrElse(throw SparqlGenerationException("Separator expected for non count query, but none given"))")
AS ?valueObjectConcat)
} else {
SELECT (count(distinct ?resource) as ?count)
}
Expand Down Expand Up @@ -133,6 +138,11 @@ WHERE {
case None => {}
}

@if(returnFiles) {
OPTIONAL {
?resource knora-base:hasFileValue ?valueObject .
}
}
}
@if(!countQuery) {
GROUP BY ?resource
Expand Down
Expand Up @@ -36,6 +36,7 @@
* @param limitToProject limit search to the given project.
* @param limitToResourceClass limit search to given resource class.
* @param limitToStandoffClass limit the search to given standoff class.
* @param returnFiles if true, return any file value attached to each matching resource.
* @param separator the separator to be used in aggregation functions.
* @param limit maximal amount of rows to be returned
* @param offset offset for paging (starts with 0)
Expand All @@ -46,6 +47,7 @@
limitToProject: Option[IRI],
limitToResourceClass: Option[IRI],
limitToStandoffClass: Option[IRI],
returnFiles: Boolean,
separator: Option[Char],
limit: Int,
offset: Int,
Expand All @@ -55,7 +57,10 @@ PREFIX knora-base: <http://www.knora.org/ontology/knora-base#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>

@if(!countQuery) {
SELECT DISTINCT ?resource (GROUP_CONCAT(IF(BOUND(?valueObject), STR(?valueObject), ""); separator="@separator.getOrElse(throw SparqlGenerationException("Separator expected for non count query, but none given"))") AS ?valueObjectConcat)
SELECT DISTINCT ?resource
(GROUP_CONCAT(IF(BOUND(?valueObject), STR(?valueObject), "");
separator="@separator.getOrElse(throw SparqlGenerationException("Separator expected for non count query, but none given"))")
AS ?valueObjectConcat)
} else {
SELECT (count(distinct ?resource) as ?count)
}
Expand Down Expand Up @@ -135,6 +140,13 @@ WHERE {
case None => {}
}

@if(returnFiles) {
OPTIONAL {
?fileValueProp rdfs:subPropertyOf* knora-base:hasFileValue .
?resource ?fileValueProp ?valueObject .
}
}

FILTER NOT EXISTS {
?resource knora-base:isDeleted true .
}
Expand Down
Expand Up @@ -216,6 +216,22 @@ class SearchRouteV2R2RSpec extends R2RSpec {
}
}

"return files attached to full-text search results" in {

Get("/v2/search/p7v?returnFiles=true") ~> searchPath ~> check {

assert(status == StatusCodes.OK, response.toString)

val expectedAnswerJSONLD =
readOrWriteTextFile(responseAs[String],
Paths.get("test_data/searchR2RV2/FulltextSearchWithImage.jsonld"),
writeTestDataFiles)

compareJSONLDForResourcesResponse(expectedJSONLD = expectedAnswerJSONLD, receivedJSONLD = responseAs[String])

}
}

"not accept a fulltext query containing http://api.knora.org" in {
val invalidSearchString: String =
URLEncoder.encode("PREFIX knora-api: <http://api.knora.org/ontology/knora-api/v2#>", "UTF-8")
Expand Down
Expand Up @@ -3,7 +3,7 @@
*
* This file is part of Knora.
*
* Knora is free software: you can redistribute it and/or modify
* Knora is free software: you can redistribute it and/or modifySearchResponderV2
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
Expand All @@ -25,6 +25,7 @@ import org.knora.webapi.messages.StringFormatter
import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
import org.knora.webapi.messages.v2.responder.resourcemessages._
import org.knora.webapi.messages.v2.responder.searchmessages._
import org.knora.webapi.messages.v2.responder.valuemessages.{ReadValueV2, StillImageFileValueContentV2}
import org.knora.webapi.responders.v2.ResourcesResponseCheckerV2.compareReadResourcesSequenceV2Response
import org.knora.webapi.sharedtestdata.SharedTestDataADM
import org.knora.webapi.{ApiV2Complex, CoreSpec, SchemaOptions}
Expand Down Expand Up @@ -58,6 +59,7 @@ class SearchResponderV2Spec extends CoreSpec() with ImplicitSender {
limitToProject = None,
limitToResourceClass = None,
limitToStandoffClass = None,
returnFiles = false,
targetSchema = ApiV2Complex,
schemaOptions = SchemaOptions.ForStandoffWithTextValues,
featureFactoryConfig = defaultFeatureFactoryConfig,
Expand All @@ -81,6 +83,7 @@ class SearchResponderV2Spec extends CoreSpec() with ImplicitSender {
limitToProject = None,
limitToResourceClass = None,
limitToStandoffClass = None,
returnFiles = false,
targetSchema = ApiV2Complex,
schemaOptions = SchemaOptions.ForStandoffWithTextValues,
featureFactoryConfig = defaultFeatureFactoryConfig,
Expand All @@ -95,6 +98,36 @@ class SearchResponderV2Spec extends CoreSpec() with ImplicitSender {

}

"return files attached to full-text search results" in {

responderManager ! FulltextSearchRequestV2(
searchValue = "p7v",
offset = 0,
limitToProject = None,
limitToResourceClass = None,
limitToStandoffClass = None,
returnFiles = true,
targetSchema = ApiV2Complex,
schemaOptions = SchemaOptions.ForStandoffWithTextValues,
featureFactoryConfig = defaultFeatureFactoryConfig,
requestingUser = SharedTestDataADM.anythingUser1
)

expectMsgPF(timeout) {
case response: ReadResourcesSequenceV2 =>
val hasImageFileValues: Boolean =
response.resources.flatMap(_.values.values.flatten).exists { readValueV2: ReadValueV2 =>
readValueV2.valueContent match {
case _: StillImageFileValueContentV2 => true
case _ => false
}
}

assert(hasImageFileValues)
}

}

"perform an extended search for books that have the title 'Zeitglöcklein des Lebens'" in {

responderManager ! GravsearchRequestV2(
Expand Down
Expand Up @@ -1566,7 +1566,6 @@ class SearchResponderV2SpecFullData(implicit stringFormatter: StringFormatter) {
))
)

// Dear Ben: I am aware of the fact that this code is not formatted properly and I know that this deeply disturbs you. But please leave it like this since otherwise I cannot possibly read and understand this query.
val constructQueryForBooksWithTitleZeitgloecklein: ConstructQuery = ConstructQuery(
constructClause = ConstructClause(
statements = Vector(
Expand Down Expand Up @@ -1708,7 +1707,6 @@ class SearchResponderV2SpecFullData(implicit stringFormatter: StringFormatter) {
)
)

// Dear Ben: please see my comment above
val constructQueryForBooksWithoutTitleZeitgloecklein: ConstructQuery = ConstructQuery(
constructClause = ConstructClause(
statements = Vector(
Expand Down

0 comments on commit 01f59bd

Please sign in to comment.