feat(api-v2): Optionally return file values in full-text search results (DSP-1191)

docs/03-apis/api-v2/reading-and-searching-resources.md

@@ -509,6 +509,9 @@ in `app/v2` in `application.conf`.
 If the parameter `limitToStandoffClass` is provided, Knora will look for search terms
 that are marked up with the indicated standoff class.
 
+If the parameter `returnFiles=true` is provided, Knora will return any
+file value attached to each matching resource.
+
 To request the number of results rather than the results themselves, you can
 do a count query:
 

webapi/src/main/scala/org/knora/webapi/messages/v2/responder/searchmessages/SearchMessagesV2.scala

@@ -60,6 +60,7 @@ case class FullTextSearchCountRequestV2(searchValue: String,
   * @param offset               the offset to be used for paging.
   * @param limitToProject       limit search to given project.
   * @param limitToResourceClass limit search to given resource class.
+  * @param returnFiles          if true, return any file value value attached to each matching resource.
   * @param targetSchema         the target API schema.
   * @param schemaOptions        the schema options submitted with the request.
   * @param featureFactoryConfig the feature factory configuration.
@@ -70,6 +71,7 @@ case class FulltextSearchRequestV2(searchValue: String,
                                    limitToProject: Option[IRI],
                                    limitToResourceClass: Option[SmartIri],
                                    limitToStandoffClass: Option[SmartIri],
+                                   returnFiles: Boolean,
                                    targetSchema: ApiV2Schema,
                                    schemaOptions: Set[SchemaOption],
                                    featureFactoryConfig: FeatureFactoryConfig,

webapi/src/main/scala/org/knora/webapi/responders/v2/SearchResponderV2.scala

@@ -80,34 +80,42 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
                             limitToStandoffClass,
                             featureFactoryConfig,
                             requestingUser)
+
     case FulltextSearchRequestV2(searchValue,
                                  offset,
                                  limitToProject,
                                  limitToResourceClass,
                                  limitToStandoffClass,
+                                 returnFiles,
                                  targetSchema,
                                  schemaOptions,
                                  featureFactoryConfig,
                                  requestingUser) =>
-      fulltextSearchV2(searchValue,
-                       offset,
-                       limitToProject,
-                       limitToResourceClass,
-                       limitToStandoffClass,
-                       targetSchema,
-                       schemaOptions,
-                       featureFactoryConfig,
-                       requestingUser)
+      fulltextSearchV2(
+        searchValue,
+        offset,
+        limitToProject,
+        limitToResourceClass,
+        limitToStandoffClass,
+        returnFiles,
+        targetSchema,
+        schemaOptions,
+        featureFactoryConfig,
+        requestingUser
+      )
+
     case GravsearchCountRequestV2(query, featureFactoryConfig, requestingUser) =>
       gravsearchCountV2(inputQuery = query,
                         featureFactoryConfig = featureFactoryConfig,
                         requestingUser = requestingUser)
+
     case GravsearchRequestV2(query, targetSchema, schemaOptions, featureFactoryConfig, requestingUser) =>
       gravsearchV2(inputQuery = query,
                    targetSchema = targetSchema,
                    schemaOptions = schemaOptions,
                    featureFactoryConfig = featureFactoryConfig,
                    requestingUser = requestingUser)
+
     case SearchResourceByLabelCountRequestV2(searchValue,
                                              limitToProject,
                                              limitToResourceClass,
@@ -118,6 +126,7 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
                                     limitToResourceClass,
                                     featureFactoryConfig,
                                     requestingUser)
+
     case SearchResourceByLabelRequestV2(searchValue,
                                         offset,
                                         limitToProject,
@@ -132,8 +141,10 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
                                targetSchema,
                                featureFactoryConfig,
                                requestingUser)
+
     case resourcesInProjectGetRequestV2: SearchResourcesByProjectAndClassRequestV2 =>
       searchResourcesByProjectAndClassV2(resourcesInProjectGetRequestV2)
+
     case other => handleUnexpectedMessage(other, log, this.getClass.getName)
   }
 
@@ -167,7 +178,8 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
             searchTerms = searchTerms,
             limitToProject = limitToProject,
             limitToResourceClass = limitToResourceClass.map(_.toString),
-            limitToStandoffClass.map(_.toString),
+            limitToStandoffClass = limitToStandoffClass.map(_.toString),
+            returnFiles = false, // not relevant for a count query
             separator = None, // no separator needed for count query
             limit = 1,
             offset = 0,
@@ -197,6 +209,7 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
     * @param offset               the offset to be used for paging.
     * @param limitToProject       limit search to given project.
     * @param limitToResourceClass limit search to given resource class.
+    * @param returnFiles          if true, return any file value attached to each matching resource.
     * @param targetSchema         the target API schema.
     * @param schemaOptions        the schema options submitted with the request.
     * @param featureFactoryConfig the feature factory configuration.
@@ -208,6 +221,7 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
                                limitToProject: Option[IRI],
                                limitToResourceClass: Option[SmartIri],
                                limitToStandoffClass: Option[SmartIri],
+                               returnFiles: Boolean,
                                targetSchema: ApiV2Schema,
                                schemaOptions: Set[SchemaOption],
                                featureFactoryConfig: FeatureFactoryConfig,
@@ -227,6 +241,7 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
             limitToProject = limitToProject,
             limitToResourceClass = limitToResourceClass.map(_.toString),
             limitToStandoffClass = limitToStandoffClass.map(_.toString),
+            returnFiles = returnFiles,
             separator = Some(groupConcatSeparator),
             limit = settings.v2ResultsPerPage,
             offset = offset * settings.v2ResultsPerPage, // determine the actual offset
@@ -364,7 +379,6 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
     * @return a [[ResourceCountV2]] representing the number of resources that have been found.
     */
   private def gravsearchCountV2(inputQuery: ConstructQuery,
-                                apiSchema: ApiV2Schema = ApiV2Simple,
                                 featureFactoryConfig: FeatureFactoryConfig,
                                 requestingUser: UserADM): Future[ResourceCountV2] = {
 

webapi/src/main/scala/org/knora/webapi/routing/v2/SearchRouteV2.scala

@@ -41,6 +41,7 @@ class SearchRouteV2(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit
   private val LIMIT_TO_RESOURCE_CLASS = "limitToResourceClass"
   private val OFFSET = "offset"
   private val LIMIT_TO_STANDOFF_CLASS = "limitToStandoffClass"
+  private val RETURN_FILES = "returnFiles"
 
   /**
     * Returns the route.
@@ -236,6 +237,11 @@ class SearchRouteV2(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit
 
           val limitToStandoffClass: Option[SmartIri] = getStandoffClass(params)
 
+          val returnFiles: Boolean = stringFormatter.optionStringToBoolean(
+            params.get(RETURN_FILES),
+            throw BadRequestException(s"Invalid boolean value for '$RETURN_FILES'")
+          )
+
           val targetSchema: ApiV2Schema = RouteUtilV2.getOntologySchema(requestContext)
           val schemaOptions: Set[SchemaOption] = RouteUtilV2.getSchemaOptions(requestContext)
 
@@ -250,7 +256,8 @@ class SearchRouteV2(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit
               offset = offset,
               limitToProject = limitToProject,
               limitToResourceClass = limitToResourceClass,
-              limitToStandoffClass,
+              limitToStandoffClass = limitToStandoffClass,
+              returnFiles = returnFiles,
               featureFactoryConfig = featureFactoryConfig,
               requestingUser = requestingUser,
               targetSchema = targetSchema,

webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v1/searchFulltextStandard.scala.txt

@@ -147,7 +147,6 @@ WHERE {
     OPTIONAL {
        ?resource knora-base:hasStillImageFileValue ?fileValue .
        ?fileValue a knora-base:StillImageFileValue .
-       ?fileValue knora-base:isPreview true .
        ?fileValue knora-base:internalFilename ?previewPath .
 
         OPTIONAL {

webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/searchFulltext.scala.txt

@@ -32,6 +32,7 @@
  * @param limitToProject limit search to the given project.
  * @param limitToResourceClass limit search to given resource class.
  * @param limitToStandoffClass limit the search to given standoff class.
+ * @param returnFiles if true, return any file value attached to each matching resource.
  * @param separator the separator to be used in aggregation functions.
  * @param limit maximal amount of rows to be returned
  * @param offset offset for paging (starts with 0)
@@ -42,6 +43,7 @@
   limitToProject: Option[IRI],
   limitToResourceClass: Option[IRI],
   limitToStandoffClass: Option[IRI],
+  returnFiles: Boolean,
   separator: Option[Char],
   limit: Int,
   offset: Int,
@@ -53,6 +55,7 @@
                                                     limitToProject = limitToProject,
                                                     limitToResourceClass = limitToResourceClass,
                                                     limitToStandoffClass = limitToStandoffClass,
+                                                    returnFiles = returnFiles,
                                                     separator = separator,
                                                     limit = limit,
                                                     offset = offset,
@@ -65,6 +68,7 @@
                                                      limitToProject = limitToProject,
                                                      limitToResourceClass = limitToResourceClass,
                                                      limitToStandoffClass = limitToStandoffClass,
+                                                     returnFiles = returnFiles,
                                                      separator = separator,
                                                      limit = limit,
                                                      offset = offset,

webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/searchFulltextGraphDB.scala.txt

@@ -36,6 +36,7 @@
  * @param limitToProject limit search to the given project.
  * @param limitToResourceClass limit search to given resource class.
  * @param limitToStandoffClass limit the search to given standoff class.
+ * @param returnFiles if true, return any file value attached to each matching resource.
  * @param limit maximal amount of rows to be returned
  * @param offset offset for paging (starts with 0)
  * @param countQuery indicates whether it is a count query or the actual resources should be returned.
@@ -44,6 +45,7 @@
   limitToProject: Option[IRI],
   limitToResourceClass: Option[IRI],
   limitToStandoffClass: Option[IRI],
+  returnFiles: Boolean,
   separator: Option[Char],
   limit: Int,
   offset: Int,
@@ -53,7 +55,10 @@ PREFIX knora-base: <http://www.knora.org/ontology/knora-base#>
 PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
 
 @if(!countQuery) {
-SELECT DISTINCT ?resource (GROUP_CONCAT(IF(BOUND(?valueObject), STR(?valueObject), ""); separator="@separator.getOrElse(throw SparqlGenerationException("Separator expected for non count query, but none given"))") AS ?valueObjectConcat)
+SELECT DISTINCT ?resource
+    (GROUP_CONCAT(IF(BOUND(?valueObject), STR(?valueObject), "");
+        separator="@separator.getOrElse(throw SparqlGenerationException("Separator expected for non count query, but none given"))")
+            AS ?valueObjectConcat)
 } else {
     SELECT (count(distinct ?resource) as ?count)
 }
@@ -133,6 +138,11 @@ WHERE {
         case None => {}
     }
 
+    @if(returnFiles) {
+        OPTIONAL {
+            ?resource knora-base:hasFileValue ?valueObject .
+        }
+    }
 }
 @if(!countQuery) {
 GROUP BY ?resource

webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/searchFulltextStandard.scala.txt

@@ -36,6 +36,7 @@
  * @param limitToProject limit search to the given project.
  * @param limitToResourceClass limit search to given resource class.
  * @param limitToStandoffClass limit the search to given standoff class.
+ * @param returnFiles if true, return any file value attached to each matching resource.
  * @param separator the separator to be used in aggregation functions.
  * @param limit maximal amount of rows to be returned
  * @param offset offset for paging (starts with 0)
@@ -46,6 +47,7 @@
   limitToProject: Option[IRI],
   limitToResourceClass: Option[IRI],
   limitToStandoffClass: Option[IRI],
+  returnFiles: Boolean,
   separator: Option[Char],
   limit: Int,
   offset: Int,
@@ -55,7 +57,10 @@ PREFIX knora-base: <http://www.knora.org/ontology/knora-base#>
 PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
 
 @if(!countQuery) {
-SELECT DISTINCT ?resource (GROUP_CONCAT(IF(BOUND(?valueObject), STR(?valueObject), ""); separator="@separator.getOrElse(throw SparqlGenerationException("Separator expected for non count query, but none given"))") AS ?valueObjectConcat)
+SELECT DISTINCT ?resource
+                (GROUP_CONCAT(IF(BOUND(?valueObject), STR(?valueObject), "");
+                    separator="@separator.getOrElse(throw SparqlGenerationException("Separator expected for non count query, but none given"))")
+                    AS ?valueObjectConcat)
 } else {
     SELECT (count(distinct ?resource) as ?count)
 }
@@ -135,6 +140,13 @@ WHERE {
         case None => {}
     }
 
+    @if(returnFiles) {
+        OPTIONAL {
+            ?fileValueProp rdfs:subPropertyOf* knora-base:hasFileValue .
+            ?resource ?fileValueProp ?valueObject .
+        }
+    }
+
     FILTER NOT EXISTS {
         ?resource knora-base:isDeleted true .
     }

webapi/src/test/scala/org/knora/webapi/e2e/v2/SearchRouteV2R2RSpec.scala

@@ -216,6 +216,22 @@ class SearchRouteV2R2RSpec extends R2RSpec {
       }
     }
 
+    "return files attached to full-text search results" in {
+
+      Get("/v2/search/p7v?returnFiles=true") ~> searchPath ~> check {
+
+        assert(status == StatusCodes.OK, response.toString)
+
+        val expectedAnswerJSONLD =
+          readOrWriteTextFile(responseAs[String],
+                              Paths.get("test_data/searchR2RV2/FulltextSearchWithImage.jsonld"),
+                              writeTestDataFiles)
+
+        compareJSONLDForResourcesResponse(expectedJSONLD = expectedAnswerJSONLD, receivedJSONLD = responseAs[String])
+
+      }
+    }
+
     "not accept a fulltext query containing http://api.knora.org" in {
       val invalidSearchString: String =
         URLEncoder.encode("PREFIX knora-api: <http://api.knora.org/ontology/knora-api/v2#>", "UTF-8")

webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2Spec.scala

@@ -3,7 +3,7 @@
  *
  * This file is part of Knora.
  *
- * Knora is free software: you can redistribute it and/or modify
+ * Knora is free software: you can redistribute it and/or modifySearchResponderV2
  * it under the terms of the GNU Affero General Public License as published
  * by the Free Software Foundation, either version 3 of the License, or
  * (at your option) any later version.
@@ -25,6 +25,7 @@ import org.knora.webapi.messages.StringFormatter
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
 import org.knora.webapi.messages.v2.responder.resourcemessages._
 import org.knora.webapi.messages.v2.responder.searchmessages._
+import org.knora.webapi.messages.v2.responder.valuemessages.{ReadValueV2, StillImageFileValueContentV2}
 import org.knora.webapi.responders.v2.ResourcesResponseCheckerV2.compareReadResourcesSequenceV2Response
 import org.knora.webapi.sharedtestdata.SharedTestDataADM
 import org.knora.webapi.{ApiV2Complex, CoreSpec, SchemaOptions}
@@ -58,6 +59,7 @@ class SearchResponderV2Spec extends CoreSpec() with ImplicitSender {
         limitToProject = None,
         limitToResourceClass = None,
         limitToStandoffClass = None,
+        returnFiles = false,
         targetSchema = ApiV2Complex,
         schemaOptions = SchemaOptions.ForStandoffWithTextValues,
         featureFactoryConfig = defaultFeatureFactoryConfig,
@@ -81,6 +83,7 @@ class SearchResponderV2Spec extends CoreSpec() with ImplicitSender {
         limitToProject = None,
         limitToResourceClass = None,
         limitToStandoffClass = None,
+        returnFiles = false,
         targetSchema = ApiV2Complex,
         schemaOptions = SchemaOptions.ForStandoffWithTextValues,
         featureFactoryConfig = defaultFeatureFactoryConfig,
@@ -95,6 +98,36 @@ class SearchResponderV2Spec extends CoreSpec() with ImplicitSender {
 
     }
 
+    "return files attached to full-text search results" in {
+
+      responderManager ! FulltextSearchRequestV2(
+        searchValue = "p7v",
+        offset = 0,
+        limitToProject = None,
+        limitToResourceClass = None,
+        limitToStandoffClass = None,
+        returnFiles = true,
+        targetSchema = ApiV2Complex,
+        schemaOptions = SchemaOptions.ForStandoffWithTextValues,
+        featureFactoryConfig = defaultFeatureFactoryConfig,
+        requestingUser = SharedTestDataADM.anythingUser1
+      )
+
+      expectMsgPF(timeout) {
+        case response: ReadResourcesSequenceV2 =>
+          val hasImageFileValues: Boolean =
+            response.resources.flatMap(_.values.values.flatten).exists { readValueV2: ReadValueV2 =>
+              readValueV2.valueContent match {
+                case _: StillImageFileValueContentV2 => true
+                case _                               => false
+              }
+            }
+
+          assert(hasImageFileValues)
+      }
+
+    }
+
     "perform an extended search for books that have the title 'Zeitglöcklein des Lebens'" in {
 
       responderManager ! GravsearchRequestV2(

webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2SpecFullData.scala

@@ -1566,7 +1566,6 @@ class SearchResponderV2SpecFullData(implicit stringFormatter: StringFormatter) {
       ))
   )
 
-  // Dear Ben: I am aware of the fact that this code is not formatted properly and I know that this deeply disturbs you. But please leave it like this since otherwise I cannot possibly read and understand this query.
   val constructQueryForBooksWithTitleZeitgloecklein: ConstructQuery = ConstructQuery(
     constructClause = ConstructClause(
       statements = Vector(
@@ -1708,7 +1707,6 @@ class SearchResponderV2SpecFullData(implicit stringFormatter: StringFormatter) {
     )
   )
 
-  // Dear Ben: please see my comment above
   val constructQueryForBooksWithoutTitleZeitgloecklein: ConstructQuery = ConstructQuery(
     constructClause = ConstructClause(
       statements = Vector(