feat(gravsearch): Optimise Gravsearch queries using topological sort (DSP-1327)

third_party/dependencies.bzl

@@ -137,6 +137,9 @@ def dependencies():
             # Additional Selenium libraries besides the ones pulled in during init
             # of io_bazel_rules_webtesting
             "org.seleniumhq.selenium:selenium-support:3.141.59",
+
+            # Graph for Scala
+            "org.scala-graph:graph-core_2.12:1.13.1",
         ],
         repositories = [
             "https://repo.maven.apache.org/maven2",
@@ -187,6 +190,7 @@ BASE_TEST_DEPENDENCIES = [
     "@maven//:org_scalatest_scalatest_shouldmatchers_2_12",
     "@maven//:org_scalatest_scalatest_compatible",
     "@maven//:org_scalactic_scalactic_2_12",
+    "@maven//:org_scala_graph_graph_core_2_12",
 ]
 
 BASE_TEST_DEPENDENCIES_WITH_JSON = BASE_TEST_DEPENDENCIES + [

webapi/src/main/resources/application.conf

@@ -292,6 +292,21 @@ app {
                 "Benjamin Geer <benjamin.geer@dasch.swiss>"
             ]
         }
+
+        gravsearch-dependency-optimisation {
+            description = "Optimise Gravsearch queries by reordering query patterns according to their dependencies."
+
+            available-versions = [ 1 ]
+            default-version = 1
+            enabled-by-default = yes
+            override-allowed = yes
+            expiration-date = "2021-12-01T00:00:00Z"
+
+            developer-emails = [
+                "Sepideh Alassi <sepideh.alassi@dasch.swiss>"
+                "Benjamin Geer <benjamin.geer@dasch.swiss>"
+            ]
+        }
     }
 
     shacl {

webapi/src/main/scala/org/knora/webapi/messages/BUILD.bazel

@@ -44,5 +44,6 @@ scala_library(
         "@maven//:org_scala_lang_scala_reflect",
         "@maven//:org_slf4j_slf4j_api",
         "@maven//:org_springframework_security_spring_security_core",
+        "@maven//:org_scala_graph_graph_core_2_12",
     ],
 )

webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala

@@ -55,10 +55,6 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
   // suffix appended to variables that are returned by a SPARQL aggregation function.
   protected val groupConcatVariableSuffix = "__Concat"
 
-  // A set of types that can be treated as dates by the knora-api:toSimpleDate function.
-  private val dateTypes: Set[IRI] =
-    Set(OntologyConstants.KnoraApiV2Complex.DateValue, OntologyConstants.KnoraApiV2Complex.StandoffTag)
-
   /**
     * A container for a generated variable representing a value literal.
     *
@@ -352,14 +348,14 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
 
     }
 
-    val (maybeSubjectTypeIri: Option[SmartIri], subjectIsResource: Boolean) =
+    val maybeSubjectType: Option[NonPropertyTypeInfo] =
       typeInspectionResult.getTypeOfEntity(statementPattern.subj) match {
-        case Some(NonPropertyTypeInfo(subjectTypeIri, isResourceType, _)) => (Some(subjectTypeIri), isResourceType)
-        case _                                                            => (None, false)
+        case Some(nonPropertyTypeInfo: NonPropertyTypeInfo) => Some(nonPropertyTypeInfo)
+        case _                                              => None
       }
 
     // Is the subject of the statement a resource?
-    if (subjectIsResource) {
+    if (maybeSubjectType.exists(_.isResourceType)) {
       // Yes. Is the object of the statement also a resource?
       if (propertyTypeInfo.objectIsResourceType) {
         // Yes. This is a link property. Make sure that the object is either an IRI or a variable (cannot be a literal).
@@ -490,7 +486,7 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
       if (querySchema == ApiV2Complex) {
         // Yes. If the subject is a standoff tag and the object is a resource, that's an error, because the client
         // has to use the knora-api:standoffLink function instead.
-        if (maybeSubjectTypeIri.contains(OntologyConstants.KnoraApiV2Complex.StandoffTag.toSmartIri) && propertyTypeInfo.objectIsResourceType) {
+        if (maybeSubjectType.exists(_.isStandoffTagType) && propertyTypeInfo.objectIsResourceType) {
           throw GravsearchException(
             s"Invalid statement pattern (use the knora-api:standoffLink function instead): ${statementPattern.toSparql.trim}")
         } else {
@@ -769,7 +765,7 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
       case xsdLiteral: XsdLiteral if xsdLiteral.datatype.toString == OntologyConstants.KnoraApiV2Simple.ListNode =>
         xsdLiteral.value
 
-      case other =>
+      case _ =>
         throw GravsearchException(s"Invalid type for literal ${OntologyConstants.KnoraApiV2Simple.ListNode}")
     }
 
@@ -1259,7 +1255,7 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
     val langLiteral: XsdLiteral = compareExpression.rightArg match {
       case strLiteral: XsdLiteral if strLiteral.datatype == OntologyConstants.Xsd.String.toSmartIri => strLiteral
 
-      case other =>
+      case _ =>
         throw GravsearchException(
           s"Right argument of comparison statement must be a string literal for use with 'lang' function call")
     }
@@ -1821,9 +1817,8 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
     val standoffTagVar = functionCallExpression.getArgAsQueryVar(pos = 1)
 
     typeInspectionResult.getTypeOfEntity(standoffTagVar) match {
-      case Some(nonPropertyTypeInfo: NonPropertyTypeInfo)
-          if nonPropertyTypeInfo.typeIri.toString == OntologyConstants.KnoraApiV2Complex.StandoffTag =>
-        ()
+      case Some(nonPropertyTypeInfo: NonPropertyTypeInfo) if nonPropertyTypeInfo.isStandoffTagType => ()
+
       case _ =>
         throw GravsearchException(
           s"The second argument of ${functionIri.toSparql} must represent a knora-api:StandoffTag")
@@ -1930,7 +1925,7 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
 
     typeInspectionResult.getTypeOfEntity(dateBaseVar) match {
       case Some(nonPropInfo: NonPropertyTypeInfo) =>
-        if (!dateTypes.contains(nonPropInfo.typeIri.toString)) {
+        if (!(nonPropInfo.isStandoffTagType || nonPropInfo.typeIri.toString == OntologyConstants.KnoraApiV2Complex.DateValue)) {
           throw GravsearchException(
             s"${dateBaseVar.toSparql} must represent a knora-api:DateValue or a knora-api:StandoffDateTag")
         }
@@ -2042,53 +2037,4 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
     }
 
   }
-
-  /**
-    * Optimises a query by removing `rdf:type` statements that are known to be redundant. A redundant
-    * `rdf:type` statement gives the type of a variable whose type is already restricted by its
-    * use with a property that can only be used with that type (unless the property
-    * statement is in an `OPTIONAL` block).
-    *
-    * @param patterns the query patterns.
-    * @return the optimised query patterns.
-    */
-  protected def removeEntitiesInferredFromProperty(patterns: Seq[QueryPattern]): Seq[QueryPattern] = {
-
-    // Collect all entities which are used as subject or object of an OptionalPattern.
-    val optionalEntities = patterns
-      .filter {
-        case OptionalPattern(_) => true
-        case _                  => false
-      }
-      .flatMap {
-        case optionalPattern: OptionalPattern =>
-          optionalPattern.patterns.flatMap {
-            case pattern: StatementPattern =>
-              GravsearchTypeInspectionUtil.maybeTypeableEntity(pattern.subj) ++ GravsearchTypeInspectionUtil
-                .maybeTypeableEntity(pattern.obj)
-            case _ => None
-          }
-        case _ => None
-      }
-
-    // remove statements whose predicate is rdf:type, type of subject is inferred from a property, and the subject is not in optionalEntities.
-    val optimisedPatterns = patterns.filter {
-      case statementPattern: StatementPattern =>
-        statementPattern.pred match {
-          case iriRef: IriRef =>
-            val subject = GravsearchTypeInspectionUtil.maybeTypeableEntity(statementPattern.subj)
-            subject match {
-              case Some(typeableEntity) =>
-                !(iriRef.iri.toString == OntologyConstants.Rdf.Type && typeInspectionResult.entitiesInferredFromProperties.keySet
-                  .contains(typeableEntity)
-                  && !optionalEntities.contains(typeableEntity))
-              case _ => true
-            }
-
-          case _ => true
-        }
-      case _ => true
-    }
-    optimisedPatterns
-  }
 }